diff options
| -rw-r--r-- | Cargo.lock | 476 | ||||
| -rw-r--r-- | crates/driver/src/driver.rs | 409 | ||||
| -rw-r--r-- | crates/driver/src/parse.rs | 95 | ||||
| -rw-r--r-- | crates/driver/src/recipe.rs | 97 | ||||
| -rw-r--r-- | crates/rebel-common/Cargo.toml (renamed from crates/common/Cargo.toml) | 0 | ||||
| -rw-r--r-- | crates/rebel-common/src/error.rs (renamed from crates/common/src/error.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-common/src/lib.rs (renamed from crates/common/src/lib.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-common/src/string_hash.rs (renamed from crates/common/src/string_hash.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-common/src/types.rs (renamed from crates/common/src/types.rs) | 25 | ||||
| -rw-r--r-- | crates/rebel-parse/Cargo.toml | 20 | ||||
| -rw-r--r-- | crates/rebel-parse/benches/recipe.rs | 21 | ||||
| -rw-r--r-- | crates/rebel-parse/examples/parse-string.rs | 66 | ||||
| -rw-r--r-- | crates/rebel-parse/src/ast.rs | 223 | ||||
| -rw-r--r-- | crates/rebel-parse/src/grammar/mod.rs | 3 | ||||
| -rw-r--r-- | crates/rebel-parse/src/grammar/recipe.rs | 166 | ||||
| -rw-r--r-- | crates/rebel-parse/src/grammar/task_ref.rs | 65 | ||||
| -rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 104 | ||||
| -rw-r--r-- | crates/rebel-parse/src/lib.rs | 8 | ||||
| -rw-r--r-- | crates/rebel-parse/src/token.rs | 68 | ||||
| -rw-r--r-- | crates/rebel-resolve/Cargo.toml | 15 | ||||
| -rw-r--r-- | crates/rebel-resolve/src/args.rs (renamed from crates/driver/src/args.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-resolve/src/context.rs (renamed from crates/driver/src/context.rs) | 124 | ||||
| -rw-r--r-- | crates/rebel-resolve/src/lib.rs (renamed from crates/driver/src/resolve.rs) | 79 | ||||
| -rw-r--r-- | crates/rebel-resolve/src/paths.rs (renamed from crates/driver/src/paths.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-resolve/src/pin.rs (renamed from crates/driver/src/pin.rs) | 12 | ||||
| -rw-r--r-- | crates/rebel-resolve/src/task.rs (renamed from crates/driver/src/task.rs) | 41 | ||||
| -rw-r--r-- | crates/rebel-runner/Cargo.toml (renamed from crates/runner/Cargo.toml) | 4 | ||||
| -rw-r--r-- | crates/rebel-runner/src/init.rs (renamed from crates/runner/src/init.rs) | 2 | ||||
| -rw-r--r-- | crates/rebel-runner/src/jobserver.rs (renamed from crates/runner/src/jobserver.rs) | 17 | ||||
| -rw-r--r-- | crates/rebel-runner/src/lib.rs (renamed from crates/runner/src/lib.rs) | 18 | ||||
| -rw-r--r-- | crates/rebel-runner/src/ns.rs (renamed from crates/runner/src/ns.rs) | 2 | ||||
| -rw-r--r-- | crates/rebel-runner/src/paths.rs (renamed from crates/runner/src/paths.rs) | 2 | ||||
| -rw-r--r-- | crates/rebel-runner/src/tar.rs (renamed from crates/runner/src/tar.rs) | 2 | ||||
| -rw-r--r-- | crates/rebel-runner/src/task.rs (renamed from crates/runner/src/task.rs) | 42 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/checkable.rs (renamed from crates/runner/src/util/checkable.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/cjson.rs (renamed from crates/runner/src/util/cjson.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/clone.rs (renamed from crates/runner/src/util/clone.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/fs.rs (renamed from crates/runner/src/util/fs.rs) | 4 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/mod.rs (renamed from crates/runner/src/util/mod.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/stack.rs (renamed from crates/runner/src/util/stack.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/steal.rs (renamed from crates/runner/src/util/steal.rs) | 0 | ||||
| -rw-r--r-- | crates/rebel-runner/src/util/unix.rs (renamed from crates/runner/src/util/unix.rs) | 13 | ||||
| -rw-r--r-- | crates/rebel/Cargo.toml (renamed from crates/driver/Cargo.toml) | 16 | ||||
| -rw-r--r-- | crates/rebel/src/driver.rs | 481 | ||||
| -rw-r--r-- | crates/rebel/src/main.rs (renamed from crates/driver/src/main.rs) | 46 | ||||
| -rw-r--r-- | crates/rebel/src/recipe.rs | 167 | ||||
| -rw-r--r-- | crates/rebel/src/template.rs (renamed from crates/driver/src/template.rs) | 29 | ||||
| -rw-r--r-- | examples/pins.yml.example | 6 | ||||
| -rw-r--r-- | examples/recipes/bar/build.yml (renamed from examples/recipes/bar.yml) | 0 | ||||
| -rw-r--r-- | examples/recipes/binutils/build.yml (renamed from examples/recipes/binutils@2.41.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/busybox/build.yml (renamed from examples/recipes/busybox@1.36.1.yml) | 28 | ||||
| -rw-r--r-- | examples/recipes/e2fsprogs/build.yml (renamed from examples/recipes/e2fsprogs@1.47.0.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/foo/build.yml (renamed from examples/recipes/foo.yml) | 0 | ||||
| -rw-r--r-- | examples/recipes/gcc/build.libgcc-initial.yml (renamed from examples/recipes/libgcc-initial@13.2.0.yml) | 16 | ||||
| -rw-r--r-- | examples/recipes/gcc/build.libgcc.yml (renamed from examples/recipes/libgcc@13.2.0.yml) | 10 | ||||
| -rw-r--r-- | examples/recipes/gcc/build.libs.yml (renamed from examples/recipes/gcc-libs@13.2.0.yml) | 12 | ||||
| -rw-r--r-- | examples/recipes/gcc/build.yml (renamed from examples/recipes/gcc@13.2.0.yml) | 26 | ||||
| -rw-r--r-- | examples/recipes/glibc/build.yml (renamed from examples/recipes/glibc@2.38.yml) | 30 | ||||
| -rw-r--r-- | examples/recipes/gmp/build.recipe | 69 | ||||
| -rw-r--r-- | examples/recipes/gmp/build.yml (renamed from examples/recipes/gmp@6.3.0.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/image/build.yml (renamed from examples/recipes/image.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/linux/build.uapi-headers.yml (renamed from examples/recipes/linux-uapi-headers@6.5.5.yml) | 10 | ||||
| -rw-r--r-- | examples/recipes/linux/build.yml (renamed from examples/recipes/linux@6.5.5.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/ls/build.yml (renamed from examples/recipes/ls.yml) | 0 | ||||
| -rw-r--r-- | examples/recipes/make_ext4fs/build.yml (renamed from examples/recipes/make_ext4fs@2020-01-05.yml) | 12 | ||||
| -rw-r--r-- | examples/recipes/mpc/build.yml (renamed from examples/recipes/mpc@1.3.1.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/mpfr/build.yml (renamed from examples/recipes/mpfr@4.2.1.yml) | 22 | ||||
| -rw-r--r-- | examples/recipes/toolchain/build.yml (renamed from examples/recipes/toolchain.yml) | 4 | ||||
| -rw-r--r-- | examples/recipes/zlib/build.yml (renamed from examples/recipes/zlib@1.3.yml) | 22 |
69 files changed, 2255 insertions, 1106 deletions
@@ -4,9 +4,9 @@ version = 3 [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -18,36 +18,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -79,15 +79,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec", @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "capctl" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bee5dac8774a3495f734563416fe6de9ac762ccf762ec7653df52914435da86" +checksum = "4a6e71767585f51c2a33fed6d67147ec0343725fc3c03bf4b89fe67fede56aa5" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -119,12 +119,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" [[package]] name = "cfg-if" @@ -133,10 +130,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] name = "clap" -version = "4.4.6" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -144,33 +147,34 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", "clap_lex", "strsim", + "terminal_size", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.60", ] [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -179,6 +183,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + +[[package]] name = "constant_time_eq" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -186,9 +196,9 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -221,6 +231,31 @@ dependencies = [ ] [[package]] +name = "divan" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d567df2c9c2870a43f3f2bd65aaeb18dbce1c18f217c3e564b4fbaeb3ee56c" +dependencies = [ + "cfg-if", + "clap", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27540baf49be0d484d8f0130d7d8da3011c32a44d4fc873368154f1510e574a2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] name = "enum-kinds" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -238,15 +273,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -261,9 +306,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "4.4.0" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39b3bc2a8f715298032cf5087e58573809374b08160aa7d750582bdb82d2683" +checksum = "d08485b96a0e6393e9e4d1b8d48cf74ad6c063cd905eb33f42c1ce3f0377539b" dependencies = [ "log", "pest", @@ -275,15 +320,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hex" @@ -296,9 +341,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", @@ -306,15 +351,15 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "lazy_static" @@ -324,50 +369,41 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] -name = "log" -version = "0.4.20" +name = "linux-raw-sys" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] -name = "memchr" -version = "2.6.3" +name = "log" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] -name = "minimal-lexical" -version = "0.2.1" +name = "memchr" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "nix" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.5.0", "cfg-if", + "cfg_aliases", "libc", ] [[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] name = "olpc-cjson" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -380,15 +416,39 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "peg" +version = "0.8.2" +source = "git+https://github.com/kevinmehall/rust-peg.git#2fc1cadaa1efcf47c867715d063ac2d7296945c6" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.8.2" +source = "git+https://github.com/kevinmehall/rust-peg.git#2fc1cadaa1efcf47c867715d063ac2d7296945c6" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.8.2" +source = "git+https://github.com/kevinmehall/rust-peg.git#2fc1cadaa1efcf47c867715d063ac2d7296945c6" [[package]] name = "pest" -version = "2.7.4" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4" +checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" dependencies = [ "memchr", "thiserror", @@ -397,9 +457,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.4" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8" +checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" dependencies = [ "pest", "pest_generator", @@ -407,22 +467,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.4" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a" +checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.60", ] [[package]] name = "pest_meta" -version = "2.7.4" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" +checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" dependencies = [ "once_cell", "pest", @@ -431,18 +491,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -452,16 +512,14 @@ name = "rebel" version = "0.1.0" dependencies = [ "clap", - "deb-version", - "enum-kinds", "handlebars", "indoc", "lazy_static", "nix", - "nom", "rebel-common", + "rebel-parse", + "rebel-resolve", "rebel-runner", - "scoped-tls-hkt", "serde", "serde_yaml", "walkdir", @@ -476,6 +534,26 @@ dependencies = [ ] [[package]] +name = "rebel-parse" +version = "0.1.0" +dependencies = [ + "clap", + "divan", + "peg", + "rebel-common", +] + +[[package]] +name = "rebel-resolve" +version = "0.1.0" +dependencies = [ + "deb-version", + "enum-kinds", + "rebel-common", + "serde", +] + +[[package]] name = "rebel-runner" version = "0.1.0" dependencies = [ @@ -497,18 +575,37 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] name = "ryu" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -520,36 +617,30 @@ dependencies = [ ] [[package]] -name = "scoped-tls-hkt" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ddc765d3410d9f6c6ca071bf0b67f6b01e3ec4595dc3892f02677e75819dddc" - -[[package]] name = "serde" -version = "1.0.188" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.60", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ "itoa", "ryu", @@ -558,9 +649,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ "indexmap", "itoa", @@ -582,9 +673,9 @@ dependencies = [ [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -605,9 +696,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -632,23 +723,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57d3e5629bb07f004134a87a2d3cb6272f0a455942401f35776e3f17ecc8d955" [[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + +[[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.60", ] [[package]] @@ -680,9 +781,9 @@ checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "uds" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3803a8c885a33e84f898c82c3e72dbd5b2e807da66443731065a509fb11c6c9f" +checksum = "885c31f06fce836457fe3ef09a59f83fe8db95d270b11cd78f40a4666c4d1661" dependencies = [ "libc", ] @@ -695,18 +796,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "utf8parse" @@ -722,52 +823,39 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "winapi-util" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-sys 0.52.0", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.6" +name = "windows-sys" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "winapi", + "windows-targets 0.48.5", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] name = "windows-sys" -version = "0.48.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.5", ] [[package]] @@ -776,13 +864,29 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -792,46 +896,96 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] name = "xattr" -version = "1.0.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", + "linux-raw-sys", + "rustix", ] diff --git a/crates/driver/src/driver.rs b/crates/driver/src/driver.rs deleted file mode 100644 index d0abbcb..0000000 --- a/crates/driver/src/driver.rs +++ /dev/null @@ -1,409 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - os::unix::{net::UnixStream, prelude::*}, -}; - -use indoc::indoc; -use nix::poll; - -use common::{error::*, string_hash::*, types::*}; -use runner::Runner; - -use crate::{ - context::{Context, TaskRef}, - paths, resolve, - task::*, - template, -}; - -#[derive(Debug)] -pub struct CompletionState<'ctx> { - ctx: &'ctx Context, - tasks_done: HashMap<TaskRef<'ctx>, TaskOutput>, -} - -impl<'ctx> CompletionState<'ctx> { - pub fn new(ctx: &'ctx Context) -> Self { - CompletionState { - ctx, - tasks_done: Default::default(), - } - } - - // Treats both "depends" and "inherit" as dependencies - fn deps_satisfied(&self, task_ref: &TaskRef) -> bool { - resolve::get_dependent_tasks(self.ctx, task_ref) - .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref))) - .unwrap() - .into_iter() - .all(|dep| self.tasks_done.contains_key(&dep)) - } - - fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result<Vec<Dependency>> { - let task_def = &self.ctx[task]; - task_def - .fetch - .iter() - .map(|Fetch { name, sha256 }| { - Ok(Dependency::Fetch { - name: template::ENGINE - .eval_raw(name, &task.args) - .with_context(|| { - format!("Failed to evaluate fetch filename for task {}", task) - })?, - target_dir: paths::TASK_DLDIR.to_string(), - sha256: *sha256, - }) - }) - .collect() - } - - fn task_deps(&self, task: &TaskRef<'ctx>) -> Result<HashSet<Dependency>> { - Ok(self - .fetch_deps(task)? - .into_iter() - .chain( - resolve::runtime_depends( - self.ctx, - self.ctx - .get_build_depends(task) - .with_context(|| format!("invalid build depends for {}", task))?, - ) - .expect("invalid runtime depends of build_depends") - .into_iter() - .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) - .map(|&output| Dependency::Task { - output, - path: "".to_string(), - }), - ) - .chain( - resolve::runtime_depends( - self.ctx, - self.ctx - .get_host_depends(task) - .with_context(|| format!("invalid depends for {}", task))?, - ) - .expect("invalid runtime depends of host_depends") - .into_iter() - .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) - .map(|&output| Dependency::Task { - output, - path: paths::TASK_SYSROOT.to_string(), - }), - ) - .collect()) - } - - fn task_inherit_chain(&self, task_ref: &TaskRef<'ctx>) -> Vec<LayerHash> { - let inherit = match self - .ctx - .get_inherit_depend(task_ref) - .expect("invalid inherit depends") - { - Some(inherit) => inherit, - None => return vec![], - }; - - let mut chain = self.task_inherit_chain(&inherit); - if let Some(layer) = self.tasks_done[&inherit].layer { - chain.push(layer); - } - chain - } - - fn print_summary(&self) { - println!(); - println!("Summary:"); - - let mut tasks: Box<[_]> = self.tasks_done.iter().collect(); - tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task)); - for (task_ref, task) in tasks.iter() { - println!(); - println!("{:#}", task_ref); - if let Some(hash) = task.input_hash { - println!(" input: {}", hash); - } - if let Some(hash) = task.layer { - println!(" layer: {}", hash); - } - if !task.outputs.is_empty() { - println!(" outputs:"); - - let mut outputs: Box<[_]> = task.outputs.iter().collect(); - outputs.sort_by_key(|(output, _)| *output); - for (output, hash) in outputs.iter() { - println!(" {}: {}", output, hash); - } - } - } - } -} - -enum SpawnResult { - Spawned(UnixStream), - Skipped(TaskOutput), -} - -#[derive(Debug)] -pub struct Driver<'ctx> { - rdeps: HashMap<TaskRef<'ctx>, Vec<TaskRef<'ctx>>>, - force_run: HashSet<TaskRef<'ctx>>, - tasks_blocked: HashSet<TaskRef<'ctx>>, - tasks_runnable: Vec<TaskRef<'ctx>>, - tasks_running: HashMap<RawFd, (UnixStream, TaskRef<'ctx>)>, - state: CompletionState<'ctx>, -} - -impl<'ctx> Driver<'ctx> { - pub fn new( - ctx: &'ctx Context, - taskset: HashSet<TaskRef<'ctx>>, - force_run: HashSet<TaskRef<'ctx>>, - ) -> Result<Self> { - let mut driver = Driver { - rdeps: Default::default(), - force_run, - tasks_blocked: Default::default(), - tasks_runnable: Default::default(), - tasks_running: Default::default(), - state: CompletionState::new(ctx), - }; - - for task in taskset { - let mut has_depends = false; - for dep in resolve::get_dependent_tasks(ctx, &task) - .map_err(|_| Error::new(format!("invalid dependency for {}", task)))? - { - let rdep = driver.rdeps.entry(dep.clone()).or_default(); - rdep.push(task.clone()); - has_depends = true; - } - - if has_depends { - driver.tasks_blocked.insert(task); - } else { - driver.tasks_runnable.push(task); - } - } - - Ok(driver) - } - - fn task_setup(task_ref: &TaskRef<'ctx>) -> Vec<&'static str> { - let mut ret = vec![indoc! {" - export PATH={{build.prefix}}/sbin:{{build.prefix}}/bin:$PATH - cd {{workdir}} - - export SOURCE_DATE_EPOCH=1 - - export AR_FOR_BUILD=ar - export AS_FOR_BUILD=as - export DLLTOOL_FOR_BUILD=dlltool - export CC_FOR_BUILD=gcc - export CXX_FOR_BUILD=g++ - export GCC_FOR_BUILD=gcc - export GFORTRAN_FOR_BUILD=gfortran - export GOC_FOR_BUILD=goc - export LD_FOR_BUILD=ld - export LIPO_FOR_BUILD=lipo - export NM_FOR_BUILD=nm - export OBJCOPY_FOR_BUILD=objcopy - export OBJDUMP_FOR_BUILD=objdump - export RANLIB_FOR_BUILD=ranlib - export STRIP_FOR_BUILD=strip - export WINDRES_FOR_BUILD=windres - export WINDMC_FOR_BUILD=windmc - "}]; - - if task_ref.args.contains_key("build_to_host") { - ret.push(indoc! {" - export AR={{build_to_host.cross_compile}}ar - export AS={{build_to_host.cross_compile}}as - export DLLTOOL={{build_to_host.cross_compile}}dlltool - export CC={{build_to_host.cross_compile}}gcc - export CXX={{build_to_host.cross_compile}}g++ - export GCC={{build_to_host.cross_compile}}gcc - export GFORTRAN={{build_to_host.cross_compile}}gfortran - export GOC={{build_to_host.cross_compile}}goc - export LD={{build_to_host.cross_compile}}ld - export LIPO={{build_to_host.cross_compile}}lipo - export NM={{build_to_host.cross_compile}}nm - export OBJCOPY={{build_to_host.cross_compile}}objcopy - export OBJDUMP={{build_to_host.cross_compile}}objdump - export RANLIB={{build_to_host.cross_compile}}ranlib - export STRIP={{build_to_host.cross_compile}}strip - export WINDRES={{build_to_host.cross_compile}}windres - export WINDMC={{build_to_host.cross_compile}}windmc - "}); - } - - if task_ref.args.contains_key("build_to_target") { - ret.push(indoc! {" - export AR_FOR_TARGET={{build_to_target.cross_compile}}ar - export AS_FOR_TARGET={{build_to_target.cross_compile}}as - export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool - export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc - export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++ - export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc - export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran - export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc - export LD_FOR_TARGET={{build_to_target.cross_compile}}ld - export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo - export NM_FOR_TARGET={{build_to_target.cross_compile}}nm - export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy - export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump - export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib - export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip - export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres - export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc - "}); - } - ret - } - - fn update_runnable(&mut self, task_ref: TaskRef<'ctx>, task_output: TaskOutput) { - let rdeps = self.rdeps.get(&task_ref); - - self.state.tasks_done.insert(task_ref, task_output); - - for rdep in rdeps.unwrap_or(&Vec::new()) { - if !self.tasks_blocked.contains(rdep) { - continue; - } - if self.state.deps_satisfied(rdep) { - self.tasks_blocked.remove(rdep); - self.tasks_runnable.push(rdep.clone()); - } - } - } - - fn spawn_task(&self, task_ref: &TaskRef<'ctx>, runner: &Runner) -> Result<SpawnResult> { - let task_def = &self.state.ctx[task_ref]; - if task_def.action.is_empty() { - println!("Skipping empty task {:#}", task_ref); - return Ok(SpawnResult::Skipped(TaskOutput::default())); - } - - let task_deps = self.state.task_deps(task_ref)?; - let task_output = task_def - .output - .iter() - .map(|(name, Output { path, .. })| { - let output_path = if let Some(path) = path { - format!("{}/{}", paths::TASK_DESTDIR, path) - } else { - paths::TASK_DESTDIR.to_string() - }; - (name.clone(), output_path) - }) - .collect(); - - let inherit_chain = self.state.task_inherit_chain(task_ref); - - let mut run = Self::task_setup(task_ref); - run.push(&task_def.action.run); - - let command = template::ENGINE - .eval(&run.concat(), &task_ref.args) - .with_context(|| { - format!("Failed to evaluate command template for task {}", task_ref) - })?; - - let rootfs = self.state.ctx.get_rootfs(); - let task = Task { - label: format!("{:#}", task_ref), - command, - workdir: paths::TASK_WORKDIR.to_string(), - rootfs: rootfs.0, - inherit: inherit_chain, - depends: task_deps, - outputs: task_output, - pins: HashMap::from([rootfs.clone()]), - force_run: self.force_run.contains(task_ref), - }; - - Ok(SpawnResult::Spawned(runner.spawn(&task))) - } - - fn run_task(&mut self, task_ref: TaskRef<'ctx>, runner: &Runner) -> Result<()> { - match self.spawn_task(&task_ref, runner)? { - SpawnResult::Spawned(socket) => { - assert!(self - .tasks_running - .insert(socket.as_raw_fd(), (socket, task_ref)) - .is_none()); - } - SpawnResult::Skipped(result) => { - self.update_runnable(task_ref, result); - } - } - Ok(()) - } - - fn run_tasks(&mut self, runner: &Runner) -> Result<()> { - while let Some(task_ref) = self.tasks_runnable.pop() { - self.run_task(task_ref, runner)?; - } - Ok(()) - } - - fn wait_for_task(&mut self) -> Result<()> { - let mut pollfds: Vec<_> = self - .tasks_running - .values() - .map(|(socket, _)| poll::PollFd::new(socket, poll::PollFlags::POLLIN)) - .collect(); - - while poll::poll(&mut pollfds, -1).context("poll()")? == 0 {} - - let pollevents: Vec<_> = pollfds - .into_iter() - .map(|pollfd| { - ( - pollfd.as_fd().as_raw_fd(), - pollfd.revents().expect("Unknown events in poll() return"), - ) - }) - .collect(); - - for (fd, events) in pollevents { - if !events.contains(poll::PollFlags::POLLIN) { - if events.intersects(!poll::PollFlags::POLLIN) { - return Err(Error::new( - "Unexpected error status for socket file descriptor", - )); - } - continue; - } - - let (socket, task_ref) = self.tasks_running.remove(&fd).unwrap(); - - let task_output = Runner::result(&socket)?; - self.update_runnable(task_ref, task_output); - } - - Ok(()) - } - - fn is_empty(&self) -> bool { - self.tasks_runnable.is_empty() && self.tasks_running.is_empty() - } - - fn is_done(&self) -> bool { - self.is_empty() && self.tasks_blocked.is_empty() - } - - pub fn run(&mut self, runner: &Runner) -> Result<()> { - while !self.is_empty() { - self.run_tasks(runner)?; - self.wait_for_task()?; - } - - assert!(self.is_done(), "No runnable tasks left"); - - self.state.print_summary(); - - Ok(()) - } -} diff --git a/crates/driver/src/parse.rs b/crates/driver/src/parse.rs deleted file mode 100644 index aad9360..0000000 --- a/crates/driver/src/parse.rs +++ /dev/null @@ -1,95 +0,0 @@ -use nom::{ - bytes::complete::{tag, take_while1}, - combinator::{all_consuming, opt}, - error::ParseError, - Err, IResult, InputLength, Parser, -}; - -#[derive(Debug, Clone, Copy)] -pub struct Task<'a> { - pub recipe: &'a str, - pub task: &'a str, - pub host: Option<&'a str>, - pub target: Option<&'a str>, -} - -#[derive(Debug, Clone, Copy)] -pub struct TaskFlags { - pub force_run: bool, -} - -fn is_name_char(c: char) -> bool { - matches!(c, 'a'..='z' | 'A' ..='Z' | '0'..='9' | '_' | '-') -} - -fn name(input: &str) -> IResult<&str, &str> { - take_while1(is_name_char)(input) -} - -fn task_id(input: &str) -> IResult<&str, (&str, &str)> { - let (input, recipe) = name(input)?; - let (input, _) = tag(":")(input)?; - let (input, task) = name(input)?; - Ok((input, (recipe, task))) -} - -fn task_arg_target(input: &str) -> IResult<&str, &str> { - let (input, _) = tag(":")(input)?; - let (input, target) = name(input)?; - Ok((input, target)) -} - -fn task_args(input: &str) -> IResult<&str, (Option<&str>, Option<&str>)> { - let (input, _) = tag("/")(input)?; - let (input, host) = opt(name)(input)?; - let (input, target) = opt(task_arg_target)(input)?; - - Ok((input, (host, target))) -} - -fn task(input: &str) -> IResult<&str, Task> { - let (input, (recipe, task)) = task_id(input)?; - let (input, args) = opt(task_args)(input)?; - - let (host, target) = args.unwrap_or_default(); - - Ok(( - input, - Task { - recipe, - task, - host, - target, - }, - )) -} - -fn task_flags(input: &str) -> IResult<&str, TaskFlags> { - let (input, force_run) = opt(tag("+"))(input)?; - - Ok(( - input, - TaskFlags { - force_run: force_run.is_some(), - }, - )) -} - -fn task_with_flags(input: &str) -> IResult<&str, (Task, TaskFlags)> { - let (input, task) = task(input)?; - let (input, flags) = task_flags(input)?; - - Ok((input, (task, flags))) -} - -fn parse_all<I, O, E: ParseError<I>, F>(f: F, input: I) -> Result<O, Err<E>> -where - I: InputLength, - F: Parser<I, O, E>, -{ - all_consuming(f)(input).map(|(_, result)| result) -} - -pub fn parse_task_with_flags(input: &str) -> Option<(Task, TaskFlags)> { - parse_all(task_with_flags, input).ok() -} diff --git a/crates/driver/src/recipe.rs b/crates/driver/src/recipe.rs deleted file mode 100644 index 474096b..0000000 --- a/crates/driver/src/recipe.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::{collections::HashMap, fs::File, path::Path, result}; - -use scoped_tls_hkt::scoped_thread_local; -use serde::{Deserialize, Deserializer}; -use walkdir::WalkDir; - -use common::{error::*, types::*}; - -use crate::task::{RecipeMeta, TaskDef}; - -scoped_thread_local!(static CURRENT_RECIPE: str); - -fn current_recipe() -> String { - CURRENT_RECIPE.with(|current| current.to_string()) -} - -pub fn deserialize_task_id<'de, D>(deserializer: D) -> result::Result<TaskID, D::Error> -where - D: Deserializer<'de>, -{ - #[derive(Deserialize)] - struct RecipeTaskID { - recipe: Option<String>, - task: String, - } - let RecipeTaskID { recipe, task } = RecipeTaskID::deserialize(deserializer)?; - Ok(TaskID { - recipe: recipe.unwrap_or_else(current_recipe), - task, - }) -} - -#[derive(Debug, Deserialize)] -struct Recipe { - #[serde(default)] - pub meta: RecipeMeta, - pub tasks: HashMap<String, TaskDef>, -} - -fn read_recipe(path: &Path) -> Result<Recipe> { - let f = File::open(path).context("IO error")?; - - let recipe: Recipe = serde_yaml::from_reader(f) - .map_err(Error::new) - .context("YAML error")?; - - Ok(recipe) -} - -fn is_yml(path: &Path) -> bool { - path.extension() == Some("yml".as_ref()) -} - -pub fn read_recipes<P: AsRef<Path>>(path: P) -> Result<HashMap<TaskID, Vec<TaskDef>>> { - let mut tasks = HashMap::<TaskID, Vec<TaskDef>>::new(); - - for entry in WalkDir::new(path) - .sort_by_file_name() - .into_iter() - .filter_map(|e| e.ok()) - { - let path = entry.path(); - if !path.is_file() || !is_yml(path) { - continue; - } - - let stem: &str = match path.file_stem().map(|n| n.to_str()) { - Some(Some(v)) => v, - _ => continue, - }; - let (basename, version) = match stem.split_once('@') { - Some((basename, version)) => (basename, Some(version)), - None => (stem, None), - }; - - let recipe = CURRENT_RECIPE.set(basename, || read_recipe(path))?; - - let mut meta = recipe.meta; - if meta.name.is_empty() { - meta.name = basename.to_string(); - } - if meta.version.is_none() { - meta.version = version.map(|v| v.to_string()); - } - - for (label, mut task) in recipe.tasks { - let task_id = TaskID { - recipe: basename.to_string(), - task: label, - }; - task.meta = meta.clone(); - tasks.entry(task_id).or_default().push(task); - } - } - - Ok(tasks) -} diff --git a/crates/common/Cargo.toml b/crates/rebel-common/Cargo.toml index 954ebe5..954ebe5 100644 --- a/crates/common/Cargo.toml +++ b/crates/rebel-common/Cargo.toml diff --git a/crates/common/src/error.rs b/crates/rebel-common/src/error.rs index ba25af4..ba25af4 100644 --- a/crates/common/src/error.rs +++ b/crates/rebel-common/src/error.rs diff --git a/crates/common/src/lib.rs b/crates/rebel-common/src/lib.rs index 8d630dd..8d630dd 100644 --- a/crates/common/src/lib.rs +++ b/crates/rebel-common/src/lib.rs diff --git a/crates/common/src/string_hash.rs b/crates/rebel-common/src/string_hash.rs index a2b00db..a2b00db 100644 --- a/crates/common/src/string_hash.rs +++ b/crates/rebel-common/src/string_hash.rs diff --git a/crates/common/src/types.rs b/crates/rebel-common/src/types.rs index 32b9182..d3beb70 100644 --- a/crates/common/src/types.rs +++ b/crates/rebel-common/src/types.rs @@ -13,9 +13,30 @@ pub struct TaskID { pub task: String, } +impl TaskID { + pub fn as_ref(&self) -> TaskIDRef<'_> { + TaskIDRef { + recipe: &self.recipe, + task: &self.task, + } + } +} + impl Display for TaskID { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}:{}", self.recipe, self.task) + self.as_ref().fmt(f) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TaskIDRef<'a> { + pub recipe: &'a str, + pub task: &'a str, +} + +impl<'a> Display for TaskIDRef<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}::{}", self.recipe, self.task) } } @@ -39,7 +60,7 @@ pub struct Task { pub command: String, pub workdir: String, pub rootfs: ArchiveHash, - pub inherit: Vec<LayerHash>, + pub ancestors: Vec<LayerHash>, pub depends: HashSet<Dependency>, pub outputs: HashMap<String, String>, pub pins: HashMap<ArchiveHash, String>, diff --git a/crates/rebel-parse/Cargo.toml b/crates/rebel-parse/Cargo.toml new file mode 100644 index 0000000..0233af8 --- /dev/null +++ b/crates/rebel-parse/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rebel-parse" +version = "0.1.0" +authors = ["Matthias Schiffer <mschiffer@universe-factory.net>"] +license = "MIT" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +peg = { git = "https://github.com/kevinmehall/rust-peg.git", version = "0.8.2" } +rebel-common = { path = "../rebel-common" } + +[dev-dependencies] +clap = { version = "4.0.0", features = ["derive"] } +divan = "0.1.14" + +[[bench]] +name = "recipe" +harness = false diff --git a/crates/rebel-parse/benches/recipe.rs b/crates/rebel-parse/benches/recipe.rs new file mode 100644 index 0000000..4cff857 --- /dev/null +++ b/crates/rebel-parse/benches/recipe.rs @@ -0,0 +1,21 @@ +use rebel_parse::{ast, token::TokenStream}; + +fn main() { + divan::main(); +} + +const RECIPE: &str = include_str!("../../../examples/recipes/gmp/build.recipe"); + +#[divan::bench] +fn tokenize() -> TokenStream<'static> { + rebel_parse::tokenize::token_stream(divan::black_box(RECIPE)).unwrap() +} + +#[divan::bench] +fn parse(bencher: divan::Bencher) { + let tokens = tokenize(); + + bencher.bench(|| -> ast::Recipe<'static> { + rebel_parse::recipe::recipe(divan::black_box(&tokens)).unwrap() + }); +} diff --git a/crates/rebel-parse/examples/parse-string.rs b/crates/rebel-parse/examples/parse-string.rs new file mode 100644 index 0000000..190d0fa --- /dev/null +++ b/crates/rebel-parse/examples/parse-string.rs @@ -0,0 +1,66 @@ +use std::{fmt::Debug, process, time::Instant}; + +use clap::{Parser, ValueEnum}; + +use rebel_parse::{recipe, tokenize}; + +#[derive(Clone, Debug, PartialEq, Eq, ValueEnum)] +enum Rule { + Tokenize, + Recipe, + RecipeStmt, + Body, + BodyStmt, + Expr, +} + +#[derive(Clone, Debug, Parser)] +struct Opts { + rule: Rule, + input: String, +} + +fn main() { + let opts: Opts = Opts::parse(); + let input = opts.input.trim(); + + fn as_debug<'a>(v: impl Debug + 'a) -> Box<dyn Debug + 'a> { + Box::new(v) + } + + let start = Instant::now(); + let result = tokenize::token_stream(input); + let dur = Instant::now().duration_since(start); + println!("Tokenization took {} µs", dur.as_micros()); + + let tokens = match result { + Ok(value) => value, + Err(err) => { + println!("{err}"); + process::exit(1); + } + }; + + let start = Instant::now(); + let result = match opts.rule { + Rule::Tokenize => Ok(as_debug(tokens)), + Rule::Recipe => recipe::recipe(&tokens).map(as_debug), + Rule::RecipeStmt => recipe::recipe_stmt(&tokens).map(as_debug), + Rule::Body => recipe::body(&tokens).map(as_debug), + Rule::BodyStmt => recipe::body_stmt(&tokens).map(as_debug), + Rule::Expr => recipe::expr(&tokens).map(as_debug), + }; + if opts.rule != Rule::Tokenize { + let dur = Instant::now().duration_since(start); + println!("Parsing took {} µs", dur.as_micros()); + } + + match result { + Ok(value) => { + println!("{value:#?}"); + } + Err(err) => { + println!("{err}"); + } + }; +} diff --git a/crates/rebel-parse/src/ast.rs b/crates/rebel-parse/src/ast.rs new file mode 100644 index 0000000..648a79e --- /dev/null +++ b/crates/rebel-parse/src/ast.rs @@ -0,0 +1,223 @@ +use crate::token; + +pub type Recipe<'a> = Vec<RecipeStmt<'a>>; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RecipeStmt<'a> { + BodyStmt(BodyStmt<'a>), + Fetch { + name: Ident<'a>, + body: Body<'a>, + }, + Task { + name: Ident<'a>, + params: Vec<FuncParam<'a>>, + body: Body<'a>, + }, +} + +pub type Body<'a> = Vec<BodyStmt<'a>>; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BodyStmt<'a> { + Assign { + left: Box<TypedExpr<'a>>, + op: Option<OpBinary>, + right: Box<Expr<'a>>, + }, +} + +impl<'a> BodyStmt<'a> { + pub(crate) fn assign(left: TypedExpr<'a>, op: Option<OpBinary>, right: Expr<'a>) -> Self { + BodyStmt::Assign { + left: Box::new(left), + op, + right: Box::new(right), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Expr<'a> { + Binary { + left: Box<Expr<'a>>, + op: OpBinary, + right: Box<Expr<'a>>, + }, + Unary { + op: OpUnary, + expr: Box<Expr<'a>>, + }, + Apply { + expr: Box<Expr<'a>>, + params: Vec<Expr<'a>>, + }, + Method { + expr: Box<Expr<'a>>, + method: Ident<'a>, + params: Vec<Expr<'a>>, + }, + Index { + expr: Box<Expr<'a>>, + index: Box<Expr<'a>>, + }, + Field { + expr: Box<Expr<'a>>, + field: Ident<'a>, + }, + Paren(Box<Expr<'a>>), + Path(Path<'a>), + Literal(Literal<'a>), +} + +impl<'a> Expr<'a> { + pub(crate) fn binary(left: Expr<'a>, op: OpBinary, right: Expr<'a>) -> Self { + Expr::Binary { + left: Box::new(left), + op, + right: Box::new(right), + } + } + + pub(crate) fn unary(op: OpUnary, expr: Expr<'a>) -> Self { + Expr::Unary { + op, + expr: Box::new(expr), + } + } + + pub(crate) fn apply(expr: Expr<'a>, params: Vec<Expr<'a>>) -> Self { + Expr::Apply { + expr: Box::new(expr), + params, + } + } + + pub(crate) fn method(expr: Expr<'a>, method: Ident<'a>, params: Vec<Expr<'a>>) -> Self { + Expr::Method { + expr: Box::new(expr), + method, + params, + } + } + + pub(crate) fn index(expr: Expr<'a>, index: Expr<'a>) -> Self { + Expr::Index { + expr: Box::new(expr), + index: Box::new(index), + } + } + + pub(crate) fn field(expr: Expr<'a>, field: Ident<'a>) -> Self { + Expr::Field { + expr: Box::new(expr), + field, + } + } + + pub(crate) fn paren(expr: Expr<'a>) -> Self { + Expr::Paren(Box::new(expr)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TypedExpr<'a> { + pub expr: Expr<'a>, + pub typ: Option<Expr<'a>>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FuncParam<'a> { + pub name: Ident<'a>, + pub typ: Expr<'a>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Literal<'a> { + Unit, + Boolean(bool), + Integer(u64), + String(Vec<StringPiece<'a>>), + Tuple(Vec<Expr<'a>>), + Array(Vec<Expr<'a>>), + Map(Vec<MapEntry<'a>>), +} + +impl<'a> Literal<'a> { + pub(crate) fn number(s: &'a str) -> Result<Self, &'static str> { + let (radix, rest) = if let Some(rest) = s.strip_prefix("0x") { + (16, rest) + } else if let Some(rest) = s.strip_prefix("0o") { + (8, rest) + } else if let Some(rest) = s.strip_prefix("0b") { + (2, rest) + } else { + (10, s) + }; + let digits = rest.replace('_', ""); + let value = u64::from_str_radix(&digits, radix).or(Err("number"))?; + Ok(Literal::Integer(value)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StringPiece<'a> { + Chars(&'a str), + Escape(char), + Interp(Expr<'a>), +} + +impl<'a> TryFrom<&token::StringPiece<'a>> for StringPiece<'a> { + type Error = &'static str; + + fn try_from(value: &token::StringPiece<'a>) -> Result<Self, Self::Error> { + use crate::recipe; + + Ok(match value { + token::StringPiece::Chars(chars) => StringPiece::Chars(chars), + token::StringPiece::Escape(c) => StringPiece::Escape(*c), + token::StringPiece::Interp(tokens) => StringPiece::Interp( + recipe::expr(tokens).or(Err("Invalid expression in string interpolation"))?, + ), + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MapEntry<'a> { + pub key: &'a str, + pub value: Expr<'a>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OpUnary { + Not, + Neg, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OpBinary { + Add, + Sub, + Mul, + Div, + Rem, + And, + Or, + Eq, + Lt, + Le, + Ne, + Ge, + Gt, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Path<'a> { + pub components: Vec<Ident<'a>>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Ident<'a> { + pub name: &'a str, +} diff --git a/crates/rebel-parse/src/grammar/mod.rs b/crates/rebel-parse/src/grammar/mod.rs new file mode 100644 index 0000000..de06991 --- /dev/null +++ b/crates/rebel-parse/src/grammar/mod.rs @@ -0,0 +1,3 @@ +pub mod recipe; +pub mod task_ref; +pub mod tokenize; diff --git a/crates/rebel-parse/src/grammar/recipe.rs b/crates/rebel-parse/src/grammar/recipe.rs new file mode 100644 index 0000000..49b1689 --- /dev/null +++ b/crates/rebel-parse/src/grammar/recipe.rs @@ -0,0 +1,166 @@ +use crate::ast::{self, Expr}; +use crate::token::*; + +pub use rules::*; + +peg::parser! { + pub grammar rules<'a>() for TokenStream<'a> { + use ast::OpBinary::*; + use ast::OpUnary::*; + + pub rule recipe() -> ast::Recipe<'a> + = recipe:recipe_stmt()* { recipe } + + pub rule recipe_stmt() -> ast::RecipeStmt<'a> + = keyword_fetch() name:ident() p('{') body:body() p('}') { + ast::RecipeStmt::Fetch { name, body: Vec::new() } + } + / keyword_task() name:ident() p('(') params:func_params() p(')') + p('{') body:body() p('}') { + ast::RecipeStmt::Task { name, params, body } + } + / stmt:body_stmt() { + ast::RecipeStmt::BodyStmt(stmt) + } + + pub rule body() -> ast::Body<'a> + = body:body_stmt()* { body } + + pub rule body_stmt() -> ast::BodyStmt<'a> + = left:typed_expr() op:assign_op() right:expr() p(';') { + ast::BodyStmt::assign(left, op, right) + } + + rule assign_op() -> Option<ast::OpBinary> + = p2('+', '=') { Some(Add) } + / p2('-', '=') { Some(Sub) } + / p2('*', '=') { Some(Mul) } + / p2('/', '=') { Some(Div) } + / p2('%', '=') { Some(Rem) } + / p('=') { None } + + rule typed_expr() -> ast::TypedExpr<'a> + = expr:expr() typ:tagged(<p(':')>, <expr()>)? { ast::TypedExpr { expr, typ } } + + pub rule expr() -> Expr<'a> = precedence! { + left:(@) p2('|', '|') right:@ { Expr::binary(left, Or, right) } + -- + left:(@) p2('&', '&') right:@ { Expr::binary(left, And, right) } + -- + left:(@) p2('=', '=') right:@ { Expr::binary(left, Eq, right) } + left:(@) p2('!', '=') right:@ { Expr::binary(left, Ne, right) } + left:(@) p('<') right:@ { Expr::binary(left, Lt, right) } + left:(@) p('>') right:@ { Expr::binary(left, Gt, right) } + left:(@) p2('<', '=') right:@ { Expr::binary(left, Le, right) } + left:(@) p2('>', '=') right:@ { Expr::binary(left, Ge, right) } + -- + left:(@) p('+') right:@ { Expr::binary(left, Add, right) } + left:(@) p('-') right:@ { Expr::binary(left, Sub, right) } + -- + left:(@) p('*') right:@ { Expr::binary(left, Mul, right) } + left:(@) p('/') right:@ { Expr::binary(left, Div, right) } + left:(@) p('%') right:@ { Expr::binary(left, Rem, right) } + -- + p('-') expr:@ { Expr::unary(Neg, expr) } + p('!') expr:@ { Expr::unary(Not, expr) } + -- + expr:@ p('(') params:call_params() p(')') { + Expr::apply(expr, params) + } + expr:@ p('[') index:expr() p(']') { Expr::index(expr, index) } + -- + expr:@ p('.') method:field() p('(') params:call_params() p(')') { + Expr::method(expr, method, params) + } + expr:@ p('.') field:field() { Expr::field(expr, field) } + -- + p('(') e:expr() p(')') { Expr::paren(e) } + e:atom() { e } + } + + rule atom() -> Expr<'a> + = lit:literal() { Expr::Literal(lit) } + / path:path() { Expr::Path(path) } + + rule call_params() -> Vec<ast::Expr<'a>> + = args:delimited(<expr()>, <p(',')>) { args } + + rule func_params() -> Vec<ast::FuncParam<'a>> + = params:delimited(<func_param()>, <p(',')>) { params } + + rule func_param() -> ast::FuncParam<'a> + = name:ident() p(':') typ:expr() { ast::FuncParam { name, typ } } + + rule literal() -> ast::Literal<'a> + = keyword_true() { ast::Literal::Boolean(true) } + / keyword_false() { ast::Literal::Boolean(false) } + / [Token::Number(content)] { ? + ast::Literal::number(content) + } + / [Token::String(String { pieces, .. })] { ? + let ast_pieces = pieces.iter().map(|piece| piece.try_into()).collect::<Result<_, _>>()?; + Ok(ast::Literal::String(ast_pieces)) + } + / p('(') p(')') { ast::Literal::Unit } + / p('(') elements:(expr() ** p(',')) p(',')? p(')') { + ast::Literal::Tuple(elements) + } + / p('[') elements:delimited(<expr()>, <p(',')>) p(']') { + ast::Literal::Array(elements) + } + / p('{') entries:delimited(<map_entry()>, <p(',')>) p('}') { + ast::Literal::Map(entries) + } + + rule map_entry() -> ast::MapEntry<'a> + = key:field() p('=') value:expr() { + ast::MapEntry { key: key.name, value } + } + + rule path() -> ast::Path<'a> + = components:ident() ++ p2(':', ':') { ast::Path { components } } + + rule field() -> ast::Ident<'a> + = ident() + / [Token::Number(content)] { + ast::Ident { name: content } + } + + rule p_(ch: char) + = [Token::Punct(Punct(c, Spacing::Joint)) if *c == ch] {} + + rule p(ch: char) -> () + = [Token::Punct(Punct(c, _)) if *c == ch] {} + + rule p2(ch1: char, ch2: char) + = p_(ch1) p(ch2) + + rule ident() -> ast::Ident<'a> + = !keyword() [Token::Ident(name)] { ast::Ident { name } } + + rule keyword() + = keyword_true() + / keyword_false() + / keyword_fetch() + / keyword_task() + + rule keyword_true() + = const_ident("true") + rule keyword_false() + = const_ident("false") + rule keyword_fetch() + = const_ident("fetch") + rule keyword_task() + = const_ident("task") + + rule const_ident(keyword: &str) + = [Token::Ident(name) if *name == keyword] + + rule delimited<T>(expr: rule<T>, delim: rule<()>) -> Vec<T> + = values:(expr() ++ delim()) delim()? { values } + / { Vec::new() } + + rule tagged<T>(tag: rule<()>, value: rule<T>) -> T + = tag() v:value() { v } + } +} diff --git a/crates/rebel-parse/src/grammar/task_ref.rs b/crates/rebel-parse/src/grammar/task_ref.rs new file mode 100644 index 0000000..77d6c5f --- /dev/null +++ b/crates/rebel-parse/src/grammar/task_ref.rs @@ -0,0 +1,65 @@ +pub use rules::*; + +use rebel_common::types::TaskIDRef; + +#[derive(Debug, Clone, Copy)] +pub struct TaskRef<'a> { + pub id: TaskIDRef<'a>, + pub args: TaskArgs<'a>, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct TaskArgs<'a> { + pub host: Option<&'a str>, + pub target: Option<&'a str>, +} + +#[derive(Debug, Clone, Copy)] +pub struct TaskFlags { + pub force_run: bool, +} + +peg::parser! { + pub grammar rules() for str { + pub rule task_ref_with_flags() -> (TaskRef<'input>, TaskFlags) + = task:task_ref() flags:task_flags() { (task, flags) } + + pub rule task_ref() -> TaskRef<'input> + = id:task_id() args:task_args() { + TaskRef { + id, + args, + } + } + + rule recipe_id() -> &'input str + = $(name() ("/" name())?) + + rule task_id() -> TaskIDRef<'input> + = recipe:recipe_id() "::" task:name() { + TaskIDRef { recipe, task } + } + + rule task_args() -> TaskArgs<'input> + = "@" host:name()? target:tagged(<":">, <name()>)? { + TaskArgs { + host, + target, + } + } + / { Default::default() } + + rule task_flags() -> TaskFlags + = force_run:force_run() { TaskFlags { force_run } } + + rule force_run() -> bool + = "+" { true } + / { false } + + rule name() -> &'input str + = $(['a'..='z' | 'A' ..='Z' | '0'..='9' | '_' | '-']+) + + rule tagged<T>(tag: rule<()>, value: rule<T>) -> T + = tag() v:value() { v } + } +} diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs new file mode 100644 index 0000000..841e61b --- /dev/null +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -0,0 +1,104 @@ +use crate::token::*; + +pub use rules::*; + +peg::parser! { + pub grammar rules() for str { + pub rule token_stream() -> TokenStream<'input> + = _ tokens:(token() ** _) _ { TokenStream(tokens) } + + pub rule token() -> Token<'input> + = number:number() { Token::Number(number) } + / string:string() { Token::String(string) } + / ident:ident() { Token::Ident(ident) } + / punct:punct() { Token::Punct(punct) } + + rule ident() -> &'input str + = $( + ['a'..='z' | 'A' ..='Z' | '_' ] + ['a'..='z' | 'A' ..='Z' | '_' | '0'..='9']* + ) + + rule punct() -> Punct + = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } + + rule punct_char() -> char + = !number() !string() !ident() !__ ch:[_] { ch } + + rule spacing() -> Spacing + = &punct_char() { Spacing::Joint } + / { Spacing::Alone } + + rule number() -> &'input str + = $(['0'..='9'] ['0'..='9' | 'a'..='z' | 'A'..='Z' | '_']*) + + rule string() -> String<'input> + = "\"" pieces:string_piece()* "\"" { + String { + pieces, + kind: StringKind::String, + } + } + / "r\"" chars:$([^'"']*) "\"" { + String { + pieces: vec![StringPiece::Chars(chars)], + kind: StringKind::RawString, + } + } + / "```" newline() pieces:script_string_piece()* "```" { + String { + pieces, + kind: StringKind::ScriptString, + } + } + + rule string_piece() -> StringPiece<'input> + = chars:$((!"{{" [^'"' | '\\'])+) { StringPiece::Chars(chars) } + / "\\" escape:string_escape() { StringPiece::Escape(escape) } + / string_interp() + + rule string_escape() -> char + = "n" { '\n' } + / "r" { '\r' } + / "t" { '\t' } + / "\\" { '\\' } + / "\"" { '"' } + / "0" { '\0' } + / "x" digits:$(['0'..='7'] hex_digit()) { + u8::from_str_radix(digits, 16).unwrap().into() + } + / "u{" digits:$(hex_digit()*<1,6>) "}" { ? + u32::from_str_radix(digits, 16).unwrap().try_into().or(Err("Invalid unicode escape")) + } + + rule script_string_piece() -> StringPiece<'input> + = chars:$((!"{{" !"```" [_])+) { StringPiece::Chars(chars) } + / string_interp() + + rule string_interp() -> StringPiece<'input> + = "{{" _ tokens:(subtoken() ++ _) _ "}}" { + StringPiece::Interp(TokenStream(tokens)) + } + + rule subtoken() -> Token<'input> + = !"}}" token:token() { token } + + rule hex_digit() + = ['0'..='9' | 'a'..='f' | 'A'..='F'] + + /// Mandatory whitespace + rule __ + = ([' ' | '\t'] / quiet!{newline()} / quiet!{comment()})+ + + /// Optional whitespace + rule _ + = quiet!{__?} + + rule comment() + = "//" (!newline() [_])* (newline() / ![_]) + / "/*" (!"*/" [_])* "*/" + + rule newline() + = ['\n' | '\r'] + } +} diff --git a/crates/rebel-parse/src/lib.rs b/crates/rebel-parse/src/lib.rs new file mode 100644 index 0000000..4a8c431 --- /dev/null +++ b/crates/rebel-parse/src/lib.rs @@ -0,0 +1,8 @@ +pub mod ast; +pub mod token; + +mod grammar; + +pub use grammar::recipe; +pub use grammar::task_ref; +pub use grammar::tokenize; diff --git a/crates/rebel-parse/src/token.rs b/crates/rebel-parse/src/token.rs new file mode 100644 index 0000000..2f2f849 --- /dev/null +++ b/crates/rebel-parse/src/token.rs @@ -0,0 +1,68 @@ +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Token<'a> { + Ident(&'a str), + Punct(Punct), + String(String<'a>), + Number(&'a str), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Punct(pub char, pub Spacing); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Spacing { + Alone, + Joint, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct String<'a> { + pub pieces: Vec<StringPiece<'a>>, + pub kind: StringKind, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StringPiece<'a> { + Chars(&'a str), + Escape(char), + Interp(TokenStream<'a>), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StringKind { + String, + RawString, + ScriptString, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TokenStream<'a>(pub Vec<Token<'a>>); + +impl<'a> peg::Parse for TokenStream<'a> { + type PositionRepr = usize; + + fn start(&self) -> usize { + 0 + } + + fn is_eof(&self, pos: usize) -> bool { + pos >= self.0.len() + } + + fn position_repr(&self, pos: usize) -> Self::PositionRepr { + pos + } +} + +impl<'input, 'a: 'input> peg::ParseElem<'input> for TokenStream<'a> { + type Element = &'input Token<'a>; + + fn parse_elem(&'input self, pos: usize) -> peg::RuleResult<Self::Element> { + use peg::RuleResult; + + match self.0[pos..].first() { + Some(c) => RuleResult::Matched(pos + 1, c), + None => RuleResult::Failed, + } + } +} diff --git a/crates/rebel-resolve/Cargo.toml b/crates/rebel-resolve/Cargo.toml new file mode 100644 index 0000000..4b3e113 --- /dev/null +++ b/crates/rebel-resolve/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rebel-resolve" +version = "0.1.0" +authors = ["Matthias Schiffer <mschiffer@universe-factory.net>"] +license = "MIT" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rebel-common = { path = "../rebel-common" } + +deb-version = "0.1.1" +enum-kinds = "0.5.1" +serde = { version = "1", features = ["derive", "rc"] } diff --git a/crates/driver/src/args.rs b/crates/rebel-resolve/src/args.rs index 805646a..805646a 100644 --- a/crates/driver/src/args.rs +++ b/crates/rebel-resolve/src/args.rs diff --git a/crates/driver/src/context.rs b/crates/rebel-resolve/src/context.rs index 9674e5f..996d981 100644 --- a/crates/driver/src/context.rs +++ b/crates/rebel-resolve/src/context.rs @@ -9,15 +9,14 @@ use std::{ result, }; -use common::{ +use rebel_common::{ error::{self, Contextualizable}, string_hash::ArchiveHash, - types::TaskID, + types::TaskIDRef, }; use crate::{ args::*, - parse::{self, TaskFlags}, paths, pin::{self, Pins}, task::*, @@ -32,7 +31,7 @@ pub enum ErrorKind<'a> { #[derive(Debug, Clone, Copy)] pub struct Error<'a> { - pub task: &'a TaskID, + pub task: TaskIDRef<'a>, pub kind: ErrorKind<'a>, } @@ -65,7 +64,7 @@ pub type Result<'a, T> = result::Result<T, Error<'a>>; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct TaskRef<'ctx> { - pub id: &'ctx TaskID, + pub id: TaskIDRef<'ctx>, pub args: Rc<TaskArgs>, } @@ -75,7 +74,7 @@ impl<'ctx> Display for TaskRef<'ctx> { return self.id.fmt(f); } - let pv_arg = match self.args.get("pv") { + let version_arg = match self.args.get("version") { Some(Arg::String(s)) => Some(s), _ => None, }; @@ -89,13 +88,13 @@ impl<'ctx> Display for TaskRef<'ctx> { }; write!(f, "{}", self.id.recipe)?; - if let Some(pv) = pv_arg { - write!(f, "@{}", pv)?; + if let Some(version) = version_arg { + write!(f, "#{}", version)?; } - write!(f, ":{}", self.id.task)?; + write!(f, "::{}", self.id.task)?; if host_arg.is_some() || target_arg.is_some() { - write!(f, "/")?; + write!(f, "@")?; } if let Some(host) = host_arg { @@ -144,12 +143,15 @@ fn platform_relation(args: &TaskArgs, from: &str, to: &str) -> Option<PlatformRe pub struct Context { platforms: HashMap<String, Arg>, globals: TaskArgs, - tasks: HashMap<TaskID, Vec<TaskDef>>, + tasks: HashMap<String, HashMap<String, Vec<TaskDef>>>, rootfs: (ArchiveHash, String), } impl Context { - pub fn new(mut tasks: HashMap<TaskID, Vec<TaskDef>>, pins: Pins) -> error::Result<Self> { + pub fn new( + mut tasks: HashMap<String, HashMap<String, Vec<TaskDef>>>, + pins: Pins, + ) -> error::Result<Self> { let platforms: HashMap<_, _> = [ arg( "build", @@ -212,7 +214,7 @@ impl Context { } fn add_rootfs_tasks( - tasks: &mut HashMap<TaskID, Vec<TaskDef>>, + tasks: &mut HashMap<String, HashMap<String, Vec<TaskDef>>>, provides: Vec<pin::Provides>, globals: &TaskArgs, ) -> error::Result<()> { @@ -255,10 +257,9 @@ impl Context { task_def.priority = i32::MAX; tasks - .entry(TaskID { - recipe: recipe.to_string(), - task: task.to_string(), - }) + .entry(recipe) + .or_default() + .entry(task) .or_default() .push(task_def); } @@ -293,9 +294,12 @@ impl Context { .max_by(|task1, task2| Self::compare_tasks(task1, task2)) } - fn get_with_args<'a>(&self, id: &'a TaskID, args: &TaskArgs) -> Result<'a, &TaskDef> { - self.tasks - .get(id) + fn get_by_ref(&self, id: TaskIDRef) -> Option<&[TaskDef]> { + Some(self.tasks.get(id.recipe)?.get(id.task)?) + } + + fn get_with_args<'a>(&self, id: TaskIDRef<'a>, args: &TaskArgs) -> Result<'a, &TaskDef> { + self.get_by_ref(id) .and_then(|tasks| Self::select_task(tasks, args)) .ok_or(Error { task: id, @@ -307,7 +311,7 @@ impl Context { self.get_with_args(task.id, task.args.as_ref()) } - fn task_ref<'ctx>(&'ctx self, id: &'ctx TaskID, args: &TaskArgs) -> Result<TaskRef> { + fn task_ref<'ctx>(&'ctx self, id: TaskIDRef<'ctx>, args: &TaskArgs) -> Result<TaskRef> { let task_def = self.get_with_args(id, args)?; let mut arg_def: HashMap<_, _> = task_def.args.iter().map(|(k, &v)| (k, v)).collect(); @@ -345,8 +349,11 @@ impl Context { new_args.set("cross_compile", cross_compile); - new_args.set("pn", Some(task_def.meta.name.clone())); - new_args.set("pv", task_def.meta.version.clone()); + new_args.set("basename", Some(task_def.meta.basename.clone())); + new_args.set("recipename", Some(task_def.meta.recipename.clone())); + new_args.set("recipe", Some(task_def.meta.recipe.clone())); + new_args.set("name", Some(task_def.meta.name.clone())); + new_args.set("version", task_def.meta.version.clone()); Ok(TaskRef { id, @@ -354,21 +361,27 @@ impl Context { }) } - pub fn parse(&self, s: &str) -> error::Result<(TaskRef, TaskFlags)> { - let (parsed, flags) = parse::parse_task_with_flags(s).context("Invalid task syntax")?; - - let recipe = parsed.recipe.to_string(); - let task = parsed.task.to_string(); - - let id = TaskID { recipe, task }; - let (ctx_id, _) = self + pub fn lookup( + &self, + id: TaskIDRef, + host: Option<&str>, + target: Option<&str>, + ) -> error::Result<TaskRef> { + let (ctx_recipe, recipe_tasks) = self .tasks - .get_key_value(&id) + .get_key_value(id.recipe) .with_context(|| format!("Task {} not found", id))?; + let (ctx_task, _) = recipe_tasks + .get_key_value(id.task) + .with_context(|| format!("Task {} not found", id))?; + let ctx_id = TaskIDRef { + recipe: ctx_recipe, + task: ctx_task, + }; let mut args = self.globals.clone(); - if let Some(host) = parsed.host { + if let Some(host) = host { let plat = self .platforms .get(host) @@ -376,7 +389,7 @@ impl Context { args.set("host", Some(plat)); args.set("target", Some(plat)); } - if let Some(target) = parsed.target { + if let Some(target) = target { let plat = self .platforms .get(target) @@ -388,11 +401,11 @@ impl Context { .task_ref(ctx_id, &args) .with_context(|| format!("Failed to instantiate task {}", id))?; - Ok((task_ref, flags)) + Ok(task_ref) } fn map_args<'ctx, 'args>( - task: &'ctx TaskID, + task: TaskIDRef<'ctx>, mapping: &'ctx ArgMapping, args: &'args TaskArgs, build_dep: bool, @@ -419,37 +432,44 @@ impl Context { Ok(Cow::Owned(ret)) } - fn inherit_ref<'ctx>(&'ctx self, dep: &'ctx InheritDep, args: &TaskArgs) -> Result<TaskRef> { - let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, false)?; - self.task_ref(&dep.dep.id, mapped_args.as_ref()) + fn parent_ref<'ctx>( + &'ctx self, + dep_of: TaskIDRef<'ctx>, + dep: &'ctx ParentDep, + args: &TaskArgs, + ) -> Result<TaskRef> { + let id = dep.dep.id(dep_of.recipe); + let mapped_args = Context::map_args(id, &dep.dep.args, args, false)?; + self.task_ref(id, mapped_args.as_ref()) } pub fn output_ref<'ctx>( &'ctx self, + dep_of: TaskIDRef<'ctx>, dep: &'ctx OutputDep, args: &TaskArgs, build_dep: bool, ) -> Result<OutputRef<'ctx>> { - let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, build_dep)?; + let id = dep.dep.id(dep_of.recipe); + let mapped_args = Context::map_args(id, &dep.dep.args, args, build_dep)?; Ok(OutputRef { - task: self.task_ref(&dep.dep.id, mapped_args.as_ref())?, + task: self.task_ref(id, mapped_args.as_ref())?, output: &dep.output, }) } - pub fn get_inherit_depend<'ctx>( + pub fn get_parent_depend<'ctx>( &'ctx self, task_ref: &TaskRef<'ctx>, ) -> Result<Option<TaskRef>> { let task = self.get(task_ref)?; - let inherit = match &task.inherit { - Some(inherit) => inherit, - None => return Ok(None), + let Some(parent) = &task.parent else { + return Ok(None); }; - Some(self.inherit_ref(inherit, &task_ref.args)).transpose() + Some(self.parent_ref(task_ref.id, parent, &task_ref.args)).transpose() } - fn inherit_iter<'ctx>( + fn ancestor_iter<'ctx>( &'ctx self, task_ref: &TaskRef<'ctx>, ) -> impl Iterator<Item = Result<TaskRef>> { @@ -463,7 +483,7 @@ impl Context { Ok(task_ref) => task_ref, Err(err) => return Some(Err(err)), }; - self.1 = self.0.get_inherit_depend(&task_ref).transpose(); + self.1 = self.0.get_parent_depend(&task_ref).transpose(); Some(Ok(task_ref)) } } @@ -478,14 +498,14 @@ impl Context { let mut ret = HashSet::new(); let mut allow_noinherit = true; - for current in self.inherit_iter(task_ref) { + for current in self.ancestor_iter(task_ref) { let current_ref = current?; let task = self.get(¤t_ref)?; let entries = task .build_depends .iter() .filter(|dep| allow_noinherit || !dep.noinherit) - .map(|dep| self.output_ref(dep, ¤t_ref.args, true)) + .map(|dep| self.output_ref(task_ref.id, dep, ¤t_ref.args, true)) .collect::<Result<Vec<_>>>()?; ret.extend(entries); @@ -502,14 +522,14 @@ impl Context { let mut ret = HashSet::new(); let mut allow_noinherit = true; - for current in self.inherit_iter(task_ref) { + for current in self.ancestor_iter(task_ref) { let current_ref = current?; let task = self.get(¤t_ref)?; let entries = task .depends .iter() .filter(|dep| allow_noinherit || !dep.noinherit) - .map(|dep| self.output_ref(dep, ¤t_ref.args, false)) + .map(|dep| self.output_ref(task_ref.id, dep, ¤t_ref.args, false)) .collect::<Result<Vec<_>>>()?; ret.extend(entries); diff --git a/crates/driver/src/resolve.rs b/crates/rebel-resolve/src/lib.rs index d03f26d..cc44de8 100644 --- a/crates/driver/src/resolve.rs +++ b/crates/rebel-resolve/src/lib.rs @@ -1,13 +1,19 @@ +pub mod args; +pub mod context; +pub mod paths; +pub mod pin; +pub mod task; + use std::collections::{HashMap, HashSet}; use std::fmt; use std::rc::Rc; -use common::types::TaskID; +use rebel_common::types::TaskIDRef; -use crate::args::TaskArgs; -use crate::context::{self, Context, OutputRef, TaskRef}; +use args::TaskArgs; +use context::{Context, OutputRef, TaskRef}; -#[derive(Debug)] +#[derive(Debug, Default)] pub struct DepChain<'ctx>(pub Vec<TaskRef<'ctx>>); impl<'ctx> fmt::Display for DepChain<'ctx> { @@ -38,8 +44,8 @@ impl<'ctx> From<&TaskRef<'ctx>> for DepChain<'ctx> { } } -impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> { - fn from(id: &'ctx TaskID) -> Self { +impl<'ctx> From<TaskIDRef<'ctx>> for DepChain<'ctx> { + fn from(id: TaskIDRef<'ctx>) -> Self { TaskRef { id, args: Rc::new(TaskArgs::default()), @@ -48,11 +54,14 @@ impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> { } } +const MAX_ERRORS: usize = 100; + #[derive(Debug)] pub enum ErrorKind<'ctx> { Context(context::Error<'ctx>), OutputNotFound(&'ctx str), DependencyCycle, + TooManyErrors, } #[derive(Debug)] @@ -76,6 +85,13 @@ impl<'ctx> Error<'ctx> { } } + fn too_many_errors() -> Self { + Error { + dep_chain: DepChain::default(), + kind: ErrorKind::TooManyErrors, + } + } + fn extend(&mut self, task: &TaskRef<'ctx>) { self.dep_chain.0.push(task.clone()); } @@ -94,6 +110,9 @@ impl<'ctx> fmt::Display for Error<'ctx> { ErrorKind::DependencyCycle => { write!(f, "Dependency Cycle: ")?; } + ErrorKind::TooManyErrors => { + write!(f, "Too many errors, stopping.")?; + } } dep_chain.fmt(f) } @@ -149,7 +168,7 @@ where let mut errors = Vec::new(); for runtime_dep in &output.runtime_depends { - match ctx.output_ref(runtime_dep, &task.args, false) { + match ctx.output_ref(task.id, runtime_dep, &task.args, false) { Ok(output_ref) => { for mut error in add_dep(ret, ctx, output_ref) { error.extend(task); @@ -201,7 +220,7 @@ pub fn get_dependent_tasks<'ctx>( task_ref: &TaskRef<'ctx>, ) -> Result<HashSet<TaskRef<'ctx>>, Vec<Error<'ctx>>> { Ok(ctx - .get_inherit_depend(task_ref) + .get_parent_depend(task_ref) .map_err(|err| vec![err.into()])? .into_iter() .chain( @@ -254,33 +273,43 @@ impl<'ctx> Resolver<'ctx> { .insert(task.clone(), ResolveState::Resolving); let mut ret = Vec::new(); - let mut handle_errors = |errors: Vec<Error<'ctx>>| { + let mut handle_errors = |errors: Vec<Error<'ctx>>| -> Result<(), ()> { for mut error in errors { error.extend(task); ret.push(error); + + if ret.len() > MAX_ERRORS { + ret.push(Error::too_many_errors()); + return Err(()); + } } + Ok(()) }; - match self.ctx.get_inherit_depend(task) { - Ok(Some(inherit)) => { - handle_errors(self.add_task(&inherit, None)); - } - Ok(None) => {} - Err(err) => { - handle_errors(vec![err.into()]); + let _ = (|| -> Result<(), ()> { + match self.ctx.get_parent_depend(task) { + Ok(Some(parent)) => { + handle_errors(self.add_task(&parent, None))?; + } + Ok(None) => {} + Err(err) => { + handle_errors(vec![err.into()])?; + } } - } - match get_dependent_outputs(self.ctx, task) { - Ok(rdeps) => { - for rdep in rdeps { - handle_errors(self.add_task(&rdep.task, Some(rdep.output))); + match get_dependent_outputs(self.ctx, task) { + Ok(rdeps) => { + for rdep in rdeps { + handle_errors(self.add_task(&rdep.task, Some(rdep.output)))?; + } + } + Err(errors) => { + handle_errors(errors)?; } } - Err(errors) => { - handle_errors(errors); - } - } + + Ok(()) + })(); if ret.is_empty() { *self diff --git a/crates/driver/src/paths.rs b/crates/rebel-resolve/src/paths.rs index 274dda1..274dda1 100644 --- a/crates/driver/src/paths.rs +++ b/crates/rebel-resolve/src/paths.rs diff --git a/crates/driver/src/pin.rs b/crates/rebel-resolve/src/pin.rs index 26e445c..bffc940 100644 --- a/crates/driver/src/pin.rs +++ b/crates/rebel-resolve/src/pin.rs @@ -1,8 +1,8 @@ -use std::{collections::HashMap, fs::File, path::Path}; +use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use common::{error::*, string_hash::*}; +use rebel_common::string_hash::*; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Args { @@ -29,11 +29,3 @@ pub struct Pin { } pub type Pins = HashMap<String, Pin>; - -pub fn read_pins<P: AsRef<Path>>(path: P) -> Result<Pins> { - let f = File::open(path)?; - let pins: Pins = serde_yaml::from_reader(f) - .map_err(Error::new) - .context("YAML error")?; - Ok(pins) -} diff --git a/crates/driver/src/task.rs b/crates/rebel-resolve/src/task.rs index df3bc68..1220d45 100644 --- a/crates/driver/src/task.rs +++ b/crates/rebel-resolve/src/task.rs @@ -2,28 +2,26 @@ use std::collections::{HashMap, HashSet}; use serde::Deserialize; -use common::{string_hash::StringHash, types::TaskID}; +use rebel_common::{string_hash::StringHash, types::TaskIDRef}; -use crate::{ - args::{ArgMapping, ArgType, TaskArgs}, - recipe, -}; - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash, Default)] -pub struct RecipeMeta { - #[serde(default)] - pub name: String, - pub version: Option<String>, -} +use crate::args::{ArgMapping, ArgType, TaskArgs}; #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] pub struct TaskDep { - #[serde(flatten, deserialize_with = "recipe::deserialize_task_id")] - pub id: TaskID, + pub recipe: Option<String>, + pub task: String, #[serde(default)] pub args: ArgMapping, } +impl TaskDep { + pub fn id<'a>(&'a self, recipe: &'a str) -> TaskIDRef<'a> { + let recipe = self.recipe.as_deref().unwrap_or(recipe); + let task = &self.task; + TaskIDRef { recipe, task } + } +} + #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] pub struct Fetch { pub name: String, @@ -35,7 +33,7 @@ fn default_output_name() -> String { } #[derive(Clone, Debug, Deserialize)] -pub struct InheritDep { +pub struct ParentDep { #[serde(flatten)] pub dep: TaskDep, } @@ -69,14 +67,23 @@ impl Action { } } +#[derive(Clone, Debug, Default)] +pub struct TaskMeta { + pub basename: String, + pub recipename: String, + pub recipe: String, + pub name: String, + pub version: Option<String>, +} + #[derive(Clone, Debug, Deserialize, Default)] pub struct TaskDef { #[serde(skip)] - pub meta: RecipeMeta, + pub meta: TaskMeta, #[serde(default)] pub args: HashMap<String, ArgType>, #[serde(default)] - pub inherit: Option<InheritDep>, + pub parent: Option<ParentDep>, #[serde(default)] pub fetch: HashSet<Fetch>, #[serde(default)] diff --git a/crates/runner/Cargo.toml b/crates/rebel-runner/Cargo.toml index bd1287e..3df06f8 100644 --- a/crates/runner/Cargo.toml +++ b/crates/rebel-runner/Cargo.toml @@ -8,14 +8,14 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -common = { path = "../common", package = "rebel-common" } +rebel-common = { path = "../rebel-common" } bincode = "1.3.3" blake3 = { version = "1.3.0", features = ["traits-preview"] } capctl = "0.2.0" digest = "0.10.1" libc = "0.2.84" -nix = { version = "0.27.1", features = ["user", "fs", "process", "mount", "sched", "poll", "signal", "hostname"] } +nix = { version = "0.28.0", features = ["user", "fs", "process", "mount", "sched", "poll", "signal", "hostname", "resource"] } olpc-cjson = "0.1.0" serde = { version = "1", features = ["derive"] } serde_json = "1.0.62" diff --git a/crates/runner/src/init.rs b/crates/rebel-runner/src/init.rs index ede8fd8..0172a01 100644 --- a/crates/runner/src/init.rs +++ b/crates/rebel-runner/src/init.rs @@ -1,6 +1,6 @@ use nix::mount::{self, MsFlags}; -use common::error::*; +use rebel_common::error::*; use crate::{paths, util::fs}; diff --git a/crates/runner/src/jobserver.rs b/crates/rebel-runner/src/jobserver.rs index 2422a75..7c3f2f7 100644 --- a/crates/runner/src/jobserver.rs +++ b/crates/rebel-runner/src/jobserver.rs @@ -1,11 +1,11 @@ use std::{ - os::fd::{AsRawFd, FromRawFd, OwnedFd}, + os::fd::{AsFd, AsRawFd, OwnedFd}, slice, }; use nix::{errno::Errno, fcntl::OFlag, poll, unistd}; -use common::error::*; +use rebel_common::error::*; use super::util::unix; @@ -18,13 +18,10 @@ pub struct Jobserver { impl Jobserver { pub fn new(tokens: usize) -> Result<Jobserver> { - let (piper, pipew) = - unistd::pipe2(OFlag::O_CLOEXEC | OFlag::O_NONBLOCK).context("pipe()")?; - let r = unsafe { OwnedFd::from_raw_fd(piper) }; - let w = unsafe { OwnedFd::from_raw_fd(pipew) }; + let (r, w) = unistd::pipe2(OFlag::O_CLOEXEC | OFlag::O_NONBLOCK).context("pipe()")?; for _ in 0..tokens { - if unistd::write(w.as_raw_fd(), b"+").is_err() { + if unistd::write(w.as_fd(), b"+").is_err() { break; } } @@ -36,8 +33,8 @@ impl Jobserver { pub fn wait(&mut self) -> u8 { loop { poll::poll( - &mut [poll::PollFd::new(&self.r, poll::PollFlags::POLLIN)], - -1, + &mut [poll::PollFd::new(self.r.as_fd(), poll::PollFlags::POLLIN)], + poll::PollTimeout::NONE, ) .expect("poll()"); @@ -59,7 +56,7 @@ impl Jobserver { } pub fn post(&mut self, token: u8) { - let n = unistd::write(self.w.as_raw_fd(), slice::from_ref(&token)).expect("write()"); + let n = unistd::write(self.w.as_fd(), slice::from_ref(&token)).expect("write()"); assert!(n == 1); } diff --git a/crates/runner/src/lib.rs b/crates/rebel-runner/src/lib.rs index 308b54c..7dde05d 100644 --- a/crates/runner/src/lib.rs +++ b/crates/rebel-runner/src/lib.rs @@ -17,6 +17,7 @@ use std::{ use capctl::prctl; use nix::{ errno::Errno, + fcntl::Flock, poll, sched::CloneFlags, sys::{ @@ -28,7 +29,7 @@ use nix::{ }; use uds::UnixSeqpacketConn; -use common::{error::*, types::*}; +use rebel_common::{error::*, types::*}; use jobserver::Jobserver; use util::{checkable::Checkable, clone, steal::Steal, unix}; @@ -99,7 +100,14 @@ fn borrow_socket_fd(socket: &UnixSeqpacketConn) -> BorrowedFd<'_> { unsafe { BorrowedFd::borrow_raw(socket.as_raw_fd()) } } -fn runner(uid: Uid, gid: Gid, socket: UnixSeqpacketConn, _lockfile: File, options: &Options) -> ! { +fn runner( + uid: Uid, + gid: Gid, + socket: UnixSeqpacketConn, + _lockfile: Flock<File>, + options: &Options, +) -> ! { + unistd::setsid().expect("setsid()"); ns::mount_proc(); ns::setup_userns(Uid::from_raw(0), Gid::from_raw(0), uid, gid); @@ -127,10 +135,10 @@ fn runner(uid: Uid, gid: Gid, socket: UnixSeqpacketConn, _lockfile: File, option loop { let socket_fd = borrow_socket_fd(&ctx.socket); let mut pollfds = [ - poll::PollFd::new(&signal_fd, poll::PollFlags::POLLIN), - poll::PollFd::new(&socket_fd, poll::PollFlags::POLLIN), + poll::PollFd::new(signal_fd.as_fd(), poll::PollFlags::POLLIN), + poll::PollFd::new(socket_fd.as_fd(), poll::PollFlags::POLLIN), ]; - poll::poll(&mut pollfds, -1).expect("poll()"); + poll::poll(&mut pollfds, poll::PollTimeout::NONE).expect("poll()"); let signal_events = pollfds[0] .revents() diff --git a/crates/runner/src/ns.rs b/crates/rebel-runner/src/ns.rs index 4a8e3e7..986aa80 100644 --- a/crates/runner/src/ns.rs +++ b/crates/rebel-runner/src/ns.rs @@ -4,7 +4,7 @@ use nix::{ unistd::{self, Gid, Pid, Uid}, }; -use common::error::*; +use rebel_common::error::*; use super::util::clone; diff --git a/crates/runner/src/paths.rs b/crates/rebel-runner/src/paths.rs index 4b3a126..84f9c4d 100644 --- a/crates/runner/src/paths.rs +++ b/crates/rebel-runner/src/paths.rs @@ -36,7 +36,7 @@ //! └── rootfs/ # rootfs overlay mountpoint //! ``` -use common::string_hash::*; +use rebel_common::string_hash::*; pub const DOWNLOADS_DIR: &str = "build/downloads"; pub const PIN_DIR: &str = "build/pinned"; diff --git a/crates/runner/src/tar.rs b/crates/rebel-runner/src/tar.rs index 1a66408..891c603 100644 --- a/crates/runner/src/tar.rs +++ b/crates/rebel-runner/src/tar.rs @@ -11,7 +11,7 @@ use nix::{ sys::wait, }; -use common::{error::*, string_hash::ArchiveHash}; +use rebel_common::{error::*, string_hash::ArchiveHash}; use super::{ ns, diff --git a/crates/runner/src/task.rs b/crates/rebel-runner/src/task.rs index b716b82..5bb253a 100644 --- a/crates/runner/src/task.rs +++ b/crates/rebel-runner/src/task.rs @@ -11,13 +11,13 @@ use capctl::prctl; use nix::{ mount::{self, MsFlags}, sched::{unshare, CloneFlags}, - sys::wait, + sys::{resource, time::TimeVal, wait}, unistd::{self, Gid, Uid}, }; use serde::Serialize; use tee_readwrite::{TeeReader, TeeWriter}; -use common::{error::*, string_hash::*, types::*}; +use rebel_common::{error::*, string_hash::*, types::*}; use walkdir::WalkDir; use super::{ @@ -52,7 +52,7 @@ fn input_hash(task: &Task) -> InputHash { pub command: &'a str, pub workdir: &'a str, pub rootfs: &'a ArchiveHash, - pub inherit: &'a [LayerHash], + pub ancestors: &'a [LayerHash], pub depends: HashMap<DependencyHash, &'a Dependency>, pub outputs: &'a HashMap<String, String>, } @@ -60,7 +60,7 @@ fn input_hash(task: &Task) -> InputHash { command: &task.command, workdir: &task.workdir, rootfs: &task.rootfs, - inherit: &task.inherit, + ancestors: &task.ancestors, depends: task .depends .iter() @@ -94,7 +94,7 @@ fn init_task(input_hash: &InputHash, task: &Task) -> Result<fs::Mount> { .with_context(|| format!("Failed to write {}", runfile))?; let mount_target = paths::join(&[&task_tmp_dir, &task.workdir]); - let mount = if task.inherit.is_empty() { + let mount = if task.ancestors.is_empty() { fs::mount(task_layer_dir, &mount_target, None, MsFlags::MS_BIND, None) .with_context(|| format!("Failed to bind mount to {:?}", mount_target))? } else { @@ -104,7 +104,7 @@ fn init_task(input_hash: &InputHash, task: &Task) -> Result<fs::Mount> { fs::fixup_permissions(&task_work_dir)?; let lower = task - .inherit + .ancestors .iter() .rev() .map(paths::layer_dir) @@ -452,7 +452,6 @@ fn run_task(input_hash: &InputHash, task: &Task, jobserver: &mut Jobserver) -> R .env_clear() .env("PATH", "/usr/sbin:/usr/bin:/sbin:/bin") .env("HOME", "/build") - .env("INPUT_HASH", input_hash.to_string()) .env("MAKEFLAGS", jobserver.to_makeflags()) .exec(); eprintln!("{}", err); @@ -569,6 +568,26 @@ fn save_cached(input_hash: &InputHash, output: &TaskOutput) -> Result<()> { Ok(()) } +trait AsSecsF32 { + fn as_secs_f32(&self) -> f32; +} + +impl AsSecsF32 for TimeVal { + fn as_secs_f32(&self) -> f32 { + self.tv_sec() as f32 + 1e-6 * (self.tv_usec() as f32) + } +} + +fn get_usage(total: f32) -> String { + let usage = resource::getrusage(resource::UsageWho::RUSAGE_CHILDREN).expect("getrusage()"); + + let user = usage.user_time().as_secs_f32(); + let system = usage.system_time().as_secs_f32(); + let cpu = (100.0 * (user + system) / total).round(); + + format!("{user:.2}s user {system:.2}s system {cpu:.0}% cpu") +} + pub fn handle(task: Task, jobserver: &mut Jobserver) -> Result<TaskOutput> { let input_hash = input_hash(&task); @@ -597,12 +616,11 @@ pub fn handle(task: Task, jobserver: &mut Jobserver) -> Result<TaskOutput> { save_cached(&input_hash, &task_output)?; - let duration = Instant::now().duration_since(start_time); + let duration = Instant::now().duration_since(start_time).as_secs_f32(); + let usage = get_usage(duration); println!( - "Finished task {} ({}) in {}", - task.label, - input_hash, - duration.as_secs_f32() + "Finished task {} ({}) in {:.2}s ({})", + task.label, input_hash, duration, usage, ); if let Ok(cached_output) = cached_output { diff --git a/crates/runner/src/util/checkable.rs b/crates/rebel-runner/src/util/checkable.rs index 8528d29..8528d29 100644 --- a/crates/runner/src/util/checkable.rs +++ b/crates/rebel-runner/src/util/checkable.rs diff --git a/crates/runner/src/util/cjson.rs b/crates/rebel-runner/src/util/cjson.rs index e3840ce..e3840ce 100644 --- a/crates/runner/src/util/cjson.rs +++ b/crates/rebel-runner/src/util/cjson.rs diff --git a/crates/runner/src/util/clone.rs b/crates/rebel-runner/src/util/clone.rs index 51a31c3..51a31c3 100644 --- a/crates/runner/src/util/clone.rs +++ b/crates/rebel-runner/src/util/clone.rs diff --git a/crates/runner/src/util/fs.rs b/crates/rebel-runner/src/util/fs.rs index 9e16648..9e33eb7 100644 --- a/crates/runner/src/util/fs.rs +++ b/crates/rebel-runner/src/util/fs.rs @@ -11,7 +11,7 @@ use nix::{ unistd, }; -use common::error::*; +use rebel_common::error::*; pub fn open<P: AsRef<Path>>(path: P) -> Result<fs::File> { fs::File::open(path.as_ref()) @@ -123,5 +123,5 @@ pub fn mount<P1: AsRef<Path>, P2: AsRef<Path>>( pub fn pipe() -> Result<(File, File)> { unistd::pipe2(OFlag::O_CLOEXEC) .context("pipe2()") - .map(|(piper, pipew)| unsafe { (File::from_raw_fd(piper), File::from_raw_fd(pipew)) }) + .map(|(piper, pipew)| (File::from(piper), File::from(pipew))) } diff --git a/crates/runner/src/util/mod.rs b/crates/rebel-runner/src/util/mod.rs index 0fbe3b5..0fbe3b5 100644 --- a/crates/runner/src/util/mod.rs +++ b/crates/rebel-runner/src/util/mod.rs diff --git a/crates/runner/src/util/stack.rs b/crates/rebel-runner/src/util/stack.rs index 15d5daf..15d5daf 100644 --- a/crates/runner/src/util/stack.rs +++ b/crates/rebel-runner/src/util/stack.rs diff --git a/crates/runner/src/util/steal.rs b/crates/rebel-runner/src/util/steal.rs index 91b2cdf..91b2cdf 100644 --- a/crates/runner/src/util/steal.rs +++ b/crates/rebel-runner/src/util/steal.rs diff --git a/crates/runner/src/util/unix.rs b/crates/rebel-runner/src/util/unix.rs index 156e441..a97b1db 100644 --- a/crates/runner/src/util/unix.rs +++ b/crates/rebel-runner/src/util/unix.rs @@ -1,12 +1,12 @@ use std::{fs::File, os::unix::prelude::*, path::Path}; use nix::{ - fcntl::{self, FcntlArg, FdFlag, OFlag}, + fcntl::{self, FcntlArg, FdFlag, Flock, OFlag}, sched, unistd::Pid, }; -use common::error::*; +use rebel_common::error::*; use super::fs; @@ -65,7 +65,7 @@ pub fn nproc() -> Result<usize> { Ok(count) } -pub fn lock<P: AsRef<Path>>(path: P, exclusive: bool, blocking: bool) -> Result<File> { +pub fn lock<P: AsRef<Path>>(path: P, exclusive: bool, blocking: bool) -> Result<Flock<File>> { use fcntl::FlockArg::*; if let Some(parent) = path.as_ref().parent() { @@ -80,8 +80,7 @@ pub fn lock<P: AsRef<Path>>(path: P, exclusive: bool, blocking: bool) -> Result< }; let file = fs::create(path.as_ref())?; - fcntl::flock(file.as_raw_fd(), arg) - .with_context(|| format!("flock failed on {:?}", path.as_ref()))?; - - Ok(file) + fcntl::Flock::lock(file, arg) + .map_err(|(_, errno)| errno) + .with_context(|| format!("flock failed on {:?}", path.as_ref())) } diff --git a/crates/driver/Cargo.toml b/crates/rebel/Cargo.toml index ecea8b8..9eba0fa 100644 --- a/crates/driver/Cargo.toml +++ b/crates/rebel/Cargo.toml @@ -8,18 +8,16 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -common = { path = "../common", package = "rebel-common" } -runner = { path = "../runner", package = "rebel-runner" } +rebel-common = { path = "../rebel-common" } +rebel-parse = { path = "../rebel-parse" } +rebel-resolve = { path = "../rebel-resolve" } +rebel-runner = { path = "../rebel-runner" } clap = { version = "4.0.0", features = ["derive"] } -deb-version = "0.1.1" -enum-kinds = "0.5.1" -handlebars = "4.1.3" +handlebars = "5.1.2" indoc = "2.0.4" lazy_static = "1.4.0" -nix = { version = "0.27.1", features = ["poll"] } -nom = "7.1.0" -scoped-tls-hkt = "0.1.2" -serde = { version = "1", features = ["derive", "rc"] } +nix = { version = "0.28.0", features = ["poll", "signal"] } +serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" walkdir = "2" diff --git a/crates/rebel/src/driver.rs b/crates/rebel/src/driver.rs new file mode 100644 index 0000000..e4de2a7 --- /dev/null +++ b/crates/rebel/src/driver.rs @@ -0,0 +1,481 @@ +use std::{ + collections::{HashMap, HashSet}, + iter, + os::unix::{net::UnixStream, prelude::*}, +}; + +use indoc::indoc; +use nix::{ + poll, + sys::{ + signal, + signalfd::{SfdFlags, SignalFd}, + }, +}; + +use rebel_common::{error::*, string_hash::*, types::*}; +use rebel_resolve::{ + self as resolve, + context::{Context, OutputRef, TaskRef}, + paths, + task::*, +}; +use rebel_runner::Runner; + +use crate::template; + +#[derive(Debug)] +pub struct CompletionState<'ctx> { + ctx: &'ctx Context, + tasks_done: HashMap<TaskRef<'ctx>, TaskOutput>, +} + +impl<'ctx> CompletionState<'ctx> { + pub fn new(ctx: &'ctx Context) -> Self { + CompletionState { + ctx, + tasks_done: Default::default(), + } + } + + // Treats both "depends" and "parent" as dependencies + fn deps_satisfied(&self, task_ref: &TaskRef) -> bool { + resolve::get_dependent_tasks(self.ctx, task_ref) + .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref))) + .unwrap() + .into_iter() + .all(|dep| self.tasks_done.contains_key(&dep)) + } + + fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result<Vec<Dependency>> { + let task_def = &self.ctx[task]; + task_def + .fetch + .iter() + .map(|Fetch { name, sha256 }| { + Ok(Dependency::Fetch { + name: template::ENGINE.eval(name, &task.args).with_context(|| { + format!("Failed to evaluate fetch filename for task {}", task) + })?, + target_dir: paths::TASK_DLDIR.to_string(), + sha256: *sha256, + }) + }) + .collect() + } + + fn dep_closure<I>(&self, deps: I, path: &'ctx str) -> impl Iterator<Item = Dependency> + '_ + where + I: IntoIterator<Item = OutputRef<'ctx>>, + { + resolve::runtime_depends(self.ctx, deps) + .expect("invalid runtime depends") + .into_iter() + .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) + .map(|&output| Dependency::Task { + output, + path: path.to_string(), + }) + } + + fn build_deps(&self, task: &TaskRef<'ctx>) -> Result<impl Iterator<Item = Dependency> + '_> { + Ok(self.dep_closure( + self.ctx + .get_build_depends(task) + .with_context(|| format!("invalid build depends for {}", task))?, + "", + )) + } + + fn host_deps(&self, task: &TaskRef<'ctx>) -> Result<impl Iterator<Item = Dependency> + '_> { + Ok(self.dep_closure( + self.ctx + .get_host_depends(task) + .with_context(|| format!("invalid depends for {}", task))?, + paths::TASK_SYSROOT, + )) + } + + fn task_deps(&self, task: &TaskRef<'ctx>) -> Result<HashSet<Dependency>> { + let fetch_deps = self.fetch_deps(task)?.into_iter(); + let build_deps = self.build_deps(task)?; + let host_deps = self.host_deps(task)?; + + Ok(fetch_deps.chain(build_deps).chain(host_deps).collect()) + } + + fn task_ancestors(&self, task_ref: &TaskRef<'ctx>) -> Vec<LayerHash> { + let Some(parent) = self + .ctx + .get_parent_depend(task_ref) + .expect("invalid parent depends") + else { + return vec![]; + }; + + let mut chain = self.task_ancestors(&parent); + if let Some(layer) = self.tasks_done[&parent].layer { + chain.push(layer); + } + chain + } + + fn print_summary(&self) { + println!(); + println!("Summary:"); + + let mut tasks: Box<[_]> = self.tasks_done.iter().collect(); + tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task)); + for (task_ref, task) in tasks.iter() { + println!(); + println!("{:#}", task_ref); + if let Some(hash) = task.input_hash { + println!(" input: {}", hash); + } + if let Some(hash) = task.layer { + println!(" layer: {}", hash); + } + if !task.outputs.is_empty() { + println!(" outputs:"); + + let mut outputs: Box<[_]> = task.outputs.iter().collect(); + outputs.sort_by_key(|(output, _)| *output); + for (output, hash) in outputs.iter() { + println!(" {}: {}", output, hash); + } + } + } + } +} + +#[derive(Debug)] +enum SpawnResult { + Spawned(UnixStream), + Skipped(TaskOutput), +} + +#[derive(Debug, PartialEq, Eq, Hash)] +enum TaskWaitResult { + Failed, + Interrupted, +} + +#[derive(Debug)] +pub struct Driver<'ctx> { + rdeps: HashMap<TaskRef<'ctx>, Vec<TaskRef<'ctx>>>, + force_run: HashSet<TaskRef<'ctx>>, + tasks_blocked: HashSet<TaskRef<'ctx>>, + tasks_runnable: Vec<TaskRef<'ctx>>, + tasks_running: HashMap<RawFd, (UnixStream, TaskRef<'ctx>)>, + state: CompletionState<'ctx>, +} + +impl<'ctx> Driver<'ctx> { + pub fn new( + ctx: &'ctx Context, + taskset: HashSet<TaskRef<'ctx>>, + force_run: HashSet<TaskRef<'ctx>>, + ) -> Result<Self> { + let mut driver = Driver { + rdeps: Default::default(), + force_run, + tasks_blocked: Default::default(), + tasks_runnable: Default::default(), + tasks_running: Default::default(), + state: CompletionState::new(ctx), + }; + + for task in taskset { + let mut has_depends = false; + for dep in resolve::get_dependent_tasks(ctx, &task) + .map_err(|_| Error::new(format!("invalid dependency for {}", task)))? + { + let rdep = driver.rdeps.entry(dep.clone()).or_default(); + rdep.push(task.clone()); + has_depends = true; + } + + if has_depends { + driver.tasks_blocked.insert(task); + } else { + driver.tasks_runnable.push(task); + } + } + + Ok(driver) + } + + const PREAMBLE: &'static str = indoc! {" + export PATH={{build.prefix}}/sbin:{{build.prefix}}/bin:$PATH + cd {{workdir}} + + export SOURCE_DATE_EPOCH=1 + + export AR_FOR_BUILD=ar + export AS_FOR_BUILD=as + export DLLTOOL_FOR_BUILD=dlltool + export CC_FOR_BUILD=gcc + export CXX_FOR_BUILD=g++ + export GCC_FOR_BUILD=gcc + export GFORTRAN_FOR_BUILD=gfortran + export GOC_FOR_BUILD=goc + export LD_FOR_BUILD=ld + export LIPO_FOR_BUILD=lipo + export NM_FOR_BUILD=nm + export OBJCOPY_FOR_BUILD=objcopy + export OBJDUMP_FOR_BUILD=objdump + export RANLIB_FOR_BUILD=ranlib + export STRIP_FOR_BUILD=strip + export WINDRES_FOR_BUILD=windres + export WINDMC_FOR_BUILD=windmc + "}; + const PREAMBLE_HOST: &'static str = indoc! {" + export AR={{build_to_host.cross_compile}}ar + export AS={{build_to_host.cross_compile}}as + export DLLTOOL={{build_to_host.cross_compile}}dlltool + export CC={{build_to_host.cross_compile}}gcc + export CXX={{build_to_host.cross_compile}}g++ + export GCC={{build_to_host.cross_compile}}gcc + export GFORTRAN={{build_to_host.cross_compile}}gfortran + export GOC={{build_to_host.cross_compile}}goc + export LD={{build_to_host.cross_compile}}ld + export LIPO={{build_to_host.cross_compile}}lipo + export NM={{build_to_host.cross_compile}}nm + export OBJCOPY={{build_to_host.cross_compile}}objcopy + export OBJDUMP={{build_to_host.cross_compile}}objdump + export RANLIB={{build_to_host.cross_compile}}ranlib + export STRIP={{build_to_host.cross_compile}}strip + export WINDRES={{build_to_host.cross_compile}}windres + export WINDMC={{build_to_host.cross_compile}}windmc + "}; + const PREAMBLE_TARGET: &'static str = indoc! {" + export AR_FOR_TARGET={{build_to_target.cross_compile}}ar + export AS_FOR_TARGET={{build_to_target.cross_compile}}as + export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool + export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc + export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++ + export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc + export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran + export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc + export LD_FOR_TARGET={{build_to_target.cross_compile}}ld + export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo + export NM_FOR_TARGET={{build_to_target.cross_compile}}nm + export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy + export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump + export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib + export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip + export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres + export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc + "}; + + fn task_preamble(task_ref: &TaskRef<'ctx>) -> Vec<&'static str> { + let mut ret = vec![Self::PREAMBLE]; + + if task_ref.args.contains_key("build_to_host") { + ret.push(Self::PREAMBLE_HOST); + } + if task_ref.args.contains_key("build_to_target") { + ret.push(Self::PREAMBLE_TARGET); + } + ret + } + + fn update_runnable(&mut self, task_ref: TaskRef<'ctx>, task_output: TaskOutput) { + let rdeps = self.rdeps.get(&task_ref); + + self.state.tasks_done.insert(task_ref, task_output); + + for rdep in rdeps.unwrap_or(&Vec::new()) { + if !self.tasks_blocked.contains(rdep) { + continue; + } + if self.state.deps_satisfied(rdep) { + self.tasks_blocked.remove(rdep); + self.tasks_runnable.push(rdep.clone()); + } + } + } + + fn spawn_task(&self, task_ref: &TaskRef<'ctx>, runner: &Runner) -> Result<SpawnResult> { + let task_def = &self.state.ctx[task_ref]; + if task_def.action.is_empty() { + println!("Skipping empty task {:#}", task_ref); + return Ok(SpawnResult::Skipped(TaskOutput::default())); + } + + let task_deps = self.state.task_deps(task_ref)?; + let task_output = task_def + .output + .iter() + .map(|(name, Output { path, .. })| { + let output_path = if let Some(path) = path { + format!("{}/{}", paths::TASK_DESTDIR, path) + } else { + paths::TASK_DESTDIR.to_string() + }; + (name.clone(), output_path) + }) + .collect(); + + let ancestors = self.state.task_ancestors(task_ref); + + let mut run = Self::task_preamble(task_ref); + run.push(&task_def.action.run); + + let command = template::ENGINE + .eval_sh(&run.concat(), &task_ref.args) + .with_context(|| { + format!("Failed to evaluate command template for task {}", task_ref) + })?; + + let rootfs = self.state.ctx.get_rootfs(); + let task = Task { + label: format!("{:#}", task_ref), + command, + workdir: paths::TASK_WORKDIR.to_string(), + rootfs: rootfs.0, + ancestors, + depends: task_deps, + outputs: task_output, + pins: HashMap::from([rootfs.clone()]), + force_run: self.force_run.contains(task_ref), + }; + + Ok(SpawnResult::Spawned(runner.spawn(&task))) + } + + fn run_task(&mut self, task_ref: TaskRef<'ctx>, runner: &Runner) -> Result<()> { + match self.spawn_task(&task_ref, runner)? { + SpawnResult::Spawned(socket) => { + assert!(self + .tasks_running + .insert(socket.as_raw_fd(), (socket, task_ref)) + .is_none()); + } + SpawnResult::Skipped(result) => { + self.update_runnable(task_ref, result); + } + } + Ok(()) + } + + fn run_tasks(&mut self, runner: &Runner) -> Result<()> { + while let Some(task_ref) = self.tasks_runnable.pop() { + self.run_task(task_ref, runner)?; + } + Ok(()) + } + + fn wait_for_task(&mut self, signal_fd: &mut SignalFd) -> Result<Option<TaskWaitResult>> { + let mut pollfds: Vec<_> = self + .tasks_running + .values() + .map(|(socket, _)| socket.as_fd()) + .chain(iter::once(signal_fd.as_fd())) + .map(|fd| poll::PollFd::new(fd, poll::PollFlags::POLLIN)) + .collect(); + + while poll::poll(&mut pollfds, poll::PollTimeout::NONE).context("poll()")? == 0 {} + + let pollevents: Vec<_> = pollfds + .into_iter() + .map(|pollfd| { + ( + pollfd.as_fd().as_raw_fd(), + pollfd.revents().expect("Unknown events in poll() return"), + ) + }) + .collect(); + + for (fd, events) in pollevents { + if !events.contains(poll::PollFlags::POLLIN) { + if events.intersects(!poll::PollFlags::POLLIN) { + return Err(Error::new( + "Unexpected error status for socket file descriptor", + )); + } + continue; + } + + if fd == signal_fd.as_raw_fd() { + let _signal = signal_fd.read_signal().expect("read_signal()").unwrap(); + return Ok(Some(TaskWaitResult::Interrupted)); + } + + let (socket, task_ref) = self.tasks_running.remove(&fd).unwrap(); + + match Runner::result(&socket) { + Ok(task_output) => { + self.update_runnable(task_ref, task_output); + } + Err(error) => { + eprintln!("{}", error); + return Ok(Some(TaskWaitResult::Failed)); + } + } + } + + Ok(None) + } + + fn is_done(&self) -> bool { + self.tasks_blocked.is_empty() + && self.tasks_runnable.is_empty() + && self.tasks_running.is_empty() + } + + fn setup_signalfd() -> Result<SignalFd> { + let mut signals = signal::SigSet::empty(); + signals.add(signal::Signal::SIGINT); + signal::pthread_sigmask(signal::SigmaskHow::SIG_BLOCK, Some(&signals), None) + .expect("pthread_sigmask()"); + SignalFd::with_flags(&signals, SfdFlags::SFD_CLOEXEC) + .context("Failed to create signal file descriptor") + } + + fn raise_sigint() { + let mut signals = signal::SigSet::empty(); + signals.add(signal::Signal::SIGINT); + signal::pthread_sigmask(signal::SigmaskHow::SIG_UNBLOCK, Some(&signals), None) + .expect("pthread_sigmask()"); + signal::raise(signal::Signal::SIGINT).expect("raise()"); + unreachable!(); + } + + pub fn run(&mut self, runner: &Runner, keep_going: bool) -> Result<bool> { + let mut success = true; + let mut interrupted = false; + + let mut signal_fd = Self::setup_signalfd()?; + + self.run_tasks(runner)?; + + while !self.tasks_running.is_empty() { + match self.wait_for_task(&mut signal_fd)? { + Some(TaskWaitResult::Failed) => { + success = false; + } + Some(TaskWaitResult::Interrupted) => { + if interrupted { + Self::raise_sigint(); + } + eprintln!("Interrupt received, not spawning new tasks. Interrupt again to stop immediately."); + interrupted = true; + } + None => {} + } + if !interrupted && (success || keep_going) { + self.run_tasks(runner)?; + } + } + + if interrupted || !success { + return Ok(false); + } + + assert!(self.is_done(), "No runnable tasks left"); + self.state.print_summary(); + + Ok(true) + } +} diff --git a/crates/driver/src/main.rs b/crates/rebel/src/main.rs index 98aa10a..625b43d 100644 --- a/crates/driver/src/main.rs +++ b/crates/rebel/src/main.rs @@ -1,19 +1,15 @@ -mod args; -mod context; mod driver; -mod parse; -mod paths; -mod pin; mod recipe; -mod resolve; -mod task; mod template; -use std::collections::HashSet; +use std::{collections::HashSet, fs::File, path::Path}; use clap::Parser; -use runner::Runner; +use rebel_common::error::*; +use rebel_parse as parse; +use rebel_resolve::{self as resolve, context, pin}; +use rebel_runner::{self as runner, Runner}; #[derive(Parser)] #[clap(version, about)] @@ -22,11 +18,22 @@ struct Opts { /// Defaults to the number of available CPUs #[clap(short, long)] jobs: Option<usize>, + /// Keep going after some tasks have failed + #[clap(short, long)] + keep_going: bool, /// The tasks to run #[clap(name = "task", required = true)] tasks: Vec<String>, } +fn read_pins<P: AsRef<Path>>(path: P) -> Result<pin::Pins> { + let f = File::open(path)?; + let pins: pin::Pins = serde_yaml::from_reader(f) + .map_err(Error::new) + .context("YAML error")?; + Ok(pins) +} + fn main() { let opts: Opts = Opts::parse(); @@ -34,7 +41,7 @@ fn main() { let ctx = context::Context::new( recipe::read_recipes("examples/recipes").unwrap(), - pin::read_pins("examples/pins.yml").unwrap(), + read_pins("examples/pins.yml").unwrap(), ) .unwrap(); @@ -42,7 +49,11 @@ fn main() { let mut force_run = HashSet::new(); for task in opts.tasks { - let (task_ref, flags) = match ctx.parse(&task) { + let Ok((parsed, flags)) = parse::task_ref::task_ref_with_flags(&task) else { + eprintln!("Invalid task syntax"); + std::process::exit(1); + }; + let task_ref = match ctx.lookup(parsed.id, parsed.args.host, parsed.args.target) { Ok(task_ref) => task_ref, Err(err) => { eprintln!("{}", err); @@ -62,8 +73,15 @@ fn main() { } let taskset = rsv.into_taskset(); let mut driver = driver::Driver::new(&ctx, taskset, force_run).unwrap(); - if let Err(error) = driver.run(&runner) { - eprintln!("{}", error); - std::process::exit(1); + match driver.run(&runner, opts.keep_going) { + Ok(success) => { + if !success { + std::process::exit(1); + } + } + Err(error) => { + eprintln!("{}", error); + std::process::exit(1); + } } } diff --git a/crates/rebel/src/recipe.rs b/crates/rebel/src/recipe.rs new file mode 100644 index 0000000..28cc84c --- /dev/null +++ b/crates/rebel/src/recipe.rs @@ -0,0 +1,167 @@ +use std::{collections::HashMap, ffi::OsStr, fs::File, path::Path}; + +use serde::{de::DeserializeOwned, Deserialize}; +use walkdir::WalkDir; + +use rebel_common::error::*; +use rebel_resolve::task::{TaskDef, TaskMeta}; + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct RecipeMeta { + pub name: Option<String>, + pub version: Option<String>, +} + +#[derive(Debug, Deserialize)] +struct Recipe { + #[serde(default)] + pub meta: RecipeMeta, + pub tasks: HashMap<String, TaskDef>, +} + +#[derive(Debug, Deserialize)] +struct Subrecipe { + pub tasks: HashMap<String, TaskDef>, +} + +fn read_yaml<T: DeserializeOwned>(path: &Path) -> Result<T> { + let f = File::open(path).context("IO error")?; + + let value: T = serde_yaml::from_reader(f) + .map_err(Error::new) + .context("YAML error")?; + + Ok(value) +} + +const RECIPE_NAME: &str = "build"; +const RECIPE_PREFIX: &str = "build."; + +fn recipe_name(path: &Path) -> Option<&str> { + if path.extension() != Some("yml".as_ref()) { + return None; + } + + let stem = path.file_stem()?.to_str()?; + if stem == RECIPE_NAME { + return Some(""); + } + stem.strip_prefix(RECIPE_PREFIX) +} + +fn handle_recipe_tasks( + tasks: &mut HashMap<String, HashMap<String, Vec<TaskDef>>>, + recipe_tasks: HashMap<String, TaskDef>, + meta: &TaskMeta, +) { + let task_map = match tasks.get_mut(&meta.recipe) { + Some(task_map) => task_map, + None => tasks.entry(meta.recipe.clone()).or_default(), + }; + + for (label, mut task) in recipe_tasks { + task.meta = meta.clone(); + task_map.entry(label).or_default().push(task); + } +} + +fn read_recipe_tasks( + path: &Path, + basename: &str, + tasks: &mut HashMap<String, HashMap<String, Vec<TaskDef>>>, +) -> Result<RecipeMeta> { + let recipe_def = read_yaml::<Recipe>(path)?; + + let name = recipe_def + .meta + .name + .as_deref() + .unwrap_or(basename) + .to_string(); + + let meta = TaskMeta { + basename: basename.to_string(), + recipename: "".to_string(), + recipe: basename.to_string(), + name, + version: recipe_def.meta.version.clone(), + }; + + handle_recipe_tasks(tasks, recipe_def.tasks, &meta); + + Ok(recipe_def.meta) +} + +fn read_subrecipe_tasks( + path: &Path, + basename: &str, + recipename: &str, + recipe_meta: &RecipeMeta, + tasks: &mut HashMap<String, HashMap<String, Vec<TaskDef>>>, +) -> Result<()> { + let recipe = format!("{basename}/{recipename}"); + let recipe_def = read_yaml::<Subrecipe>(path)?; + + let name = recipe_meta.name.as_deref().unwrap_or(basename).to_string(); + + let meta = TaskMeta { + basename: basename.to_string(), + recipename: recipename.to_string(), + recipe: recipe.clone(), + name, + version: recipe_meta.version.clone(), + }; + + handle_recipe_tasks(tasks, recipe_def.tasks, &meta); + + Ok(()) +} + +pub fn read_recipes<P: AsRef<Path>>( + path: P, +) -> Result<HashMap<String, HashMap<String, Vec<TaskDef>>>> { + let mut tasks = HashMap::<String, HashMap<String, Vec<TaskDef>>>::new(); + let mut recipe_metas = HashMap::<String, RecipeMeta>::new(); + + for entry in WalkDir::new(path) + .sort_by(|a, b| { + // Files are sorted first by stem, then by extension, so that + // recipe.yml will always be read before recipe.NAME.yml + let stem_cmp = a.path().file_stem().cmp(&b.path().file_stem()); + let ext_cmp = a.path().extension().cmp(&b.path().extension()); + stem_cmp.then(ext_cmp) + }) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if !path.is_file() { + continue; + } + + let Some(recipename) = recipe_name(path) else { + continue; + }; + let Some(basename) = path + .parent() + .and_then(Path::file_name) + .and_then(OsStr::to_str) + else { + continue; + }; + + if recipename.is_empty() { + recipe_metas.insert( + basename.to_string(), + read_recipe_tasks(path, basename, &mut tasks)?, + ); + } else { + let Some(recipe_meta) = recipe_metas.get(basename) else { + continue; + }; + read_subrecipe_tasks(path, basename, recipename, recipe_meta, &mut tasks)?; + } + } + + Ok(tasks) +} diff --git a/crates/driver/src/template.rs b/crates/rebel/src/template.rs index 7bb089c..50fb334 100644 --- a/crates/driver/src/template.rs +++ b/crates/rebel/src/template.rs @@ -1,42 +1,39 @@ use handlebars::Handlebars; use lazy_static::lazy_static; -use common::error::*; +use rebel_common::error::*; +use rebel_resolve::args::TaskArgs; -use crate::args::TaskArgs; - -fn escape(s: &str) -> String { +fn escape_sh(s: &str) -> String { format!("'{}'", s.replace('\'', "'\\''")) } #[derive(Debug)] pub struct TemplateEngine { tpl: Handlebars<'static>, - tpl_raw: Handlebars<'static>, + tpl_sh: Handlebars<'static>, } impl TemplateEngine { pub fn new() -> Self { let mut tpl = Handlebars::new(); tpl.set_strict_mode(true); - tpl.register_escape_fn(escape); - - let mut tpl_raw = Handlebars::new(); - tpl_raw.set_strict_mode(true); - tpl_raw.register_escape_fn(handlebars::no_escape); + tpl.register_escape_fn(handlebars::no_escape); - TemplateEngine { tpl, tpl_raw } - } + let mut tpl_sh = Handlebars::new(); + tpl_sh.set_strict_mode(true); + tpl_sh.register_escape_fn(escape_sh); - pub fn eval_raw(&self, input: &str, args: &TaskArgs) -> Result<String> { - self.tpl_raw - .render_template(input, args) - .map_err(Error::new) + TemplateEngine { tpl, tpl_sh } } pub fn eval(&self, input: &str, args: &TaskArgs) -> Result<String> { self.tpl.render_template(input, args).map_err(Error::new) } + + pub fn eval_sh(&self, input: &str, args: &TaskArgs) -> Result<String> { + self.tpl_sh.render_template(input, args).map_err(Error::new) + } } lazy_static! { diff --git a/examples/pins.yml.example b/examples/pins.yml.example index f4dc98f..c16cedb 100644 --- a/examples/pins.yml.example +++ b/examples/pins.yml.example @@ -2,12 +2,12 @@ rootfs-8d1fb3274ebc: hash: '8d1fb3274ebcb0f7ae88f3ce5810ad20014d08efca29e617a873670949de729d' is-rootfs: true provides: - - recipe: 'gcc-libs' + - recipe: 'gcc/libs' task: 'install' output: ['default'] args: host: 'build' - - recipe: 'linux-uapi-headers' + - recipe: 'linux/uapi-headers' task: 'install' output: ['default'] args: @@ -54,7 +54,7 @@ rootfs-8d1fb3274ebc: args: host: 'build' target: 'build' - - recipe: 'libgcc' + - recipe: 'gcc/libgcc' task: 'install' output: ['default'] args: diff --git a/examples/recipes/bar.yml b/examples/recipes/bar/build.yml index f25322e..f25322e 100644 --- a/examples/recipes/bar.yml +++ b/examples/recipes/bar/build.yml diff --git a/examples/recipes/binutils@2.41.yml b/examples/recipes/binutils/build.yml index d6271a9..dd82d2e 100644 --- a/examples/recipes/binutils@2.41.yml +++ b/examples/recipes/binutils/build.yml @@ -1,16 +1,18 @@ +meta: + version: '2.41' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: 'ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz configure: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -19,9 +21,9 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --target={{target.gnu_triplet}} \ @@ -42,10 +44,10 @@ tasks: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make find -name config.log -delete @@ -53,7 +55,7 @@ tasks: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -61,5 +63,5 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install diff --git a/examples/recipes/busybox@1.36.1.yml b/examples/recipes/busybox/build.yml index abed443..fa87bc3 100644 --- a/examples/recipes/busybox@1.36.1.yml +++ b/examples/recipes/busybox/build.yml @@ -1,27 +1,29 @@ +meta: + version: '1.36.1' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.bz2' + - name: '{{name}}-{{version}}.tar.bz2' sha256: 'b8cc24c9574d809e7279c3be349795c5d5ceb6fdf19ca709f80cde50e47de314' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.bz2 + tar xf {{dldir}}/{{name}}-{{version}}.tar.bz2 configure: - inherit: + parent: task: 'unpack' run: | - mkdir {{pn}}-build - cd {{pn}}-build + mkdir {{name}}-build + cd {{name}}-build make \ - -f ../{{pn}}-{{pv}}/Makefile \ - KBUILD_SRC=../{{pn}}-{{pv}} \ + -f ../{{name}}-{{version}}/Makefile \ + KBUILD_SRC=../{{name}}-{{version}} \ defconfig compile: args: host: 'platform' - inherit: + parent: task: 'configure' build_depends: - recipe: 'toolchain' @@ -30,21 +32,21 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - cd {{pn}}-build + cd {{name}}-build make \ ARCH={{host.karch}} \ CROSS_COMPILE={{cross_compile}} \ EXTRA_CFLAGS='-DBB_EXTRA_VERSION=""' \ - {{pn}} busybox.links + {{name}} busybox.links sed -i \ -e 's@^/usr@@' \ -e 's@^/sbin@/bin@' \ - {{pn}}.links + {{name}}.links install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -52,7 +54,7 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - cd {{pn}}-build + cd {{name}}-build make \ ARCH={{host.karch}} \ CROSS_COMPILE={{cross_compile}} \ diff --git a/examples/recipes/e2fsprogs@1.47.0.yml b/examples/recipes/e2fsprogs/build.yml index 9925772..508d1d4 100644 --- a/examples/recipes/e2fsprogs@1.47.0.yml +++ b/examples/recipes/e2fsprogs/build.yml @@ -1,15 +1,17 @@ +meta: + version: '1.47.0' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: '144af53f2bbd921cef6f8bea88bb9faddca865da3fbc657cc9b4d2001097d5db' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -18,9 +20,9 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --prefix={{host.prefix}} @@ -29,17 +31,17 @@ tasks: compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make find -name config.log -delete install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -47,6 +49,6 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install diff --git a/examples/recipes/foo.yml b/examples/recipes/foo/build.yml index 2982795..2982795 100644 --- a/examples/recipes/foo.yml +++ b/examples/recipes/foo/build.yml diff --git a/examples/recipes/libgcc-initial@13.2.0.yml b/examples/recipes/gcc/build.libgcc-initial.yml index 43f6f45..ed230ba 100644 --- a/examples/recipes/libgcc-initial@13.2.0.yml +++ b/examples/recipes/gcc/build.libgcc-initial.yml @@ -1,18 +1,16 @@ -meta: - name: 'gcc' tasks: configure: args: host: 'platform' target: 'platform' - inherit: + parent: recipe: 'gcc' task: 'compile' depends: - recipe: 'gcc' task: 'header-stubs' run: | - cd {{pn}}-build + cd {{name}}-build make configure-target-libgcc # A hack borrowed from OpenEmbedded: @@ -30,22 +28,22 @@ tasks: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make all-target-libgcc install: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'compile' output: default: {} run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install-target-libgcc - ln -s libgcc.a {{destdir}}{{host.prefix}}/lib/gcc/{{target.gnu_triplet}}/"$(cat ../{{pn}}-{{pv}}/gcc/BASE-VER)"/libgcc_eh.a + ln -s libgcc.a {{destdir}}{{host.prefix}}/lib/gcc/{{target.gnu_triplet}}/"$(cat ../{{name}}-{{version}}/gcc/BASE-VER)"/libgcc_eh.a diff --git a/examples/recipes/libgcc@13.2.0.yml b/examples/recipes/gcc/build.libgcc.yml index 96a2ce8..67e21f7 100644 --- a/examples/recipes/libgcc@13.2.0.yml +++ b/examples/recipes/gcc/build.libgcc.yml @@ -1,11 +1,9 @@ -meta: - name: 'gcc' tasks: compile: args: host: 'platform' target: 'platform' - inherit: + parent: recipe: 'gcc' task: 'compile' depends: @@ -14,19 +12,19 @@ tasks: args: host: 'target' run: | - cd {{pn}}-build + cd {{name}}-build make all-target-libgcc install: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'compile' output: default: {} run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install-target-libgcc rm {{destdir}}{{host_to_target.sysroot}}{{target.prefix}}/lib/libgcc_s.so* rmdir -p --ignore-fail-on-non-empty {{destdir}}{{host_to_target.sysroot}}{{target.prefix}}/lib diff --git a/examples/recipes/gcc-libs@13.2.0.yml b/examples/recipes/gcc/build.libs.yml index 5320d29..856fd1c 100644 --- a/examples/recipes/gcc-libs@13.2.0.yml +++ b/examples/recipes/gcc/build.libs.yml @@ -1,29 +1,27 @@ -meta: - name: 'gcc' tasks: compile: args: host: 'platform' - inherit: - recipe: 'libgcc' + parent: + recipe: 'gcc/libgcc' task: 'compile' args: host: 'build' target: 'host' run: | - cd {{pn}}-build + cd {{name}}-build make all-target-libatomic all-target-libgomp all-target-libquadmath all-target-libstdc++-v3 install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: path: 'sysroot' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}}/install install-target-libgcc install-target-libatomic install-target-libgomp install-target-libquadmath install-target-libstdc++-v3 rm -r {{destdir}}/install{{build.prefix}}/lib/gcc mv -T {{destdir}}/install/{{build_to_host.sysroot}} {{destdir}}/sysroot diff --git a/examples/recipes/gcc@13.2.0.yml b/examples/recipes/gcc/build.yml index f59fd41..3352b2d 100644 --- a/examples/recipes/gcc@13.2.0.yml +++ b/examples/recipes/gcc/build.yml @@ -1,12 +1,14 @@ +meta: + version: '13.2.0' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: 'e275e76442a6067341a27f04c5c6b83d8613144004c0413528863dc6b5c743da' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz - sed -i -e 's@^MULTILIB_OSDIRNAMES@# &@' {{pn}}-{{pv}}/gcc/config/*/t-* + sed -i -e 's@^MULTILIB_OSDIRNAMES@# &@' {{name}}-{{version}}/gcc/config/*/t-* header-stubs: args: @@ -24,7 +26,7 @@ tasks: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -45,9 +47,9 @@ tasks: run: | export CXX_FOR_TARGET="$CXX_FOR_TARGET -nostdinc++" - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --target={{target.gnu_triplet}} \ @@ -58,7 +60,7 @@ tasks: --with-build-sysroot={{sysroot}} \ --with-native-system-header-dir={{target.prefix}}/include \ --with-toolexeclibdir={{host_to_target.sysroot}}{{target.prefix}}/lib \ - --with-gxx-include-dir={{host_to_target.sysroot}}{{target.prefix}}/include/c++/"$(cat ../{{pn}}-{{pv}}/gcc/BASE-VER)" \ + --with-gxx-include-dir={{host_to_target.sysroot}}{{target.prefix}}/include/c++/"$(cat ../{{name}}-{{version}}/gcc/BASE-VER)" \ --with-gnu-ld \ --enable-shared \ --enable-languages=c,c++ \ @@ -93,13 +95,13 @@ tasks: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'configure' depends: - task: 'header-stubs' noinherit: true run: | - cd {{pn}}-build + cd {{name}}-build make all-host find -name config.log -delete @@ -107,7 +109,7 @@ tasks: args: host: 'platform' target: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -119,6 +121,6 @@ tasks: - recipe: 'zlib' task: 'install' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install-host diff --git a/examples/recipes/glibc@2.38.yml b/examples/recipes/glibc/build.yml index 12fb7a4..f1bf890 100644 --- a/examples/recipes/glibc@2.38.yml +++ b/examples/recipes/glibc/build.yml @@ -1,30 +1,32 @@ +meta: + version: '2.38-25-gf6445dc94da1' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' - sha256: 'fb82998998b2b29965467bc1b69d152e9c307d2cf301c9eafb4555b770ef3fd2' + - name: '{{name}}-{{version}}.tar.xz' + sha256: '92594ade540761f70d57de4f3a150128e7b7f7ed653a59d1acc23468ab70fae9' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'gcc' task: 'install' - - recipe: 'libgcc-initial' + - recipe: 'gcc/libgcc-initial' task: 'install' depends: - - recipe: 'linux-uapi-headers' + - recipe: 'linux/uapi-headers' task: 'install' run: | export BUILD_CC="$CC_FOR_BUILD" - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --prefix={{host.prefix}} \ @@ -45,22 +47,22 @@ tasks: compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: runtime_depends: - - recipe: 'linux-uapi-headers' + - recipe: 'linux/uapi-headers' task: 'install' run: | - cd {{pn}}-build + cd {{name}}-build make install_root={{destdir}} install diff --git a/examples/recipes/gmp/build.recipe b/examples/recipes/gmp/build.recipe new file mode 100644 index 0000000..44e87d6 --- /dev/null +++ b/examples/recipes/gmp/build.recipe @@ -0,0 +1,69 @@ +// External definitions used by this recipe: +// +// workdir: String +// name: String +// destdir: String +// build: Platform +// build_depend: (task: TaskID) -> TaskDep +// host_depend: (task: TaskID) -> TaskDep + +version = "6.3.0"; +sourcedir = "{{workdir}}/{{name}}-{{version}}"; +builddir = "{{workdir}}/{{name}}-build"; + +fetch source { + url = ["https://invalid/{{name}}-{{version}}.tar.xz"]; + // TODO: Move to lockfile + sha256 = "a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898"; +} + +task unpack() { + depends = [source]; + + run = ``` + tar xf {{source.path}} + ```; +} + +task configure(host: Platform) { + parent = unpack(); + depends = [ + build_depend(toolchain::build_depends), + host_depend(toolchain::depends), + ]; + + run = ``` + mkdir {{builddir}} + cd {{builddir}} + {{sourcedir}}/configure \ + --build={{build.gnu_triplet}} \ + --host={{host.gnu_triplet}} \ + --prefix={{host.prefix}} + ```; +} + +task compile(host: Platform) { + parent = configure(host); + + run = ``` + cd {{builddir}} + make + ```; +} + +task install(host: Platform) { + parent = compile(host); + + output = { + default = { + runtime_depends = [host_depend(toolchain::depends)], + }, + }; + + run = ``` + cd {{builddir}} + make DESTDIR={{destdir}} install + rm {{destdir}}{{host.prefix}}/lib/*.a + rm {{destdir}}{{host.prefix}}/lib/*.la + ```; +} diff --git a/examples/recipes/gmp@6.3.0.yml b/examples/recipes/gmp/build.yml index 8ff3385..322f0f0 100644 --- a/examples/recipes/gmp@6.3.0.yml +++ b/examples/recipes/gmp/build.yml @@ -1,15 +1,17 @@ +meta: + version: '6.3.0' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: 'a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -18,9 +20,9 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --prefix={{host.prefix}} @@ -29,16 +31,16 @@ tasks: compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -46,7 +48,7 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install rm {{destdir}}{{host.prefix}}/lib/*.a rm {{destdir}}{{host.prefix}}/lib/*.la diff --git a/examples/recipes/image.yml b/examples/recipes/image/build.yml index 883fa3b..a42899f 100644 --- a/examples/recipes/image.yml +++ b/examples/recipes/image/build.yml @@ -15,12 +15,13 @@ tasks: mkdir -p \ {{destdir}}/dev \ {{destdir}}/proc \ - {{destdir}}/sys \ {{destdir}}/root \ {{destdir}}/run \ - {{destdir}}/var \ + {{destdir}}/sys \ + {{destdir}}/tmp \ {{destdir}}/usr/bin \ - {{destdir}}/usr/lib + {{destdir}}/usr/lib \ + {{destdir}}/var ln -s bin {{destdir}}/usr/sbin ln -s usr/bin {{destdir}}/bin @@ -44,13 +45,16 @@ tasks: output: default: {} run: | - IMAGE={{destdir}}/{{pn}}.ext4 - UUID="$( - python3 \ - -c 'import sys; import uuid; print(uuid.UUID(hex=sys.argv[1][:32], version=4))' \ - "$INPUT_HASH" - )" + IMAGE={{destdir}}/{{name}}.ext4 mkdir {{destdir}} make_ext4fs -l 2G -T "$SOURCE_DATE_EPOCH" -L root "$IMAGE" {{sysroot}} + E2FSPROGS_FAKE_TIME="$SOURCE_DATE_EPOCH" tune2fs -U '00000000-0000-4000-8000-000000000000' "$IMAGE" + + HASH=$(sha256sum "$IMAGE") + UUID=$( + python3 \ + -c 'import sys; import uuid; print(uuid.UUID(hex=sys.argv[1][:32], version=4))' \ + "$HASH" + ) E2FSPROGS_FAKE_TIME="$SOURCE_DATE_EPOCH" tune2fs -U "$UUID" "$IMAGE" diff --git a/examples/recipes/linux-uapi-headers@6.5.5.yml b/examples/recipes/linux/build.uapi-headers.yml index c431105..a302aa9 100644 --- a/examples/recipes/linux-uapi-headers@6.5.5.yml +++ b/examples/recipes/linux/build.uapi-headers.yml @@ -1,21 +1,19 @@ -meta: - name: 'linux' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: '8cf10379f7df8ea731e09bff3d0827414e4b643dd41dc99d0af339669646ef95' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz install: args: host: 'platform' - inherit: + parent: task: 'unpack' output: default: {} run: | - cd {{pn}}-{{pv}} + cd {{name}}-{{version}} make INSTALL_HDR_PATH={{destdir}}{{host.prefix}} ARCH={{host.karch}} headers_install diff --git a/examples/recipes/linux@6.5.5.yml b/examples/recipes/linux/build.yml index 77f5879..3f84291 100644 --- a/examples/recipes/linux@6.5.5.yml +++ b/examples/recipes/linux/build.yml @@ -1,24 +1,26 @@ +meta: + version: '6.5.5' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: '8cf10379f7df8ea731e09bff3d0827414e4b643dd41dc99d0af339669646ef95' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'gcc' task: 'install' run: | - mkdir {{pn}}-build - cd {{pn}}-build + mkdir {{name}}-build + cd {{name}}-build make \ - -f ../{{pn}}-{{pv}}/Makefile \ + -f ../{{name}}-{{version}}/Makefile \ ARCH={{host.karch}} \ CROSS_COMPILE={{cross_compile}} \ defconfig @@ -26,12 +28,12 @@ tasks: compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | export KBUILD_BUILD_TIMESTAMP="@${SOURCE_DATE_EPOCH}" - cd {{pn}}-build + cd {{name}}-build make \ ARCH={{host.karch}} \ CROSS_COMPILE={{cross_compile}} \ @@ -40,7 +42,7 @@ tasks: install: args: host: 'platform' - inherit: + parent: task: 'compile' output: boot: @@ -48,7 +50,7 @@ tasks: modules: path: 'modules' run: | - cd {{pn}}-build + cd {{name}}-build mkdir -p {{destdir}}/boot make \ ARCH={{host.karch}} \ diff --git a/examples/recipes/ls.yml b/examples/recipes/ls/build.yml index 633336d..633336d 100644 --- a/examples/recipes/ls.yml +++ b/examples/recipes/ls/build.yml diff --git a/examples/recipes/make_ext4fs@2020-01-05.yml b/examples/recipes/make_ext4fs/build.yml index 53fed98..183389a 100644 --- a/examples/recipes/make_ext4fs@2020-01-05.yml +++ b/examples/recipes/make_ext4fs/build.yml @@ -3,10 +3,10 @@ meta: tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.gz' + - name: '{{name}}-{{version}}.tar.gz' sha256: 'bfe984cc757c676639090a5b34bdfc359cdef9b723de77efbce590872dce4132' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.gz + tar xf {{dldir}}/{{name}}-{{version}}.tar.gz compile: args: @@ -19,16 +19,16 @@ tasks: task: 'depends' - recipe: 'zlib' task: 'install' - inherit: + parent: task: 'unpack' run: | - cd {{pn}}-{{pv}} + cd {{name}}-{{version}} make install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -38,7 +38,7 @@ tasks: - recipe: 'zlib' task: 'install' run: | - cd {{pn}}-{{pv}} + cd {{name}}-{{version}} install -d {{destdir}}{{host.prefix}}/bin install -m755 make_ext4fs {{destdir}}{{host.prefix}}/bin/ diff --git a/examples/recipes/mpc@1.3.1.yml b/examples/recipes/mpc/build.yml index 835b0af..e415c60 100644 --- a/examples/recipes/mpc@1.3.1.yml +++ b/examples/recipes/mpc/build.yml @@ -1,15 +1,17 @@ +meta: + version: '1.3.1' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.gz' + - name: '{{name}}-{{version}}.tar.gz' sha256: 'ab642492f5cf882b74aa0cb730cd410a81edcdbec895183ce930e706c1c759b8' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.gz + tar xf {{dldir}}/{{name}}-{{version}}.tar.gz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -20,9 +22,9 @@ tasks: - recipe: 'mpfr' task: 'install' run: | - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --prefix={{host.prefix}} @@ -31,16 +33,16 @@ tasks: compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -50,7 +52,7 @@ tasks: - recipe: 'mpfr' task: 'install' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install rm {{destdir}}{{host.prefix}}/lib/*.a rm {{destdir}}{{host.prefix}}/lib/*.la diff --git a/examples/recipes/mpfr@4.2.1.yml b/examples/recipes/mpfr/build.yml index 5747932..619bec7 100644 --- a/examples/recipes/mpfr@4.2.1.yml +++ b/examples/recipes/mpfr/build.yml @@ -1,15 +1,17 @@ +meta: + version: '4.2.1' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.xz' + - name: '{{name}}-{{version}}.tar.xz' sha256: '277807353a6726978996945af13e52829e3abd7a9a5b7fb2793894e18f1fcbb2' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.xz + tar xf {{dldir}}/{{name}}-{{version}}.tar.xz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -20,9 +22,9 @@ tasks: - recipe: 'gmp' task: 'install' run: | - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --build={{build.gnu_triplet}} \ --host={{host.gnu_triplet}} \ --prefix={{host.prefix}} @@ -31,16 +33,16 @@ tasks: compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -50,7 +52,7 @@ tasks: - recipe: 'gmp' task: 'install' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install rm {{destdir}}{{host.prefix}}/lib/*.a rm {{destdir}}{{host.prefix}}/lib/*.la diff --git a/examples/recipes/toolchain.yml b/examples/recipes/toolchain/build.yml index ab8e6de..d612261 100644 --- a/examples/recipes/toolchain.yml +++ b/examples/recipes/toolchain/build.yml @@ -8,7 +8,7 @@ tasks: runtime_depends: - recipe: 'gcc' task: 'install' - - recipe: 'libgcc' + - recipe: 'gcc/libgcc' task: 'install' depends: @@ -19,7 +19,7 @@ tasks: runtime_depends: - recipe: 'glibc' task: 'install' - - recipe: 'gcc-libs' + - recipe: 'gcc/libs' task: 'install' build: diff --git a/examples/recipes/zlib@1.3.yml b/examples/recipes/zlib/build.yml index 0102e74..3e72844 100644 --- a/examples/recipes/zlib@1.3.yml +++ b/examples/recipes/zlib/build.yml @@ -1,15 +1,17 @@ +meta: + version: '1.3' tasks: unpack: fetch: - - name: '{{pn}}-{{pv}}.tar.gz' + - name: '{{name}}-{{version}}.tar.gz' sha256: 'ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e' run: | - tar xf {{dldir}}/{{pn}}-{{pv}}.tar.gz + tar xf {{dldir}}/{{name}}-{{version}}.tar.gz configure: args: host: 'platform' - inherit: + parent: task: 'unpack' build_depends: - recipe: 'toolchain' @@ -18,25 +20,25 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - mkdir {{pn}}-build - cd {{pn}}-build - ../{{pn}}-{{pv}}/configure \ + mkdir {{name}}-build + cd {{name}}-build + ../{{name}}-{{version}}/configure \ --prefix={{host.prefix}} find -name configure.log -delete compile: args: host: 'platform' - inherit: + parent: task: 'configure' run: | - cd {{pn}}-build + cd {{name}}-build make install: args: host: 'platform' - inherit: + parent: task: 'compile' output: default: @@ -44,7 +46,7 @@ tasks: - recipe: 'toolchain' task: 'depends' run: | - cd {{pn}}-build + cd {{name}}-build make DESTDIR={{destdir}} install rm {{destdir}}{{host.prefix}}/lib/*.a |