From 0fd43866480b19b2feb27c1f36978e3de3660d2d Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sun, 19 Sep 2021 13:33:27 +0200 Subject: Add template engine to evaluate task commands The field "run" is renamed to "command" in a few places. We only use the evaluated command string for TaskInput now, not the TaskID or arguments. --- Cargo.lock | 161 ++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/executor.rs | 17 +++-- src/main.rs | 1 + src/runner/mod.rs | 2 +- src/runner/runc/run.rs | 2 +- src/runner/runc/spec.rs | 4 +- src/template.rs | 27 ++++++++ 8 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 src/template.rs diff --git a/Cargo.lock b/Cargo.lock index 3842c9e..68b4cad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,15 +34,48 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.4", +] + [[package]] name = "block-buffer" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array", + "generic-array 0.14.4", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", ] +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "cc" version = "1.0.70" @@ -188,13 +221,22 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.4", +] + [[package]] name = "digest" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array", + "generic-array 0.14.4", ] [[package]] @@ -203,6 +245,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "filetime" version = "0.2.15" @@ -237,6 +285,15 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -281,6 +338,20 @@ dependencies = [ "syn", ] +[[package]] +name = "handlebars" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b09e2322d20d14bc2572401ce7c1d60b4748580a76c230ed9c1f8938f0c833" +dependencies = [ + "log", + "pest", + "pest_derive", + "quick-error", + "serde", + "serde_json", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -400,6 +471,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -494,6 +571,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -506,6 +589,49 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6acbef58a60fe69ab50510a55bc8cdd4d6cf2283d27ad338f54cb52747a9cf2d" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -554,6 +680,12 @@ dependencies = [ "cc", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.9" @@ -649,6 +781,7 @@ name = "rebel" version = "0.1.0" dependencies = [ "clap", + "handlebars", "hex", "ipc-channel", "libc", @@ -751,17 +884,29 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + [[package]] name = "sha2" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", - "opaque-debug", + "digest 0.9.0", + "opaque-debug 0.3.0", ] [[package]] @@ -890,6 +1035,12 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + [[package]] name = "unicode-normalization" version = "0.1.19" diff --git a/Cargo.toml b/Cargo.toml index e746584..5af429f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" [dependencies] clap = "3.0.0-beta.2" +handlebars = "4.1.3" hex = { version = "0.4.3", features = ["std", "serde"] } ipc-channel = { git = "https://github.com/servo/ipc-channel.git" } libc = "0.2.84" diff --git a/src/executor.rs b/src/executor.rs index 2440d51..b35f6a4 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -6,6 +6,7 @@ use crate::{ context::*, paths, resolve, runner, task::*, + template::TemplateEngine, types::*, util::{cjson, error::*, fs}, }; @@ -24,11 +25,10 @@ const TASK_ENV: &[(&str, &str)] = &[ #[derive(Clone, Debug, Serialize)] struct TaskInput<'ctx> { - pub task: &'ctx TaskRef<'ctx>, pub inherit: Option<&'ctx InputHash>, pub depends: &'ctx HashMap, pub output: &'ctx HashMap, - pub run: &'ctx str, + pub command: &'ctx str, pub env: &'ctx HashMap, } @@ -80,6 +80,7 @@ pub struct Executor<'ctx> { tasks_done: HashMap, TaskMeta>, rdeps: HashMap, Vec>>, env: HashMap, + tpl: TemplateEngine, } impl<'ctx> Executor<'ctx> { @@ -96,6 +97,7 @@ impl<'ctx> Executor<'ctx> { tasks_done: HashMap::new(), rdeps: HashMap::new(), env, + tpl: TemplateEngine::new(), }; for task in taskset { @@ -154,7 +156,7 @@ impl<'ctx> Executor<'ctx> { } fn run_one(&self, task_ref: &TaskRef, runner: &impl runner::Runner) -> Result { - let task_def = &self.ctx[&task_ref.id]; + let task_def = &self.ctx[task_ref.id]; let task_deps = self.task_deps(task_ref); let task_output = task_def .output @@ -178,12 +180,15 @@ impl<'ctx> Executor<'ctx> { (Vec::new(), None) }; + let command = self.tpl.eval(&task_def, &task_ref.args).with_context(|| { + format!("Failed to evaluate command template for task {}", task_ref) + })?; + let input_hash = TaskInput { - task: &task_ref, inherit: inherit_hash, depends: &task_deps, output: &task_output, - run: &task_def.action.run, + command: &command, env: &env, } .input_hash(); @@ -202,7 +207,7 @@ impl<'ctx> Executor<'ctx> { let task = runner::Task { id: task_ref.id.clone(), - run: task_def.action.run.clone(), + command, input_hash, inherit: inherit_chain.clone(), depends: task_deps.clone(), diff --git a/src/main.rs b/src/main.rs index f731cc4..27714a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ mod recipe; mod resolve; mod runner; mod task; +mod template; mod types; mod unshare; mod util; diff --git a/src/runner/mod.rs b/src/runner/mod.rs index fec1550..1c169f4 100644 --- a/src/runner/mod.rs +++ b/src/runner/mod.rs @@ -8,7 +8,7 @@ use crate::{types::*, util::error::*}; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Task { pub id: TaskID, - pub run: String, + pub command: String, pub input_hash: InputHash, pub inherit: Vec, pub depends: HashMap, diff --git a/src/runner/runc/run.rs b/src/runner/runc/run.rs index ede6a89..a8d579e 100644 --- a/src/runner/runc/run.rs +++ b/src/runner/runc/run.rs @@ -218,7 +218,7 @@ pub fn handle_task(task: runner::Task) -> Result> { let task_tmp_dir = paths::task_tmp_dir(&task.input_hash); - spec::generate_spec(task.run.as_str(), &task.env) + spec::generate_spec(task.command.as_str(), &task.env) .save(paths::join(&[&task_tmp_dir, "config.json"])) .map_err(Error::new) .context("Failed to save runtime config")?; diff --git a/src/runner/runc/spec.rs b/src/runner/runc/spec.rs index 0b0d4a0..051efc4 100644 --- a/src/runner/runc/spec.rs +++ b/src/runner/runc/spec.rs @@ -6,7 +6,7 @@ use serde_json::json; use crate::{paths, unshare}; -pub fn generate_spec(run: &str, env: &HashMap) -> runtime::Spec { +pub fn generate_spec(command: &str, env: &HashMap) -> runtime::Spec { let env_entries: Vec = env.iter().map(|(k, v)| format!("{}={}", k, v)).collect(); runtime::Spec::deserialize(json!({ "ociVersion": "1.0.2", @@ -19,7 +19,7 @@ pub fn generate_spec(run: &str, env: &HashMap) -> runtime::Spec "args": [ "sh", "-ec", - run + command ], "env": env_entries, "cwd": paths::abs(paths::WORKDIR_PREFIX), diff --git a/src/template.rs b/src/template.rs new file mode 100644 index 0000000..035e09a --- /dev/null +++ b/src/template.rs @@ -0,0 +1,27 @@ +use handlebars::Handlebars; + +use crate::{task::*, util::error::*}; + +fn escape(s: &str) -> String { + format!("'{}'", s.replace("'", "'\\''")) +} + +#[derive(Debug)] +pub struct TemplateEngine(Handlebars<'static>); + +impl TemplateEngine { + pub fn new() -> Self { + let mut tpl = Handlebars::new(); + + tpl.set_strict_mode(true); + tpl.register_escape_fn(escape); + + TemplateEngine(tpl) + } + + pub fn eval(&self, task: &TaskDef, args: &TaskArgs) -> Result { + self.0 + .render_template(&task.action.run, args) + .map_err(Error::new) + } +} -- cgit v1.2.3