From e9bf0fc40c0eb7e9d4228b804d62f31b0a136528 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sat, 20 Apr 2024 14:28:05 +0200 Subject: Rename directories to match crate names --- crates/rebel/Cargo.toml | 25 ++ crates/rebel/src/args.rs | 122 ++++++++++ crates/rebel/src/context.rs | 533 +++++++++++++++++++++++++++++++++++++++++++ crates/rebel/src/driver.rs | 480 ++++++++++++++++++++++++++++++++++++++ crates/rebel/src/main.rs | 79 +++++++ crates/rebel/src/parse.rs | 72 ++++++ crates/rebel/src/paths.rs | 4 + crates/rebel/src/pin.rs | 39 ++++ crates/rebel/src/recipe.rs | 188 +++++++++++++++ crates/rebel/src/resolve.rs | 334 +++++++++++++++++++++++++++ crates/rebel/src/task.rs | 96 ++++++++ crates/rebel/src/template.rs | 42 ++++ 12 files changed, 2014 insertions(+) create mode 100644 crates/rebel/Cargo.toml create mode 100644 crates/rebel/src/args.rs create mode 100644 crates/rebel/src/context.rs create mode 100644 crates/rebel/src/driver.rs create mode 100644 crates/rebel/src/main.rs create mode 100644 crates/rebel/src/parse.rs create mode 100644 crates/rebel/src/paths.rs create mode 100644 crates/rebel/src/pin.rs create mode 100644 crates/rebel/src/recipe.rs create mode 100644 crates/rebel/src/resolve.rs create mode 100644 crates/rebel/src/task.rs create mode 100644 crates/rebel/src/template.rs (limited to 'crates/rebel') diff --git a/crates/rebel/Cargo.toml b/crates/rebel/Cargo.toml new file mode 100644 index 0000000..7164ca6 --- /dev/null +++ b/crates/rebel/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "rebel" +version = "0.1.0" +authors = ["Matthias Schiffer "] +license = "MIT" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../rebel-common", package = "rebel-common" } +runner = { path = "../rebel-runner", package = "rebel-runner" } + +clap = { version = "4.0.0", features = ["derive"] } +deb-version = "0.1.1" +enum-kinds = "0.5.1" +handlebars = "5.1.2" +indoc = "2.0.4" +lazy_static = "1.4.0" +nix = { version = "0.28.0", features = ["poll", "signal"] } +scoped-tls-hkt = "0.1.2" +serde = { version = "1", features = ["derive", "rc"] } +serde_yaml = "0.9" +walkdir = "2" +peg = "0.8.2" diff --git a/crates/rebel/src/args.rs b/crates/rebel/src/args.rs new file mode 100644 index 0000000..805646a --- /dev/null +++ b/crates/rebel/src/args.rs @@ -0,0 +1,122 @@ +use std::{ + collections::{hash_map, HashMap}, + hash, + rc::Rc, +}; + +use enum_kinds::EnumKind; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Platform { + #[serde(skip)] + pub short: String, + pub gnu_triplet: String, + pub karch: String, + pub prefix: String, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct PlatformRelation { + pub is_same: bool, + pub sysroot: String, + pub cross_compile: String, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, EnumKind)] +#[serde(untagged)] +#[enum_kind(ArgType, derive(Deserialize), serde(rename_all = "snake_case"))] +pub enum Arg { + String(Rc), + Platform(Rc), + PlatformRelation(Rc), +} + +impl From<&Arg> for Arg { + fn from(value: &Arg) -> Self { + value.clone() + } +} + +impl From for Arg { + fn from(value: String) -> Self { + Arg::String(Rc::new(value)) + } +} + +impl From for Arg { + fn from(value: Platform) -> Self { + Arg::Platform(Rc::new(value)) + } +} + +impl From for Arg { + fn from(value: PlatformRelation) -> Self { + Arg::PlatformRelation(Rc::new(value)) + } +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, Default)] +pub struct TaskArgs(HashMap); + +impl TaskArgs { + pub fn contains_key(&self, key: &str) -> bool { + self.0.contains_key(key) + } + + pub fn get(&self, key: &str) -> Option<&Arg> { + self.0.get(key) + } + + pub fn set(&mut self, key: &str, value: Option) + where + T: Into, + { + if let Some(v) = value { + self.0.insert(key.to_string(), v.into()); + } else { + self.0.remove(key); + } + } + + pub fn iter(&self) -> hash_map::Iter { + self.into_iter() + } +} + +impl FromIterator<(String, Arg)> for TaskArgs { + fn from_iter>(iter: T) -> Self { + TaskArgs(HashMap::from_iter(iter)) + } +} + +impl<'a> IntoIterator for &'a TaskArgs { + type Item = (&'a String, &'a Arg); + + type IntoIter = hash_map::Iter<'a, String, Arg>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +#[allow(clippy::derived_hash_with_manual_eq)] +impl hash::Hash for TaskArgs { + fn hash(&self, _state: &mut H) { + // Don't do anything: Properly hashing the task args is likely to cost + // much more performance than the hash collisions caused by TaskRefs + // that only differ by the args + } +} + +pub fn arg>(key: &str, value: A) -> (String, Arg) { + (key.to_string(), value.into()) +} + +#[derive(Clone, Debug, Deserialize, Default, PartialEq, Eq)] +pub struct ArgMapping(pub HashMap); + +#[allow(clippy::derived_hash_with_manual_eq)] +impl hash::Hash for ArgMapping { + fn hash(&self, _state: &mut H) {} +} diff --git a/crates/rebel/src/context.rs b/crates/rebel/src/context.rs new file mode 100644 index 0000000..be98813 --- /dev/null +++ b/crates/rebel/src/context.rs @@ -0,0 +1,533 @@ +use std::{ + borrow::Cow, + cmp::Ordering, + collections::{HashMap, HashSet}, + fmt::Display, + hash::Hash, + ops::Index, + rc::Rc, + result, +}; + +use common::{ + error::{self, Contextualizable}, + string_hash::ArchiveHash, + types::TaskID, +}; + +use crate::{ + args::*, + parse::{self, TaskFlags}, + paths, + pin::{self, Pins}, + task::*, +}; + +#[derive(Debug, Clone, Copy)] +pub enum ErrorKind<'a> { + TaskNotFound, + InvalidArgument(&'a str), + InvalidArgRef(&'a str), +} + +#[derive(Debug, Clone, Copy)] +pub struct Error<'a> { + pub task: &'a TaskID, + pub kind: ErrorKind<'a>, +} + +impl<'a> Display for Error<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Error { task, kind } = self; + match kind { + ErrorKind::TaskNotFound => write!(f, "Task '{}' not found", task), + ErrorKind::InvalidArgument(arg) => write!( + f, + "Invalid or missing argument '{}' for task '{}'", + arg, task + ), + ErrorKind::InvalidArgRef(arg) => write!( + f, + "Invalid reference for argument '{}' of task '{}'", + arg, task + ), + } + } +} + +impl<'a> From> for error::Error { + fn from(err: Error) -> Self { + error::Error::new(err) + } +} + +pub type Result<'a, T> = result::Result>; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TaskRef<'ctx> { + pub id: &'ctx TaskID, + pub args: Rc, +} + +impl<'ctx> Display for TaskRef<'ctx> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !f.alternate() { + return self.id.fmt(f); + } + + let version_arg = match self.args.get("version") { + Some(Arg::String(s)) => Some(s), + _ => None, + }; + let host_arg = match self.args.get("host") { + Some(Arg::Platform(platform)) => Some(platform), + _ => None, + }; + let target_arg = match self.args.get("target") { + Some(Arg::Platform(platform)) => Some(platform), + _ => None, + }; + + write!(f, "{}", self.id.recipe)?; + if let Some(version) = version_arg { + write!(f, "#{}", version)?; + } + write!(f, "::{}", self.id.task)?; + + if host_arg.is_some() || target_arg.is_some() { + write!(f, "@")?; + } + + if let Some(host) = host_arg { + write!(f, "{}", host.short)?; + } + if let Some(target) = target_arg { + write!(f, ":{}", target.short)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct OutputRef<'ctx> { + pub task: TaskRef<'ctx>, + pub output: &'ctx str, +} + +fn platform_relation(args: &TaskArgs, from: &str, to: &str) -> Option { + let plat_from = match args.get(from)? { + Arg::Platform(plat) => plat, + _ => return None, + }; + let plat_to = match args.get(to)? { + Arg::Platform(plat) => plat, + _ => return None, + }; + + let plat_rel = if plat_from == plat_to { + PlatformRelation { + is_same: true, + sysroot: "".to_string(), + cross_compile: "".to_string(), + } + } else { + PlatformRelation { + is_same: false, + sysroot: paths::TASK_SYSROOT.to_string(), + cross_compile: format!("{}/bin/{}-", plat_from.prefix, plat_to.gnu_triplet), + } + }; + Some(plat_rel) +} + +#[derive(Debug)] +pub struct Context { + platforms: HashMap, + globals: TaskArgs, + tasks: HashMap>, + rootfs: (ArchiveHash, String), +} + +impl Context { + pub fn new(mut tasks: HashMap>, pins: Pins) -> error::Result { + let platforms: HashMap<_, _> = [ + arg( + "build", + Platform { + short: "build".to_string(), + gnu_triplet: "x86_64-linux-gnu".to_string(), + karch: "x86_64".to_string(), + prefix: "/opt/toolchain".to_string(), + }, + ), + arg( + "aarch64", + Platform { + short: "aarch64".to_string(), + gnu_triplet: "aarch64-linux-gnu".to_string(), + karch: "arm64".to_string(), + prefix: "/usr".to_string(), + }, + ), + ] + .into_iter() + .collect(); + + let globals = TaskArgs::from_iter([ + ("build".to_string(), platforms["build"].clone()), + arg("workdir", paths::TASK_WORKDIR.to_string()), + arg("dldir", paths::TASK_DLDIR.to_string()), + arg("destdir", paths::TASK_DESTDIR.to_string()), + arg("sysroot", paths::TASK_SYSROOT.to_string()), + ]); + let (rootfs, rootfs_provides) = + Context::handle_pins(pins).context("Failed to process pin list")?; + + Context::add_rootfs_tasks(&mut tasks, rootfs_provides, &globals) + .context("Failed to determine rootfs-provided tasks from pin list")?; + + Ok(Context { + platforms, + globals, + tasks, + rootfs, + }) + } + + fn handle_pins(pins: Pins) -> error::Result<((ArchiveHash, String), Vec)> { + let mut ret = None; + + for (name, pin) in pins { + if pin.is_rootfs { + if ret.is_some() { + return Err(error::Error::new("Multiple is-rootfs pins")); + } + let hash = pin.hash.context("is-rootfs pin without hash")?; + + ret = Some(((hash, name), pin.provides)); + } + } + + ret.context("No is-rootfs pins") + } + + fn add_rootfs_tasks( + tasks: &mut HashMap>, + provides: Vec, + globals: &TaskArgs, + ) -> error::Result<()> { + let build = globals.get("build").unwrap(); + + for pin::Provides { + recipe, + task, + output, + args, + } in provides + { + let mut task_def = TaskDef::default(); + + if let Some(host) = args.host { + if host != "build" { + return Err(error::Error::new(format!("Invalid host value '{}'", host))); + } + task_def.args.insert("host".to_string(), build.into()); + task_def.arg_match.set("host", Some(build)); + } + + if let Some(target) = args.target { + if target != "build" { + return Err(error::Error::new(format!( + "Invalid target value '{}'", + target + ))); + } + task_def.args.insert("target".to_string(), build.into()); + task_def.arg_match.set("target", Some(build)); + } + + for output_entry in output { + task_def + .output + .insert(output_entry.to_string(), Output::default()); + } + + task_def.priority = i32::MAX; + + tasks + .entry(TaskID { + recipe: recipe.to_string(), + task: task.to_string(), + }) + .or_default() + .push(task_def); + } + + Ok(()) + } + + pub fn get_rootfs(&self) -> &(ArchiveHash, String) { + &self.rootfs + } + + fn match_task(task: &TaskDef, args: &TaskArgs) -> bool { + task.arg_match + .iter() + .all(|(key, value)| args.get(key) == Some(value)) + } + + fn compare_tasks(task1: &TaskDef, task2: &TaskDef) -> Ordering { + task1 + .priority + .cmp(&task2.priority) + .then(deb_version::compare_versions( + task1.meta.version.as_deref().unwrap_or_default(), + task2.meta.version.as_deref().unwrap_or_default(), + )) + } + + fn select_task<'ctx>(tasks: &'ctx [TaskDef], args: &TaskArgs) -> Option<&'ctx TaskDef> { + tasks + .iter() + .filter(|task| Self::match_task(task, args)) + .max_by(|task1, task2| Self::compare_tasks(task1, task2)) + } + + fn get_with_args<'a>(&self, id: &'a TaskID, args: &TaskArgs) -> Result<'a, &TaskDef> { + self.tasks + .get(id) + .and_then(|tasks| Self::select_task(tasks, args)) + .ok_or(Error { + task: id, + kind: ErrorKind::TaskNotFound, + }) + } + + pub fn get<'a>(&self, task: &TaskRef<'a>) -> Result<'a, &TaskDef> { + self.get_with_args(task.id, task.args.as_ref()) + } + + fn task_ref<'ctx>(&'ctx self, id: &'ctx TaskID, args: &TaskArgs) -> Result { + let task_def = self.get_with_args(id, args)?; + + let mut arg_def: HashMap<_, _> = task_def.args.iter().map(|(k, &v)| (k, v)).collect(); + for (key, arg) in &self.globals { + // TODO: Handle conflicts between explicit args and globals + arg_def.insert(key, ArgType::from(arg)); + } + + let mut new_args = TaskArgs::default(); + + for (key, typ) in arg_def { + if let Some(arg) = args.get(key) { + if ArgType::from(arg) == typ { + new_args.set(key, Some(arg)); + continue; + } + } + return Err(Error { + task: id, + kind: ErrorKind::InvalidArgument(key), + }); + } + + let build_to_host = platform_relation(&new_args, "build", "host"); + let host_to_target = platform_relation(&new_args, "host", "target"); + let build_to_target = platform_relation(&new_args, "build", "target"); + + let cross_compile = build_to_host + .as_ref() + .map(|build_to_host| build_to_host.cross_compile.clone()); + + new_args.set("build_to_host", build_to_host); + new_args.set("host_to_target", host_to_target); + new_args.set("build_to_target", build_to_target); + + new_args.set("cross_compile", cross_compile); + + new_args.set("basename", Some(task_def.meta.basename.clone())); + new_args.set("recipename", Some(task_def.meta.recipename.clone())); + new_args.set("recipe", Some(task_def.meta.recipe.clone())); + new_args.set("name", Some(task_def.meta.name.clone())); + new_args.set("version", task_def.meta.version.clone()); + + Ok(TaskRef { + id, + args: Rc::new(new_args), + }) + } + + pub fn parse(&self, s: &str) -> error::Result<(TaskRef, TaskFlags)> { + let (parsed, flags) = parse::task_with_flags(s) + .ok() + .context("Invalid task syntax")?; + + let recipe = parsed.id.recipe.to_string(); + let task = parsed.id.task.to_string(); + + let id = TaskID { recipe, task }; + let (ctx_id, _) = self + .tasks + .get_key_value(&id) + .with_context(|| format!("Task {} not found", id))?; + + let mut args = self.globals.clone(); + + if let Some(host) = parsed.args.host { + let plat = self + .platforms + .get(host) + .with_context(|| format!("Platform '{}' not found", host))?; + args.set("host", Some(plat)); + args.set("target", Some(plat)); + } + if let Some(target) = parsed.args.target { + let plat = self + .platforms + .get(target) + .with_context(|| format!("Platform '{}' not found", target))?; + args.set("target", Some(plat)); + } + + let task_ref = self + .task_ref(ctx_id, &args) + .with_context(|| format!("Failed to instantiate task {}", id))?; + + Ok((task_ref, flags)) + } + + fn map_args<'ctx, 'args>( + task: &'ctx TaskID, + mapping: &'ctx ArgMapping, + args: &'args TaskArgs, + build_dep: bool, + ) -> Result<'ctx, Cow<'args, TaskArgs>> { + if mapping.0.is_empty() && !build_dep { + return Ok(Cow::Borrowed(args)); + } + + let mut ret = args.clone(); + + if build_dep { + ret.set("host", args.get("build")); + ret.set("target", args.get("host")); + } + + for (to, from) in &mapping.0 { + let value = args.get(from).ok_or(Error { + task, + kind: ErrorKind::InvalidArgRef(to), + })?; + ret.set(to, Some(value.clone())); + } + + Ok(Cow::Owned(ret)) + } + + fn parent_ref<'ctx>(&'ctx self, dep: &'ctx ParentDep, args: &TaskArgs) -> Result { + let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, false)?; + self.task_ref(&dep.dep.id, mapped_args.as_ref()) + } + + pub fn output_ref<'ctx>( + &'ctx self, + dep: &'ctx OutputDep, + args: &TaskArgs, + build_dep: bool, + ) -> Result> { + let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, build_dep)?; + Ok(OutputRef { + task: self.task_ref(&dep.dep.id, mapped_args.as_ref())?, + output: &dep.output, + }) + } + + pub fn get_parent_depend<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> Result> { + let task = self.get(task_ref)?; + let Some(parent) = &task.parent else { + return Ok(None); + }; + Some(self.parent_ref(parent, &task_ref.args)).transpose() + } + + fn ancestor_iter<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> impl Iterator> { + struct Iter<'ctx>(&'ctx Context, Option>>); + + impl<'ctx> Iterator for Iter<'ctx> { + type Item = Result<'ctx, TaskRef<'ctx>>; + + fn next(&mut self) -> Option { + let task_ref = match self.1.take()? { + Ok(task_ref) => task_ref, + Err(err) => return Some(Err(err)), + }; + self.1 = self.0.get_parent_depend(&task_ref).transpose(); + Some(Ok(task_ref)) + } + } + + Iter(self, Some(Ok(task_ref.clone()))) + } + + pub fn get_build_depends<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> Result> { + let mut ret = HashSet::new(); + let mut allow_noinherit = true; + + for current in self.ancestor_iter(task_ref) { + let current_ref = current?; + let task = self.get(¤t_ref)?; + let entries = task + .build_depends + .iter() + .filter(|dep| allow_noinherit || !dep.noinherit) + .map(|dep| self.output_ref(dep, ¤t_ref.args, true)) + .collect::>>()?; + ret.extend(entries); + + allow_noinherit = false; + } + + Ok(ret) + } + + pub fn get_host_depends<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> Result> { + let mut ret = HashSet::new(); + let mut allow_noinherit = true; + + for current in self.ancestor_iter(task_ref) { + let current_ref = current?; + let task = self.get(¤t_ref)?; + let entries = task + .depends + .iter() + .filter(|dep| allow_noinherit || !dep.noinherit) + .map(|dep| self.output_ref(dep, ¤t_ref.args, false)) + .collect::>>()?; + ret.extend(entries); + + allow_noinherit = false; + } + + Ok(ret) + } +} + +impl Index<&TaskRef<'_>> for Context { + type Output = TaskDef; + + fn index(&self, index: &TaskRef) -> &TaskDef { + self.get(index).expect("Invalid TaskRef") + } +} diff --git a/crates/rebel/src/driver.rs b/crates/rebel/src/driver.rs new file mode 100644 index 0000000..b2655c6 --- /dev/null +++ b/crates/rebel/src/driver.rs @@ -0,0 +1,480 @@ +use std::{ + collections::{HashMap, HashSet}, + iter, + os::unix::{net::UnixStream, prelude::*}, +}; + +use indoc::indoc; +use nix::{ + poll, + sys::{ + signal, + signalfd::{SfdFlags, SignalFd}, + }, +}; + +use common::{error::*, string_hash::*, types::*}; +use runner::Runner; + +use crate::{ + context::{Context, OutputRef, TaskRef}, + paths, resolve, + task::*, + template, +}; + +#[derive(Debug)] +pub struct CompletionState<'ctx> { + ctx: &'ctx Context, + tasks_done: HashMap, TaskOutput>, +} + +impl<'ctx> CompletionState<'ctx> { + pub fn new(ctx: &'ctx Context) -> Self { + CompletionState { + ctx, + tasks_done: Default::default(), + } + } + + // Treats both "depends" and "parent" as dependencies + fn deps_satisfied(&self, task_ref: &TaskRef) -> bool { + resolve::get_dependent_tasks(self.ctx, task_ref) + .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref))) + .unwrap() + .into_iter() + .all(|dep| self.tasks_done.contains_key(&dep)) + } + + fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result> { + let task_def = &self.ctx[task]; + task_def + .fetch + .iter() + .map(|Fetch { name, sha256 }| { + Ok(Dependency::Fetch { + name: template::ENGINE.eval(name, &task.args).with_context(|| { + format!("Failed to evaluate fetch filename for task {}", task) + })?, + target_dir: paths::TASK_DLDIR.to_string(), + sha256: *sha256, + }) + }) + .collect() + } + + fn dep_closure(&self, deps: I, path: &'ctx str) -> impl Iterator + '_ + where + I: IntoIterator>, + { + resolve::runtime_depends(self.ctx, deps) + .expect("invalid runtime depends") + .into_iter() + .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) + .map(|&output| Dependency::Task { + output, + path: path.to_string(), + }) + } + + fn build_deps(&self, task: &TaskRef<'ctx>) -> Result + '_> { + Ok(self.dep_closure( + self.ctx + .get_build_depends(task) + .with_context(|| format!("invalid build depends for {}", task))?, + "", + )) + } + + fn host_deps(&self, task: &TaskRef<'ctx>) -> Result + '_> { + Ok(self.dep_closure( + self.ctx + .get_host_depends(task) + .with_context(|| format!("invalid depends for {}", task))?, + paths::TASK_SYSROOT, + )) + } + + fn task_deps(&self, task: &TaskRef<'ctx>) -> Result> { + let fetch_deps = self.fetch_deps(task)?.into_iter(); + let build_deps = self.build_deps(task)?; + let host_deps = self.host_deps(task)?; + + Ok(fetch_deps.chain(build_deps).chain(host_deps).collect()) + } + + fn task_ancestors(&self, task_ref: &TaskRef<'ctx>) -> Vec { + let Some(parent) = self + .ctx + .get_parent_depend(task_ref) + .expect("invalid parent depends") + else { + return vec![]; + }; + + let mut chain = self.task_ancestors(&parent); + if let Some(layer) = self.tasks_done[&parent].layer { + chain.push(layer); + } + chain + } + + fn print_summary(&self) { + println!(); + println!("Summary:"); + + let mut tasks: Box<[_]> = self.tasks_done.iter().collect(); + tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task)); + for (task_ref, task) in tasks.iter() { + println!(); + println!("{:#}", task_ref); + if let Some(hash) = task.input_hash { + println!(" input: {}", hash); + } + if let Some(hash) = task.layer { + println!(" layer: {}", hash); + } + if !task.outputs.is_empty() { + println!(" outputs:"); + + let mut outputs: Box<[_]> = task.outputs.iter().collect(); + outputs.sort_by_key(|(output, _)| *output); + for (output, hash) in outputs.iter() { + println!(" {}: {}", output, hash); + } + } + } + } +} + +#[derive(Debug)] +enum SpawnResult { + Spawned(UnixStream), + Skipped(TaskOutput), +} + +#[derive(Debug, PartialEq, Eq, Hash)] +enum TaskWaitResult { + Failed, + Interrupted, +} + +#[derive(Debug)] +pub struct Driver<'ctx> { + rdeps: HashMap, Vec>>, + force_run: HashSet>, + tasks_blocked: HashSet>, + tasks_runnable: Vec>, + tasks_running: HashMap)>, + state: CompletionState<'ctx>, +} + +impl<'ctx> Driver<'ctx> { + pub fn new( + ctx: &'ctx Context, + taskset: HashSet>, + force_run: HashSet>, + ) -> Result { + let mut driver = Driver { + rdeps: Default::default(), + force_run, + tasks_blocked: Default::default(), + tasks_runnable: Default::default(), + tasks_running: Default::default(), + state: CompletionState::new(ctx), + }; + + for task in taskset { + let mut has_depends = false; + for dep in resolve::get_dependent_tasks(ctx, &task) + .map_err(|_| Error::new(format!("invalid dependency for {}", task)))? + { + let rdep = driver.rdeps.entry(dep.clone()).or_default(); + rdep.push(task.clone()); + has_depends = true; + } + + if has_depends { + driver.tasks_blocked.insert(task); + } else { + driver.tasks_runnable.push(task); + } + } + + Ok(driver) + } + + const PREAMBLE: &'static str = indoc! {" + export PATH={{build.prefix}}/sbin:{{build.prefix}}/bin:$PATH + cd {{workdir}} + + export SOURCE_DATE_EPOCH=1 + + export AR_FOR_BUILD=ar + export AS_FOR_BUILD=as + export DLLTOOL_FOR_BUILD=dlltool + export CC_FOR_BUILD=gcc + export CXX_FOR_BUILD=g++ + export GCC_FOR_BUILD=gcc + export GFORTRAN_FOR_BUILD=gfortran + export GOC_FOR_BUILD=goc + export LD_FOR_BUILD=ld + export LIPO_FOR_BUILD=lipo + export NM_FOR_BUILD=nm + export OBJCOPY_FOR_BUILD=objcopy + export OBJDUMP_FOR_BUILD=objdump + export RANLIB_FOR_BUILD=ranlib + export STRIP_FOR_BUILD=strip + export WINDRES_FOR_BUILD=windres + export WINDMC_FOR_BUILD=windmc + "}; + const PREAMBLE_HOST: &'static str = indoc! {" + export AR={{build_to_host.cross_compile}}ar + export AS={{build_to_host.cross_compile}}as + export DLLTOOL={{build_to_host.cross_compile}}dlltool + export CC={{build_to_host.cross_compile}}gcc + export CXX={{build_to_host.cross_compile}}g++ + export GCC={{build_to_host.cross_compile}}gcc + export GFORTRAN={{build_to_host.cross_compile}}gfortran + export GOC={{build_to_host.cross_compile}}goc + export LD={{build_to_host.cross_compile}}ld + export LIPO={{build_to_host.cross_compile}}lipo + export NM={{build_to_host.cross_compile}}nm + export OBJCOPY={{build_to_host.cross_compile}}objcopy + export OBJDUMP={{build_to_host.cross_compile}}objdump + export RANLIB={{build_to_host.cross_compile}}ranlib + export STRIP={{build_to_host.cross_compile}}strip + export WINDRES={{build_to_host.cross_compile}}windres + export WINDMC={{build_to_host.cross_compile}}windmc + "}; + const PREAMBLE_TARGET: &'static str = indoc! {" + export AR_FOR_TARGET={{build_to_target.cross_compile}}ar + export AS_FOR_TARGET={{build_to_target.cross_compile}}as + export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool + export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc + export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++ + export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc + export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran + export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc + export LD_FOR_TARGET={{build_to_target.cross_compile}}ld + export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo + export NM_FOR_TARGET={{build_to_target.cross_compile}}nm + export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy + export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump + export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib + export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip + export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres + export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc + "}; + + fn task_preamble(task_ref: &TaskRef<'ctx>) -> Vec<&'static str> { + let mut ret = vec![Self::PREAMBLE]; + + if task_ref.args.contains_key("build_to_host") { + ret.push(Self::PREAMBLE_HOST); + } + if task_ref.args.contains_key("build_to_target") { + ret.push(Self::PREAMBLE_TARGET); + } + ret + } + + fn update_runnable(&mut self, task_ref: TaskRef<'ctx>, task_output: TaskOutput) { + let rdeps = self.rdeps.get(&task_ref); + + self.state.tasks_done.insert(task_ref, task_output); + + for rdep in rdeps.unwrap_or(&Vec::new()) { + if !self.tasks_blocked.contains(rdep) { + continue; + } + if self.state.deps_satisfied(rdep) { + self.tasks_blocked.remove(rdep); + self.tasks_runnable.push(rdep.clone()); + } + } + } + + fn spawn_task(&self, task_ref: &TaskRef<'ctx>, runner: &Runner) -> Result { + let task_def = &self.state.ctx[task_ref]; + if task_def.action.is_empty() { + println!("Skipping empty task {:#}", task_ref); + return Ok(SpawnResult::Skipped(TaskOutput::default())); + } + + let task_deps = self.state.task_deps(task_ref)?; + let task_output = task_def + .output + .iter() + .map(|(name, Output { path, .. })| { + let output_path = if let Some(path) = path { + format!("{}/{}", paths::TASK_DESTDIR, path) + } else { + paths::TASK_DESTDIR.to_string() + }; + (name.clone(), output_path) + }) + .collect(); + + let ancestors = self.state.task_ancestors(task_ref); + + let mut run = Self::task_preamble(task_ref); + run.push(&task_def.action.run); + + let command = template::ENGINE + .eval_sh(&run.concat(), &task_ref.args) + .with_context(|| { + format!("Failed to evaluate command template for task {}", task_ref) + })?; + + let rootfs = self.state.ctx.get_rootfs(); + let task = Task { + label: format!("{:#}", task_ref), + command, + workdir: paths::TASK_WORKDIR.to_string(), + rootfs: rootfs.0, + ancestors, + depends: task_deps, + outputs: task_output, + pins: HashMap::from([rootfs.clone()]), + force_run: self.force_run.contains(task_ref), + }; + + Ok(SpawnResult::Spawned(runner.spawn(&task))) + } + + fn run_task(&mut self, task_ref: TaskRef<'ctx>, runner: &Runner) -> Result<()> { + match self.spawn_task(&task_ref, runner)? { + SpawnResult::Spawned(socket) => { + assert!(self + .tasks_running + .insert(socket.as_raw_fd(), (socket, task_ref)) + .is_none()); + } + SpawnResult::Skipped(result) => { + self.update_runnable(task_ref, result); + } + } + Ok(()) + } + + fn run_tasks(&mut self, runner: &Runner) -> Result<()> { + while let Some(task_ref) = self.tasks_runnable.pop() { + self.run_task(task_ref, runner)?; + } + Ok(()) + } + + fn wait_for_task(&mut self, signal_fd: &mut SignalFd) -> Result> { + let mut pollfds: Vec<_> = self + .tasks_running + .values() + .map(|(socket, _)| socket.as_fd()) + .chain(iter::once(signal_fd.as_fd())) + .map(|fd| poll::PollFd::new(fd, poll::PollFlags::POLLIN)) + .collect(); + + while poll::poll(&mut pollfds, poll::PollTimeout::NONE).context("poll()")? == 0 {} + + let pollevents: Vec<_> = pollfds + .into_iter() + .map(|pollfd| { + ( + pollfd.as_fd().as_raw_fd(), + pollfd.revents().expect("Unknown events in poll() return"), + ) + }) + .collect(); + + for (fd, events) in pollevents { + if !events.contains(poll::PollFlags::POLLIN) { + if events.intersects(!poll::PollFlags::POLLIN) { + return Err(Error::new( + "Unexpected error status for socket file descriptor", + )); + } + continue; + } + + if fd == signal_fd.as_raw_fd() { + let _signal = signal_fd.read_signal().expect("read_signal()").unwrap(); + return Ok(Some(TaskWaitResult::Interrupted)); + } + + let (socket, task_ref) = self.tasks_running.remove(&fd).unwrap(); + + match Runner::result(&socket) { + Ok(task_output) => { + self.update_runnable(task_ref, task_output); + } + Err(error) => { + eprintln!("{}", error); + return Ok(Some(TaskWaitResult::Failed)); + } + } + } + + Ok(None) + } + + fn is_done(&self) -> bool { + self.tasks_blocked.is_empty() + && self.tasks_runnable.is_empty() + && self.tasks_running.is_empty() + } + + fn setup_signalfd() -> Result { + let mut signals = signal::SigSet::empty(); + signals.add(signal::Signal::SIGINT); + signal::pthread_sigmask(signal::SigmaskHow::SIG_BLOCK, Some(&signals), None) + .expect("pthread_sigmask()"); + SignalFd::with_flags(&signals, SfdFlags::SFD_CLOEXEC) + .context("Failed to create signal file descriptor") + } + + fn raise_sigint() { + let mut signals = signal::SigSet::empty(); + signals.add(signal::Signal::SIGINT); + signal::pthread_sigmask(signal::SigmaskHow::SIG_UNBLOCK, Some(&signals), None) + .expect("pthread_sigmask()"); + signal::raise(signal::Signal::SIGINT).expect("raise()"); + unreachable!(); + } + + pub fn run(&mut self, runner: &Runner, keep_going: bool) -> Result { + let mut success = true; + let mut interrupted = false; + + let mut signal_fd = Self::setup_signalfd()?; + + self.run_tasks(runner)?; + + while !self.tasks_running.is_empty() { + match self.wait_for_task(&mut signal_fd)? { + Some(TaskWaitResult::Failed) => { + success = false; + } + Some(TaskWaitResult::Interrupted) => { + if interrupted { + Self::raise_sigint(); + } + eprintln!("Interrupt received, not spawning new tasks. Interrupt again to stop immediately."); + interrupted = true; + } + None => {} + } + if !interrupted && (success || keep_going) { + self.run_tasks(runner)?; + } + } + + if interrupted || !success { + return Ok(false); + } + + assert!(self.is_done(), "No runnable tasks left"); + self.state.print_summary(); + + Ok(true) + } +} diff --git a/crates/rebel/src/main.rs b/crates/rebel/src/main.rs new file mode 100644 index 0000000..bd08f18 --- /dev/null +++ b/crates/rebel/src/main.rs @@ -0,0 +1,79 @@ +mod args; +mod context; +mod driver; +mod parse; +mod paths; +mod pin; +mod recipe; +mod resolve; +mod task; +mod template; + +use std::collections::HashSet; + +use clap::Parser; + +use runner::Runner; + +#[derive(Parser)] +#[clap(version, about)] +struct Opts { + /// Allow N jobs at once. + /// Defaults to the number of available CPUs + #[clap(short, long)] + jobs: Option, + /// Keep going after some tasks have failed + #[clap(short, long)] + keep_going: bool, + /// The tasks to run + #[clap(name = "task", required = true)] + tasks: Vec, +} + +fn main() { + let opts: Opts = Opts::parse(); + + let runner = unsafe { Runner::new(&runner::Options { jobs: opts.jobs }) }.unwrap(); + + let ctx = context::Context::new( + recipe::read_recipes("examples/recipes").unwrap(), + pin::read_pins("examples/pins.yml").unwrap(), + ) + .unwrap(); + + let mut rsv = resolve::Resolver::new(&ctx); + let mut force_run = HashSet::new(); + + for task in opts.tasks { + let (task_ref, flags) = match ctx.parse(&task) { + Ok(task_ref) => task_ref, + Err(err) => { + eprintln!("{}", err); + std::process::exit(1); + } + }; + let errors = rsv.add_goal(&task_ref); + if !errors.is_empty() { + for error in errors { + eprintln!("{}", error); + } + std::process::exit(1); + } + if flags.force_run { + force_run.insert(task_ref); + } + } + let taskset = rsv.into_taskset(); + let mut driver = driver::Driver::new(&ctx, taskset, force_run).unwrap(); + match driver.run(&runner, opts.keep_going) { + Ok(success) => { + if !success { + std::process::exit(1); + } + } + Err(error) => { + eprintln!("{}", error); + std::process::exit(1); + } + } +} diff --git a/crates/rebel/src/parse.rs b/crates/rebel/src/parse.rs new file mode 100644 index 0000000..5857efb --- /dev/null +++ b/crates/rebel/src/parse.rs @@ -0,0 +1,72 @@ +#[derive(Debug, Clone, Copy)] +pub struct TaskID<'a> { + pub recipe: &'a str, + pub task: &'a str, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct TaskArgs<'a> { + pub host: Option<&'a str>, + pub target: Option<&'a str>, +} + +#[derive(Debug, Clone, Copy)] +pub struct Task<'a> { + pub id: TaskID<'a>, + pub args: TaskArgs<'a>, +} + +#[derive(Debug, Clone, Copy)] +pub struct TaskFlags { + pub force_run: bool, +} + +peg::parser! { + grammar rules() for str { + rule t(tag: rule<()>, value: rule) -> T + = tag() v:value() { v } + + rule name_char() + = ['a'..='z' | 'A' ..='Z' | '0'..='9' | '_' | '-'] + + rule name() -> &'input str + = $(name_char()+) + + rule recipe_id() -> &'input str + = $(name() ("/" name())?) + + rule task_id() -> TaskID<'input> + = recipe:recipe_id() "::" task:name() { + TaskID { recipe, task } + } + + rule task_args() -> TaskArgs<'input> + = "@" host:name()? target:t(<":">, )? { + TaskArgs { + host, + target, + } + } + / { Default::default() } + + pub rule task() -> Task<'input> + = id:task_id() args:task_args() { + Task { + id, + args, + } + } + + rule force_run() -> bool + = "+" { true } + / { false } + + rule task_flags() -> TaskFlags + = force_run:force_run() { TaskFlags { force_run } } + + pub rule task_with_flags() -> (Task<'input>, TaskFlags) + = task:task() flags:task_flags() { (task, flags) } + } +} + +pub use rules::*; diff --git a/crates/rebel/src/paths.rs b/crates/rebel/src/paths.rs new file mode 100644 index 0000000..274dda1 --- /dev/null +++ b/crates/rebel/src/paths.rs @@ -0,0 +1,4 @@ +pub const TASK_DESTDIR: &str = "/build/dest"; +pub const TASK_DLDIR: &str = "/build/downloads"; +pub const TASK_WORKDIR: &str = "/build/work"; +pub const TASK_SYSROOT: &str = "/opt/toolchain/sysroot"; diff --git a/crates/rebel/src/pin.rs b/crates/rebel/src/pin.rs new file mode 100644 index 0000000..26e445c --- /dev/null +++ b/crates/rebel/src/pin.rs @@ -0,0 +1,39 @@ +use std::{collections::HashMap, fs::File, path::Path}; + +use serde::{Deserialize, Serialize}; + +use common::{error::*, string_hash::*}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Args { + pub host: Option, + pub target: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Provides { + pub recipe: String, + pub task: String, + pub output: Vec, + pub args: Args, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Pin { + pub hash: Option, + #[serde(default)] + pub provides: Vec, + #[serde(default)] + pub is_rootfs: bool, +} + +pub type Pins = HashMap; + +pub fn read_pins>(path: P) -> Result { + let f = File::open(path)?; + let pins: Pins = serde_yaml::from_reader(f) + .map_err(Error::new) + .context("YAML error")?; + Ok(pins) +} diff --git a/crates/rebel/src/recipe.rs b/crates/rebel/src/recipe.rs new file mode 100644 index 0000000..16d3751 --- /dev/null +++ b/crates/rebel/src/recipe.rs @@ -0,0 +1,188 @@ +use std::{collections::HashMap, ffi::OsStr, fs::File, path::Path, result}; + +use scoped_tls_hkt::scoped_thread_local; +use serde::{de::DeserializeOwned, Deserialize, Deserializer}; +use walkdir::WalkDir; + +use common::{error::*, types::*}; + +use crate::task::{TaskDef, TaskMeta}; + +scoped_thread_local!(static CURRENT_RECIPE: str); + +fn current_recipe() -> String { + CURRENT_RECIPE.with(|current| current.to_string()) +} + +pub fn deserialize_task_id<'de, D>(deserializer: D) -> result::Result +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + struct RecipeTaskID { + recipe: Option, + task: String, + } + let RecipeTaskID { recipe, task } = RecipeTaskID::deserialize(deserializer)?; + Ok(TaskID { + recipe: recipe.unwrap_or_else(current_recipe), + task, + }) +} + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct RecipeMeta { + pub name: Option, + pub version: Option, +} + +#[derive(Debug, Deserialize)] +struct Recipe { + #[serde(default)] + pub meta: RecipeMeta, + pub tasks: HashMap, +} + +#[derive(Debug, Deserialize)] +struct Subrecipe { + pub tasks: HashMap, +} + +fn read_yaml(path: &Path) -> Result { + let f = File::open(path).context("IO error")?; + + let value: T = serde_yaml::from_reader(f) + .map_err(Error::new) + .context("YAML error")?; + + Ok(value) +} + +const RECIPE_NAME: &str = "build"; +const RECIPE_PREFIX: &str = "build."; + +fn recipe_name(path: &Path) -> Option<&str> { + if path.extension() != Some("yml".as_ref()) { + return None; + } + + let stem = path.file_stem()?.to_str()?; + if stem == RECIPE_NAME { + return Some(""); + } + stem.strip_prefix(RECIPE_PREFIX) +} + +fn handle_recipe_tasks( + tasks: &mut HashMap>, + recipe_tasks: HashMap, + meta: &TaskMeta, +) { + for (label, mut task) in recipe_tasks { + let task_id = TaskID { + recipe: meta.recipe.clone(), + task: label, + }; + task.meta = meta.clone(); + tasks.entry(task_id).or_default().push(task); + } +} + +fn read_recipe_tasks( + path: &Path, + basename: &str, + tasks: &mut HashMap>, +) -> Result { + let recipe_def = CURRENT_RECIPE.set(basename, || read_yaml::(path))?; + + let name = recipe_def + .meta + .name + .as_deref() + .unwrap_or(basename) + .to_string(); + + let meta = TaskMeta { + basename: basename.to_string(), + recipename: "".to_string(), + recipe: basename.to_string(), + name, + version: recipe_def.meta.version.clone(), + }; + + handle_recipe_tasks(tasks, recipe_def.tasks, &meta); + + Ok(recipe_def.meta) +} + +fn read_subrecipe_tasks( + path: &Path, + basename: &str, + recipename: &str, + recipe_meta: &RecipeMeta, + tasks: &mut HashMap>, +) -> Result<()> { + let recipe = format!("{basename}/{recipename}"); + let recipe_def = CURRENT_RECIPE.set(&recipe, || read_yaml::(path))?; + + let name = recipe_meta.name.as_deref().unwrap_or(basename).to_string(); + + let meta = TaskMeta { + basename: basename.to_string(), + recipename: recipename.to_string(), + recipe: recipe.clone(), + name, + version: recipe_meta.version.clone(), + }; + + handle_recipe_tasks(tasks, recipe_def.tasks, &meta); + + Ok(()) +} + +pub fn read_recipes>(path: P) -> Result>> { + let mut tasks = HashMap::>::new(); + let mut recipe_metas = HashMap::::new(); + + for entry in WalkDir::new(path) + .sort_by(|a, b| { + // Files are sorted first by stem, then by extension, so that + // recipe.yml will always be read before recipe.NAME.yml + let stem_cmp = a.path().file_stem().cmp(&b.path().file_stem()); + let ext_cmp = a.path().extension().cmp(&b.path().extension()); + stem_cmp.then(ext_cmp) + }) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + if !path.is_file() { + continue; + } + + let Some(recipename) = recipe_name(path) else { + continue; + }; + let Some(basename) = path + .parent() + .and_then(Path::file_name) + .and_then(OsStr::to_str) + else { + continue; + }; + + if recipename.is_empty() { + recipe_metas.insert( + basename.to_string(), + read_recipe_tasks(path, basename, &mut tasks)?, + ); + } else { + let Some(recipe_meta) = recipe_metas.get(basename) else { + continue; + }; + read_subrecipe_tasks(path, basename, recipename, recipe_meta, &mut tasks)?; + } + } + + Ok(tasks) +} diff --git a/crates/rebel/src/resolve.rs b/crates/rebel/src/resolve.rs new file mode 100644 index 0000000..102c483 --- /dev/null +++ b/crates/rebel/src/resolve.rs @@ -0,0 +1,334 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::rc::Rc; + +use common::types::TaskID; + +use crate::args::TaskArgs; +use crate::context::{self, Context, OutputRef, TaskRef}; + +#[derive(Debug, Default)] +pub struct DepChain<'ctx>(pub Vec>); + +impl<'ctx> fmt::Display for DepChain<'ctx> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut first = true; + for task in self.0.iter().rev() { + if !first { + write!(f, " -> ")?; + } + write!(f, "{}", task)?; + + first = false; + } + + Ok(()) + } +} + +impl<'ctx> From> for DepChain<'ctx> { + fn from(task: TaskRef<'ctx>) -> Self { + DepChain(vec![task]) + } +} + +impl<'ctx> From<&TaskRef<'ctx>> for DepChain<'ctx> { + fn from(task: &TaskRef<'ctx>) -> Self { + task.clone().into() + } +} + +impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> { + fn from(id: &'ctx TaskID) -> Self { + TaskRef { + id, + args: Rc::new(TaskArgs::default()), + } + .into() + } +} + +const MAX_ERRORS: usize = 100; + +#[derive(Debug)] +pub enum ErrorKind<'ctx> { + Context(context::Error<'ctx>), + OutputNotFound(&'ctx str), + DependencyCycle, + TooManyErrors, +} + +#[derive(Debug)] +pub struct Error<'ctx> { + pub dep_chain: DepChain<'ctx>, + pub kind: ErrorKind<'ctx>, +} + +impl<'ctx> Error<'ctx> { + fn output_not_found(task: &TaskRef<'ctx>, output: &'ctx str) -> Self { + Error { + dep_chain: task.into(), + kind: ErrorKind::OutputNotFound(output), + } + } + + fn dependency_cycle(task: &TaskRef<'ctx>) -> Self { + Error { + dep_chain: task.into(), + kind: ErrorKind::DependencyCycle, + } + } + + fn too_many_errors() -> Self { + Error { + dep_chain: DepChain::default(), + kind: ErrorKind::TooManyErrors, + } + } + + fn extend(&mut self, task: &TaskRef<'ctx>) { + self.dep_chain.0.push(task.clone()); + } +} + +impl<'ctx> fmt::Display for Error<'ctx> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let Error { dep_chain, kind } = self; + match kind { + ErrorKind::Context(err) => { + write!(f, "{}: ", err)?; + } + ErrorKind::OutputNotFound(output) => { + write!(f, "Output '{}' not found: ", output)?; + } + ErrorKind::DependencyCycle => { + write!(f, "Dependency Cycle: ")?; + } + ErrorKind::TooManyErrors => { + write!(f, "Too many errors, stopping.")?; + } + } + dep_chain.fmt(f) + } +} + +impl<'ctx> From> for Error<'ctx> { + fn from(err: context::Error<'ctx>) -> Self { + Error { + dep_chain: err.task.into(), + kind: ErrorKind::Context(err), + } + } +} + +impl<'ctx> std::error::Error for Error<'ctx> {} + +#[derive(Debug, PartialEq)] +enum ResolveState { + Resolving, + Resolved, +} + +pub fn runtime_depends<'ctx, I>( + ctx: &'ctx Context, + deps: I, +) -> Result, Vec> +where + I: IntoIterator>, +{ + fn add_dep<'ctx>( + ret: &mut HashSet>, + ctx: &'ctx Context, + dep: OutputRef<'ctx>, + ) -> Vec> { + if ret.contains(&dep) { + return Vec::new(); + } + + let task = &dep.task; + let task_def = match ctx.get(task) { + Ok(task) => task, + Err(err) => return vec![err.into()], + }; + + let output = match task_def.output.get(dep.output) { + Some(output) => output, + None => { + return vec![Error::output_not_found(task, dep.output)]; + } + }; + + ret.insert(dep.clone()); + + let mut errors = Vec::new(); + for runtime_dep in &output.runtime_depends { + match ctx.output_ref(runtime_dep, &task.args, false) { + Ok(output_ref) => { + for mut error in add_dep(ret, ctx, output_ref) { + error.extend(task); + errors.push(error); + } + } + Err(err) => { + let mut err: Error = err.into(); + err.extend(task); + errors.push(err); + } + }; + } + errors + } + + let mut ret = HashSet::new(); + let mut errors = Vec::new(); + + for dep in deps { + errors.extend(add_dep(&mut ret, ctx, dep)); + } + + if !errors.is_empty() { + return Err(errors); + } + + Ok(ret) +} + +pub fn get_dependent_outputs<'ctx>( + ctx: &'ctx Context, + task_ref: &TaskRef<'ctx>, +) -> Result>, Vec>> { + let deps: HashSet<_> = ctx + .get_build_depends(task_ref) + .map_err(|err| vec![err.into()])? + .into_iter() + .chain( + ctx.get_host_depends(task_ref) + .map_err(|err| vec![err.into()])?, + ) + .collect(); + runtime_depends(ctx, deps) +} + +pub fn get_dependent_tasks<'ctx>( + ctx: &'ctx Context, + task_ref: &TaskRef<'ctx>, +) -> Result>, Vec>> { + Ok(ctx + .get_parent_depend(task_ref) + .map_err(|err| vec![err.into()])? + .into_iter() + .chain( + get_dependent_outputs(ctx, task_ref)? + .into_iter() + .map(|dep| dep.task), + ) + .collect()) +} + +#[derive(Debug)] +pub struct Resolver<'ctx> { + ctx: &'ctx Context, + resolve_state: HashMap, ResolveState>, +} + +impl<'ctx> Resolver<'ctx> { + pub fn new(ctx: &'ctx Context) -> Self { + Resolver { + ctx, + resolve_state: HashMap::new(), + } + } + + fn tasks_resolved(&self) -> bool { + self.resolve_state + .values() + .all(|resolved| *resolved == ResolveState::Resolved) + } + + fn add_task(&mut self, task: &TaskRef<'ctx>, output: Option<&'ctx str>) -> Vec> { + match self.resolve_state.get(task) { + Some(ResolveState::Resolving) => return vec![Error::dependency_cycle(task)], + Some(ResolveState::Resolved) => return vec![], + None => (), + } + + let task_def = match self.ctx.get(task) { + Ok(task_def) => task_def, + Err(err) => return vec![err.into()], + }; + + if let Some(task_output) = output { + if !task_def.output.contains_key(task_output) { + return vec![Error::output_not_found(task, task_output)]; + } + } + + self.resolve_state + .insert(task.clone(), ResolveState::Resolving); + + let mut ret = Vec::new(); + let mut handle_errors = |errors: Vec>| -> Result<(), ()> { + for mut error in errors { + error.extend(task); + ret.push(error); + + if ret.len() > MAX_ERRORS { + ret.push(Error::too_many_errors()); + return Err(()); + } + } + Ok(()) + }; + + let _ = (|| -> Result<(), ()> { + match self.ctx.get_parent_depend(task) { + Ok(Some(parent)) => { + handle_errors(self.add_task(&parent, None))?; + } + Ok(None) => {} + Err(err) => { + handle_errors(vec![err.into()])?; + } + } + + match get_dependent_outputs(self.ctx, task) { + Ok(rdeps) => { + for rdep in rdeps { + handle_errors(self.add_task(&rdep.task, Some(rdep.output)))?; + } + } + Err(errors) => { + handle_errors(errors)?; + } + } + + Ok(()) + })(); + + if ret.is_empty() { + *self + .resolve_state + .get_mut(task) + .expect("Missing resolve_state") = ResolveState::Resolved; + } else { + self.resolve_state.remove(task); + } + + ret + } + + pub fn add_goal(&mut self, task: &TaskRef<'ctx>) -> Vec> { + let ret = self.add_task(task, None); + debug_assert!(self.tasks_resolved()); + ret + } + + pub fn into_taskset(self) -> HashSet> { + debug_assert!(self.tasks_resolved()); + + self.resolve_state + .into_iter() + .map(|entry| entry.0) + .collect() + } +} diff --git a/crates/rebel/src/task.rs b/crates/rebel/src/task.rs new file mode 100644 index 0000000..e84766e --- /dev/null +++ b/crates/rebel/src/task.rs @@ -0,0 +1,96 @@ +use std::collections::{HashMap, HashSet}; + +use serde::Deserialize; + +use common::{string_hash::StringHash, types::TaskID}; + +use crate::{ + args::{ArgMapping, ArgType, TaskArgs}, + recipe, +}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] +pub struct TaskDep { + #[serde(flatten, deserialize_with = "recipe::deserialize_task_id")] + pub id: TaskID, + #[serde(default)] + pub args: ArgMapping, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] +pub struct Fetch { + pub name: String, + pub sha256: StringHash, +} + +fn default_output_name() -> String { + "default".to_string() +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ParentDep { + #[serde(flatten)] + pub dep: TaskDep, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] +pub struct OutputDep { + #[serde(flatten)] + pub dep: TaskDep, + #[serde(default)] + pub noinherit: bool, + #[serde(default = "default_output_name")] + pub output: String, +} + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct Output { + pub path: Option, + #[serde(default)] + pub runtime_depends: HashSet, +} + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct Action { + #[serde(default)] + pub run: String, +} + +impl Action { + pub fn is_empty(&self) -> bool { + self.run.is_empty() + } +} + +#[derive(Clone, Debug, Default)] +pub struct TaskMeta { + pub basename: String, + pub recipename: String, + pub recipe: String, + pub name: String, + pub version: Option, +} + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct TaskDef { + #[serde(skip)] + pub meta: TaskMeta, + #[serde(default)] + pub args: HashMap, + #[serde(default)] + pub parent: Option, + #[serde(default)] + pub fetch: HashSet, + #[serde(default)] + pub build_depends: HashSet, + #[serde(default)] + pub depends: HashSet, + #[serde(default)] + pub output: HashMap, + #[serde(flatten)] + pub action: Action, + #[serde(default)] + pub priority: i32, + #[serde(skip)] + pub arg_match: TaskArgs, +} diff --git a/crates/rebel/src/template.rs b/crates/rebel/src/template.rs new file mode 100644 index 0000000..1a091ed --- /dev/null +++ b/crates/rebel/src/template.rs @@ -0,0 +1,42 @@ +use handlebars::Handlebars; +use lazy_static::lazy_static; + +use common::error::*; + +use crate::args::TaskArgs; + +fn escape_sh(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) +} + +#[derive(Debug)] +pub struct TemplateEngine { + tpl: Handlebars<'static>, + tpl_sh: Handlebars<'static>, +} + +impl TemplateEngine { + pub fn new() -> Self { + let mut tpl = Handlebars::new(); + tpl.set_strict_mode(true); + tpl.register_escape_fn(handlebars::no_escape); + + let mut tpl_sh = Handlebars::new(); + tpl_sh.set_strict_mode(true); + tpl_sh.register_escape_fn(escape_sh); + + TemplateEngine { tpl, tpl_sh } + } + + pub fn eval(&self, input: &str, args: &TaskArgs) -> Result { + self.tpl.render_template(input, args).map_err(Error::new) + } + + pub fn eval_sh(&self, input: &str, args: &TaskArgs) -> Result { + self.tpl_sh.render_template(input, args).map_err(Error::new) + } +} + +lazy_static! { + pub static ref ENGINE: TemplateEngine = TemplateEngine::new(); +} -- cgit v1.2.3