From 86f4c08b81e2129b5d1012c1350e68e3c0560282 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 28 Oct 2021 18:29:57 +0200 Subject: Rename executor to driver --- crates/driver/Cargo.toml | 24 +++ crates/driver/src/args.rs | 122 +++++++++++ crates/driver/src/context.rs | 445 ++++++++++++++++++++++++++++++++++++++++ crates/driver/src/driver.rs | 367 +++++++++++++++++++++++++++++++++ crates/driver/src/main.rs | 56 +++++ crates/driver/src/recipe.rs | 115 +++++++++++ crates/driver/src/resolve.rs | 312 ++++++++++++++++++++++++++++ crates/driver/src/task.rs | 84 ++++++++ crates/driver/src/template.rs | 44 ++++ crates/executor/Cargo.toml | 24 --- crates/executor/src/args.rs | 122 ----------- crates/executor/src/context.rs | 445 ---------------------------------------- crates/executor/src/executor.rs | 367 --------------------------------- crates/executor/src/main.rs | 56 ----- crates/executor/src/recipe.rs | 115 ----------- crates/executor/src/resolve.rs | 312 ---------------------------- crates/executor/src/task.rs | 84 -------- crates/executor/src/template.rs | 44 ---- 18 files changed, 1569 insertions(+), 1569 deletions(-) create mode 100644 crates/driver/Cargo.toml create mode 100644 crates/driver/src/args.rs create mode 100644 crates/driver/src/context.rs create mode 100644 crates/driver/src/driver.rs create mode 100644 crates/driver/src/main.rs create mode 100644 crates/driver/src/recipe.rs create mode 100644 crates/driver/src/resolve.rs create mode 100644 crates/driver/src/task.rs create mode 100644 crates/driver/src/template.rs delete mode 100644 crates/executor/Cargo.toml delete mode 100644 crates/executor/src/args.rs delete mode 100644 crates/executor/src/context.rs delete mode 100644 crates/executor/src/executor.rs delete mode 100644 crates/executor/src/main.rs delete mode 100644 crates/executor/src/recipe.rs delete mode 100644 crates/executor/src/resolve.rs delete mode 100644 crates/executor/src/task.rs delete mode 100644 crates/executor/src/template.rs diff --git a/crates/driver/Cargo.toml b/crates/driver/Cargo.toml new file mode 100644 index 0000000..215c451 --- /dev/null +++ b/crates/driver/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "rebel" +version = "0.1.0" +authors = ["Matthias Schiffer "] +license = "MIT" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../common", package = "rebel-common" } +runner = { path = "../runner", package = "rebel-runner" } + +clap = "3.0.0-beta.2" +enum-kinds = "0.5.1" +handlebars = "4.1.3" +indoc = "1.0.3" +lazy_static = "1.4.0" +nix = "0.23.0" +regex = "1.5.4" +scoped-tls-hkt = "0.1.2" +serde = { version = "1", features = ["derive", "rc"] } +serde_yaml = "0.8" +walkdir = "2" diff --git a/crates/driver/src/args.rs b/crates/driver/src/args.rs new file mode 100644 index 0000000..510a156 --- /dev/null +++ b/crates/driver/src/args.rs @@ -0,0 +1,122 @@ +use std::{ + collections::{hash_map, HashMap}, + hash, + rc::Rc, +}; + +use enum_kinds::EnumKind; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Platform { + #[serde(skip)] + pub short: String, + pub gnu_triplet: String, + pub karch: String, + pub prefix: String, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct PlatformRelation { + pub is_same: bool, + pub sysroot: String, + pub cross_compile: String, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, EnumKind)] +#[serde(untagged)] +#[enum_kind(ArgType, derive(Deserialize), serde(rename_all = "snake_case"))] +pub enum Arg { + String(Rc), + Platform(Rc), + PlatformRelation(Rc), +} + +impl From<&Arg> for Arg { + fn from(value: &Arg) -> Self { + value.clone() + } +} + +impl From for Arg { + fn from(value: String) -> Self { + Arg::String(Rc::new(value)) + } +} + +impl From for Arg { + fn from(value: Platform) -> Self { + Arg::Platform(Rc::new(value)) + } +} + +impl From for Arg { + fn from(value: PlatformRelation) -> Self { + Arg::PlatformRelation(Rc::new(value)) + } +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, Default)] +pub struct TaskArgs(HashMap); + +impl TaskArgs { + pub fn contains_key(&self, key: &str) -> bool { + self.0.contains_key(key) + } + + pub fn get(&self, key: &str) -> Option<&Arg> { + self.0.get(key) + } + + pub fn set(&mut self, key: &str, value: Option) + where + T: Into, + { + if let Some(v) = value { + self.0.insert(key.to_string(), v.into()); + } else { + self.0.remove(key); + } + } + + pub fn iter(&self) -> hash_map::Iter { + self.into_iter() + } +} + +impl FromIterator<(String, Arg)> for TaskArgs { + fn from_iter>(iter: T) -> Self { + TaskArgs(HashMap::from_iter(iter)) + } +} + +impl<'a> IntoIterator for &'a TaskArgs { + type Item = (&'a String, &'a Arg); + + type IntoIter = hash_map::Iter<'a, String, Arg>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +#[allow(clippy::derive_hash_xor_eq)] +impl hash::Hash for TaskArgs { + fn hash(&self, _state: &mut H) { + // Don't do anything: Properly hashing the task args is likely to cost + // much more performance than the hash collisions caused by TaskRefs + // that only differ by the args + } +} + +pub fn arg>(key: &str, value: A) -> (String, Arg) { + (key.to_string(), value.into()) +} + +#[derive(Clone, Debug, Deserialize, Default, PartialEq, Eq)] +pub struct ArgMapping(pub HashMap); + +#[allow(clippy::derive_hash_xor_eq)] +impl hash::Hash for ArgMapping { + fn hash(&self, _state: &mut H) {} +} diff --git a/crates/driver/src/context.rs b/crates/driver/src/context.rs new file mode 100644 index 0000000..daed230 --- /dev/null +++ b/crates/driver/src/context.rs @@ -0,0 +1,445 @@ +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + fmt::Display, + hash::Hash, + ops::Index, + rc::Rc, + result, +}; + +use lazy_static::lazy_static; +use regex::Regex; + +use common::{ + error::{self, Contextualizable}, + types::TaskID, +}; +use runner::paths; + +use crate::{ + args::{self, arg, Arg, ArgMapping, ArgType, PlatformRelation, TaskArgs}, + task::*, +}; + +#[derive(Debug, Clone, Copy)] +pub enum ErrorKind<'ctx> { + TaskNotFound, + InvalidArgument(&'ctx str), + InvalidArgRef(&'ctx str), +} + +#[derive(Debug, Clone, Copy)] +pub struct Error<'ctx> { + pub task: &'ctx TaskID, + pub kind: ErrorKind<'ctx>, +} + +impl<'ctx> Display for Error<'ctx> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Error { task, kind } = self; + match kind { + ErrorKind::TaskNotFound => write!(f, "Task '{}' not found", task), + ErrorKind::InvalidArgument(arg) => write!( + f, + "Invalid or missing argument '{}' for task '{}'", + arg, task + ), + ErrorKind::InvalidArgRef(arg) => write!( + f, + "Invalid reference for argument '{}' of task '{}'", + arg, task + ), + } + } +} + +impl<'ctx> From> for error::Error { + fn from(err: Error) -> Self { + error::Error::new(err) + } +} + +pub type Result<'ctx, T> = result::Result>; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TaskRef<'ctx> { + pub id: &'ctx TaskID, + pub args: Rc, +} + +impl<'ctx> Display for TaskRef<'ctx> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !f.alternate() { + return self.id.fmt(f); + } + + let pv_arg = match self.args.get("pv") { + Some(Arg::String(s)) => Some(s), + _ => None, + }; + let host_arg = match self.args.get("host") { + Some(Arg::Platform(platform)) => Some(platform), + _ => None, + }; + let target_arg = match self.args.get("target") { + Some(Arg::Platform(platform)) => Some(platform), + _ => None, + }; + + write!(f, "{}", self.id.recipe)?; + if let Some(pv) = pv_arg { + write!(f, "-{}", pv)?; + } + write!(f, ":{}", self.id.task)?; + + if let Some(host) = host_arg { + write!(f, "@{}", host.short)?; + } + if let Some(target) = target_arg { + write!(f, "/{}", target.short)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct OutputRef<'ctx> { + pub task: TaskRef<'ctx>, + pub output: &'ctx str, +} + +fn platform_relation(args: &TaskArgs, from: &str, to: &str) -> Option { + let plat_from = match args.get(from)? { + Arg::Platform(plat) => plat, + _ => return None, + }; + let plat_to = match args.get(to)? { + Arg::Platform(plat) => plat, + _ => return None, + }; + + let plat_rel = if plat_from == plat_to { + PlatformRelation { + is_same: true, + sysroot: "".to_string(), + cross_compile: "".to_string(), + } + } else { + PlatformRelation { + is_same: false, + sysroot: paths::abs(paths::TASK_SYSROOT), + cross_compile: format!("{}/bin/{}-", plat_from.prefix, plat_to.gnu_triplet), + } + }; + Some(plat_rel) +} + +#[derive(Debug)] +pub struct Context { + platforms: HashMap, + globals: TaskArgs, + tasks: HashMap, +} + +impl Context { + pub fn new(tasks: HashMap) -> Self { + let platforms: HashMap<_, _> = [ + arg( + "build", + args::Platform { + short: "build".to_string(), + gnu_triplet: "x86_64-linux-gnu".to_string(), + karch: "x86_64".to_string(), + prefix: "/opt/toolchain".to_string(), + }, + ), + arg( + "aarch64", + args::Platform { + short: "aarch64".to_string(), + gnu_triplet: "aarch64-linux-gnu".to_string(), + karch: "arm64".to_string(), + prefix: "/usr".to_string(), + }, + ), + ] + .into_iter() + .collect(); + + let globals = TaskArgs::from_iter([ + ("build".to_string(), platforms["build"].clone()), + arg("workdir", paths::abs(paths::TASK_WORKDIR)), + arg("dldir", paths::abs(paths::TASK_DLDIR)), + arg("destdir", paths::abs(paths::TASK_DESTDIR)), + arg("sysroot", paths::abs(paths::TASK_SYSROOT)), + ]); + + Context { + platforms, + globals, + tasks, + } + } + + pub fn get<'ctx>(&'ctx self, id: &'ctx TaskID) -> Result<&TaskDef> { + self.tasks.get(id).ok_or(Error { + task: id, + kind: ErrorKind::TaskNotFound, + }) + } + + fn task_ref<'ctx>(&'ctx self, id: &'ctx TaskID, args: &TaskArgs) -> Result { + let task_def = self.get(id)?; + + let mut arg_def: HashMap<_, _> = task_def.args.iter().map(|(k, &v)| (k, v)).collect(); + for (key, arg) in &self.globals { + // TODO: Handle conflicts between explicit args and globals + arg_def.insert(key, ArgType::from(arg)); + } + + let mut new_args = TaskArgs::default(); + + for (key, typ) in arg_def { + if let Some(arg) = args.get(key) { + if ArgType::from(arg) == typ { + new_args.set(key, Some(arg)); + continue; + } + } + return Err(Error { + task: id, + kind: ErrorKind::InvalidArgument(key), + }); + } + + let build_to_host = platform_relation(&new_args, "build", "host"); + let host_to_target = platform_relation(&new_args, "host", "target"); + let build_to_target = platform_relation(&new_args, "build", "target"); + + let cross_compile = build_to_host + .as_ref() + .map(|build_to_host| build_to_host.cross_compile.clone()); + + new_args.set("build_to_host", build_to_host); + new_args.set("host_to_target", host_to_target); + new_args.set("build_to_target", build_to_target); + + new_args.set("cross_compile", cross_compile); + + new_args.set("pn", Some(task_def.meta.name.clone())); + new_args.set("pv", task_def.meta.version.clone()); + + Ok(TaskRef { + id, + args: Rc::new(new_args), + }) + } + + pub fn parse<'ctx>(&'ctx self, s: &str) -> error::Result { + lazy_static! { + static ref RE: Regex = Regex::new( + r"^(?P[[:word:]-]+):(?P[[:word:]-]+)(?:@(?P[[:word:]-]+))?(?:/(?P[[:word:]-]+))?$", + ).unwrap(); + } + + let cap = RE.captures(s).context("Invalid task syntax")?; + + let recipe = cap["recipe"].to_string(); + let task = cap["task"].to_string(); + + let id = TaskID { recipe, task }; + let (ctx_id, _) = self + .tasks + .get_key_value(&id) + .with_context(|| format!("Task {}:{} not found", id.recipe, id.task))?; + + let mut args = self.globals.clone(); + + if let Some(host) = cap.name("host") { + let plat = self + .platforms + .get(host.as_str()) + .with_context(|| format!("Platform '{}' not found", host.as_str()))?; + args.set("host", Some(plat)); + args.set("target", Some(plat)); + } + if let Some(target) = cap.name("target") { + let plat = self + .platforms + .get(target.as_str()) + .with_context(|| format!("Platform '{}' not found", target.as_str()))?; + args.set("target", Some(plat)); + } + + self.task_ref(ctx_id, &args) + .with_context(|| format!("Failed to instantiate task {}:{}", id.recipe, id.task)) + } + + fn map_args<'ctx, 'args>( + task: &'ctx TaskID, + mapping: &'ctx ArgMapping, + args: &'args TaskArgs, + build_dep: bool, + ) -> Result<'ctx, Cow<'args, TaskArgs>> { + if mapping.0.is_empty() && !build_dep { + return Ok(Cow::Borrowed(args)); + } + + let mut ret = args.clone(); + + if build_dep { + ret.set("host", args.get("build")); + ret.set("target", args.get("host")); + } + + for (to, from) in &mapping.0 { + let value = args.get(from).ok_or(Error { + task, + kind: ErrorKind::InvalidArgRef(to), + })?; + ret.set(to, Some(value.clone())); + } + + Ok(Cow::Owned(ret)) + } + + fn inherit_ref<'ctx>(&'ctx self, dep: &'ctx InheritDep, args: &TaskArgs) -> Result { + let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, false)?; + self.task_ref(&dep.dep.id, mapped_args.as_ref()) + } + + pub fn output_ref<'ctx>( + &'ctx self, + dep: &'ctx OutputDep, + args: &TaskArgs, + build_dep: bool, + ) -> Result> { + let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, build_dep)?; + Ok(OutputRef { + task: self.task_ref(&dep.dep.id, mapped_args.as_ref())?, + output: &dep.output, + }) + } + + pub fn get_inherit_depend<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> Result> { + let task = self.get(task_ref.id)?; + let inherit = match &task.inherit { + Some(inherit) => inherit, + None => return Ok(None), + }; + Some(self.inherit_ref(inherit, &task_ref.args)).transpose() + } + + fn inherit_iter<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> impl Iterator> { + struct Iter<'ctx>(&'ctx Context, Option>>); + + impl<'ctx> Iterator for Iter<'ctx> { + type Item = Result<'ctx, TaskRef<'ctx>>; + + fn next(&mut self) -> Option { + let task_ref = match self.1.take()? { + Ok(task_ref) => task_ref, + Err(err) => return Some(Err(err)), + }; + self.1 = self.0.get_inherit_depend(&task_ref).transpose(); + Some(Ok(task_ref)) + } + } + + Iter(self, Some(Ok(task_ref.clone()))) + } + + pub fn get_build_depends<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> Result> { + let mut ret = HashSet::new(); + let mut allow_noinherit = true; + + for current in self.inherit_iter(task_ref) { + let task_ref = current?; + let task = self.get(task_ref.id)?; + let entries = task + .build_depends + .iter() + .filter(|dep| allow_noinherit || !dep.noinherit) + .map(|dep| self.output_ref(dep, &task_ref.args, true)) + .collect::>>()?; + ret.extend(entries); + + allow_noinherit = false; + } + + Ok(ret) + } + + pub fn get_host_depends<'ctx>( + &'ctx self, + task_ref: &TaskRef<'ctx>, + ) -> Result> { + let mut ret = HashSet::new(); + let mut allow_noinherit = true; + + for current in self.inherit_iter(task_ref) { + let task_ref = current?; + let task = self.get(task_ref.id)?; + let entries = task + .depends + .iter() + .filter(|dep| allow_noinherit || !dep.noinherit) + .map(|dep| self.output_ref(dep, &task_ref.args, false)) + .collect::>>()?; + ret.extend(entries); + + allow_noinherit = false; + } + + Ok(ret) + } + + pub fn in_rootfs<'ctx>(&'ctx self, output: &OutputRef<'ctx>) -> bool { + let build = self.globals.get("build").unwrap(); + if output.task.args.get("host") != Some(build) { + return false; + } + if let Some(target) = output.task.args.get("target") { + if target != build { + return false; + } + } + + // TODO: Do not hardcode this + match ( + output.task.id.recipe.as_str(), + output.task.id.task.as_str(), + output.output, + ) { + ("gmp", "install", "default") => true, + ("mpfr", "install", "default") => true, + ("mpc", "install", "default") => true, + ("zlib", "install", "default") => true, + ("binutils", "install", "default") => true, + ("gcc", "install", "default") => true, + ("libgcc", "install", "default") => true, + ("gcc-libs", "install", "default") => true, + ("linux-uapi-headers", "install", "default") => true, + ("glibc", "install", "default") => true, + _ => false, + } + } +} + +impl Index<&TaskID> for Context { + type Output = TaskDef; + + fn index(&self, index: &TaskID) -> &TaskDef { + self.tasks.get(index).expect("Invalid TaskID") + } +} diff --git a/crates/driver/src/driver.rs b/crates/driver/src/driver.rs new file mode 100644 index 0000000..f43f8e8 --- /dev/null +++ b/crates/driver/src/driver.rs @@ -0,0 +1,367 @@ +use std::{ + collections::{HashMap, HashSet}, + os::unix::{net::UnixStream, prelude::*}, +}; + +use indoc::indoc; +use nix::poll; + +use common::{error::*, string_hash::*, types::*}; +use runner::{paths, Runner}; + +use crate::{ + context::{Context, TaskRef}, + resolve, + task::*, + template, +}; + +#[derive(Debug)] +pub struct CompletionState<'ctx> { + ctx: &'ctx Context, + tasks_done: HashMap, TaskOutput>, +} + +impl<'ctx> CompletionState<'ctx> { + pub fn new(ctx: &'ctx Context) -> Self { + CompletionState { + ctx, + tasks_done: Default::default(), + } + } + + // Treats both "depends" and "inherit" as dependencies + fn deps_satisfied(&self, task_ref: &TaskRef) -> bool { + resolve::get_dependent_tasks(self.ctx, task_ref) + .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref))) + .unwrap() + .into_iter() + .all(|dep| self.tasks_done.contains_key(&dep)) + } + + fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result> { + let task_def = &self.ctx[task.id]; + task_def + .fetch + .iter() + .map(|Fetch { name, sha256 }| { + Ok(Dependency::Fetch { + name: template::ENGINE + .eval_raw(name, &task.args) + .with_context(|| { + format!("Failed to evaluate fetch filename for task {}", task) + })?, + sha256: *sha256, + }) + }) + .collect() + } + + fn task_deps(&self, task: &TaskRef<'ctx>) -> Result> { + Ok(self + .fetch_deps(task)? + .into_iter() + .chain( + resolve::runtime_depends( + self.ctx, + self.ctx + .get_build_depends(task) + .with_context(|| format!("invalid build depends for {}", task))?, + ) + .expect("invalid runtime depends of build_depends") + .into_iter() + .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) + .map(|&output| Dependency::Task { + output, + path: "".to_string(), + }), + ) + .chain( + resolve::runtime_depends( + self.ctx, + self.ctx + .get_host_depends(task) + .with_context(|| format!("invalid depends for {}", task))?, + ) + .expect("invalid runtime depends of host_depends") + .into_iter() + .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) + .map(|&output| Dependency::Task { + output, + path: paths::abs(paths::TASK_SYSROOT), + }), + ) + .collect()) + } + + fn task_inherit_chain(&self, task_ref: &TaskRef<'ctx>) -> Vec { + let inherit = match self + .ctx + .get_inherit_depend(task_ref) + .expect("invalid inherit depends") + { + Some(inherit) => inherit, + None => return vec![], + }; + + let mut chain = self.task_inherit_chain(&inherit); + if let Some(layer) = self.tasks_done[&inherit].layer { + chain.push(layer); + } + chain + } + + fn print_summary(&self) { + println!(); + println!("Summary:"); + + let mut tasks: Box<[_]> = self.tasks_done.iter().collect(); + tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task)); + for (task_ref, task) in tasks.iter() { + println!(); + println!("{:#}", task_ref); + println!(" input: {}", task.input_hash); + if let Some(hash) = task.layer { + println!(" layer: {}", hash); + } + if !task.outputs.is_empty() { + println!(" outputs:"); + + let mut outputs: Box<[_]> = task.outputs.iter().collect(); + outputs.sort_by_key(|(output, _)| *output); + for (output, hash) in outputs.iter() { + println!(" {}: {}", output, hash); + } + } + } + } +} + +#[derive(Debug)] +pub struct Driver<'ctx> { + rdeps: HashMap, Vec>>, + tasks_blocked: HashSet>, + tasks_runnable: Vec>, + tasks_running: HashMap)>, + state: CompletionState<'ctx>, +} + +impl<'ctx> Driver<'ctx> { + pub fn new(ctx: &'ctx Context, taskset: HashSet>) -> Result { + let mut driver = Driver { + rdeps: Default::default(), + tasks_blocked: Default::default(), + tasks_runnable: Default::default(), + tasks_running: Default::default(), + state: CompletionState::new(ctx), + }; + + for task in taskset { + let mut has_depends = false; + for dep in resolve::get_dependent_tasks(ctx, &task) + .map_err(|_| Error::new(format!("invalid dependency for {}", task)))? + { + let rdep = driver.rdeps.entry(dep.clone()).or_default(); + rdep.push(task.clone()); + has_depends = true; + } + + if has_depends { + driver.tasks_blocked.insert(task); + } else { + driver.tasks_runnable.push(task); + } + } + + Ok(driver) + } + + fn task_setup(task_ref: &TaskRef<'ctx>) -> Vec<&'static str> { + let mut ret = vec![indoc! {" + export SOURCE_DATE_EPOCH=1 + + export AR_FOR_BUILD=ar + export AS_FOR_BUILD=as + export DLLTOOL_FOR_BUILD=dlltool + export CC_FOR_BUILD=gcc + export CXX_FOR_BUILD=g++ + export GCC_FOR_BUILD=gcc + export GFORTRAN_FOR_BUILD=gfortran + export GOC_FOR_BUILD=goc + export LD_FOR_BUILD=ld + export LIPO_FOR_BUILD=lipo + export NM_FOR_BUILD=nm + export OBJCOPY_FOR_BUILD=objcopy + export OBJDUMP_FOR_BUILD=objdump + export RANLIB_FOR_BUILD=ranlib + export STRIP_FOR_BUILD=strip + export WINDRES_FOR_BUILD=windres + export WINDMC_FOR_BUILD=windmc + "}]; + + if task_ref.args.contains_key("build_to_host") { + ret.push(indoc! {" + export AR={{build_to_host.cross_compile}}ar + export AS={{build_to_host.cross_compile}}as + export DLLTOOL={{build_to_host.cross_compile}}dlltool + export CC={{build_to_host.cross_compile}}gcc + export CXX={{build_to_host.cross_compile}}g++ + export GCC={{build_to_host.cross_compile}}gcc + export GFORTRAN={{build_to_host.cross_compile}}gfortran + export GOC={{build_to_host.cross_compile}}goc + export LD={{build_to_host.cross_compile}}ld + export LIPO={{build_to_host.cross_compile}}lipo + export NM={{build_to_host.cross_compile}}nm + export OBJCOPY={{build_to_host.cross_compile}}objcopy + export OBJDUMP={{build_to_host.cross_compile}}objdump + export RANLIB={{build_to_host.cross_compile}}ranlib + export STRIP={{build_to_host.cross_compile}}strip + export WINDRES={{build_to_host.cross_compile}}windres + export WINDMC={{build_to_host.cross_compile}}windmc + "}); + } + + if task_ref.args.contains_key("build_to_target") { + ret.push(indoc! {" + export AR_FOR_TARGET={{build_to_target.cross_compile}}ar + export AS_FOR_TARGET={{build_to_target.cross_compile}}as + export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool + export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc + export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++ + export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc + export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran + export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc + export LD_FOR_TARGET={{build_to_target.cross_compile}}ld + export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo + export NM_FOR_TARGET={{build_to_target.cross_compile}}nm + export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy + export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump + export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib + export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip + export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres + export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc + "}); + } + ret + } + + fn update_runnable(&mut self, task_ref: TaskRef<'ctx>, task_output: TaskOutput) { + let rdeps = self.rdeps.get(&task_ref); + + self.state.tasks_done.insert(task_ref, task_output); + + for rdep in rdeps.unwrap_or(&Vec::new()) { + if !self.tasks_blocked.contains(rdep) { + continue; + } + if self.state.deps_satisfied(rdep) { + self.tasks_blocked.remove(rdep); + self.tasks_runnable.push(rdep.clone()); + } + } + } + + fn spawn_task(&self, task_ref: &TaskRef<'ctx>, runner: &Runner) -> Result { + let task_def = &self.state.ctx[task_ref.id]; + let task_deps = self.state.task_deps(task_ref)?; + let task_output = task_def + .output + .iter() + .map(|(name, Output { path, .. })| { + ( + name.clone(), + path.as_ref().map(String::as_str).unwrap_or(".").to_string(), + ) + }) + .collect(); + + let inherit_chain = self.state.task_inherit_chain(task_ref); + + let mut run = Self::task_setup(task_ref); + run.push(&task_def.action.run); + + let command = template::ENGINE + .eval(&run.concat(), &task_ref.args) + .with_context(|| { + format!("Failed to evaluate command template for task {}", task_ref) + })?; + + let task = Task { + label: format!("{:#}", task_ref), + command, + inherit: inherit_chain, + depends: task_deps, + outputs: task_output, + }; + + Ok(runner.spawn(&task)) + } + + fn run_task(&mut self, task_ref: TaskRef<'ctx>, runner: &Runner) -> Result<()> { + let socket = self.spawn_task(&task_ref, runner)?; + assert!(self + .tasks_running + .insert(socket.as_raw_fd(), (socket, task_ref)) + .is_none()); + Ok(()) + } + + fn run_tasks(&mut self, runner: &Runner) -> Result<()> { + while let Some(task_ref) = self.tasks_runnable.pop() { + self.run_task(task_ref, runner)?; + } + Ok(()) + } + + fn wait_for_task(&mut self) -> Result<()> { + let mut pollfds: Box<[_]> = self + .tasks_running + .keys() + .copied() + .map(|fd| poll::PollFd::new(fd, poll::PollFlags::POLLIN)) + .collect(); + + while poll::poll(&mut pollfds, -1).context("poll()")? == 0 {} + + for pollfd in &*pollfds { + let events = pollfd.revents().expect("Unknown events in poll() return"); + if !events.contains(poll::PollFlags::POLLIN) { + if events.intersects(!poll::PollFlags::POLLIN) { + return Err(Error::new( + "Unexpected error status for socket file descriptor", + )); + } + continue; + } + + let fd = pollfd.as_raw_fd(); + let (socket, task_ref) = self.tasks_running.remove(&fd).unwrap(); + + let task_output = Runner::result(&socket)?; + self.update_runnable(task_ref, task_output); + } + + Ok(()) + } + + fn is_empty(&self) -> bool { + self.tasks_runnable.is_empty() && self.tasks_running.is_empty() + } + + fn is_done(&self) -> bool { + self.is_empty() && self.tasks_blocked.is_empty() + } + + pub fn run(&mut self, runner: &Runner) -> Result<()> { + while !self.is_empty() { + self.run_tasks(runner)?; + self.wait_for_task()?; + } + + assert!(self.is_done(), "No runnable tasks left"); + + self.state.print_summary(); + + Ok(()) + } +} diff --git a/crates/driver/src/main.rs b/crates/driver/src/main.rs new file mode 100644 index 0000000..286480d --- /dev/null +++ b/crates/driver/src/main.rs @@ -0,0 +1,56 @@ +mod args; +mod context; +mod driver; +mod recipe; +mod resolve; +mod task; +mod template; + +use clap::Parser; + +use runner::Runner; + +#[derive(Parser)] +#[clap(version = clap::crate_version!())] +struct Opts { + /// Allow N jobs at once. + /// Defaults to the number of available CPUs + #[clap(short, long)] + jobs: Option, + /// The tasks to run + #[clap(name = "task", required = true)] + tasks: Vec, +} + +fn main() { + let opts: Opts = Opts::parse(); + + let runner = unsafe { Runner::new(&runner::Options { jobs: opts.jobs }) }.unwrap(); + + let ctx = context::Context::new(recipe::read_recipes("examples").unwrap()); + + let mut rsv = resolve::Resolver::new(&ctx); + + for task in opts.tasks { + let task_ref = match ctx.parse(&task) { + Ok(task_ref) => task_ref, + Err(err) => { + eprintln!("{}", err); + std::process::exit(1); + } + }; + let errors = rsv.add_goal(&task_ref); + if !errors.is_empty() { + for error in errors { + eprintln!("{}", error); + } + std::process::exit(1); + } + } + let taskset = rsv.into_taskset(); + let mut driver = driver::Driver::new(&ctx, taskset).unwrap(); + if let Err(error) = driver.run(&runner) { + eprintln!("{}", error); + std::process::exit(1); + } +} diff --git a/crates/driver/src/recipe.rs b/crates/driver/src/recipe.rs new file mode 100644 index 0000000..04f356b --- /dev/null +++ b/crates/driver/src/recipe.rs @@ -0,0 +1,115 @@ +use std::{collections::HashMap, fmt, fs::File, io, path::Path, result}; + +use scoped_tls_hkt::scoped_thread_local; +use serde::{Deserialize, Deserializer}; +use walkdir::WalkDir; + +use common::types::*; + +use crate::task::{RecipeMeta, TaskDef}; + +scoped_thread_local!(static CURRENT_RECIPE: str); + +fn current_recipe() -> String { + CURRENT_RECIPE.with(|current| current.to_string()) +} + +pub fn deserialize_task_id<'de, D>(deserializer: D) -> result::Result +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + struct RecipeTaskID { + recipe: Option, + task: String, + } + let RecipeTaskID { recipe, task } = RecipeTaskID::deserialize(deserializer)?; + Ok(TaskID { + recipe: recipe.unwrap_or_else(current_recipe), + task, + }) +} + +#[derive(Debug, Deserialize)] +struct Recipe { + #[serde(default)] + pub meta: RecipeMeta, + pub tasks: HashMap, +} + +#[derive(Debug)] +pub enum Error { + IOError(io::Error), + YAMLError(serde_yaml::Error), +} + +impl From for Error { + fn from(err: io::Error) -> Self { + Error::IOError(err) + } +} + +impl From for Error { + fn from(err: serde_yaml::Error) -> Self { + Error::YAMLError(err) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::IOError(err) => write!(f, "IO error: {}", err), + Error::YAMLError(err) => write!(f, "YAML error: {}", err), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +fn read_recipe(path: &Path) -> Result { + let f = File::open(path)?; + + let recipe: Recipe = serde_yaml::from_reader(f)?; + + Ok(recipe) +} + +fn is_yml(path: &Path) -> bool { + path.extension() == Some("yml".as_ref()) +} + +pub fn read_recipes>(path: P) -> Result> { + let mut tasks = HashMap::new(); + + for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) { + let path = entry.path(); + if !path.is_file() || !is_yml(path) { + continue; + } + + let basename: &str = match path.file_stem().map(|n| n.to_str()) { + Some(Some(v)) => v, + _ => continue, + }; + + let recipe = CURRENT_RECIPE.set(basename, || read_recipe(path))?; + + let mut meta = recipe.meta; + if meta.name.is_empty() { + meta.name = basename.to_string(); + } + + for (label, mut task) in recipe.tasks { + let task_id = TaskID { + recipe: basename.to_string(), + task: label, + }; + task.meta = meta.clone(); + tasks.insert(task_id, task); + } + } + + Ok(tasks) +} diff --git a/crates/driver/src/resolve.rs b/crates/driver/src/resolve.rs new file mode 100644 index 0000000..338ce3f --- /dev/null +++ b/crates/driver/src/resolve.rs @@ -0,0 +1,312 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::rc::Rc; + +use common::types::TaskID; + +use crate::args::TaskArgs; +use crate::context::{self, Context, OutputRef, TaskRef}; + +#[derive(Debug)] +pub struct DepChain<'ctx>(pub Vec>); + +impl<'ctx> fmt::Display for DepChain<'ctx> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut first = true; + for task in self.0.iter().rev() { + if !first { + write!(f, " -> ")?; + } + write!(f, "{}", task)?; + + first = false; + } + + Ok(()) + } +} + +impl<'ctx> From> for DepChain<'ctx> { + fn from(task: TaskRef<'ctx>) -> Self { + DepChain(vec![task]) + } +} + +impl<'ctx> From<&TaskRef<'ctx>> for DepChain<'ctx> { + fn from(task: &TaskRef<'ctx>) -> Self { + task.clone().into() + } +} + +impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> { + fn from(id: &'ctx TaskID) -> Self { + TaskRef { + id, + args: Rc::new(TaskArgs::default()), + } + .into() + } +} + +#[derive(Debug)] +pub enum ErrorKind<'ctx> { + Context(context::Error<'ctx>), + OutputNotFound(&'ctx str), + DependencyCycle, +} + +#[derive(Debug)] +pub struct Error<'ctx> { + pub dep_chain: DepChain<'ctx>, + pub kind: ErrorKind<'ctx>, +} + +impl<'ctx> Error<'ctx> { + fn output_not_found(task: &TaskRef<'ctx>, output: &'ctx str) -> Self { + Error { + dep_chain: task.into(), + kind: ErrorKind::OutputNotFound(output), + } + } + + fn dependency_cycle(task: &TaskRef<'ctx>) -> Self { + Error { + dep_chain: task.into(), + kind: ErrorKind::DependencyCycle, + } + } + + fn extend(&mut self, task: &TaskRef<'ctx>) { + self.dep_chain.0.push(task.clone()); + } +} + +impl<'ctx> fmt::Display for Error<'ctx> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let Error { dep_chain, kind } = self; + match kind { + ErrorKind::Context(err) => { + write!(f, "{}: ", err)?; + } + ErrorKind::OutputNotFound(output) => { + write!(f, "Output '{}' not found: ", output)?; + } + ErrorKind::DependencyCycle => { + write!(f, "Dependency Cycle: ")?; + } + } + dep_chain.fmt(f) + } +} + +impl<'ctx> From> for Error<'ctx> { + fn from(err: context::Error<'ctx>) -> Self { + Error { + dep_chain: err.task.into(), + kind: ErrorKind::Context(err), + } + } +} + +impl<'ctx> std::error::Error for Error<'ctx> {} + +#[derive(Debug, PartialEq)] +enum ResolveState { + Resolving, + Resolved, +} + +pub fn runtime_depends<'ctx, I>( + ctx: &'ctx Context, + deps: I, +) -> Result, Vec> +where + I: IntoIterator>, +{ + fn add_dep<'ctx>( + ret: &mut HashSet>, + ctx: &'ctx Context, + dep: OutputRef<'ctx>, + ) -> Vec> { + if ret.contains(&dep) || ctx.in_rootfs(&dep) { + return Vec::new(); + } + + let task = &dep.task; + let task_def = match ctx.get(task.id) { + Ok(task) => task, + Err(err) => return vec![err.into()], + }; + + let output = match task_def.output.get(dep.output) { + Some(output) => output, + None => { + return vec![Error::output_not_found(task, dep.output)]; + } + }; + + ret.insert(dep.clone()); + + let mut errors = Vec::new(); + for runtime_dep in &output.runtime_depends { + match ctx.output_ref(runtime_dep, &task.args, false) { + Ok(output_ref) => { + for mut error in add_dep(ret, ctx, output_ref) { + error.extend(task); + errors.push(error); + } + } + Err(err) => { + let mut err: Error = err.into(); + err.extend(task); + errors.push(err); + } + }; + } + errors + } + + let mut ret = HashSet::new(); + let mut errors = Vec::new(); + + for dep in deps { + errors.extend(add_dep(&mut ret, ctx, dep)); + } + + if !errors.is_empty() { + return Err(errors); + } + + Ok(ret) +} + +pub fn get_dependent_outputs<'ctx>( + ctx: &'ctx Context, + task_ref: &TaskRef<'ctx>, +) -> Result>, Vec>> { + let deps: HashSet<_> = ctx + .get_build_depends(task_ref) + .map_err(|err| vec![err.into()])? + .into_iter() + .chain( + ctx.get_host_depends(task_ref) + .map_err(|err| vec![err.into()])? + .into_iter(), + ) + .collect(); + runtime_depends(ctx, deps) +} + +pub fn get_dependent_tasks<'ctx>( + ctx: &'ctx Context, + task_ref: &TaskRef<'ctx>, +) -> Result>, Vec>> { + Ok(ctx + .get_inherit_depend(task_ref) + .map_err(|err| vec![err.into()])? + .into_iter() + .chain( + get_dependent_outputs(ctx, task_ref)? + .into_iter() + .map(|dep| dep.task), + ) + .collect()) +} + +#[derive(Debug)] +pub struct Resolver<'ctx> { + ctx: &'ctx Context, + resolve_state: HashMap, ResolveState>, +} + +impl<'ctx> Resolver<'ctx> { + pub fn new(ctx: &'ctx Context) -> Self { + Resolver { + ctx, + resolve_state: HashMap::new(), + } + } + + fn tasks_resolved(&self) -> bool { + self.resolve_state + .values() + .all(|resolved| *resolved == ResolveState::Resolved) + } + + fn add_task(&mut self, task: &TaskRef<'ctx>, output: Option<&'ctx str>) -> Vec> { + match self.resolve_state.get(task) { + Some(ResolveState::Resolving) => return vec![Error::dependency_cycle(task)], + Some(ResolveState::Resolved) => return vec![], + None => (), + } + + let task_def = match self.ctx.get(task.id) { + Ok(task_def) => task_def, + Err(err) => return vec![err.into()], + }; + + if let Some(task_output) = output { + if !task_def.output.contains_key(task_output) { + return vec![Error::output_not_found(task, task_output)]; + } + } + + self.resolve_state + .insert(task.clone(), ResolveState::Resolving); + + let mut ret = Vec::new(); + let mut handle_errors = |errors: Vec>| { + for mut error in errors { + error.extend(task); + ret.push(error); + } + }; + + match self.ctx.get_inherit_depend(task) { + Ok(Some(inherit)) => { + handle_errors(self.add_task(&inherit, None)); + } + Ok(None) => {} + Err(err) => { + handle_errors(vec![err.into()]); + } + } + + match get_dependent_outputs(self.ctx, task) { + Ok(rdeps) => { + for rdep in rdeps { + handle_errors(self.add_task(&rdep.task, Some(rdep.output))); + } + } + Err(errors) => { + handle_errors(errors); + } + } + + if ret.is_empty() { + *self + .resolve_state + .get_mut(task) + .expect("Missing resolve_state") = ResolveState::Resolved; + } else { + self.resolve_state.remove(task); + } + + ret + } + + pub fn add_goal(&mut self, task: &TaskRef<'ctx>) -> Vec> { + let ret = self.add_task(task, None); + debug_assert!(self.tasks_resolved()); + ret + } + + pub fn into_taskset(self) -> HashSet> { + debug_assert!(self.tasks_resolved()); + + self.resolve_state + .into_iter() + .map(|entry| entry.0) + .collect() + } +} diff --git a/crates/driver/src/task.rs b/crates/driver/src/task.rs new file mode 100644 index 0000000..fe9572c --- /dev/null +++ b/crates/driver/src/task.rs @@ -0,0 +1,84 @@ +use std::collections::{HashMap, HashSet}; + +use serde::Deserialize; + +use common::{string_hash::StringHash, types::TaskID}; + +use crate::{ + args::{ArgMapping, ArgType}, + recipe, +}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash, Default)] +pub struct RecipeMeta { + #[serde(default)] + pub name: String, + pub version: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] +pub struct TaskDep { + #[serde(flatten, deserialize_with = "recipe::deserialize_task_id")] + pub id: TaskID, + #[serde(default)] + pub args: ArgMapping, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] +pub struct Fetch { + pub name: String, + pub sha256: StringHash, +} + +fn default_output_name() -> String { + "default".to_string() +} + +#[derive(Clone, Debug, Deserialize)] +pub struct InheritDep { + #[serde(flatten)] + pub dep: TaskDep, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] +pub struct OutputDep { + #[serde(flatten)] + pub dep: TaskDep, + #[serde(default)] + pub noinherit: bool, + #[serde(default = "default_output_name")] + pub output: String, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct Output { + pub path: Option, + #[serde(default)] + pub runtime_depends: HashSet, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct Action { + #[serde(default)] + pub run: String, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct TaskDef { + #[serde(skip)] + pub meta: RecipeMeta, + #[serde(default)] + pub args: HashMap, + #[serde(default)] + pub inherit: Option, + #[serde(default)] + pub fetch: HashSet, + #[serde(default)] + pub build_depends: HashSet, + #[serde(default)] + pub depends: HashSet, + #[serde(default)] + pub output: HashMap, + #[serde(flatten)] + pub action: Action, +} diff --git a/crates/driver/src/template.rs b/crates/driver/src/template.rs new file mode 100644 index 0000000..b663e7d --- /dev/null +++ b/crates/driver/src/template.rs @@ -0,0 +1,44 @@ +use handlebars::Handlebars; +use lazy_static::lazy_static; + +use common::error::*; + +use crate::args::TaskArgs; + +fn escape(s: &str) -> String { + format!("'{}'", s.replace("'", "'\\''")) +} + +#[derive(Debug)] +pub struct TemplateEngine { + tpl: Handlebars<'static>, + tpl_raw: Handlebars<'static>, +} + +impl TemplateEngine { + pub fn new() -> Self { + let mut tpl = Handlebars::new(); + tpl.set_strict_mode(true); + tpl.register_escape_fn(escape); + + let mut tpl_raw = Handlebars::new(); + tpl_raw.set_strict_mode(true); + tpl_raw.register_escape_fn(handlebars::no_escape); + + TemplateEngine { tpl, tpl_raw } + } + + pub fn eval_raw(&self, input: &str, args: &TaskArgs) -> Result { + self.tpl_raw + .render_template(input, args) + .map_err(Error::new) + } + + pub fn eval(&self, input: &str, args: &TaskArgs) -> Result { + self.tpl.render_template(input, args).map_err(Error::new) + } +} + +lazy_static! { + pub static ref ENGINE: TemplateEngine = TemplateEngine::new(); +} diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml deleted file mode 100644 index 215c451..0000000 --- a/crates/executor/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "rebel" -version = "0.1.0" -authors = ["Matthias Schiffer "] -license = "MIT" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -common = { path = "../common", package = "rebel-common" } -runner = { path = "../runner", package = "rebel-runner" } - -clap = "3.0.0-beta.2" -enum-kinds = "0.5.1" -handlebars = "4.1.3" -indoc = "1.0.3" -lazy_static = "1.4.0" -nix = "0.23.0" -regex = "1.5.4" -scoped-tls-hkt = "0.1.2" -serde = { version = "1", features = ["derive", "rc"] } -serde_yaml = "0.8" -walkdir = "2" diff --git a/crates/executor/src/args.rs b/crates/executor/src/args.rs deleted file mode 100644 index 510a156..0000000 --- a/crates/executor/src/args.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{ - collections::{hash_map, HashMap}, - hash, - rc::Rc, -}; - -use enum_kinds::EnumKind; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, PartialEq, Eq)] -pub struct Platform { - #[serde(skip)] - pub short: String, - pub gnu_triplet: String, - pub karch: String, - pub prefix: String, -} - -#[derive(Debug, Serialize, PartialEq, Eq)] -pub struct PlatformRelation { - pub is_same: bool, - pub sysroot: String, - pub cross_compile: String, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq, EnumKind)] -#[serde(untagged)] -#[enum_kind(ArgType, derive(Deserialize), serde(rename_all = "snake_case"))] -pub enum Arg { - String(Rc), - Platform(Rc), - PlatformRelation(Rc), -} - -impl From<&Arg> for Arg { - fn from(value: &Arg) -> Self { - value.clone() - } -} - -impl From for Arg { - fn from(value: String) -> Self { - Arg::String(Rc::new(value)) - } -} - -impl From for Arg { - fn from(value: Platform) -> Self { - Arg::Platform(Rc::new(value)) - } -} - -impl From for Arg { - fn from(value: PlatformRelation) -> Self { - Arg::PlatformRelation(Rc::new(value)) - } -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq, Default)] -pub struct TaskArgs(HashMap); - -impl TaskArgs { - pub fn contains_key(&self, key: &str) -> bool { - self.0.contains_key(key) - } - - pub fn get(&self, key: &str) -> Option<&Arg> { - self.0.get(key) - } - - pub fn set(&mut self, key: &str, value: Option) - where - T: Into, - { - if let Some(v) = value { - self.0.insert(key.to_string(), v.into()); - } else { - self.0.remove(key); - } - } - - pub fn iter(&self) -> hash_map::Iter { - self.into_iter() - } -} - -impl FromIterator<(String, Arg)> for TaskArgs { - fn from_iter>(iter: T) -> Self { - TaskArgs(HashMap::from_iter(iter)) - } -} - -impl<'a> IntoIterator for &'a TaskArgs { - type Item = (&'a String, &'a Arg); - - type IntoIter = hash_map::Iter<'a, String, Arg>; - - fn into_iter(self) -> Self::IntoIter { - self.0.iter() - } -} - -#[allow(clippy::derive_hash_xor_eq)] -impl hash::Hash for TaskArgs { - fn hash(&self, _state: &mut H) { - // Don't do anything: Properly hashing the task args is likely to cost - // much more performance than the hash collisions caused by TaskRefs - // that only differ by the args - } -} - -pub fn arg>(key: &str, value: A) -> (String, Arg) { - (key.to_string(), value.into()) -} - -#[derive(Clone, Debug, Deserialize, Default, PartialEq, Eq)] -pub struct ArgMapping(pub HashMap); - -#[allow(clippy::derive_hash_xor_eq)] -impl hash::Hash for ArgMapping { - fn hash(&self, _state: &mut H) {} -} diff --git a/crates/executor/src/context.rs b/crates/executor/src/context.rs deleted file mode 100644 index daed230..0000000 --- a/crates/executor/src/context.rs +++ /dev/null @@ -1,445 +0,0 @@ -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - fmt::Display, - hash::Hash, - ops::Index, - rc::Rc, - result, -}; - -use lazy_static::lazy_static; -use regex::Regex; - -use common::{ - error::{self, Contextualizable}, - types::TaskID, -}; -use runner::paths; - -use crate::{ - args::{self, arg, Arg, ArgMapping, ArgType, PlatformRelation, TaskArgs}, - task::*, -}; - -#[derive(Debug, Clone, Copy)] -pub enum ErrorKind<'ctx> { - TaskNotFound, - InvalidArgument(&'ctx str), - InvalidArgRef(&'ctx str), -} - -#[derive(Debug, Clone, Copy)] -pub struct Error<'ctx> { - pub task: &'ctx TaskID, - pub kind: ErrorKind<'ctx>, -} - -impl<'ctx> Display for Error<'ctx> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let Error { task, kind } = self; - match kind { - ErrorKind::TaskNotFound => write!(f, "Task '{}' not found", task), - ErrorKind::InvalidArgument(arg) => write!( - f, - "Invalid or missing argument '{}' for task '{}'", - arg, task - ), - ErrorKind::InvalidArgRef(arg) => write!( - f, - "Invalid reference for argument '{}' of task '{}'", - arg, task - ), - } - } -} - -impl<'ctx> From> for error::Error { - fn from(err: Error) -> Self { - error::Error::new(err) - } -} - -pub type Result<'ctx, T> = result::Result>; - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct TaskRef<'ctx> { - pub id: &'ctx TaskID, - pub args: Rc, -} - -impl<'ctx> Display for TaskRef<'ctx> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if !f.alternate() { - return self.id.fmt(f); - } - - let pv_arg = match self.args.get("pv") { - Some(Arg::String(s)) => Some(s), - _ => None, - }; - let host_arg = match self.args.get("host") { - Some(Arg::Platform(platform)) => Some(platform), - _ => None, - }; - let target_arg = match self.args.get("target") { - Some(Arg::Platform(platform)) => Some(platform), - _ => None, - }; - - write!(f, "{}", self.id.recipe)?; - if let Some(pv) = pv_arg { - write!(f, "-{}", pv)?; - } - write!(f, ":{}", self.id.task)?; - - if let Some(host) = host_arg { - write!(f, "@{}", host.short)?; - } - if let Some(target) = target_arg { - write!(f, "/{}", target.short)?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct OutputRef<'ctx> { - pub task: TaskRef<'ctx>, - pub output: &'ctx str, -} - -fn platform_relation(args: &TaskArgs, from: &str, to: &str) -> Option { - let plat_from = match args.get(from)? { - Arg::Platform(plat) => plat, - _ => return None, - }; - let plat_to = match args.get(to)? { - Arg::Platform(plat) => plat, - _ => return None, - }; - - let plat_rel = if plat_from == plat_to { - PlatformRelation { - is_same: true, - sysroot: "".to_string(), - cross_compile: "".to_string(), - } - } else { - PlatformRelation { - is_same: false, - sysroot: paths::abs(paths::TASK_SYSROOT), - cross_compile: format!("{}/bin/{}-", plat_from.prefix, plat_to.gnu_triplet), - } - }; - Some(plat_rel) -} - -#[derive(Debug)] -pub struct Context { - platforms: HashMap, - globals: TaskArgs, - tasks: HashMap, -} - -impl Context { - pub fn new(tasks: HashMap) -> Self { - let platforms: HashMap<_, _> = [ - arg( - "build", - args::Platform { - short: "build".to_string(), - gnu_triplet: "x86_64-linux-gnu".to_string(), - karch: "x86_64".to_string(), - prefix: "/opt/toolchain".to_string(), - }, - ), - arg( - "aarch64", - args::Platform { - short: "aarch64".to_string(), - gnu_triplet: "aarch64-linux-gnu".to_string(), - karch: "arm64".to_string(), - prefix: "/usr".to_string(), - }, - ), - ] - .into_iter() - .collect(); - - let globals = TaskArgs::from_iter([ - ("build".to_string(), platforms["build"].clone()), - arg("workdir", paths::abs(paths::TASK_WORKDIR)), - arg("dldir", paths::abs(paths::TASK_DLDIR)), - arg("destdir", paths::abs(paths::TASK_DESTDIR)), - arg("sysroot", paths::abs(paths::TASK_SYSROOT)), - ]); - - Context { - platforms, - globals, - tasks, - } - } - - pub fn get<'ctx>(&'ctx self, id: &'ctx TaskID) -> Result<&TaskDef> { - self.tasks.get(id).ok_or(Error { - task: id, - kind: ErrorKind::TaskNotFound, - }) - } - - fn task_ref<'ctx>(&'ctx self, id: &'ctx TaskID, args: &TaskArgs) -> Result { - let task_def = self.get(id)?; - - let mut arg_def: HashMap<_, _> = task_def.args.iter().map(|(k, &v)| (k, v)).collect(); - for (key, arg) in &self.globals { - // TODO: Handle conflicts between explicit args and globals - arg_def.insert(key, ArgType::from(arg)); - } - - let mut new_args = TaskArgs::default(); - - for (key, typ) in arg_def { - if let Some(arg) = args.get(key) { - if ArgType::from(arg) == typ { - new_args.set(key, Some(arg)); - continue; - } - } - return Err(Error { - task: id, - kind: ErrorKind::InvalidArgument(key), - }); - } - - let build_to_host = platform_relation(&new_args, "build", "host"); - let host_to_target = platform_relation(&new_args, "host", "target"); - let build_to_target = platform_relation(&new_args, "build", "target"); - - let cross_compile = build_to_host - .as_ref() - .map(|build_to_host| build_to_host.cross_compile.clone()); - - new_args.set("build_to_host", build_to_host); - new_args.set("host_to_target", host_to_target); - new_args.set("build_to_target", build_to_target); - - new_args.set("cross_compile", cross_compile); - - new_args.set("pn", Some(task_def.meta.name.clone())); - new_args.set("pv", task_def.meta.version.clone()); - - Ok(TaskRef { - id, - args: Rc::new(new_args), - }) - } - - pub fn parse<'ctx>(&'ctx self, s: &str) -> error::Result { - lazy_static! { - static ref RE: Regex = Regex::new( - r"^(?P[[:word:]-]+):(?P[[:word:]-]+)(?:@(?P[[:word:]-]+))?(?:/(?P[[:word:]-]+))?$", - ).unwrap(); - } - - let cap = RE.captures(s).context("Invalid task syntax")?; - - let recipe = cap["recipe"].to_string(); - let task = cap["task"].to_string(); - - let id = TaskID { recipe, task }; - let (ctx_id, _) = self - .tasks - .get_key_value(&id) - .with_context(|| format!("Task {}:{} not found", id.recipe, id.task))?; - - let mut args = self.globals.clone(); - - if let Some(host) = cap.name("host") { - let plat = self - .platforms - .get(host.as_str()) - .with_context(|| format!("Platform '{}' not found", host.as_str()))?; - args.set("host", Some(plat)); - args.set("target", Some(plat)); - } - if let Some(target) = cap.name("target") { - let plat = self - .platforms - .get(target.as_str()) - .with_context(|| format!("Platform '{}' not found", target.as_str()))?; - args.set("target", Some(plat)); - } - - self.task_ref(ctx_id, &args) - .with_context(|| format!("Failed to instantiate task {}:{}", id.recipe, id.task)) - } - - fn map_args<'ctx, 'args>( - task: &'ctx TaskID, - mapping: &'ctx ArgMapping, - args: &'args TaskArgs, - build_dep: bool, - ) -> Result<'ctx, Cow<'args, TaskArgs>> { - if mapping.0.is_empty() && !build_dep { - return Ok(Cow::Borrowed(args)); - } - - let mut ret = args.clone(); - - if build_dep { - ret.set("host", args.get("build")); - ret.set("target", args.get("host")); - } - - for (to, from) in &mapping.0 { - let value = args.get(from).ok_or(Error { - task, - kind: ErrorKind::InvalidArgRef(to), - })?; - ret.set(to, Some(value.clone())); - } - - Ok(Cow::Owned(ret)) - } - - fn inherit_ref<'ctx>(&'ctx self, dep: &'ctx InheritDep, args: &TaskArgs) -> Result { - let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, false)?; - self.task_ref(&dep.dep.id, mapped_args.as_ref()) - } - - pub fn output_ref<'ctx>( - &'ctx self, - dep: &'ctx OutputDep, - args: &TaskArgs, - build_dep: bool, - ) -> Result> { - let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, build_dep)?; - Ok(OutputRef { - task: self.task_ref(&dep.dep.id, mapped_args.as_ref())?, - output: &dep.output, - }) - } - - pub fn get_inherit_depend<'ctx>( - &'ctx self, - task_ref: &TaskRef<'ctx>, - ) -> Result> { - let task = self.get(task_ref.id)?; - let inherit = match &task.inherit { - Some(inherit) => inherit, - None => return Ok(None), - }; - Some(self.inherit_ref(inherit, &task_ref.args)).transpose() - } - - fn inherit_iter<'ctx>( - &'ctx self, - task_ref: &TaskRef<'ctx>, - ) -> impl Iterator> { - struct Iter<'ctx>(&'ctx Context, Option>>); - - impl<'ctx> Iterator for Iter<'ctx> { - type Item = Result<'ctx, TaskRef<'ctx>>; - - fn next(&mut self) -> Option { - let task_ref = match self.1.take()? { - Ok(task_ref) => task_ref, - Err(err) => return Some(Err(err)), - }; - self.1 = self.0.get_inherit_depend(&task_ref).transpose(); - Some(Ok(task_ref)) - } - } - - Iter(self, Some(Ok(task_ref.clone()))) - } - - pub fn get_build_depends<'ctx>( - &'ctx self, - task_ref: &TaskRef<'ctx>, - ) -> Result> { - let mut ret = HashSet::new(); - let mut allow_noinherit = true; - - for current in self.inherit_iter(task_ref) { - let task_ref = current?; - let task = self.get(task_ref.id)?; - let entries = task - .build_depends - .iter() - .filter(|dep| allow_noinherit || !dep.noinherit) - .map(|dep| self.output_ref(dep, &task_ref.args, true)) - .collect::>>()?; - ret.extend(entries); - - allow_noinherit = false; - } - - Ok(ret) - } - - pub fn get_host_depends<'ctx>( - &'ctx self, - task_ref: &TaskRef<'ctx>, - ) -> Result> { - let mut ret = HashSet::new(); - let mut allow_noinherit = true; - - for current in self.inherit_iter(task_ref) { - let task_ref = current?; - let task = self.get(task_ref.id)?; - let entries = task - .depends - .iter() - .filter(|dep| allow_noinherit || !dep.noinherit) - .map(|dep| self.output_ref(dep, &task_ref.args, false)) - .collect::>>()?; - ret.extend(entries); - - allow_noinherit = false; - } - - Ok(ret) - } - - pub fn in_rootfs<'ctx>(&'ctx self, output: &OutputRef<'ctx>) -> bool { - let build = self.globals.get("build").unwrap(); - if output.task.args.get("host") != Some(build) { - return false; - } - if let Some(target) = output.task.args.get("target") { - if target != build { - return false; - } - } - - // TODO: Do not hardcode this - match ( - output.task.id.recipe.as_str(), - output.task.id.task.as_str(), - output.output, - ) { - ("gmp", "install", "default") => true, - ("mpfr", "install", "default") => true, - ("mpc", "install", "default") => true, - ("zlib", "install", "default") => true, - ("binutils", "install", "default") => true, - ("gcc", "install", "default") => true, - ("libgcc", "install", "default") => true, - ("gcc-libs", "install", "default") => true, - ("linux-uapi-headers", "install", "default") => true, - ("glibc", "install", "default") => true, - _ => false, - } - } -} - -impl Index<&TaskID> for Context { - type Output = TaskDef; - - fn index(&self, index: &TaskID) -> &TaskDef { - self.tasks.get(index).expect("Invalid TaskID") - } -} diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs deleted file mode 100644 index 748ff89..0000000 --- a/crates/executor/src/executor.rs +++ /dev/null @@ -1,367 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - os::unix::{net::UnixStream, prelude::*}, -}; - -use indoc::indoc; -use nix::poll; - -use common::{error::*, string_hash::*, types::*}; -use runner::{paths, Runner}; - -use crate::{ - context::{Context, TaskRef}, - resolve, - task::*, - template, -}; - -#[derive(Debug)] -pub struct CompletionState<'ctx> { - ctx: &'ctx Context, - tasks_done: HashMap, TaskOutput>, -} - -impl<'ctx> CompletionState<'ctx> { - pub fn new(ctx: &'ctx Context) -> Self { - CompletionState { - ctx, - tasks_done: Default::default(), - } - } - - // Treats both "depends" and "inherit" as dependencies - fn deps_satisfied(&self, task_ref: &TaskRef) -> bool { - resolve::get_dependent_tasks(self.ctx, task_ref) - .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref))) - .unwrap() - .into_iter() - .all(|dep| self.tasks_done.contains_key(&dep)) - } - - fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result> { - let task_def = &self.ctx[task.id]; - task_def - .fetch - .iter() - .map(|Fetch { name, sha256 }| { - Ok(Dependency::Fetch { - name: template::ENGINE - .eval_raw(name, &task.args) - .with_context(|| { - format!("Failed to evaluate fetch filename for task {}", task) - })?, - sha256: *sha256, - }) - }) - .collect() - } - - fn task_deps(&self, task: &TaskRef<'ctx>) -> Result> { - Ok(self - .fetch_deps(task)? - .into_iter() - .chain( - resolve::runtime_depends( - self.ctx, - self.ctx - .get_build_depends(task) - .with_context(|| format!("invalid build depends for {}", task))?, - ) - .expect("invalid runtime depends of build_depends") - .into_iter() - .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) - .map(|&output| Dependency::Task { - output, - path: "".to_string(), - }), - ) - .chain( - resolve::runtime_depends( - self.ctx, - self.ctx - .get_host_depends(task) - .with_context(|| format!("invalid depends for {}", task))?, - ) - .expect("invalid runtime depends of host_depends") - .into_iter() - .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output)) - .map(|&output| Dependency::Task { - output, - path: paths::abs(paths::TASK_SYSROOT), - }), - ) - .collect()) - } - - fn task_inherit_chain(&self, task_ref: &TaskRef<'ctx>) -> Vec { - let inherit = match self - .ctx - .get_inherit_depend(task_ref) - .expect("invalid inherit depends") - { - Some(inherit) => inherit, - None => return vec![], - }; - - let mut chain = self.task_inherit_chain(&inherit); - if let Some(layer) = self.tasks_done[&inherit].layer { - chain.push(layer); - } - chain - } - - fn print_summary(&self) { - println!(); - println!("Summary:"); - - let mut tasks: Box<[_]> = self.tasks_done.iter().collect(); - tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task)); - for (task_ref, task) in tasks.iter() { - println!(); - println!("{:#}", task_ref); - println!(" input: {}", task.input_hash); - if let Some(hash) = task.layer { - println!(" layer: {}", hash); - } - if !task.outputs.is_empty() { - println!(" outputs:"); - - let mut outputs: Box<[_]> = task.outputs.iter().collect(); - outputs.sort_by_key(|(output, _)| *output); - for (output, hash) in outputs.iter() { - println!(" {}: {}", output, hash); - } - } - } - } -} - -#[derive(Debug)] -pub struct Executor<'ctx> { - rdeps: HashMap, Vec>>, - tasks_blocked: HashSet>, - tasks_runnable: Vec>, - tasks_running: HashMap)>, - state: CompletionState<'ctx>, -} - -impl<'ctx> Executor<'ctx> { - pub fn new(ctx: &'ctx Context, taskset: HashSet>) -> Result { - let mut exc = Executor { - rdeps: Default::default(), - tasks_blocked: Default::default(), - tasks_runnable: Default::default(), - tasks_running: Default::default(), - state: CompletionState::new(ctx), - }; - - for task in taskset { - let mut has_depends = false; - for dep in resolve::get_dependent_tasks(ctx, &task) - .map_err(|_| Error::new(format!("invalid dependency for {}", task)))? - { - let rdep = exc.rdeps.entry(dep.clone()).or_default(); - rdep.push(task.clone()); - has_depends = true; - } - - if has_depends { - exc.tasks_blocked.insert(task); - } else { - exc.tasks_runnable.push(task); - } - } - - Ok(exc) - } - - fn task_setup(task_ref: &TaskRef<'ctx>) -> Vec<&'static str> { - let mut ret = vec![indoc! {" - export SOURCE_DATE_EPOCH=1 - - export AR_FOR_BUILD=ar - export AS_FOR_BUILD=as - export DLLTOOL_FOR_BUILD=dlltool - export CC_FOR_BUILD=gcc - export CXX_FOR_BUILD=g++ - export GCC_FOR_BUILD=gcc - export GFORTRAN_FOR_BUILD=gfortran - export GOC_FOR_BUILD=goc - export LD_FOR_BUILD=ld - export LIPO_FOR_BUILD=lipo - export NM_FOR_BUILD=nm - export OBJCOPY_FOR_BUILD=objcopy - export OBJDUMP_FOR_BUILD=objdump - export RANLIB_FOR_BUILD=ranlib - export STRIP_FOR_BUILD=strip - export WINDRES_FOR_BUILD=windres - export WINDMC_FOR_BUILD=windmc - "}]; - - if task_ref.args.contains_key("build_to_host") { - ret.push(indoc! {" - export AR={{build_to_host.cross_compile}}ar - export AS={{build_to_host.cross_compile}}as - export DLLTOOL={{build_to_host.cross_compile}}dlltool - export CC={{build_to_host.cross_compile}}gcc - export CXX={{build_to_host.cross_compile}}g++ - export GCC={{build_to_host.cross_compile}}gcc - export GFORTRAN={{build_to_host.cross_compile}}gfortran - export GOC={{build_to_host.cross_compile}}goc - export LD={{build_to_host.cross_compile}}ld - export LIPO={{build_to_host.cross_compile}}lipo - export NM={{build_to_host.cross_compile}}nm - export OBJCOPY={{build_to_host.cross_compile}}objcopy - export OBJDUMP={{build_to_host.cross_compile}}objdump - export RANLIB={{build_to_host.cross_compile}}ranlib - export STRIP={{build_to_host.cross_compile}}strip - export WINDRES={{build_to_host.cross_compile}}windres - export WINDMC={{build_to_host.cross_compile}}windmc - "}); - } - - if task_ref.args.contains_key("build_to_target") { - ret.push(indoc! {" - export AR_FOR_TARGET={{build_to_target.cross_compile}}ar - export AS_FOR_TARGET={{build_to_target.cross_compile}}as - export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool - export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc - export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++ - export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc - export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran - export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc - export LD_FOR_TARGET={{build_to_target.cross_compile}}ld - export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo - export NM_FOR_TARGET={{build_to_target.cross_compile}}nm - export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy - export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump - export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib - export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip - export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres - export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc - "}); - } - ret - } - - fn update_runnable(&mut self, task_ref: TaskRef<'ctx>, task_output: TaskOutput) { - let rdeps = self.rdeps.get(&task_ref); - - self.state.tasks_done.insert(task_ref, task_output); - - for rdep in rdeps.unwrap_or(&Vec::new()) { - if !self.tasks_blocked.contains(rdep) { - continue; - } - if self.state.deps_satisfied(rdep) { - self.tasks_blocked.remove(rdep); - self.tasks_runnable.push(rdep.clone()); - } - } - } - - fn spawn_task(&self, task_ref: &TaskRef<'ctx>, runner: &Runner) -> Result { - let task_def = &self.state.ctx[task_ref.id]; - let task_deps = self.state.task_deps(task_ref)?; - let task_output = task_def - .output - .iter() - .map(|(name, Output { path, .. })| { - ( - name.clone(), - path.as_ref().map(String::as_str).unwrap_or(".").to_string(), - ) - }) - .collect(); - - let inherit_chain = self.state.task_inherit_chain(task_ref); - - let mut run = Self::task_setup(task_ref); - run.push(&task_def.action.run); - - let command = template::ENGINE - .eval(&run.concat(), &task_ref.args) - .with_context(|| { - format!("Failed to evaluate command template for task {}", task_ref) - })?; - - let task = Task { - label: format!("{:#}", task_ref), - command, - inherit: inherit_chain, - depends: task_deps, - outputs: task_output, - }; - - Ok(runner.spawn(&task)) - } - - fn run_task(&mut self, task_ref: TaskRef<'ctx>, runner: &Runner) -> Result<()> { - let socket = self.spawn_task(&task_ref, runner)?; - assert!(self - .tasks_running - .insert(socket.as_raw_fd(), (socket, task_ref)) - .is_none()); - Ok(()) - } - - fn run_tasks(&mut self, runner: &Runner) -> Result<()> { - while let Some(task_ref) = self.tasks_runnable.pop() { - self.run_task(task_ref, runner)?; - } - Ok(()) - } - - fn wait_for_task(&mut self) -> Result<()> { - let mut pollfds: Box<[_]> = self - .tasks_running - .keys() - .copied() - .map(|fd| poll::PollFd::new(fd, poll::PollFlags::POLLIN)) - .collect(); - - while poll::poll(&mut pollfds, -1).context("poll()")? == 0 {} - - for pollfd in &*pollfds { - let events = pollfd.revents().expect("Unknown events in poll() return"); - if !events.contains(poll::PollFlags::POLLIN) { - if events.intersects(!poll::PollFlags::POLLIN) { - return Err(Error::new( - "Unexpected error status for socket file descriptor", - )); - } - continue; - } - - let fd = pollfd.as_raw_fd(); - let (socket, task_ref) = self.tasks_running.remove(&fd).unwrap(); - - let task_output = Runner::result(&socket)?; - self.update_runnable(task_ref, task_output); - } - - Ok(()) - } - - fn is_empty(&self) -> bool { - self.tasks_runnable.is_empty() && self.tasks_running.is_empty() - } - - fn is_done(&self) -> bool { - self.is_empty() && self.tasks_blocked.is_empty() - } - - pub fn run(&mut self, runner: &Runner) -> Result<()> { - while !self.is_empty() { - self.run_tasks(runner)?; - self.wait_for_task()?; - } - - assert!(self.is_done(), "No runnable tasks left"); - - self.state.print_summary(); - - Ok(()) - } -} diff --git a/crates/executor/src/main.rs b/crates/executor/src/main.rs deleted file mode 100644 index 6360e69..0000000 --- a/crates/executor/src/main.rs +++ /dev/null @@ -1,56 +0,0 @@ -mod args; -mod context; -mod executor; -mod recipe; -mod resolve; -mod task; -mod template; - -use clap::Parser; - -use runner::Runner; - -#[derive(Parser)] -#[clap(version = clap::crate_version!())] -struct Opts { - /// Allow N jobs at once. - /// Defaults to the number of available CPUs - #[clap(short, long)] - jobs: Option, - /// The tasks to run - #[clap(name = "task", required = true)] - tasks: Vec, -} - -fn main() { - let opts: Opts = Opts::parse(); - - let runner = unsafe { Runner::new(&runner::Options { jobs: opts.jobs }) }.unwrap(); - - let ctx = context::Context::new(recipe::read_recipes("examples").unwrap()); - - let mut rsv = resolve::Resolver::new(&ctx); - - for task in opts.tasks { - let task_ref = match ctx.parse(&task) { - Ok(task_ref) => task_ref, - Err(err) => { - eprintln!("{}", err); - std::process::exit(1); - } - }; - let errors = rsv.add_goal(&task_ref); - if !errors.is_empty() { - for error in errors { - eprintln!("{}", error); - } - std::process::exit(1); - } - } - let taskset = rsv.into_taskset(); - let mut exc = executor::Executor::new(&ctx, taskset).unwrap(); - if let Err(error) = exc.run(&runner) { - eprintln!("{}", error); - std::process::exit(1); - } -} diff --git a/crates/executor/src/recipe.rs b/crates/executor/src/recipe.rs deleted file mode 100644 index 04f356b..0000000 --- a/crates/executor/src/recipe.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::{collections::HashMap, fmt, fs::File, io, path::Path, result}; - -use scoped_tls_hkt::scoped_thread_local; -use serde::{Deserialize, Deserializer}; -use walkdir::WalkDir; - -use common::types::*; - -use crate::task::{RecipeMeta, TaskDef}; - -scoped_thread_local!(static CURRENT_RECIPE: str); - -fn current_recipe() -> String { - CURRENT_RECIPE.with(|current| current.to_string()) -} - -pub fn deserialize_task_id<'de, D>(deserializer: D) -> result::Result -where - D: Deserializer<'de>, -{ - #[derive(Deserialize)] - struct RecipeTaskID { - recipe: Option, - task: String, - } - let RecipeTaskID { recipe, task } = RecipeTaskID::deserialize(deserializer)?; - Ok(TaskID { - recipe: recipe.unwrap_or_else(current_recipe), - task, - }) -} - -#[derive(Debug, Deserialize)] -struct Recipe { - #[serde(default)] - pub meta: RecipeMeta, - pub tasks: HashMap, -} - -#[derive(Debug)] -pub enum Error { - IOError(io::Error), - YAMLError(serde_yaml::Error), -} - -impl From for Error { - fn from(err: io::Error) -> Self { - Error::IOError(err) - } -} - -impl From for Error { - fn from(err: serde_yaml::Error) -> Self { - Error::YAMLError(err) - } -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::IOError(err) => write!(f, "IO error: {}", err), - Error::YAMLError(err) => write!(f, "YAML error: {}", err), - } - } -} - -impl std::error::Error for Error {} - -pub type Result = std::result::Result; - -fn read_recipe(path: &Path) -> Result { - let f = File::open(path)?; - - let recipe: Recipe = serde_yaml::from_reader(f)?; - - Ok(recipe) -} - -fn is_yml(path: &Path) -> bool { - path.extension() == Some("yml".as_ref()) -} - -pub fn read_recipes>(path: P) -> Result> { - let mut tasks = HashMap::new(); - - for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) { - let path = entry.path(); - if !path.is_file() || !is_yml(path) { - continue; - } - - let basename: &str = match path.file_stem().map(|n| n.to_str()) { - Some(Some(v)) => v, - _ => continue, - }; - - let recipe = CURRENT_RECIPE.set(basename, || read_recipe(path))?; - - let mut meta = recipe.meta; - if meta.name.is_empty() { - meta.name = basename.to_string(); - } - - for (label, mut task) in recipe.tasks { - let task_id = TaskID { - recipe: basename.to_string(), - task: label, - }; - task.meta = meta.clone(); - tasks.insert(task_id, task); - } - } - - Ok(tasks) -} diff --git a/crates/executor/src/resolve.rs b/crates/executor/src/resolve.rs deleted file mode 100644 index 338ce3f..0000000 --- a/crates/executor/src/resolve.rs +++ /dev/null @@ -1,312 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::fmt; -use std::rc::Rc; - -use common::types::TaskID; - -use crate::args::TaskArgs; -use crate::context::{self, Context, OutputRef, TaskRef}; - -#[derive(Debug)] -pub struct DepChain<'ctx>(pub Vec>); - -impl<'ctx> fmt::Display for DepChain<'ctx> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut first = true; - for task in self.0.iter().rev() { - if !first { - write!(f, " -> ")?; - } - write!(f, "{}", task)?; - - first = false; - } - - Ok(()) - } -} - -impl<'ctx> From> for DepChain<'ctx> { - fn from(task: TaskRef<'ctx>) -> Self { - DepChain(vec![task]) - } -} - -impl<'ctx> From<&TaskRef<'ctx>> for DepChain<'ctx> { - fn from(task: &TaskRef<'ctx>) -> Self { - task.clone().into() - } -} - -impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> { - fn from(id: &'ctx TaskID) -> Self { - TaskRef { - id, - args: Rc::new(TaskArgs::default()), - } - .into() - } -} - -#[derive(Debug)] -pub enum ErrorKind<'ctx> { - Context(context::Error<'ctx>), - OutputNotFound(&'ctx str), - DependencyCycle, -} - -#[derive(Debug)] -pub struct Error<'ctx> { - pub dep_chain: DepChain<'ctx>, - pub kind: ErrorKind<'ctx>, -} - -impl<'ctx> Error<'ctx> { - fn output_not_found(task: &TaskRef<'ctx>, output: &'ctx str) -> Self { - Error { - dep_chain: task.into(), - kind: ErrorKind::OutputNotFound(output), - } - } - - fn dependency_cycle(task: &TaskRef<'ctx>) -> Self { - Error { - dep_chain: task.into(), - kind: ErrorKind::DependencyCycle, - } - } - - fn extend(&mut self, task: &TaskRef<'ctx>) { - self.dep_chain.0.push(task.clone()); - } -} - -impl<'ctx> fmt::Display for Error<'ctx> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let Error { dep_chain, kind } = self; - match kind { - ErrorKind::Context(err) => { - write!(f, "{}: ", err)?; - } - ErrorKind::OutputNotFound(output) => { - write!(f, "Output '{}' not found: ", output)?; - } - ErrorKind::DependencyCycle => { - write!(f, "Dependency Cycle: ")?; - } - } - dep_chain.fmt(f) - } -} - -impl<'ctx> From> for Error<'ctx> { - fn from(err: context::Error<'ctx>) -> Self { - Error { - dep_chain: err.task.into(), - kind: ErrorKind::Context(err), - } - } -} - -impl<'ctx> std::error::Error for Error<'ctx> {} - -#[derive(Debug, PartialEq)] -enum ResolveState { - Resolving, - Resolved, -} - -pub fn runtime_depends<'ctx, I>( - ctx: &'ctx Context, - deps: I, -) -> Result, Vec> -where - I: IntoIterator>, -{ - fn add_dep<'ctx>( - ret: &mut HashSet>, - ctx: &'ctx Context, - dep: OutputRef<'ctx>, - ) -> Vec> { - if ret.contains(&dep) || ctx.in_rootfs(&dep) { - return Vec::new(); - } - - let task = &dep.task; - let task_def = match ctx.get(task.id) { - Ok(task) => task, - Err(err) => return vec![err.into()], - }; - - let output = match task_def.output.get(dep.output) { - Some(output) => output, - None => { - return vec![Error::output_not_found(task, dep.output)]; - } - }; - - ret.insert(dep.clone()); - - let mut errors = Vec::new(); - for runtime_dep in &output.runtime_depends { - match ctx.output_ref(runtime_dep, &task.args, false) { - Ok(output_ref) => { - for mut error in add_dep(ret, ctx, output_ref) { - error.extend(task); - errors.push(error); - } - } - Err(err) => { - let mut err: Error = err.into(); - err.extend(task); - errors.push(err); - } - }; - } - errors - } - - let mut ret = HashSet::new(); - let mut errors = Vec::new(); - - for dep in deps { - errors.extend(add_dep(&mut ret, ctx, dep)); - } - - if !errors.is_empty() { - return Err(errors); - } - - Ok(ret) -} - -pub fn get_dependent_outputs<'ctx>( - ctx: &'ctx Context, - task_ref: &TaskRef<'ctx>, -) -> Result>, Vec>> { - let deps: HashSet<_> = ctx - .get_build_depends(task_ref) - .map_err(|err| vec![err.into()])? - .into_iter() - .chain( - ctx.get_host_depends(task_ref) - .map_err(|err| vec![err.into()])? - .into_iter(), - ) - .collect(); - runtime_depends(ctx, deps) -} - -pub fn get_dependent_tasks<'ctx>( - ctx: &'ctx Context, - task_ref: &TaskRef<'ctx>, -) -> Result>, Vec>> { - Ok(ctx - .get_inherit_depend(task_ref) - .map_err(|err| vec![err.into()])? - .into_iter() - .chain( - get_dependent_outputs(ctx, task_ref)? - .into_iter() - .map(|dep| dep.task), - ) - .collect()) -} - -#[derive(Debug)] -pub struct Resolver<'ctx> { - ctx: &'ctx Context, - resolve_state: HashMap, ResolveState>, -} - -impl<'ctx> Resolver<'ctx> { - pub fn new(ctx: &'ctx Context) -> Self { - Resolver { - ctx, - resolve_state: HashMap::new(), - } - } - - fn tasks_resolved(&self) -> bool { - self.resolve_state - .values() - .all(|resolved| *resolved == ResolveState::Resolved) - } - - fn add_task(&mut self, task: &TaskRef<'ctx>, output: Option<&'ctx str>) -> Vec> { - match self.resolve_state.get(task) { - Some(ResolveState::Resolving) => return vec![Error::dependency_cycle(task)], - Some(ResolveState::Resolved) => return vec![], - None => (), - } - - let task_def = match self.ctx.get(task.id) { - Ok(task_def) => task_def, - Err(err) => return vec![err.into()], - }; - - if let Some(task_output) = output { - if !task_def.output.contains_key(task_output) { - return vec![Error::output_not_found(task, task_output)]; - } - } - - self.resolve_state - .insert(task.clone(), ResolveState::Resolving); - - let mut ret = Vec::new(); - let mut handle_errors = |errors: Vec>| { - for mut error in errors { - error.extend(task); - ret.push(error); - } - }; - - match self.ctx.get_inherit_depend(task) { - Ok(Some(inherit)) => { - handle_errors(self.add_task(&inherit, None)); - } - Ok(None) => {} - Err(err) => { - handle_errors(vec![err.into()]); - } - } - - match get_dependent_outputs(self.ctx, task) { - Ok(rdeps) => { - for rdep in rdeps { - handle_errors(self.add_task(&rdep.task, Some(rdep.output))); - } - } - Err(errors) => { - handle_errors(errors); - } - } - - if ret.is_empty() { - *self - .resolve_state - .get_mut(task) - .expect("Missing resolve_state") = ResolveState::Resolved; - } else { - self.resolve_state.remove(task); - } - - ret - } - - pub fn add_goal(&mut self, task: &TaskRef<'ctx>) -> Vec> { - let ret = self.add_task(task, None); - debug_assert!(self.tasks_resolved()); - ret - } - - pub fn into_taskset(self) -> HashSet> { - debug_assert!(self.tasks_resolved()); - - self.resolve_state - .into_iter() - .map(|entry| entry.0) - .collect() - } -} diff --git a/crates/executor/src/task.rs b/crates/executor/src/task.rs deleted file mode 100644 index fe9572c..0000000 --- a/crates/executor/src/task.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use serde::Deserialize; - -use common::{string_hash::StringHash, types::TaskID}; - -use crate::{ - args::{ArgMapping, ArgType}, - recipe, -}; - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash, Default)] -pub struct RecipeMeta { - #[serde(default)] - pub name: String, - pub version: Option, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] -pub struct TaskDep { - #[serde(flatten, deserialize_with = "recipe::deserialize_task_id")] - pub id: TaskID, - #[serde(default)] - pub args: ArgMapping, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] -pub struct Fetch { - pub name: String, - pub sha256: StringHash, -} - -fn default_output_name() -> String { - "default".to_string() -} - -#[derive(Clone, Debug, Deserialize)] -pub struct InheritDep { - #[serde(flatten)] - pub dep: TaskDep, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] -pub struct OutputDep { - #[serde(flatten)] - pub dep: TaskDep, - #[serde(default)] - pub noinherit: bool, - #[serde(default = "default_output_name")] - pub output: String, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct Output { - pub path: Option, - #[serde(default)] - pub runtime_depends: HashSet, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct Action { - #[serde(default)] - pub run: String, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct TaskDef { - #[serde(skip)] - pub meta: RecipeMeta, - #[serde(default)] - pub args: HashMap, - #[serde(default)] - pub inherit: Option, - #[serde(default)] - pub fetch: HashSet, - #[serde(default)] - pub build_depends: HashSet, - #[serde(default)] - pub depends: HashSet, - #[serde(default)] - pub output: HashMap, - #[serde(flatten)] - pub action: Action, -} diff --git a/crates/executor/src/template.rs b/crates/executor/src/template.rs deleted file mode 100644 index b663e7d..0000000 --- a/crates/executor/src/template.rs +++ /dev/null @@ -1,44 +0,0 @@ -use handlebars::Handlebars; -use lazy_static::lazy_static; - -use common::error::*; - -use crate::args::TaskArgs; - -fn escape(s: &str) -> String { - format!("'{}'", s.replace("'", "'\\''")) -} - -#[derive(Debug)] -pub struct TemplateEngine { - tpl: Handlebars<'static>, - tpl_raw: Handlebars<'static>, -} - -impl TemplateEngine { - pub fn new() -> Self { - let mut tpl = Handlebars::new(); - tpl.set_strict_mode(true); - tpl.register_escape_fn(escape); - - let mut tpl_raw = Handlebars::new(); - tpl_raw.set_strict_mode(true); - tpl_raw.register_escape_fn(handlebars::no_escape); - - TemplateEngine { tpl, tpl_raw } - } - - pub fn eval_raw(&self, input: &str, args: &TaskArgs) -> Result { - self.tpl_raw - .render_template(input, args) - .map_err(Error::new) - } - - pub fn eval(&self, input: &str, args: &TaskArgs) -> Result { - self.tpl.render_template(input, args).map_err(Error::new) - } -} - -lazy_static! { - pub static ref ENGINE: TemplateEngine = TemplateEngine::new(); -} -- cgit v1.2.3