summaryrefslogtreecommitdiffstats
path: root/crates/driver
diff options
context:
space:
mode:
authorMatthias Schiffer <mschiffer@universe-factory.net>2024-04-20 14:28:05 +0200
committerMatthias Schiffer <mschiffer@universe-factory.net>2024-04-20 14:38:17 +0200
commite9bf0fc40c0eb7e9d4228b804d62f31b0a136528 (patch)
tree7872f587782d5635eadbf82ae861d474d4da2efe /crates/driver
parent35e68444dd5e9d3d5fc39409c48be6eb3fa05e07 (diff)
downloadrebel-e9bf0fc40c0eb7e9d4228b804d62f31b0a136528.tar
rebel-e9bf0fc40c0eb7e9d4228b804d62f31b0a136528.zip
Rename directories to match crate names
Diffstat (limited to 'crates/driver')
-rw-r--r--crates/driver/Cargo.toml25
-rw-r--r--crates/driver/src/args.rs122
-rw-r--r--crates/driver/src/context.rs533
-rw-r--r--crates/driver/src/driver.rs480
-rw-r--r--crates/driver/src/main.rs79
-rw-r--r--crates/driver/src/parse.rs72
-rw-r--r--crates/driver/src/paths.rs4
-rw-r--r--crates/driver/src/pin.rs39
-rw-r--r--crates/driver/src/recipe.rs188
-rw-r--r--crates/driver/src/resolve.rs334
-rw-r--r--crates/driver/src/task.rs96
-rw-r--r--crates/driver/src/template.rs42
12 files changed, 0 insertions, 2014 deletions
diff --git a/crates/driver/Cargo.toml b/crates/driver/Cargo.toml
deleted file mode 100644
index df1fb19..0000000
--- a/crates/driver/Cargo.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-[package]
-name = "rebel"
-version = "0.1.0"
-authors = ["Matthias Schiffer <mschiffer@universe-factory.net>"]
-license = "MIT"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-common = { path = "../common", package = "rebel-common" }
-runner = { path = "../runner", package = "rebel-runner" }
-
-clap = { version = "4.0.0", features = ["derive"] }
-deb-version = "0.1.1"
-enum-kinds = "0.5.1"
-handlebars = "5.1.2"
-indoc = "2.0.4"
-lazy_static = "1.4.0"
-nix = { version = "0.28.0", features = ["poll", "signal"] }
-scoped-tls-hkt = "0.1.2"
-serde = { version = "1", features = ["derive", "rc"] }
-serde_yaml = "0.9"
-walkdir = "2"
-peg = "0.8.2"
diff --git a/crates/driver/src/args.rs b/crates/driver/src/args.rs
deleted file mode 100644
index 805646a..0000000
--- a/crates/driver/src/args.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-use std::{
- collections::{hash_map, HashMap},
- hash,
- rc::Rc,
-};
-
-use enum_kinds::EnumKind;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Serialize, PartialEq, Eq)]
-pub struct Platform {
- #[serde(skip)]
- pub short: String,
- pub gnu_triplet: String,
- pub karch: String,
- pub prefix: String,
-}
-
-#[derive(Debug, Serialize, PartialEq, Eq)]
-pub struct PlatformRelation {
- pub is_same: bool,
- pub sysroot: String,
- pub cross_compile: String,
-}
-
-#[derive(Clone, Debug, Serialize, PartialEq, Eq, EnumKind)]
-#[serde(untagged)]
-#[enum_kind(ArgType, derive(Deserialize), serde(rename_all = "snake_case"))]
-pub enum Arg {
- String(Rc<String>),
- Platform(Rc<Platform>),
- PlatformRelation(Rc<PlatformRelation>),
-}
-
-impl From<&Arg> for Arg {
- fn from(value: &Arg) -> Self {
- value.clone()
- }
-}
-
-impl From<String> for Arg {
- fn from(value: String) -> Self {
- Arg::String(Rc::new(value))
- }
-}
-
-impl From<Platform> for Arg {
- fn from(value: Platform) -> Self {
- Arg::Platform(Rc::new(value))
- }
-}
-
-impl From<PlatformRelation> for Arg {
- fn from(value: PlatformRelation) -> Self {
- Arg::PlatformRelation(Rc::new(value))
- }
-}
-
-#[derive(Clone, Debug, Serialize, PartialEq, Eq, Default)]
-pub struct TaskArgs(HashMap<String, Arg>);
-
-impl TaskArgs {
- pub fn contains_key(&self, key: &str) -> bool {
- self.0.contains_key(key)
- }
-
- pub fn get(&self, key: &str) -> Option<&Arg> {
- self.0.get(key)
- }
-
- pub fn set<T>(&mut self, key: &str, value: Option<T>)
- where
- T: Into<Arg>,
- {
- if let Some(v) = value {
- self.0.insert(key.to_string(), v.into());
- } else {
- self.0.remove(key);
- }
- }
-
- pub fn iter(&self) -> hash_map::Iter<String, Arg> {
- self.into_iter()
- }
-}
-
-impl FromIterator<(String, Arg)> for TaskArgs {
- fn from_iter<T: IntoIterator<Item = (String, Arg)>>(iter: T) -> Self {
- TaskArgs(HashMap::from_iter(iter))
- }
-}
-
-impl<'a> IntoIterator for &'a TaskArgs {
- type Item = (&'a String, &'a Arg);
-
- type IntoIter = hash_map::Iter<'a, String, Arg>;
-
- fn into_iter(self) -> Self::IntoIter {
- self.0.iter()
- }
-}
-
-#[allow(clippy::derived_hash_with_manual_eq)]
-impl hash::Hash for TaskArgs {
- fn hash<H: hash::Hasher>(&self, _state: &mut H) {
- // Don't do anything: Properly hashing the task args is likely to cost
- // much more performance than the hash collisions caused by TaskRefs
- // that only differ by the args
- }
-}
-
-pub fn arg<A: Into<Arg>>(key: &str, value: A) -> (String, Arg) {
- (key.to_string(), value.into())
-}
-
-#[derive(Clone, Debug, Deserialize, Default, PartialEq, Eq)]
-pub struct ArgMapping(pub HashMap<String, String>);
-
-#[allow(clippy::derived_hash_with_manual_eq)]
-impl hash::Hash for ArgMapping {
- fn hash<H: hash::Hasher>(&self, _state: &mut H) {}
-}
diff --git a/crates/driver/src/context.rs b/crates/driver/src/context.rs
deleted file mode 100644
index be98813..0000000
--- a/crates/driver/src/context.rs
+++ /dev/null
@@ -1,533 +0,0 @@
-use std::{
- borrow::Cow,
- cmp::Ordering,
- collections::{HashMap, HashSet},
- fmt::Display,
- hash::Hash,
- ops::Index,
- rc::Rc,
- result,
-};
-
-use common::{
- error::{self, Contextualizable},
- string_hash::ArchiveHash,
- types::TaskID,
-};
-
-use crate::{
- args::*,
- parse::{self, TaskFlags},
- paths,
- pin::{self, Pins},
- task::*,
-};
-
-#[derive(Debug, Clone, Copy)]
-pub enum ErrorKind<'a> {
- TaskNotFound,
- InvalidArgument(&'a str),
- InvalidArgRef(&'a str),
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Error<'a> {
- pub task: &'a TaskID,
- pub kind: ErrorKind<'a>,
-}
-
-impl<'a> Display for Error<'a> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- let Error { task, kind } = self;
- match kind {
- ErrorKind::TaskNotFound => write!(f, "Task '{}' not found", task),
- ErrorKind::InvalidArgument(arg) => write!(
- f,
- "Invalid or missing argument '{}' for task '{}'",
- arg, task
- ),
- ErrorKind::InvalidArgRef(arg) => write!(
- f,
- "Invalid reference for argument '{}' of task '{}'",
- arg, task
- ),
- }
- }
-}
-
-impl<'a> From<Error<'a>> for error::Error {
- fn from(err: Error) -> Self {
- error::Error::new(err)
- }
-}
-
-pub type Result<'a, T> = result::Result<T, Error<'a>>;
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct TaskRef<'ctx> {
- pub id: &'ctx TaskID,
- pub args: Rc<TaskArgs>,
-}
-
-impl<'ctx> Display for TaskRef<'ctx> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- if !f.alternate() {
- return self.id.fmt(f);
- }
-
- let version_arg = match self.args.get("version") {
- Some(Arg::String(s)) => Some(s),
- _ => None,
- };
- let host_arg = match self.args.get("host") {
- Some(Arg::Platform(platform)) => Some(platform),
- _ => None,
- };
- let target_arg = match self.args.get("target") {
- Some(Arg::Platform(platform)) => Some(platform),
- _ => None,
- };
-
- write!(f, "{}", self.id.recipe)?;
- if let Some(version) = version_arg {
- write!(f, "#{}", version)?;
- }
- write!(f, "::{}", self.id.task)?;
-
- if host_arg.is_some() || target_arg.is_some() {
- write!(f, "@")?;
- }
-
- if let Some(host) = host_arg {
- write!(f, "{}", host.short)?;
- }
- if let Some(target) = target_arg {
- write!(f, ":{}", target.short)?;
- }
- Ok(())
- }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct OutputRef<'ctx> {
- pub task: TaskRef<'ctx>,
- pub output: &'ctx str,
-}
-
-fn platform_relation(args: &TaskArgs, from: &str, to: &str) -> Option<PlatformRelation> {
- let plat_from = match args.get(from)? {
- Arg::Platform(plat) => plat,
- _ => return None,
- };
- let plat_to = match args.get(to)? {
- Arg::Platform(plat) => plat,
- _ => return None,
- };
-
- let plat_rel = if plat_from == plat_to {
- PlatformRelation {
- is_same: true,
- sysroot: "".to_string(),
- cross_compile: "".to_string(),
- }
- } else {
- PlatformRelation {
- is_same: false,
- sysroot: paths::TASK_SYSROOT.to_string(),
- cross_compile: format!("{}/bin/{}-", plat_from.prefix, plat_to.gnu_triplet),
- }
- };
- Some(plat_rel)
-}
-
-#[derive(Debug)]
-pub struct Context {
- platforms: HashMap<String, Arg>,
- globals: TaskArgs,
- tasks: HashMap<TaskID, Vec<TaskDef>>,
- rootfs: (ArchiveHash, String),
-}
-
-impl Context {
- pub fn new(mut tasks: HashMap<TaskID, Vec<TaskDef>>, pins: Pins) -> error::Result<Self> {
- let platforms: HashMap<_, _> = [
- arg(
- "build",
- Platform {
- short: "build".to_string(),
- gnu_triplet: "x86_64-linux-gnu".to_string(),
- karch: "x86_64".to_string(),
- prefix: "/opt/toolchain".to_string(),
- },
- ),
- arg(
- "aarch64",
- Platform {
- short: "aarch64".to_string(),
- gnu_triplet: "aarch64-linux-gnu".to_string(),
- karch: "arm64".to_string(),
- prefix: "/usr".to_string(),
- },
- ),
- ]
- .into_iter()
- .collect();
-
- let globals = TaskArgs::from_iter([
- ("build".to_string(), platforms["build"].clone()),
- arg("workdir", paths::TASK_WORKDIR.to_string()),
- arg("dldir", paths::TASK_DLDIR.to_string()),
- arg("destdir", paths::TASK_DESTDIR.to_string()),
- arg("sysroot", paths::TASK_SYSROOT.to_string()),
- ]);
- let (rootfs, rootfs_provides) =
- Context::handle_pins(pins).context("Failed to process pin list")?;
-
- Context::add_rootfs_tasks(&mut tasks, rootfs_provides, &globals)
- .context("Failed to determine rootfs-provided tasks from pin list")?;
-
- Ok(Context {
- platforms,
- globals,
- tasks,
- rootfs,
- })
- }
-
- fn handle_pins(pins: Pins) -> error::Result<((ArchiveHash, String), Vec<pin::Provides>)> {
- let mut ret = None;
-
- for (name, pin) in pins {
- if pin.is_rootfs {
- if ret.is_some() {
- return Err(error::Error::new("Multiple is-rootfs pins"));
- }
- let hash = pin.hash.context("is-rootfs pin without hash")?;
-
- ret = Some(((hash, name), pin.provides));
- }
- }
-
- ret.context("No is-rootfs pins")
- }
-
- fn add_rootfs_tasks(
- tasks: &mut HashMap<TaskID, Vec<TaskDef>>,
- provides: Vec<pin::Provides>,
- globals: &TaskArgs,
- ) -> error::Result<()> {
- let build = globals.get("build").unwrap();
-
- for pin::Provides {
- recipe,
- task,
- output,
- args,
- } in provides
- {
- let mut task_def = TaskDef::default();
-
- if let Some(host) = args.host {
- if host != "build" {
- return Err(error::Error::new(format!("Invalid host value '{}'", host)));
- }
- task_def.args.insert("host".to_string(), build.into());
- task_def.arg_match.set("host", Some(build));
- }
-
- if let Some(target) = args.target {
- if target != "build" {
- return Err(error::Error::new(format!(
- "Invalid target value '{}'",
- target
- )));
- }
- task_def.args.insert("target".to_string(), build.into());
- task_def.arg_match.set("target", Some(build));
- }
-
- for output_entry in output {
- task_def
- .output
- .insert(output_entry.to_string(), Output::default());
- }
-
- task_def.priority = i32::MAX;
-
- tasks
- .entry(TaskID {
- recipe: recipe.to_string(),
- task: task.to_string(),
- })
- .or_default()
- .push(task_def);
- }
-
- Ok(())
- }
-
- pub fn get_rootfs(&self) -> &(ArchiveHash, String) {
- &self.rootfs
- }
-
- fn match_task(task: &TaskDef, args: &TaskArgs) -> bool {
- task.arg_match
- .iter()
- .all(|(key, value)| args.get(key) == Some(value))
- }
-
- fn compare_tasks(task1: &TaskDef, task2: &TaskDef) -> Ordering {
- task1
- .priority
- .cmp(&task2.priority)
- .then(deb_version::compare_versions(
- task1.meta.version.as_deref().unwrap_or_default(),
- task2.meta.version.as_deref().unwrap_or_default(),
- ))
- }
-
- fn select_task<'ctx>(tasks: &'ctx [TaskDef], args: &TaskArgs) -> Option<&'ctx TaskDef> {
- tasks
- .iter()
- .filter(|task| Self::match_task(task, args))
- .max_by(|task1, task2| Self::compare_tasks(task1, task2))
- }
-
- fn get_with_args<'a>(&self, id: &'a TaskID, args: &TaskArgs) -> Result<'a, &TaskDef> {
- self.tasks
- .get(id)
- .and_then(|tasks| Self::select_task(tasks, args))
- .ok_or(Error {
- task: id,
- kind: ErrorKind::TaskNotFound,
- })
- }
-
- pub fn get<'a>(&self, task: &TaskRef<'a>) -> Result<'a, &TaskDef> {
- self.get_with_args(task.id, task.args.as_ref())
- }
-
- fn task_ref<'ctx>(&'ctx self, id: &'ctx TaskID, args: &TaskArgs) -> Result<TaskRef> {
- let task_def = self.get_with_args(id, args)?;
-
- let mut arg_def: HashMap<_, _> = task_def.args.iter().map(|(k, &v)| (k, v)).collect();
- for (key, arg) in &self.globals {
- // TODO: Handle conflicts between explicit args and globals
- arg_def.insert(key, ArgType::from(arg));
- }
-
- let mut new_args = TaskArgs::default();
-
- for (key, typ) in arg_def {
- if let Some(arg) = args.get(key) {
- if ArgType::from(arg) == typ {
- new_args.set(key, Some(arg));
- continue;
- }
- }
- return Err(Error {
- task: id,
- kind: ErrorKind::InvalidArgument(key),
- });
- }
-
- let build_to_host = platform_relation(&new_args, "build", "host");
- let host_to_target = platform_relation(&new_args, "host", "target");
- let build_to_target = platform_relation(&new_args, "build", "target");
-
- let cross_compile = build_to_host
- .as_ref()
- .map(|build_to_host| build_to_host.cross_compile.clone());
-
- new_args.set("build_to_host", build_to_host);
- new_args.set("host_to_target", host_to_target);
- new_args.set("build_to_target", build_to_target);
-
- new_args.set("cross_compile", cross_compile);
-
- new_args.set("basename", Some(task_def.meta.basename.clone()));
- new_args.set("recipename", Some(task_def.meta.recipename.clone()));
- new_args.set("recipe", Some(task_def.meta.recipe.clone()));
- new_args.set("name", Some(task_def.meta.name.clone()));
- new_args.set("version", task_def.meta.version.clone());
-
- Ok(TaskRef {
- id,
- args: Rc::new(new_args),
- })
- }
-
- pub fn parse(&self, s: &str) -> error::Result<(TaskRef, TaskFlags)> {
- let (parsed, flags) = parse::task_with_flags(s)
- .ok()
- .context("Invalid task syntax")?;
-
- let recipe = parsed.id.recipe.to_string();
- let task = parsed.id.task.to_string();
-
- let id = TaskID { recipe, task };
- let (ctx_id, _) = self
- .tasks
- .get_key_value(&id)
- .with_context(|| format!("Task {} not found", id))?;
-
- let mut args = self.globals.clone();
-
- if let Some(host) = parsed.args.host {
- let plat = self
- .platforms
- .get(host)
- .with_context(|| format!("Platform '{}' not found", host))?;
- args.set("host", Some(plat));
- args.set("target", Some(plat));
- }
- if let Some(target) = parsed.args.target {
- let plat = self
- .platforms
- .get(target)
- .with_context(|| format!("Platform '{}' not found", target))?;
- args.set("target", Some(plat));
- }
-
- let task_ref = self
- .task_ref(ctx_id, &args)
- .with_context(|| format!("Failed to instantiate task {}", id))?;
-
- Ok((task_ref, flags))
- }
-
- fn map_args<'ctx, 'args>(
- task: &'ctx TaskID,
- mapping: &'ctx ArgMapping,
- args: &'args TaskArgs,
- build_dep: bool,
- ) -> Result<'ctx, Cow<'args, TaskArgs>> {
- if mapping.0.is_empty() && !build_dep {
- return Ok(Cow::Borrowed(args));
- }
-
- let mut ret = args.clone();
-
- if build_dep {
- ret.set("host", args.get("build"));
- ret.set("target", args.get("host"));
- }
-
- for (to, from) in &mapping.0 {
- let value = args.get(from).ok_or(Error {
- task,
- kind: ErrorKind::InvalidArgRef(to),
- })?;
- ret.set(to, Some(value.clone()));
- }
-
- Ok(Cow::Owned(ret))
- }
-
- fn parent_ref<'ctx>(&'ctx self, dep: &'ctx ParentDep, args: &TaskArgs) -> Result<TaskRef> {
- let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, false)?;
- self.task_ref(&dep.dep.id, mapped_args.as_ref())
- }
-
- pub fn output_ref<'ctx>(
- &'ctx self,
- dep: &'ctx OutputDep,
- args: &TaskArgs,
- build_dep: bool,
- ) -> Result<OutputRef<'ctx>> {
- let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, build_dep)?;
- Ok(OutputRef {
- task: self.task_ref(&dep.dep.id, mapped_args.as_ref())?,
- output: &dep.output,
- })
- }
-
- pub fn get_parent_depend<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> Result<Option<TaskRef>> {
- let task = self.get(task_ref)?;
- let Some(parent) = &task.parent else {
- return Ok(None);
- };
- Some(self.parent_ref(parent, &task_ref.args)).transpose()
- }
-
- fn ancestor_iter<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> impl Iterator<Item = Result<TaskRef>> {
- struct Iter<'ctx>(&'ctx Context, Option<Result<'ctx, TaskRef<'ctx>>>);
-
- impl<'ctx> Iterator for Iter<'ctx> {
- type Item = Result<'ctx, TaskRef<'ctx>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- let task_ref = match self.1.take()? {
- Ok(task_ref) => task_ref,
- Err(err) => return Some(Err(err)),
- };
- self.1 = self.0.get_parent_depend(&task_ref).transpose();
- Some(Ok(task_ref))
- }
- }
-
- Iter(self, Some(Ok(task_ref.clone())))
- }
-
- pub fn get_build_depends<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> Result<HashSet<OutputRef>> {
- let mut ret = HashSet::new();
- let mut allow_noinherit = true;
-
- for current in self.ancestor_iter(task_ref) {
- let current_ref = current?;
- let task = self.get(&current_ref)?;
- let entries = task
- .build_depends
- .iter()
- .filter(|dep| allow_noinherit || !dep.noinherit)
- .map(|dep| self.output_ref(dep, &current_ref.args, true))
- .collect::<Result<Vec<_>>>()?;
- ret.extend(entries);
-
- allow_noinherit = false;
- }
-
- Ok(ret)
- }
-
- pub fn get_host_depends<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> Result<HashSet<OutputRef>> {
- let mut ret = HashSet::new();
- let mut allow_noinherit = true;
-
- for current in self.ancestor_iter(task_ref) {
- let current_ref = current?;
- let task = self.get(&current_ref)?;
- let entries = task
- .depends
- .iter()
- .filter(|dep| allow_noinherit || !dep.noinherit)
- .map(|dep| self.output_ref(dep, &current_ref.args, false))
- .collect::<Result<Vec<_>>>()?;
- ret.extend(entries);
-
- allow_noinherit = false;
- }
-
- Ok(ret)
- }
-}
-
-impl Index<&TaskRef<'_>> for Context {
- type Output = TaskDef;
-
- fn index(&self, index: &TaskRef) -> &TaskDef {
- self.get(index).expect("Invalid TaskRef")
- }
-}
diff --git a/crates/driver/src/driver.rs b/crates/driver/src/driver.rs
deleted file mode 100644
index b2655c6..0000000
--- a/crates/driver/src/driver.rs
+++ /dev/null
@@ -1,480 +0,0 @@
-use std::{
- collections::{HashMap, HashSet},
- iter,
- os::unix::{net::UnixStream, prelude::*},
-};
-
-use indoc::indoc;
-use nix::{
- poll,
- sys::{
- signal,
- signalfd::{SfdFlags, SignalFd},
- },
-};
-
-use common::{error::*, string_hash::*, types::*};
-use runner::Runner;
-
-use crate::{
- context::{Context, OutputRef, TaskRef},
- paths, resolve,
- task::*,
- template,
-};
-
-#[derive(Debug)]
-pub struct CompletionState<'ctx> {
- ctx: &'ctx Context,
- tasks_done: HashMap<TaskRef<'ctx>, TaskOutput>,
-}
-
-impl<'ctx> CompletionState<'ctx> {
- pub fn new(ctx: &'ctx Context) -> Self {
- CompletionState {
- ctx,
- tasks_done: Default::default(),
- }
- }
-
- // Treats both "depends" and "parent" as dependencies
- fn deps_satisfied(&self, task_ref: &TaskRef) -> bool {
- resolve::get_dependent_tasks(self.ctx, task_ref)
- .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref)))
- .unwrap()
- .into_iter()
- .all(|dep| self.tasks_done.contains_key(&dep))
- }
-
- fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result<Vec<Dependency>> {
- let task_def = &self.ctx[task];
- task_def
- .fetch
- .iter()
- .map(|Fetch { name, sha256 }| {
- Ok(Dependency::Fetch {
- name: template::ENGINE.eval(name, &task.args).with_context(|| {
- format!("Failed to evaluate fetch filename for task {}", task)
- })?,
- target_dir: paths::TASK_DLDIR.to_string(),
- sha256: *sha256,
- })
- })
- .collect()
- }
-
- fn dep_closure<I>(&self, deps: I, path: &'ctx str) -> impl Iterator<Item = Dependency> + '_
- where
- I: IntoIterator<Item = OutputRef<'ctx>>,
- {
- resolve::runtime_depends(self.ctx, deps)
- .expect("invalid runtime depends")
- .into_iter()
- .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output))
- .map(|&output| Dependency::Task {
- output,
- path: path.to_string(),
- })
- }
-
- fn build_deps(&self, task: &TaskRef<'ctx>) -> Result<impl Iterator<Item = Dependency> + '_> {
- Ok(self.dep_closure(
- self.ctx
- .get_build_depends(task)
- .with_context(|| format!("invalid build depends for {}", task))?,
- "",
- ))
- }
-
- fn host_deps(&self, task: &TaskRef<'ctx>) -> Result<impl Iterator<Item = Dependency> + '_> {
- Ok(self.dep_closure(
- self.ctx
- .get_host_depends(task)
- .with_context(|| format!("invalid depends for {}", task))?,
- paths::TASK_SYSROOT,
- ))
- }
-
- fn task_deps(&self, task: &TaskRef<'ctx>) -> Result<HashSet<Dependency>> {
- let fetch_deps = self.fetch_deps(task)?.into_iter();
- let build_deps = self.build_deps(task)?;
- let host_deps = self.host_deps(task)?;
-
- Ok(fetch_deps.chain(build_deps).chain(host_deps).collect())
- }
-
- fn task_ancestors(&self, task_ref: &TaskRef<'ctx>) -> Vec<LayerHash> {
- let Some(parent) = self
- .ctx
- .get_parent_depend(task_ref)
- .expect("invalid parent depends")
- else {
- return vec![];
- };
-
- let mut chain = self.task_ancestors(&parent);
- if let Some(layer) = self.tasks_done[&parent].layer {
- chain.push(layer);
- }
- chain
- }
-
- fn print_summary(&self) {
- println!();
- println!("Summary:");
-
- let mut tasks: Box<[_]> = self.tasks_done.iter().collect();
- tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task));
- for (task_ref, task) in tasks.iter() {
- println!();
- println!("{:#}", task_ref);
- if let Some(hash) = task.input_hash {
- println!(" input: {}", hash);
- }
- if let Some(hash) = task.layer {
- println!(" layer: {}", hash);
- }
- if !task.outputs.is_empty() {
- println!(" outputs:");
-
- let mut outputs: Box<[_]> = task.outputs.iter().collect();
- outputs.sort_by_key(|(output, _)| *output);
- for (output, hash) in outputs.iter() {
- println!(" {}: {}", output, hash);
- }
- }
- }
- }
-}
-
-#[derive(Debug)]
-enum SpawnResult {
- Spawned(UnixStream),
- Skipped(TaskOutput),
-}
-
-#[derive(Debug, PartialEq, Eq, Hash)]
-enum TaskWaitResult {
- Failed,
- Interrupted,
-}
-
-#[derive(Debug)]
-pub struct Driver<'ctx> {
- rdeps: HashMap<TaskRef<'ctx>, Vec<TaskRef<'ctx>>>,
- force_run: HashSet<TaskRef<'ctx>>,
- tasks_blocked: HashSet<TaskRef<'ctx>>,
- tasks_runnable: Vec<TaskRef<'ctx>>,
- tasks_running: HashMap<RawFd, (UnixStream, TaskRef<'ctx>)>,
- state: CompletionState<'ctx>,
-}
-
-impl<'ctx> Driver<'ctx> {
- pub fn new(
- ctx: &'ctx Context,
- taskset: HashSet<TaskRef<'ctx>>,
- force_run: HashSet<TaskRef<'ctx>>,
- ) -> Result<Self> {
- let mut driver = Driver {
- rdeps: Default::default(),
- force_run,
- tasks_blocked: Default::default(),
- tasks_runnable: Default::default(),
- tasks_running: Default::default(),
- state: CompletionState::new(ctx),
- };
-
- for task in taskset {
- let mut has_depends = false;
- for dep in resolve::get_dependent_tasks(ctx, &task)
- .map_err(|_| Error::new(format!("invalid dependency for {}", task)))?
- {
- let rdep = driver.rdeps.entry(dep.clone()).or_default();
- rdep.push(task.clone());
- has_depends = true;
- }
-
- if has_depends {
- driver.tasks_blocked.insert(task);
- } else {
- driver.tasks_runnable.push(task);
- }
- }
-
- Ok(driver)
- }
-
- const PREAMBLE: &'static str = indoc! {"
- export PATH={{build.prefix}}/sbin:{{build.prefix}}/bin:$PATH
- cd {{workdir}}
-
- export SOURCE_DATE_EPOCH=1
-
- export AR_FOR_BUILD=ar
- export AS_FOR_BUILD=as
- export DLLTOOL_FOR_BUILD=dlltool
- export CC_FOR_BUILD=gcc
- export CXX_FOR_BUILD=g++
- export GCC_FOR_BUILD=gcc
- export GFORTRAN_FOR_BUILD=gfortran
- export GOC_FOR_BUILD=goc
- export LD_FOR_BUILD=ld
- export LIPO_FOR_BUILD=lipo
- export NM_FOR_BUILD=nm
- export OBJCOPY_FOR_BUILD=objcopy
- export OBJDUMP_FOR_BUILD=objdump
- export RANLIB_FOR_BUILD=ranlib
- export STRIP_FOR_BUILD=strip
- export WINDRES_FOR_BUILD=windres
- export WINDMC_FOR_BUILD=windmc
- "};
- const PREAMBLE_HOST: &'static str = indoc! {"
- export AR={{build_to_host.cross_compile}}ar
- export AS={{build_to_host.cross_compile}}as
- export DLLTOOL={{build_to_host.cross_compile}}dlltool
- export CC={{build_to_host.cross_compile}}gcc
- export CXX={{build_to_host.cross_compile}}g++
- export GCC={{build_to_host.cross_compile}}gcc
- export GFORTRAN={{build_to_host.cross_compile}}gfortran
- export GOC={{build_to_host.cross_compile}}goc
- export LD={{build_to_host.cross_compile}}ld
- export LIPO={{build_to_host.cross_compile}}lipo
- export NM={{build_to_host.cross_compile}}nm
- export OBJCOPY={{build_to_host.cross_compile}}objcopy
- export OBJDUMP={{build_to_host.cross_compile}}objdump
- export RANLIB={{build_to_host.cross_compile}}ranlib
- export STRIP={{build_to_host.cross_compile}}strip
- export WINDRES={{build_to_host.cross_compile}}windres
- export WINDMC={{build_to_host.cross_compile}}windmc
- "};
- const PREAMBLE_TARGET: &'static str = indoc! {"
- export AR_FOR_TARGET={{build_to_target.cross_compile}}ar
- export AS_FOR_TARGET={{build_to_target.cross_compile}}as
- export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool
- export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc
- export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++
- export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc
- export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran
- export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc
- export LD_FOR_TARGET={{build_to_target.cross_compile}}ld
- export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo
- export NM_FOR_TARGET={{build_to_target.cross_compile}}nm
- export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy
- export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump
- export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib
- export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip
- export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres
- export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc
- "};
-
- fn task_preamble(task_ref: &TaskRef<'ctx>) -> Vec<&'static str> {
- let mut ret = vec![Self::PREAMBLE];
-
- if task_ref.args.contains_key("build_to_host") {
- ret.push(Self::PREAMBLE_HOST);
- }
- if task_ref.args.contains_key("build_to_target") {
- ret.push(Self::PREAMBLE_TARGET);
- }
- ret
- }
-
- fn update_runnable(&mut self, task_ref: TaskRef<'ctx>, task_output: TaskOutput) {
- let rdeps = self.rdeps.get(&task_ref);
-
- self.state.tasks_done.insert(task_ref, task_output);
-
- for rdep in rdeps.unwrap_or(&Vec::new()) {
- if !self.tasks_blocked.contains(rdep) {
- continue;
- }
- if self.state.deps_satisfied(rdep) {
- self.tasks_blocked.remove(rdep);
- self.tasks_runnable.push(rdep.clone());
- }
- }
- }
-
- fn spawn_task(&self, task_ref: &TaskRef<'ctx>, runner: &Runner) -> Result<SpawnResult> {
- let task_def = &self.state.ctx[task_ref];
- if task_def.action.is_empty() {
- println!("Skipping empty task {:#}", task_ref);
- return Ok(SpawnResult::Skipped(TaskOutput::default()));
- }
-
- let task_deps = self.state.task_deps(task_ref)?;
- let task_output = task_def
- .output
- .iter()
- .map(|(name, Output { path, .. })| {
- let output_path = if let Some(path) = path {
- format!("{}/{}", paths::TASK_DESTDIR, path)
- } else {
- paths::TASK_DESTDIR.to_string()
- };
- (name.clone(), output_path)
- })
- .collect();
-
- let ancestors = self.state.task_ancestors(task_ref);
-
- let mut run = Self::task_preamble(task_ref);
- run.push(&task_def.action.run);
-
- let command = template::ENGINE
- .eval_sh(&run.concat(), &task_ref.args)
- .with_context(|| {
- format!("Failed to evaluate command template for task {}", task_ref)
- })?;
-
- let rootfs = self.state.ctx.get_rootfs();
- let task = Task {
- label: format!("{:#}", task_ref),
- command,
- workdir: paths::TASK_WORKDIR.to_string(),
- rootfs: rootfs.0,
- ancestors,
- depends: task_deps,
- outputs: task_output,
- pins: HashMap::from([rootfs.clone()]),
- force_run: self.force_run.contains(task_ref),
- };
-
- Ok(SpawnResult::Spawned(runner.spawn(&task)))
- }
-
- fn run_task(&mut self, task_ref: TaskRef<'ctx>, runner: &Runner) -> Result<()> {
- match self.spawn_task(&task_ref, runner)? {
- SpawnResult::Spawned(socket) => {
- assert!(self
- .tasks_running
- .insert(socket.as_raw_fd(), (socket, task_ref))
- .is_none());
- }
- SpawnResult::Skipped(result) => {
- self.update_runnable(task_ref, result);
- }
- }
- Ok(())
- }
-
- fn run_tasks(&mut self, runner: &Runner) -> Result<()> {
- while let Some(task_ref) = self.tasks_runnable.pop() {
- self.run_task(task_ref, runner)?;
- }
- Ok(())
- }
-
- fn wait_for_task(&mut self, signal_fd: &mut SignalFd) -> Result<Option<TaskWaitResult>> {
- let mut pollfds: Vec<_> = self
- .tasks_running
- .values()
- .map(|(socket, _)| socket.as_fd())
- .chain(iter::once(signal_fd.as_fd()))
- .map(|fd| poll::PollFd::new(fd, poll::PollFlags::POLLIN))
- .collect();
-
- while poll::poll(&mut pollfds, poll::PollTimeout::NONE).context("poll()")? == 0 {}
-
- let pollevents: Vec<_> = pollfds
- .into_iter()
- .map(|pollfd| {
- (
- pollfd.as_fd().as_raw_fd(),
- pollfd.revents().expect("Unknown events in poll() return"),
- )
- })
- .collect();
-
- for (fd, events) in pollevents {
- if !events.contains(poll::PollFlags::POLLIN) {
- if events.intersects(!poll::PollFlags::POLLIN) {
- return Err(Error::new(
- "Unexpected error status for socket file descriptor",
- ));
- }
- continue;
- }
-
- if fd == signal_fd.as_raw_fd() {
- let _signal = signal_fd.read_signal().expect("read_signal()").unwrap();
- return Ok(Some(TaskWaitResult::Interrupted));
- }
-
- let (socket, task_ref) = self.tasks_running.remove(&fd).unwrap();
-
- match Runner::result(&socket) {
- Ok(task_output) => {
- self.update_runnable(task_ref, task_output);
- }
- Err(error) => {
- eprintln!("{}", error);
- return Ok(Some(TaskWaitResult::Failed));
- }
- }
- }
-
- Ok(None)
- }
-
- fn is_done(&self) -> bool {
- self.tasks_blocked.is_empty()
- && self.tasks_runnable.is_empty()
- && self.tasks_running.is_empty()
- }
-
- fn setup_signalfd() -> Result<SignalFd> {
- let mut signals = signal::SigSet::empty();
- signals.add(signal::Signal::SIGINT);
- signal::pthread_sigmask(signal::SigmaskHow::SIG_BLOCK, Some(&signals), None)
- .expect("pthread_sigmask()");
- SignalFd::with_flags(&signals, SfdFlags::SFD_CLOEXEC)
- .context("Failed to create signal file descriptor")
- }
-
- fn raise_sigint() {
- let mut signals = signal::SigSet::empty();
- signals.add(signal::Signal::SIGINT);
- signal::pthread_sigmask(signal::SigmaskHow::SIG_UNBLOCK, Some(&signals), None)
- .expect("pthread_sigmask()");
- signal::raise(signal::Signal::SIGINT).expect("raise()");
- unreachable!();
- }
-
- pub fn run(&mut self, runner: &Runner, keep_going: bool) -> Result<bool> {
- let mut success = true;
- let mut interrupted = false;
-
- let mut signal_fd = Self::setup_signalfd()?;
-
- self.run_tasks(runner)?;
-
- while !self.tasks_running.is_empty() {
- match self.wait_for_task(&mut signal_fd)? {
- Some(TaskWaitResult::Failed) => {
- success = false;
- }
- Some(TaskWaitResult::Interrupted) => {
- if interrupted {
- Self::raise_sigint();
- }
- eprintln!("Interrupt received, not spawning new tasks. Interrupt again to stop immediately.");
- interrupted = true;
- }
- None => {}
- }
- if !interrupted && (success || keep_going) {
- self.run_tasks(runner)?;
- }
- }
-
- if interrupted || !success {
- return Ok(false);
- }
-
- assert!(self.is_done(), "No runnable tasks left");
- self.state.print_summary();
-
- Ok(true)
- }
-}
diff --git a/crates/driver/src/main.rs b/crates/driver/src/main.rs
deleted file mode 100644
index bd08f18..0000000
--- a/crates/driver/src/main.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-mod args;
-mod context;
-mod driver;
-mod parse;
-mod paths;
-mod pin;
-mod recipe;
-mod resolve;
-mod task;
-mod template;
-
-use std::collections::HashSet;
-
-use clap::Parser;
-
-use runner::Runner;
-
-#[derive(Parser)]
-#[clap(version, about)]
-struct Opts {
- /// Allow N jobs at once.
- /// Defaults to the number of available CPUs
- #[clap(short, long)]
- jobs: Option<usize>,
- /// Keep going after some tasks have failed
- #[clap(short, long)]
- keep_going: bool,
- /// The tasks to run
- #[clap(name = "task", required = true)]
- tasks: Vec<String>,
-}
-
-fn main() {
- let opts: Opts = Opts::parse();
-
- let runner = unsafe { Runner::new(&runner::Options { jobs: opts.jobs }) }.unwrap();
-
- let ctx = context::Context::new(
- recipe::read_recipes("examples/recipes").unwrap(),
- pin::read_pins("examples/pins.yml").unwrap(),
- )
- .unwrap();
-
- let mut rsv = resolve::Resolver::new(&ctx);
- let mut force_run = HashSet::new();
-
- for task in opts.tasks {
- let (task_ref, flags) = match ctx.parse(&task) {
- Ok(task_ref) => task_ref,
- Err(err) => {
- eprintln!("{}", err);
- std::process::exit(1);
- }
- };
- let errors = rsv.add_goal(&task_ref);
- if !errors.is_empty() {
- for error in errors {
- eprintln!("{}", error);
- }
- std::process::exit(1);
- }
- if flags.force_run {
- force_run.insert(task_ref);
- }
- }
- let taskset = rsv.into_taskset();
- let mut driver = driver::Driver::new(&ctx, taskset, force_run).unwrap();
- match driver.run(&runner, opts.keep_going) {
- Ok(success) => {
- if !success {
- std::process::exit(1);
- }
- }
- Err(error) => {
- eprintln!("{}", error);
- std::process::exit(1);
- }
- }
-}
diff --git a/crates/driver/src/parse.rs b/crates/driver/src/parse.rs
deleted file mode 100644
index 5857efb..0000000
--- a/crates/driver/src/parse.rs
+++ /dev/null
@@ -1,72 +0,0 @@
-#[derive(Debug, Clone, Copy)]
-pub struct TaskID<'a> {
- pub recipe: &'a str,
- pub task: &'a str,
-}
-
-#[derive(Debug, Clone, Copy, Default)]
-pub struct TaskArgs<'a> {
- pub host: Option<&'a str>,
- pub target: Option<&'a str>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Task<'a> {
- pub id: TaskID<'a>,
- pub args: TaskArgs<'a>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct TaskFlags {
- pub force_run: bool,
-}
-
-peg::parser! {
- grammar rules() for str {
- rule t<T>(tag: rule<()>, value: rule<T>) -> T
- = tag() v:value() { v }
-
- rule name_char()
- = ['a'..='z' | 'A' ..='Z' | '0'..='9' | '_' | '-']
-
- rule name() -> &'input str
- = $(name_char()+)
-
- rule recipe_id() -> &'input str
- = $(name() ("/" name())?)
-
- rule task_id() -> TaskID<'input>
- = recipe:recipe_id() "::" task:name() {
- TaskID { recipe, task }
- }
-
- rule task_args() -> TaskArgs<'input>
- = "@" host:name()? target:t(<":">, <name()>)? {
- TaskArgs {
- host,
- target,
- }
- }
- / { Default::default() }
-
- pub rule task() -> Task<'input>
- = id:task_id() args:task_args() {
- Task {
- id,
- args,
- }
- }
-
- rule force_run() -> bool
- = "+" { true }
- / { false }
-
- rule task_flags() -> TaskFlags
- = force_run:force_run() { TaskFlags { force_run } }
-
- pub rule task_with_flags() -> (Task<'input>, TaskFlags)
- = task:task() flags:task_flags() { (task, flags) }
- }
-}
-
-pub use rules::*;
diff --git a/crates/driver/src/paths.rs b/crates/driver/src/paths.rs
deleted file mode 100644
index 274dda1..0000000
--- a/crates/driver/src/paths.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub const TASK_DESTDIR: &str = "/build/dest";
-pub const TASK_DLDIR: &str = "/build/downloads";
-pub const TASK_WORKDIR: &str = "/build/work";
-pub const TASK_SYSROOT: &str = "/opt/toolchain/sysroot";
diff --git a/crates/driver/src/pin.rs b/crates/driver/src/pin.rs
deleted file mode 100644
index 26e445c..0000000
--- a/crates/driver/src/pin.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use std::{collections::HashMap, fs::File, path::Path};
-
-use serde::{Deserialize, Serialize};
-
-use common::{error::*, string_hash::*};
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct Args {
- pub host: Option<String>,
- pub target: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct Provides {
- pub recipe: String,
- pub task: String,
- pub output: Vec<String>,
- pub args: Args,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "kebab-case")]
-pub struct Pin {
- pub hash: Option<ArchiveHash>,
- #[serde(default)]
- pub provides: Vec<Provides>,
- #[serde(default)]
- pub is_rootfs: bool,
-}
-
-pub type Pins = HashMap<String, Pin>;
-
-pub fn read_pins<P: AsRef<Path>>(path: P) -> Result<Pins> {
- let f = File::open(path)?;
- let pins: Pins = serde_yaml::from_reader(f)
- .map_err(Error::new)
- .context("YAML error")?;
- Ok(pins)
-}
diff --git a/crates/driver/src/recipe.rs b/crates/driver/src/recipe.rs
deleted file mode 100644
index 16d3751..0000000
--- a/crates/driver/src/recipe.rs
+++ /dev/null
@@ -1,188 +0,0 @@
-use std::{collections::HashMap, ffi::OsStr, fs::File, path::Path, result};
-
-use scoped_tls_hkt::scoped_thread_local;
-use serde::{de::DeserializeOwned, Deserialize, Deserializer};
-use walkdir::WalkDir;
-
-use common::{error::*, types::*};
-
-use crate::task::{TaskDef, TaskMeta};
-
-scoped_thread_local!(static CURRENT_RECIPE: str);
-
-fn current_recipe() -> String {
- CURRENT_RECIPE.with(|current| current.to_string())
-}
-
-pub fn deserialize_task_id<'de, D>(deserializer: D) -> result::Result<TaskID, D::Error>
-where
- D: Deserializer<'de>,
-{
- #[derive(Deserialize)]
- struct RecipeTaskID {
- recipe: Option<String>,
- task: String,
- }
- let RecipeTaskID { recipe, task } = RecipeTaskID::deserialize(deserializer)?;
- Ok(TaskID {
- recipe: recipe.unwrap_or_else(current_recipe),
- task,
- })
-}
-
-#[derive(Clone, Debug, Deserialize, Default)]
-pub struct RecipeMeta {
- pub name: Option<String>,
- pub version: Option<String>,
-}
-
-#[derive(Debug, Deserialize)]
-struct Recipe {
- #[serde(default)]
- pub meta: RecipeMeta,
- pub tasks: HashMap<String, TaskDef>,
-}
-
-#[derive(Debug, Deserialize)]
-struct Subrecipe {
- pub tasks: HashMap<String, TaskDef>,
-}
-
-fn read_yaml<T: DeserializeOwned>(path: &Path) -> Result<T> {
- let f = File::open(path).context("IO error")?;
-
- let value: T = serde_yaml::from_reader(f)
- .map_err(Error::new)
- .context("YAML error")?;
-
- Ok(value)
-}
-
-const RECIPE_NAME: &str = "build";
-const RECIPE_PREFIX: &str = "build.";
-
-fn recipe_name(path: &Path) -> Option<&str> {
- if path.extension() != Some("yml".as_ref()) {
- return None;
- }
-
- let stem = path.file_stem()?.to_str()?;
- if stem == RECIPE_NAME {
- return Some("");
- }
- stem.strip_prefix(RECIPE_PREFIX)
-}
-
-fn handle_recipe_tasks(
- tasks: &mut HashMap<TaskID, Vec<TaskDef>>,
- recipe_tasks: HashMap<String, TaskDef>,
- meta: &TaskMeta,
-) {
- for (label, mut task) in recipe_tasks {
- let task_id = TaskID {
- recipe: meta.recipe.clone(),
- task: label,
- };
- task.meta = meta.clone();
- tasks.entry(task_id).or_default().push(task);
- }
-}
-
-fn read_recipe_tasks(
- path: &Path,
- basename: &str,
- tasks: &mut HashMap<TaskID, Vec<TaskDef>>,
-) -> Result<RecipeMeta> {
- let recipe_def = CURRENT_RECIPE.set(basename, || read_yaml::<Recipe>(path))?;
-
- let name = recipe_def
- .meta
- .name
- .as_deref()
- .unwrap_or(basename)
- .to_string();
-
- let meta = TaskMeta {
- basename: basename.to_string(),
- recipename: "".to_string(),
- recipe: basename.to_string(),
- name,
- version: recipe_def.meta.version.clone(),
- };
-
- handle_recipe_tasks(tasks, recipe_def.tasks, &meta);
-
- Ok(recipe_def.meta)
-}
-
-fn read_subrecipe_tasks(
- path: &Path,
- basename: &str,
- recipename: &str,
- recipe_meta: &RecipeMeta,
- tasks: &mut HashMap<TaskID, Vec<TaskDef>>,
-) -> Result<()> {
- let recipe = format!("{basename}/{recipename}");
- let recipe_def = CURRENT_RECIPE.set(&recipe, || read_yaml::<Subrecipe>(path))?;
-
- let name = recipe_meta.name.as_deref().unwrap_or(basename).to_string();
-
- let meta = TaskMeta {
- basename: basename.to_string(),
- recipename: recipename.to_string(),
- recipe: recipe.clone(),
- name,
- version: recipe_meta.version.clone(),
- };
-
- handle_recipe_tasks(tasks, recipe_def.tasks, &meta);
-
- Ok(())
-}
-
-pub fn read_recipes<P: AsRef<Path>>(path: P) -> Result<HashMap<TaskID, Vec<TaskDef>>> {
- let mut tasks = HashMap::<TaskID, Vec<TaskDef>>::new();
- let mut recipe_metas = HashMap::<String, RecipeMeta>::new();
-
- for entry in WalkDir::new(path)
- .sort_by(|a, b| {
- // Files are sorted first by stem, then by extension, so that
- // recipe.yml will always be read before recipe.NAME.yml
- let stem_cmp = a.path().file_stem().cmp(&b.path().file_stem());
- let ext_cmp = a.path().extension().cmp(&b.path().extension());
- stem_cmp.then(ext_cmp)
- })
- .into_iter()
- .filter_map(|e| e.ok())
- {
- let path = entry.path();
- if !path.is_file() {
- continue;
- }
-
- let Some(recipename) = recipe_name(path) else {
- continue;
- };
- let Some(basename) = path
- .parent()
- .and_then(Path::file_name)
- .and_then(OsStr::to_str)
- else {
- continue;
- };
-
- if recipename.is_empty() {
- recipe_metas.insert(
- basename.to_string(),
- read_recipe_tasks(path, basename, &mut tasks)?,
- );
- } else {
- let Some(recipe_meta) = recipe_metas.get(basename) else {
- continue;
- };
- read_subrecipe_tasks(path, basename, recipename, recipe_meta, &mut tasks)?;
- }
- }
-
- Ok(tasks)
-}
diff --git a/crates/driver/src/resolve.rs b/crates/driver/src/resolve.rs
deleted file mode 100644
index 102c483..0000000
--- a/crates/driver/src/resolve.rs
+++ /dev/null
@@ -1,334 +0,0 @@
-use std::collections::{HashMap, HashSet};
-use std::fmt;
-use std::rc::Rc;
-
-use common::types::TaskID;
-
-use crate::args::TaskArgs;
-use crate::context::{self, Context, OutputRef, TaskRef};
-
-#[derive(Debug, Default)]
-pub struct DepChain<'ctx>(pub Vec<TaskRef<'ctx>>);
-
-impl<'ctx> fmt::Display for DepChain<'ctx> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let mut first = true;
- for task in self.0.iter().rev() {
- if !first {
- write!(f, " -> ")?;
- }
- write!(f, "{}", task)?;
-
- first = false;
- }
-
- Ok(())
- }
-}
-
-impl<'ctx> From<TaskRef<'ctx>> for DepChain<'ctx> {
- fn from(task: TaskRef<'ctx>) -> Self {
- DepChain(vec![task])
- }
-}
-
-impl<'ctx> From<&TaskRef<'ctx>> for DepChain<'ctx> {
- fn from(task: &TaskRef<'ctx>) -> Self {
- task.clone().into()
- }
-}
-
-impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> {
- fn from(id: &'ctx TaskID) -> Self {
- TaskRef {
- id,
- args: Rc::new(TaskArgs::default()),
- }
- .into()
- }
-}
-
-const MAX_ERRORS: usize = 100;
-
-#[derive(Debug)]
-pub enum ErrorKind<'ctx> {
- Context(context::Error<'ctx>),
- OutputNotFound(&'ctx str),
- DependencyCycle,
- TooManyErrors,
-}
-
-#[derive(Debug)]
-pub struct Error<'ctx> {
- pub dep_chain: DepChain<'ctx>,
- pub kind: ErrorKind<'ctx>,
-}
-
-impl<'ctx> Error<'ctx> {
- fn output_not_found(task: &TaskRef<'ctx>, output: &'ctx str) -> Self {
- Error {
- dep_chain: task.into(),
- kind: ErrorKind::OutputNotFound(output),
- }
- }
-
- fn dependency_cycle(task: &TaskRef<'ctx>) -> Self {
- Error {
- dep_chain: task.into(),
- kind: ErrorKind::DependencyCycle,
- }
- }
-
- fn too_many_errors() -> Self {
- Error {
- dep_chain: DepChain::default(),
- kind: ErrorKind::TooManyErrors,
- }
- }
-
- fn extend(&mut self, task: &TaskRef<'ctx>) {
- self.dep_chain.0.push(task.clone());
- }
-}
-
-impl<'ctx> fmt::Display for Error<'ctx> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let Error { dep_chain, kind } = self;
- match kind {
- ErrorKind::Context(err) => {
- write!(f, "{}: ", err)?;
- }
- ErrorKind::OutputNotFound(output) => {
- write!(f, "Output '{}' not found: ", output)?;
- }
- ErrorKind::DependencyCycle => {
- write!(f, "Dependency Cycle: ")?;
- }
- ErrorKind::TooManyErrors => {
- write!(f, "Too many errors, stopping.")?;
- }
- }
- dep_chain.fmt(f)
- }
-}
-
-impl<'ctx> From<context::Error<'ctx>> for Error<'ctx> {
- fn from(err: context::Error<'ctx>) -> Self {
- Error {
- dep_chain: err.task.into(),
- kind: ErrorKind::Context(err),
- }
- }
-}
-
-impl<'ctx> std::error::Error for Error<'ctx> {}
-
-#[derive(Debug, PartialEq)]
-enum ResolveState {
- Resolving,
- Resolved,
-}
-
-pub fn runtime_depends<'ctx, I>(
- ctx: &'ctx Context,
- deps: I,
-) -> Result<HashSet<OutputRef>, Vec<Error>>
-where
- I: IntoIterator<Item = OutputRef<'ctx>>,
-{
- fn add_dep<'ctx>(
- ret: &mut HashSet<OutputRef<'ctx>>,
- ctx: &'ctx Context,
- dep: OutputRef<'ctx>,
- ) -> Vec<Error<'ctx>> {
- if ret.contains(&dep) {
- return Vec::new();
- }
-
- let task = &dep.task;
- let task_def = match ctx.get(task) {
- Ok(task) => task,
- Err(err) => return vec![err.into()],
- };
-
- let output = match task_def.output.get(dep.output) {
- Some(output) => output,
- None => {
- return vec![Error::output_not_found(task, dep.output)];
- }
- };
-
- ret.insert(dep.clone());
-
- let mut errors = Vec::new();
- for runtime_dep in &output.runtime_depends {
- match ctx.output_ref(runtime_dep, &task.args, false) {
- Ok(output_ref) => {
- for mut error in add_dep(ret, ctx, output_ref) {
- error.extend(task);
- errors.push(error);
- }
- }
- Err(err) => {
- let mut err: Error = err.into();
- err.extend(task);
- errors.push(err);
- }
- };
- }
- errors
- }
-
- let mut ret = HashSet::new();
- let mut errors = Vec::new();
-
- for dep in deps {
- errors.extend(add_dep(&mut ret, ctx, dep));
- }
-
- if !errors.is_empty() {
- return Err(errors);
- }
-
- Ok(ret)
-}
-
-pub fn get_dependent_outputs<'ctx>(
- ctx: &'ctx Context,
- task_ref: &TaskRef<'ctx>,
-) -> Result<HashSet<OutputRef<'ctx>>, Vec<Error<'ctx>>> {
- let deps: HashSet<_> = ctx
- .get_build_depends(task_ref)
- .map_err(|err| vec![err.into()])?
- .into_iter()
- .chain(
- ctx.get_host_depends(task_ref)
- .map_err(|err| vec![err.into()])?,
- )
- .collect();
- runtime_depends(ctx, deps)
-}
-
-pub fn get_dependent_tasks<'ctx>(
- ctx: &'ctx Context,
- task_ref: &TaskRef<'ctx>,
-) -> Result<HashSet<TaskRef<'ctx>>, Vec<Error<'ctx>>> {
- Ok(ctx
- .get_parent_depend(task_ref)
- .map_err(|err| vec![err.into()])?
- .into_iter()
- .chain(
- get_dependent_outputs(ctx, task_ref)?
- .into_iter()
- .map(|dep| dep.task),
- )
- .collect())
-}
-
-#[derive(Debug)]
-pub struct Resolver<'ctx> {
- ctx: &'ctx Context,
- resolve_state: HashMap<TaskRef<'ctx>, ResolveState>,
-}
-
-impl<'ctx> Resolver<'ctx> {
- pub fn new(ctx: &'ctx Context) -> Self {
- Resolver {
- ctx,
- resolve_state: HashMap::new(),
- }
- }
-
- fn tasks_resolved(&self) -> bool {
- self.resolve_state
- .values()
- .all(|resolved| *resolved == ResolveState::Resolved)
- }
-
- fn add_task(&mut self, task: &TaskRef<'ctx>, output: Option<&'ctx str>) -> Vec<Error<'ctx>> {
- match self.resolve_state.get(task) {
- Some(ResolveState::Resolving) => return vec![Error::dependency_cycle(task)],
- Some(ResolveState::Resolved) => return vec![],
- None => (),
- }
-
- let task_def = match self.ctx.get(task) {
- Ok(task_def) => task_def,
- Err(err) => return vec![err.into()],
- };
-
- if let Some(task_output) = output {
- if !task_def.output.contains_key(task_output) {
- return vec![Error::output_not_found(task, task_output)];
- }
- }
-
- self.resolve_state
- .insert(task.clone(), ResolveState::Resolving);
-
- let mut ret = Vec::new();
- let mut handle_errors = |errors: Vec<Error<'ctx>>| -> Result<(), ()> {
- for mut error in errors {
- error.extend(task);
- ret.push(error);
-
- if ret.len() > MAX_ERRORS {
- ret.push(Error::too_many_errors());
- return Err(());
- }
- }
- Ok(())
- };
-
- let _ = (|| -> Result<(), ()> {
- match self.ctx.get_parent_depend(task) {
- Ok(Some(parent)) => {
- handle_errors(self.add_task(&parent, None))?;
- }
- Ok(None) => {}
- Err(err) => {
- handle_errors(vec![err.into()])?;
- }
- }
-
- match get_dependent_outputs(self.ctx, task) {
- Ok(rdeps) => {
- for rdep in rdeps {
- handle_errors(self.add_task(&rdep.task, Some(rdep.output)))?;
- }
- }
- Err(errors) => {
- handle_errors(errors)?;
- }
- }
-
- Ok(())
- })();
-
- if ret.is_empty() {
- *self
- .resolve_state
- .get_mut(task)
- .expect("Missing resolve_state") = ResolveState::Resolved;
- } else {
- self.resolve_state.remove(task);
- }
-
- ret
- }
-
- pub fn add_goal(&mut self, task: &TaskRef<'ctx>) -> Vec<Error<'ctx>> {
- let ret = self.add_task(task, None);
- debug_assert!(self.tasks_resolved());
- ret
- }
-
- pub fn into_taskset(self) -> HashSet<TaskRef<'ctx>> {
- debug_assert!(self.tasks_resolved());
-
- self.resolve_state
- .into_iter()
- .map(|entry| entry.0)
- .collect()
- }
-}
diff --git a/crates/driver/src/task.rs b/crates/driver/src/task.rs
deleted file mode 100644
index e84766e..0000000
--- a/crates/driver/src/task.rs
+++ /dev/null
@@ -1,96 +0,0 @@
-use std::collections::{HashMap, HashSet};
-
-use serde::Deserialize;
-
-use common::{string_hash::StringHash, types::TaskID};
-
-use crate::{
- args::{ArgMapping, ArgType, TaskArgs},
- recipe,
-};
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
-pub struct TaskDep {
- #[serde(flatten, deserialize_with = "recipe::deserialize_task_id")]
- pub id: TaskID,
- #[serde(default)]
- pub args: ArgMapping,
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
-pub struct Fetch {
- pub name: String,
- pub sha256: StringHash,
-}
-
-fn default_output_name() -> String {
- "default".to_string()
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct ParentDep {
- #[serde(flatten)]
- pub dep: TaskDep,
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
-pub struct OutputDep {
- #[serde(flatten)]
- pub dep: TaskDep,
- #[serde(default)]
- pub noinherit: bool,
- #[serde(default = "default_output_name")]
- pub output: String,
-}
-
-#[derive(Clone, Debug, Deserialize, Default)]
-pub struct Output {
- pub path: Option<String>,
- #[serde(default)]
- pub runtime_depends: HashSet<OutputDep>,
-}
-
-#[derive(Clone, Debug, Deserialize, Default)]
-pub struct Action {
- #[serde(default)]
- pub run: String,
-}
-
-impl Action {
- pub fn is_empty(&self) -> bool {
- self.run.is_empty()
- }
-}
-
-#[derive(Clone, Debug, Default)]
-pub struct TaskMeta {
- pub basename: String,
- pub recipename: String,
- pub recipe: String,
- pub name: String,
- pub version: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, Default)]
-pub struct TaskDef {
- #[serde(skip)]
- pub meta: TaskMeta,
- #[serde(default)]
- pub args: HashMap<String, ArgType>,
- #[serde(default)]
- pub parent: Option<ParentDep>,
- #[serde(default)]
- pub fetch: HashSet<Fetch>,
- #[serde(default)]
- pub build_depends: HashSet<OutputDep>,
- #[serde(default)]
- pub depends: HashSet<OutputDep>,
- #[serde(default)]
- pub output: HashMap<String, Output>,
- #[serde(flatten)]
- pub action: Action,
- #[serde(default)]
- pub priority: i32,
- #[serde(skip)]
- pub arg_match: TaskArgs,
-}
diff --git a/crates/driver/src/template.rs b/crates/driver/src/template.rs
deleted file mode 100644
index 1a091ed..0000000
--- a/crates/driver/src/template.rs
+++ /dev/null
@@ -1,42 +0,0 @@
-use handlebars::Handlebars;
-use lazy_static::lazy_static;
-
-use common::error::*;
-
-use crate::args::TaskArgs;
-
-fn escape_sh(s: &str) -> String {
- format!("'{}'", s.replace('\'', "'\\''"))
-}
-
-#[derive(Debug)]
-pub struct TemplateEngine {
- tpl: Handlebars<'static>,
- tpl_sh: Handlebars<'static>,
-}
-
-impl TemplateEngine {
- pub fn new() -> Self {
- let mut tpl = Handlebars::new();
- tpl.set_strict_mode(true);
- tpl.register_escape_fn(handlebars::no_escape);
-
- let mut tpl_sh = Handlebars::new();
- tpl_sh.set_strict_mode(true);
- tpl_sh.register_escape_fn(escape_sh);
-
- TemplateEngine { tpl, tpl_sh }
- }
-
- pub fn eval(&self, input: &str, args: &TaskArgs) -> Result<String> {
- self.tpl.render_template(input, args).map_err(Error::new)
- }
-
- pub fn eval_sh(&self, input: &str, args: &TaskArgs) -> Result<String> {
- self.tpl_sh.render_template(input, args).map_err(Error::new)
- }
-}
-
-lazy_static! {
- pub static ref ENGINE: TemplateEngine = TemplateEngine::new();
-}