summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorMatthias Schiffer <mschiffer@universe-factory.net>2021-10-25 00:30:15 +0200
committerMatthias Schiffer <mschiffer@universe-factory.net>2021-10-25 00:30:15 +0200
commit5e2ab049dc3c8514401d1aef8fd4564759352ec3 (patch)
tree66d10592d6a6b35089c79d72ea22d9d147d3d206 /src
parent34ac18d20c13a78914d447fee83204811a27b1e4 (diff)
downloadrebel-5e2ab049dc3c8514401d1aef8fd4564759352ec3.tar
rebel-5e2ab049dc3c8514401d1aef8fd4564759352ec3.zip
Move main crate to subdirectory
Diffstat (limited to 'src')
-rw-r--r--src/args.rs123
-rw-r--r--src/context.rs446
-rw-r--r--src/executor.rs347
-rw-r--r--src/main.rs51
-rw-r--r--src/recipe.rs115
-rw-r--r--src/resolve.rs312
-rw-r--r--src/task.rs84
-rw-r--r--src/template.rs39
8 files changed, 0 insertions, 1517 deletions
diff --git a/src/args.rs b/src/args.rs
deleted file mode 100644
index 527231d..0000000
--- a/src/args.rs
+++ /dev/null
@@ -1,123 +0,0 @@
-use std::{
- collections::{hash_map, HashMap},
- hash,
- iter::FromIterator,
- rc::Rc,
-};
-
-use enum_kinds::EnumKind;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Serialize, PartialEq, Eq)]
-pub struct Platform {
- #[serde(skip)]
- pub short: String,
- pub gnu_triplet: String,
- pub karch: String,
- pub prefix: String,
-}
-
-#[derive(Debug, Serialize, PartialEq, Eq)]
-pub struct PlatformRelation {
- pub is_same: bool,
- pub sysroot: String,
- pub cross_compile: String,
-}
-
-#[derive(Clone, Debug, Serialize, PartialEq, Eq, EnumKind)]
-#[serde(untagged)]
-#[enum_kind(ArgType, derive(Deserialize), serde(rename_all = "snake_case"))]
-pub enum Arg {
- String(Rc<String>),
- Platform(Rc<Platform>),
- PlatformRelation(Rc<PlatformRelation>),
-}
-
-impl From<&Arg> for Arg {
- fn from(value: &Arg) -> Self {
- value.clone()
- }
-}
-
-impl From<String> for Arg {
- fn from(value: String) -> Self {
- Arg::String(Rc::new(value))
- }
-}
-
-impl From<Platform> for Arg {
- fn from(value: Platform) -> Self {
- Arg::Platform(Rc::new(value))
- }
-}
-
-impl From<PlatformRelation> for Arg {
- fn from(value: PlatformRelation) -> Self {
- Arg::PlatformRelation(Rc::new(value))
- }
-}
-
-#[derive(Clone, Debug, Serialize, PartialEq, Eq, Default)]
-pub struct TaskArgs(HashMap<String, Arg>);
-
-impl TaskArgs {
- pub fn contains_key(&self, key: &str) -> bool {
- self.0.contains_key(key)
- }
-
- pub fn get(&self, key: &str) -> Option<&Arg> {
- self.0.get(key)
- }
-
- pub fn set<T>(&mut self, key: &str, value: Option<T>)
- where
- T: Into<Arg>,
- {
- if let Some(v) = value {
- self.0.insert(key.to_string(), v.into());
- } else {
- self.0.remove(key);
- }
- }
-
- pub fn iter(&self) -> hash_map::Iter<String, Arg> {
- self.into_iter()
- }
-}
-
-impl FromIterator<(String, Arg)> for TaskArgs {
- fn from_iter<T: IntoIterator<Item = (String, Arg)>>(iter: T) -> Self {
- TaskArgs(HashMap::from_iter(iter))
- }
-}
-
-impl<'a> IntoIterator for &'a TaskArgs {
- type Item = (&'a String, &'a Arg);
-
- type IntoIter = hash_map::Iter<'a, String, Arg>;
-
- fn into_iter(self) -> Self::IntoIter {
- self.0.iter()
- }
-}
-
-#[allow(clippy::derive_hash_xor_eq)]
-impl hash::Hash for TaskArgs {
- fn hash<H: hash::Hasher>(&self, _state: &mut H) {
- // Don't do anything: Properly hashing the task args is likely to cost
- // much more performance than the hash collisions caused by TaskRefs
- // that only differ by the args
- }
-}
-
-pub fn arg<A: Into<Arg>>(key: &str, value: A) -> (String, Arg) {
- (key.to_string(), value.into())
-}
-
-#[derive(Clone, Debug, Deserialize, Default, PartialEq, Eq)]
-pub struct ArgMapping(pub HashMap<String, String>);
-
-#[allow(clippy::derive_hash_xor_eq)]
-impl hash::Hash for ArgMapping {
- fn hash<H: hash::Hasher>(&self, _state: &mut H) {}
-}
diff --git a/src/context.rs b/src/context.rs
deleted file mode 100644
index 7a46e8d..0000000
--- a/src/context.rs
+++ /dev/null
@@ -1,446 +0,0 @@
-use std::{
- borrow::Cow,
- collections::{HashMap, HashSet},
- fmt::Display,
- hash::Hash,
- iter::FromIterator,
- ops::Index,
- rc::Rc,
- result,
-};
-
-use lazy_static::lazy_static;
-use regex::Regex;
-use serde::Serialize;
-
-use common::{
- error::{self, Contextualizable},
- types::TaskID,
-};
-use runner::paths;
-
-use crate::{
- args::{self, arg, Arg, ArgMapping, ArgType, PlatformRelation, TaskArgs},
- task::*,
-};
-
-#[derive(Debug, Clone, Copy)]
-pub enum ErrorKind<'ctx> {
- TaskNotFound,
- InvalidArgument(&'ctx str),
- InvalidArgRef(&'ctx str),
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct Error<'ctx> {
- pub task: &'ctx TaskID,
- pub kind: ErrorKind<'ctx>,
-}
-
-impl<'ctx> Display for Error<'ctx> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- let Error { task, kind } = self;
- match kind {
- ErrorKind::TaskNotFound => write!(f, "Task '{}' not found", task),
- ErrorKind::InvalidArgument(arg) => write!(
- f,
- "Invalid or missing argument '{}' for task '{}'",
- arg, task
- ),
- ErrorKind::InvalidArgRef(arg) => write!(
- f,
- "Invalid reference for argument '{}' of task '{}'",
- arg, task
- ),
- }
- }
-}
-
-impl<'ctx> From<Error<'ctx>> for error::Error {
- fn from(err: Error) -> Self {
- error::Error::new(err)
- }
-}
-
-pub type Result<'ctx, T> = result::Result<T, Error<'ctx>>;
-
-#[derive(Clone, Debug, Serialize, PartialEq, Eq, Hash)]
-pub struct TaskRef<'ctx> {
- pub id: &'ctx TaskID,
- pub args: Rc<TaskArgs>,
-}
-
-impl<'ctx> Display for TaskRef<'ctx> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- if !f.alternate() {
- return self.id.fmt(f);
- }
-
- let pv_arg = match self.args.get("pv") {
- Some(Arg::String(s)) => Some(s),
- _ => None,
- };
- let host_arg = match self.args.get("host") {
- Some(Arg::Platform(platform)) => Some(platform),
- _ => None,
- };
- let target_arg = match self.args.get("target") {
- Some(Arg::Platform(platform)) => Some(platform),
- _ => None,
- };
-
- write!(f, "{}", self.id.recipe)?;
- if let Some(pv) = pv_arg {
- write!(f, "-{}", pv)?;
- }
- write!(f, ":{}", self.id.task)?;
-
- if let Some(host) = host_arg {
- write!(f, "@{}", host.short)?;
- }
- if let Some(target) = target_arg {
- write!(f, "/{}", target.short)?;
- }
- Ok(())
- }
-}
-
-#[derive(Clone, Debug, Serialize, PartialEq, Eq, Hash)]
-pub struct OutputRef<'ctx> {
- pub task: TaskRef<'ctx>,
- pub output: &'ctx str,
-}
-
-fn platform_relation(args: &TaskArgs, from: &str, to: &str) -> Option<PlatformRelation> {
- let plat_from = match args.get(from)? {
- Arg::Platform(plat) => plat,
- _ => return None,
- };
- let plat_to = match args.get(to)? {
- Arg::Platform(plat) => plat,
- _ => return None,
- };
-
- let plat_rel = if plat_from == plat_to {
- PlatformRelation {
- is_same: true,
- sysroot: "".to_string(),
- cross_compile: "".to_string(),
- }
- } else {
- PlatformRelation {
- is_same: false,
- sysroot: paths::abs(paths::TASK_SYSROOT),
- cross_compile: format!("{}/bin/{}-", plat_from.prefix, plat_to.gnu_triplet),
- }
- };
- Some(plat_rel)
-}
-
-#[derive(Debug)]
-pub struct Context {
- platforms: HashMap<String, args::Arg>,
- globals: TaskArgs,
- tasks: HashMap<TaskID, TaskDef>,
-}
-
-impl Context {
- pub fn new(tasks: HashMap<TaskID, TaskDef>) -> Self {
- let platforms: HashMap<_, _> = IntoIterator::into_iter([
- arg(
- "build",
- args::Platform {
- short: "build".to_string(),
- gnu_triplet: "x86_64-linux-gnu".to_string(),
- karch: "x86_64".to_string(),
- prefix: "/opt/toolchain".to_string(),
- },
- ),
- arg(
- "aarch64",
- args::Platform {
- short: "aarch64".to_string(),
- gnu_triplet: "aarch64-linux-gnu".to_string(),
- karch: "arm64".to_string(),
- prefix: "/usr".to_string(),
- },
- ),
- ])
- .collect();
-
- let globals = TaskArgs::from_iter([
- ("build".to_string(), platforms["build"].clone()),
- arg("workdir", paths::abs(paths::TASK_WORKDIR)),
- arg("dldir", paths::abs(paths::TASK_DLDIR)),
- arg("destdir", paths::abs(paths::TASK_DESTDIR)),
- arg("sysroot", paths::abs(paths::TASK_SYSROOT)),
- ]);
-
- Context {
- platforms,
- globals,
- tasks,
- }
- }
-
- pub fn get<'ctx>(&'ctx self, id: &'ctx TaskID) -> Result<&TaskDef> {
- self.tasks.get(id).ok_or(Error {
- task: id,
- kind: ErrorKind::TaskNotFound,
- })
- }
-
- fn task_ref<'ctx>(&'ctx self, id: &'ctx TaskID, args: &TaskArgs) -> Result<TaskRef> {
- let task_def = self.get(id)?;
-
- let mut arg_def: HashMap<_, _> = task_def.args.iter().map(|(k, &v)| (k, v)).collect();
- for (key, arg) in &self.globals {
- // TODO: Handle conflicts between explicit args and globals
- arg_def.insert(key, ArgType::from(arg));
- }
-
- let mut new_args = TaskArgs::default();
-
- for (key, typ) in arg_def {
- if let Some(arg) = args.get(key) {
- if ArgType::from(arg) == typ {
- new_args.set(key, Some(arg));
- continue;
- }
- }
- return Err(Error {
- task: id,
- kind: ErrorKind::InvalidArgument(key),
- });
- }
-
- let build_to_host = platform_relation(&new_args, "build", "host");
- let host_to_target = platform_relation(&new_args, "host", "target");
- let build_to_target = platform_relation(&new_args, "build", "target");
-
- let cross_compile = build_to_host
- .as_ref()
- .map(|build_to_host| build_to_host.cross_compile.clone());
-
- new_args.set("build_to_host", build_to_host);
- new_args.set("host_to_target", host_to_target);
- new_args.set("build_to_target", build_to_target);
-
- new_args.set("cross_compile", cross_compile);
-
- new_args.set("pn", Some(task_def.meta.name.clone()));
- new_args.set("pv", task_def.meta.version.clone());
-
- Ok(TaskRef {
- id,
- args: Rc::new(new_args),
- })
- }
-
- pub fn parse<'ctx>(&'ctx self, s: &str) -> error::Result<TaskRef> {
- lazy_static! {
- static ref RE: Regex = Regex::new(
- r"^(?P<recipe>[[:word:]-]+):(?P<task>[[:word:]-]+)(?:@(?P<host>[[:word:]-]+))?(?:/(?P<target>[[:word:]-]+))?$",
- ).unwrap();
- }
-
- let cap = RE.captures(s).context("Invalid task syntax")?;
-
- let recipe = cap["recipe"].to_string();
- let task = cap["task"].to_string();
-
- let id = TaskID { recipe, task };
- let (ctx_id, _) = self
- .tasks
- .get_key_value(&id)
- .with_context(|| format!("Task {}:{} not found", id.recipe, id.task))?;
-
- let mut args = self.globals.clone();
-
- if let Some(host) = cap.name("host") {
- let plat = self
- .platforms
- .get(host.as_str())
- .with_context(|| format!("Platform '{}' not found", host.as_str()))?;
- args.set("host", Some(plat));
- args.set("target", Some(plat));
- }
- if let Some(target) = cap.name("target") {
- let plat = self
- .platforms
- .get(target.as_str())
- .with_context(|| format!("Platform '{}' not found", target.as_str()))?;
- args.set("target", Some(plat));
- }
-
- self.task_ref(ctx_id, &args)
- .with_context(|| format!("Failed to instantiate task {}:{}", id.recipe, id.task))
- }
-
- fn map_args<'ctx, 'args>(
- task: &'ctx TaskID,
- mapping: &'ctx ArgMapping,
- args: &'args TaskArgs,
- build_dep: bool,
- ) -> Result<'ctx, Cow<'args, TaskArgs>> {
- if mapping.0.is_empty() && !build_dep {
- return Ok(Cow::Borrowed(args));
- }
-
- let mut ret = args.clone();
-
- if build_dep {
- ret.set("host", args.get("build"));
- ret.set("target", args.get("host"));
- }
-
- for (to, from) in &mapping.0 {
- let value = args.get(from).ok_or(Error {
- task,
- kind: ErrorKind::InvalidArgRef(to),
- })?;
- ret.set(to, Some(value.clone()));
- }
-
- Ok(Cow::Owned(ret))
- }
-
- fn inherit_ref<'ctx>(&'ctx self, dep: &'ctx InheritDep, args: &TaskArgs) -> Result<TaskRef> {
- let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, false)?;
- self.task_ref(&dep.dep.id, mapped_args.as_ref())
- }
-
- pub fn output_ref<'ctx>(
- &'ctx self,
- dep: &'ctx OutputDep,
- args: &TaskArgs,
- build_dep: bool,
- ) -> Result<OutputRef<'ctx>> {
- let mapped_args = Context::map_args(&dep.dep.id, &dep.dep.args, args, build_dep)?;
- Ok(OutputRef {
- task: self.task_ref(&dep.dep.id, mapped_args.as_ref())?,
- output: &dep.output,
- })
- }
-
- pub fn get_inherit_depend<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> Result<Option<TaskRef>> {
- let task = self.get(task_ref.id)?;
- let inherit = match &task.inherit {
- Some(inherit) => inherit,
- None => return Ok(None),
- };
- Some(self.inherit_ref(inherit, &task_ref.args)).transpose()
- }
-
- fn inherit_iter<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> impl Iterator<Item = Result<TaskRef>> {
- struct Iter<'ctx>(&'ctx Context, Option<Result<'ctx, TaskRef<'ctx>>>);
-
- impl<'ctx> Iterator for Iter<'ctx> {
- type Item = Result<'ctx, TaskRef<'ctx>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- let task_ref = match self.1.take()? {
- Ok(task_ref) => task_ref,
- Err(err) => return Some(Err(err)),
- };
- self.1 = self.0.get_inherit_depend(&task_ref).transpose();
- Some(Ok(task_ref))
- }
- }
-
- Iter(self, Some(Ok(task_ref.clone())))
- }
-
- pub fn get_build_depends<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> Result<HashSet<OutputRef>> {
- let mut ret = HashSet::new();
- let mut allow_noinherit = true;
-
- for current in self.inherit_iter(task_ref) {
- let task_ref = current?;
- let task = self.get(task_ref.id)?;
- let entries = task
- .build_depends
- .iter()
- .filter(|dep| allow_noinherit || !dep.noinherit)
- .map(|dep| self.output_ref(dep, &task_ref.args, true))
- .collect::<Result<Vec<_>>>()?;
- ret.extend(entries);
-
- allow_noinherit = false;
- }
-
- Ok(ret)
- }
-
- pub fn get_host_depends<'ctx>(
- &'ctx self,
- task_ref: &TaskRef<'ctx>,
- ) -> Result<HashSet<OutputRef>> {
- let mut ret = HashSet::new();
- let mut allow_noinherit = true;
-
- for current in self.inherit_iter(task_ref) {
- let task_ref = current?;
- let task = self.get(task_ref.id)?;
- let entries = task
- .depends
- .iter()
- .filter(|dep| allow_noinherit || !dep.noinherit)
- .map(|dep| self.output_ref(dep, &task_ref.args, false))
- .collect::<Result<Vec<_>>>()?;
- ret.extend(entries);
-
- allow_noinherit = false;
- }
-
- Ok(ret)
- }
-
- pub fn in_rootfs<'ctx>(&'ctx self, output: &OutputRef<'ctx>) -> bool {
- let build = self.globals.get("build").unwrap();
- if output.task.args.get("host") != Some(build) {
- return false;
- }
- if let Some(target) = output.task.args.get("target") {
- if target != build {
- return false;
- }
- }
-
- // TODO: Do not hardcode this
- match (
- output.task.id.recipe.as_str(),
- output.task.id.task.as_str(),
- output.output,
- ) {
- ("gmp", "install", "default") => true,
- ("mpfr", "install", "default") => true,
- ("mpc", "install", "default") => true,
- ("zlib", "install", "default") => true,
- ("binutils", "install", "default") => true,
- ("gcc", "install", "default") => true,
- ("libgcc", "install", "default") => true,
- ("gcc-libs", "install", "default") => true,
- ("linux-uapi-headers", "install", "default") => true,
- ("glibc", "install", "default") => true,
- _ => false,
- }
- }
-}
-
-impl Index<&TaskID> for Context {
- type Output = TaskDef;
-
- fn index(&self, index: &TaskID) -> &TaskDef {
- self.tasks.get(index).expect("Invalid TaskID")
- }
-}
diff --git a/src/executor.rs b/src/executor.rs
deleted file mode 100644
index 1d6ee44..0000000
--- a/src/executor.rs
+++ /dev/null
@@ -1,347 +0,0 @@
-use std::collections::{HashMap, HashSet};
-
-use indoc::indoc;
-use ipc_channel::ipc;
-
-use common::{error::*, string_hash::*, types::*};
-use runner::{paths, Runner};
-
-use crate::{
- context::{Context, TaskRef},
- resolve,
- task::*,
- template::TemplateEngine,
-};
-
-pub struct Executor<'ctx> {
- ctx: &'ctx Context,
- receiver_set: ipc::IpcReceiverSet,
- tasks_blocked: HashSet<TaskRef<'ctx>>,
- tasks_runnable: Vec<TaskRef<'ctx>>,
- tasks_running: HashMap<u64, TaskRef<'ctx>>,
- tasks_done: HashMap<TaskRef<'ctx>, TaskOutput>,
- rdeps: HashMap<TaskRef<'ctx>, Vec<TaskRef<'ctx>>>,
- tpl: TemplateEngine,
-}
-
-impl<'ctx> Executor<'ctx> {
- pub fn new(ctx: &'ctx Context, taskset: HashSet<TaskRef<'ctx>>) -> Result<Self> {
- let mut exc = Executor {
- ctx,
- receiver_set: ipc::IpcReceiverSet::new().expect("IpcReceiverSet::new()"),
- tasks_blocked: HashSet::new(),
- tasks_runnable: Vec::new(),
- tasks_running: HashMap::new(),
- tasks_done: HashMap::new(),
- rdeps: HashMap::new(),
- tpl: TemplateEngine::new(),
- };
-
- for task in taskset {
- let mut has_depends = false;
- for dep in resolve::get_dependent_tasks(ctx, &task)
- .map_err(|_| Error::new(format!("invalid dependency for {}", task)))?
- {
- let rdep = exc.rdeps.entry(dep.clone()).or_default();
- rdep.push(task.clone());
- has_depends = true;
- }
-
- if has_depends {
- exc.tasks_blocked.insert(task);
- } else {
- exc.tasks_runnable.push(task);
- }
- }
-
- Ok(exc)
- }
-
- // Treats both "depends" and "inherit" as dependencies
- fn deps_satisfied(&self, task_ref: &TaskRef) -> bool {
- resolve::get_dependent_tasks(self.ctx, task_ref)
- .map_err(|_| Error::new(format!("invalid dependency for {}", task_ref)))
- .unwrap()
- .into_iter()
- .all(|dep| self.tasks_done.contains_key(&dep))
- }
-
- fn fetch_deps(&self, task: &TaskRef<'ctx>) -> Result<Vec<Dependency>> {
- let task_def = &self.ctx[task.id];
- task_def
- .fetch
- .iter()
- .map(|Fetch { name, sha256 }| {
- Ok(Dependency::Fetch {
- name: self.tpl.eval_raw(name, &task.args).with_context(|| {
- format!("Failed to evaluate fetch filename for task {}", task)
- })?,
- sha256: *sha256,
- })
- })
- .collect()
- }
-
- fn task_deps(&self, task: &TaskRef<'ctx>) -> Result<HashSet<Dependency>> {
- Ok(self
- .fetch_deps(task)?
- .into_iter()
- .chain(
- resolve::runtime_depends(
- self.ctx,
- self.ctx
- .get_build_depends(task)
- .with_context(|| format!("invalid build depends for {}", task))?,
- )
- .expect("invalid runtime depends of build_depends")
- .into_iter()
- .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output))
- .map(|&output| Dependency::Task {
- output,
- path: "".to_string(),
- }),
- )
- .chain(
- resolve::runtime_depends(
- self.ctx,
- self.ctx
- .get_host_depends(task)
- .with_context(|| format!("invalid depends for {}", task))?,
- )
- .expect("invalid runtime depends of host_depends")
- .into_iter()
- .filter_map(|dep| self.tasks_done[&dep.task].outputs.get(dep.output))
- .map(|&output| Dependency::Task {
- output,
- path: paths::abs(paths::TASK_SYSROOT),
- }),
- )
- .collect())
- }
-
- fn task_inherit_chain(&self, task_ref: &TaskRef<'ctx>) -> Vec<LayerHash> {
- let inherit = match self
- .ctx
- .get_inherit_depend(task_ref)
- .expect("invalid inherit depends")
- {
- Some(inherit) => inherit,
- None => return vec![],
- };
-
- let mut chain = self.task_inherit_chain(&inherit);
- if let Some(layer) = self.tasks_done[&inherit].layer {
- chain.push(layer);
- }
- chain
- }
-
- fn task_setup(&self, task_ref: &TaskRef<'ctx>) -> Vec<&'static str> {
- let mut ret = vec![indoc! {"
- export SOURCE_DATE_EPOCH=1
-
- export AR_FOR_BUILD=ar
- export AS_FOR_BUILD=as
- export DLLTOOL_FOR_BUILD=dlltool
- export CC_FOR_BUILD=gcc
- export CXX_FOR_BUILD=g++
- export GCC_FOR_BUILD=gcc
- export GFORTRAN_FOR_BUILD=gfortran
- export GOC_FOR_BUILD=goc
- export LD_FOR_BUILD=ld
- export LIPO_FOR_BUILD=lipo
- export NM_FOR_BUILD=nm
- export OBJCOPY_FOR_BUILD=objcopy
- export OBJDUMP_FOR_BUILD=objdump
- export RANLIB_FOR_BUILD=ranlib
- export STRIP_FOR_BUILD=strip
- export WINDRES_FOR_BUILD=windres
- export WINDMC_FOR_BUILD=windmc
- "}];
-
- if task_ref.args.contains_key("build_to_host") {
- ret.push(indoc! {"
- export AR={{build_to_host.cross_compile}}ar
- export AS={{build_to_host.cross_compile}}as
- export DLLTOOL={{build_to_host.cross_compile}}dlltool
- export CC={{build_to_host.cross_compile}}gcc
- export CXX={{build_to_host.cross_compile}}g++
- export GCC={{build_to_host.cross_compile}}gcc
- export GFORTRAN={{build_to_host.cross_compile}}gfortran
- export GOC={{build_to_host.cross_compile}}goc
- export LD={{build_to_host.cross_compile}}ld
- export LIPO={{build_to_host.cross_compile}}lipo
- export NM={{build_to_host.cross_compile}}nm
- export OBJCOPY={{build_to_host.cross_compile}}objcopy
- export OBJDUMP={{build_to_host.cross_compile}}objdump
- export RANLIB={{build_to_host.cross_compile}}ranlib
- export STRIP={{build_to_host.cross_compile}}strip
- export WINDRES={{build_to_host.cross_compile}}windres
- export WINDMC={{build_to_host.cross_compile}}windmc
- "});
- }
-
- if task_ref.args.contains_key("build_to_target") {
- ret.push(indoc! {"
- export AR_FOR_TARGET={{build_to_target.cross_compile}}ar
- export AS_FOR_TARGET={{build_to_target.cross_compile}}as
- export DLLTOOL_FOR_TARGET={{build_to_target.cross_compile}}dlltool
- export CC_FOR_TARGET={{build_to_target.cross_compile}}gcc
- export CXX_FOR_TARGET={{build_to_target.cross_compile}}g++
- export GCC_FOR_TARGET={{build_to_target.cross_compile}}gcc
- export GFORTRAN_FOR_TARGET={{build_to_target.cross_compile}}gfortran
- export GOC_FOR_TARGET={{build_to_target.cross_compile}}goc
- export LD_FOR_TARGET={{build_to_target.cross_compile}}ld
- export LIPO_FOR_TARGET={{build_to_target.cross_compile}}lipo
- export NM_FOR_TARGET={{build_to_target.cross_compile}}nm
- export OBJCOPY_FOR_TARGET={{build_to_target.cross_compile}}objcopy
- export OBJDUMP_FOR_TARGET={{build_to_target.cross_compile}}objdump
- export RANLIB_FOR_TARGET={{build_to_target.cross_compile}}ranlib
- export STRIP_FOR_TARGET={{build_to_target.cross_compile}}strip
- export WINDRES_FOR_TARGET={{build_to_target.cross_compile}}windres
- export WINDMC_FOR_TARGET={{build_to_target.cross_compile}}windmc
- "});
- }
- ret
- }
-
- fn spawn_one(
- &self,
- task_ref: &TaskRef<'ctx>,
- runner: &Runner,
- ) -> Result<ipc::IpcReceiver<Result<TaskOutput>>> {
- let task_def = &self.ctx[task_ref.id];
- let task_deps = self.task_deps(task_ref)?;
- let task_output = task_def
- .output
- .iter()
- .map(|(name, Output { path, .. })| {
- (
- name.clone(),
- path.as_ref().map(String::as_str).unwrap_or(".").to_string(),
- )
- })
- .collect();
-
- let inherit_chain = self.task_inherit_chain(task_ref);
-
- let mut run = self.task_setup(task_ref);
- run.push(&task_def.action.run);
-
- let command = self
- .tpl
- .eval(&run.concat(), &task_ref.args)
- .with_context(|| {
- format!("Failed to evaluate command template for task {}", task_ref)
- })?;
-
- let task = Task {
- label: format!("{:#}", task_ref),
- command,
- inherit: inherit_chain,
- depends: task_deps,
- outputs: task_output,
- };
-
- Ok(runner.spawn(&task))
- }
-
- fn run_tasks(&mut self, runner: &Runner) -> Result<()> {
- while let Some(task_ref) = self.tasks_runnable.pop() {
- let channel = self.spawn_one(&task_ref, runner)?;
- let id = self
- .receiver_set
- .add(channel)
- .expect("Failed to add channel to receiver set");
- self.tasks_running.insert(id, task_ref);
- }
-
- Ok(())
- }
-
- fn update_runnable(&mut self, task_ref: &TaskRef) {
- let rdeps = self.rdeps.get(task_ref);
-
- for rdep in rdeps.unwrap_or(&Vec::new()) {
- if !self.tasks_blocked.contains(rdep) {
- continue;
- }
- if self.deps_satisfied(rdep) {
- self.tasks_blocked.remove(rdep);
- self.tasks_runnable.push(rdep.clone());
- }
- }
- }
-
- fn wait_for_task(&mut self) -> Result<()> {
- let mut progress = false;
-
- while !progress {
- let events = self
- .receiver_set
- .select()
- .expect("Failed to get messages from receiver set");
- for event in events {
- match event {
- ipc::IpcSelectionResult::MessageReceived(id, msg) => {
- let task_ref = self
- .tasks_running
- .remove(&id)
- .expect("Received message for unknown task");
- let task_output = msg
- .to::<Result<TaskOutput>>()
- .expect("Failed to decode message from runner")?;
-
- self.tasks_done.insert(task_ref.clone(), task_output);
- self.update_runnable(&task_ref);
-
- progress = true;
- }
- ipc::IpcSelectionResult::ChannelClosed(id) => {
- if let Some(task) = self.tasks_running.remove(&id) {
- return Err(Error::new(format!(
- "Unexpectedly got no result for task {:#}",
- task
- )));
- }
- }
- }
- }
- }
-
- Ok(())
- }
-
- pub fn run(&mut self, runner: &Runner) -> Result<()> {
- while !(self.tasks_runnable.is_empty() && self.tasks_running.is_empty()) {
- self.run_tasks(runner)?;
- self.wait_for_task()?;
- }
-
- assert!(self.tasks_blocked.is_empty(), "No runnable tasks left");
-
- println!();
- println!("Summary:");
-
- let mut tasks: Box<[_]> = self.tasks_done.iter().collect();
- tasks.sort_by_cached_key(|(task, _)| format!("{:#}", task));
- for (task_ref, task) in tasks.iter() {
- println!();
- println!("{:#}", task_ref);
- println!(" input: {}", task.input_hash);
- if let Some(hash) = task.layer {
- println!(" layer: {}", hash);
- }
- if !task.outputs.is_empty() {
- println!(" outputs:");
-
- let mut outputs: Box<[_]> = task.outputs.iter().collect();
- outputs.sort_by_key(|(output, _)| *output);
- for (output, hash) in outputs.iter() {
- println!(" {}: {}", output, hash);
- }
- }
- }
- Ok(())
- }
-}
diff --git a/src/main.rs b/src/main.rs
deleted file mode 100644
index 4a045be..0000000
--- a/src/main.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-mod args;
-mod context;
-mod executor;
-mod recipe;
-mod resolve;
-mod task;
-mod template;
-
-use clap::Parser;
-
-use runner::Runner;
-
-#[derive(Parser)]
-#[clap(version = clap::crate_version!())]
-struct Opts {
- #[clap(name = "task", required = true)]
- tasks: Vec<String>,
-}
-
-fn main() {
- let opts: Opts = Opts::parse();
-
- let runner = unsafe { Runner::new() }.unwrap();
-
- let ctx = context::Context::new(recipe::read_recipes("examples").unwrap());
-
- let mut rsv = resolve::Resolver::new(&ctx);
-
- for task in opts.tasks {
- let task_ref = match ctx.parse(&task) {
- Ok(task_ref) => task_ref,
- Err(err) => {
- eprintln!("{}", err);
- std::process::exit(1);
- }
- };
- let errors = rsv.add_goal(&task_ref);
- if !errors.is_empty() {
- for error in errors {
- eprintln!("{}", error);
- }
- std::process::exit(1);
- }
- }
- let taskset = rsv.into_taskset();
- let mut exc = executor::Executor::new(&ctx, taskset).unwrap();
- if let Err(error) = exc.run(&runner) {
- eprintln!("{}", error);
- std::process::exit(1);
- }
-}
diff --git a/src/recipe.rs b/src/recipe.rs
deleted file mode 100644
index 04f356b..0000000
--- a/src/recipe.rs
+++ /dev/null
@@ -1,115 +0,0 @@
-use std::{collections::HashMap, fmt, fs::File, io, path::Path, result};
-
-use scoped_tls_hkt::scoped_thread_local;
-use serde::{Deserialize, Deserializer};
-use walkdir::WalkDir;
-
-use common::types::*;
-
-use crate::task::{RecipeMeta, TaskDef};
-
-scoped_thread_local!(static CURRENT_RECIPE: str);
-
-fn current_recipe() -> String {
- CURRENT_RECIPE.with(|current| current.to_string())
-}
-
-pub fn deserialize_task_id<'de, D>(deserializer: D) -> result::Result<TaskID, D::Error>
-where
- D: Deserializer<'de>,
-{
- #[derive(Deserialize)]
- struct RecipeTaskID {
- recipe: Option<String>,
- task: String,
- }
- let RecipeTaskID { recipe, task } = RecipeTaskID::deserialize(deserializer)?;
- Ok(TaskID {
- recipe: recipe.unwrap_or_else(current_recipe),
- task,
- })
-}
-
-#[derive(Debug, Deserialize)]
-struct Recipe {
- #[serde(default)]
- pub meta: RecipeMeta,
- pub tasks: HashMap<String, TaskDef>,
-}
-
-#[derive(Debug)]
-pub enum Error {
- IOError(io::Error),
- YAMLError(serde_yaml::Error),
-}
-
-impl From<io::Error> for Error {
- fn from(err: io::Error) -> Self {
- Error::IOError(err)
- }
-}
-
-impl From<serde_yaml::Error> for Error {
- fn from(err: serde_yaml::Error) -> Self {
- Error::YAMLError(err)
- }
-}
-
-impl fmt::Display for Error {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- match self {
- Error::IOError(err) => write!(f, "IO error: {}", err),
- Error::YAMLError(err) => write!(f, "YAML error: {}", err),
- }
- }
-}
-
-impl std::error::Error for Error {}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-fn read_recipe(path: &Path) -> Result<Recipe> {
- let f = File::open(path)?;
-
- let recipe: Recipe = serde_yaml::from_reader(f)?;
-
- Ok(recipe)
-}
-
-fn is_yml(path: &Path) -> bool {
- path.extension() == Some("yml".as_ref())
-}
-
-pub fn read_recipes<P: AsRef<Path>>(path: P) -> Result<HashMap<TaskID, TaskDef>> {
- let mut tasks = HashMap::new();
-
- for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) {
- let path = entry.path();
- if !path.is_file() || !is_yml(path) {
- continue;
- }
-
- let basename: &str = match path.file_stem().map(|n| n.to_str()) {
- Some(Some(v)) => v,
- _ => continue,
- };
-
- let recipe = CURRENT_RECIPE.set(basename, || read_recipe(path))?;
-
- let mut meta = recipe.meta;
- if meta.name.is_empty() {
- meta.name = basename.to_string();
- }
-
- for (label, mut task) in recipe.tasks {
- let task_id = TaskID {
- recipe: basename.to_string(),
- task: label,
- };
- task.meta = meta.clone();
- tasks.insert(task_id, task);
- }
- }
-
- Ok(tasks)
-}
diff --git a/src/resolve.rs b/src/resolve.rs
deleted file mode 100644
index 338ce3f..0000000
--- a/src/resolve.rs
+++ /dev/null
@@ -1,312 +0,0 @@
-use std::collections::{HashMap, HashSet};
-use std::fmt;
-use std::rc::Rc;
-
-use common::types::TaskID;
-
-use crate::args::TaskArgs;
-use crate::context::{self, Context, OutputRef, TaskRef};
-
-#[derive(Debug)]
-pub struct DepChain<'ctx>(pub Vec<TaskRef<'ctx>>);
-
-impl<'ctx> fmt::Display for DepChain<'ctx> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let mut first = true;
- for task in self.0.iter().rev() {
- if !first {
- write!(f, " -> ")?;
- }
- write!(f, "{}", task)?;
-
- first = false;
- }
-
- Ok(())
- }
-}
-
-impl<'ctx> From<TaskRef<'ctx>> for DepChain<'ctx> {
- fn from(task: TaskRef<'ctx>) -> Self {
- DepChain(vec![task])
- }
-}
-
-impl<'ctx> From<&TaskRef<'ctx>> for DepChain<'ctx> {
- fn from(task: &TaskRef<'ctx>) -> Self {
- task.clone().into()
- }
-}
-
-impl<'ctx> From<&'ctx TaskID> for DepChain<'ctx> {
- fn from(id: &'ctx TaskID) -> Self {
- TaskRef {
- id,
- args: Rc::new(TaskArgs::default()),
- }
- .into()
- }
-}
-
-#[derive(Debug)]
-pub enum ErrorKind<'ctx> {
- Context(context::Error<'ctx>),
- OutputNotFound(&'ctx str),
- DependencyCycle,
-}
-
-#[derive(Debug)]
-pub struct Error<'ctx> {
- pub dep_chain: DepChain<'ctx>,
- pub kind: ErrorKind<'ctx>,
-}
-
-impl<'ctx> Error<'ctx> {
- fn output_not_found(task: &TaskRef<'ctx>, output: &'ctx str) -> Self {
- Error {
- dep_chain: task.into(),
- kind: ErrorKind::OutputNotFound(output),
- }
- }
-
- fn dependency_cycle(task: &TaskRef<'ctx>) -> Self {
- Error {
- dep_chain: task.into(),
- kind: ErrorKind::DependencyCycle,
- }
- }
-
- fn extend(&mut self, task: &TaskRef<'ctx>) {
- self.dep_chain.0.push(task.clone());
- }
-}
-
-impl<'ctx> fmt::Display for Error<'ctx> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let Error { dep_chain, kind } = self;
- match kind {
- ErrorKind::Context(err) => {
- write!(f, "{}: ", err)?;
- }
- ErrorKind::OutputNotFound(output) => {
- write!(f, "Output '{}' not found: ", output)?;
- }
- ErrorKind::DependencyCycle => {
- write!(f, "Dependency Cycle: ")?;
- }
- }
- dep_chain.fmt(f)
- }
-}
-
-impl<'ctx> From<context::Error<'ctx>> for Error<'ctx> {
- fn from(err: context::Error<'ctx>) -> Self {
- Error {
- dep_chain: err.task.into(),
- kind: ErrorKind::Context(err),
- }
- }
-}
-
-impl<'ctx> std::error::Error for Error<'ctx> {}
-
-#[derive(Debug, PartialEq)]
-enum ResolveState {
- Resolving,
- Resolved,
-}
-
-pub fn runtime_depends<'ctx, I>(
- ctx: &'ctx Context,
- deps: I,
-) -> Result<HashSet<OutputRef>, Vec<Error>>
-where
- I: IntoIterator<Item = OutputRef<'ctx>>,
-{
- fn add_dep<'ctx>(
- ret: &mut HashSet<OutputRef<'ctx>>,
- ctx: &'ctx Context,
- dep: OutputRef<'ctx>,
- ) -> Vec<Error<'ctx>> {
- if ret.contains(&dep) || ctx.in_rootfs(&dep) {
- return Vec::new();
- }
-
- let task = &dep.task;
- let task_def = match ctx.get(task.id) {
- Ok(task) => task,
- Err(err) => return vec![err.into()],
- };
-
- let output = match task_def.output.get(dep.output) {
- Some(output) => output,
- None => {
- return vec![Error::output_not_found(task, dep.output)];
- }
- };
-
- ret.insert(dep.clone());
-
- let mut errors = Vec::new();
- for runtime_dep in &output.runtime_depends {
- match ctx.output_ref(runtime_dep, &task.args, false) {
- Ok(output_ref) => {
- for mut error in add_dep(ret, ctx, output_ref) {
- error.extend(task);
- errors.push(error);
- }
- }
- Err(err) => {
- let mut err: Error = err.into();
- err.extend(task);
- errors.push(err);
- }
- };
- }
- errors
- }
-
- let mut ret = HashSet::new();
- let mut errors = Vec::new();
-
- for dep in deps {
- errors.extend(add_dep(&mut ret, ctx, dep));
- }
-
- if !errors.is_empty() {
- return Err(errors);
- }
-
- Ok(ret)
-}
-
-pub fn get_dependent_outputs<'ctx>(
- ctx: &'ctx Context,
- task_ref: &TaskRef<'ctx>,
-) -> Result<HashSet<OutputRef<'ctx>>, Vec<Error<'ctx>>> {
- let deps: HashSet<_> = ctx
- .get_build_depends(task_ref)
- .map_err(|err| vec![err.into()])?
- .into_iter()
- .chain(
- ctx.get_host_depends(task_ref)
- .map_err(|err| vec![err.into()])?
- .into_iter(),
- )
- .collect();
- runtime_depends(ctx, deps)
-}
-
-pub fn get_dependent_tasks<'ctx>(
- ctx: &'ctx Context,
- task_ref: &TaskRef<'ctx>,
-) -> Result<HashSet<TaskRef<'ctx>>, Vec<Error<'ctx>>> {
- Ok(ctx
- .get_inherit_depend(task_ref)
- .map_err(|err| vec![err.into()])?
- .into_iter()
- .chain(
- get_dependent_outputs(ctx, task_ref)?
- .into_iter()
- .map(|dep| dep.task),
- )
- .collect())
-}
-
-#[derive(Debug)]
-pub struct Resolver<'ctx> {
- ctx: &'ctx Context,
- resolve_state: HashMap<TaskRef<'ctx>, ResolveState>,
-}
-
-impl<'ctx> Resolver<'ctx> {
- pub fn new(ctx: &'ctx Context) -> Self {
- Resolver {
- ctx,
- resolve_state: HashMap::new(),
- }
- }
-
- fn tasks_resolved(&self) -> bool {
- self.resolve_state
- .values()
- .all(|resolved| *resolved == ResolveState::Resolved)
- }
-
- fn add_task(&mut self, task: &TaskRef<'ctx>, output: Option<&'ctx str>) -> Vec<Error<'ctx>> {
- match self.resolve_state.get(task) {
- Some(ResolveState::Resolving) => return vec![Error::dependency_cycle(task)],
- Some(ResolveState::Resolved) => return vec![],
- None => (),
- }
-
- let task_def = match self.ctx.get(task.id) {
- Ok(task_def) => task_def,
- Err(err) => return vec![err.into()],
- };
-
- if let Some(task_output) = output {
- if !task_def.output.contains_key(task_output) {
- return vec![Error::output_not_found(task, task_output)];
- }
- }
-
- self.resolve_state
- .insert(task.clone(), ResolveState::Resolving);
-
- let mut ret = Vec::new();
- let mut handle_errors = |errors: Vec<Error<'ctx>>| {
- for mut error in errors {
- error.extend(task);
- ret.push(error);
- }
- };
-
- match self.ctx.get_inherit_depend(task) {
- Ok(Some(inherit)) => {
- handle_errors(self.add_task(&inherit, None));
- }
- Ok(None) => {}
- Err(err) => {
- handle_errors(vec![err.into()]);
- }
- }
-
- match get_dependent_outputs(self.ctx, task) {
- Ok(rdeps) => {
- for rdep in rdeps {
- handle_errors(self.add_task(&rdep.task, Some(rdep.output)));
- }
- }
- Err(errors) => {
- handle_errors(errors);
- }
- }
-
- if ret.is_empty() {
- *self
- .resolve_state
- .get_mut(task)
- .expect("Missing resolve_state") = ResolveState::Resolved;
- } else {
- self.resolve_state.remove(task);
- }
-
- ret
- }
-
- pub fn add_goal(&mut self, task: &TaskRef<'ctx>) -> Vec<Error<'ctx>> {
- let ret = self.add_task(task, None);
- debug_assert!(self.tasks_resolved());
- ret
- }
-
- pub fn into_taskset(self) -> HashSet<TaskRef<'ctx>> {
- debug_assert!(self.tasks_resolved());
-
- self.resolve_state
- .into_iter()
- .map(|entry| entry.0)
- .collect()
- }
-}
diff --git a/src/task.rs b/src/task.rs
deleted file mode 100644
index fe9572c..0000000
--- a/src/task.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-use std::collections::{HashMap, HashSet};
-
-use serde::Deserialize;
-
-use common::{string_hash::StringHash, types::TaskID};
-
-use crate::{
- args::{ArgMapping, ArgType},
- recipe,
-};
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash, Default)]
-pub struct RecipeMeta {
- #[serde(default)]
- pub name: String,
- pub version: Option<String>,
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
-pub struct TaskDep {
- #[serde(flatten, deserialize_with = "recipe::deserialize_task_id")]
- pub id: TaskID,
- #[serde(default)]
- pub args: ArgMapping,
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
-pub struct Fetch {
- pub name: String,
- pub sha256: StringHash,
-}
-
-fn default_output_name() -> String {
- "default".to_string()
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct InheritDep {
- #[serde(flatten)]
- pub dep: TaskDep,
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
-pub struct OutputDep {
- #[serde(flatten)]
- pub dep: TaskDep,
- #[serde(default)]
- pub noinherit: bool,
- #[serde(default = "default_output_name")]
- pub output: String,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct Output {
- pub path: Option<String>,
- #[serde(default)]
- pub runtime_depends: HashSet<OutputDep>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct Action {
- #[serde(default)]
- pub run: String,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct TaskDef {
- #[serde(skip)]
- pub meta: RecipeMeta,
- #[serde(default)]
- pub args: HashMap<String, ArgType>,
- #[serde(default)]
- pub inherit: Option<InheritDep>,
- #[serde(default)]
- pub fetch: HashSet<Fetch>,
- #[serde(default)]
- pub build_depends: HashSet<OutputDep>,
- #[serde(default)]
- pub depends: HashSet<OutputDep>,
- #[serde(default)]
- pub output: HashMap<String, Output>,
- #[serde(flatten)]
- pub action: Action,
-}
diff --git a/src/template.rs b/src/template.rs
deleted file mode 100644
index 0caa30d..0000000
--- a/src/template.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use handlebars::Handlebars;
-
-use common::error::*;
-
-use crate::args::TaskArgs;
-
-fn escape(s: &str) -> String {
- format!("'{}'", s.replace("'", "'\\''"))
-}
-
-#[derive(Debug)]
-pub struct TemplateEngine {
- tpl: Handlebars<'static>,
- tpl_raw: Handlebars<'static>,
-}
-
-impl TemplateEngine {
- pub fn new() -> Self {
- let mut tpl = Handlebars::new();
- tpl.set_strict_mode(true);
- tpl.register_escape_fn(escape);
-
- let mut tpl_raw = Handlebars::new();
- tpl_raw.set_strict_mode(true);
- tpl_raw.register_escape_fn(handlebars::no_escape);
-
- TemplateEngine { tpl, tpl_raw }
- }
-
- pub fn eval_raw(&self, input: &str, args: &TaskArgs) -> Result<String> {
- self.tpl_raw
- .render_template(input, args)
- .map_err(Error::new)
- }
-
- pub fn eval(&self, input: &str, args: &TaskArgs) -> Result<String> {
- self.tpl.render_template(input, args).map_err(Error::new)
- }
-}