diff options
| -rw-r--r-- | examples/bar.yml | 4 | ||||
| -rw-r--r-- | examples/binutils.yml | 17 | ||||
| -rw-r--r-- | examples/foo.yml | 4 | ||||
| -rw-r--r-- | examples/gcc-libs.yml | 10 | ||||
| -rw-r--r-- | examples/gcc.yml | 30 | ||||
| -rw-r--r-- | examples/glibc.yml | 27 | ||||
| -rw-r--r-- | examples/libgcc-initial.yml | 8 | ||||
| -rw-r--r-- | examples/linux-uapi-headers.yml | 6 | ||||
| -rw-r--r-- | examples/ls.yml | 2 | ||||
| -rw-r--r-- | src/args.rs | 27 | ||||
| -rw-r--r-- | src/context.rs | 88 | ||||
| -rw-r--r-- | src/executor.rs | 33 | ||||
| -rw-r--r-- | src/main.rs | 7 | ||||
| -rw-r--r-- | src/recipe.rs | 5 | ||||
| -rw-r--r-- | src/task.rs | 4 | ||||
| -rw-r--r-- | src/template.rs | 2 | ||||
| -rw-r--r-- | src/types.rs | 14 |
17 files changed, 169 insertions, 119 deletions
diff --git a/examples/bar.yml b/examples/bar.yml index ece6fd1..f25322e 100644 --- a/examples/bar.yml +++ b/examples/bar.yml @@ -4,6 +4,6 @@ tasks: default: {} run: | findmnt -o +PROPAGATION - mkdir -p "${DESTDIR}" - echo bar > "${DESTDIR}/bar" + mkdir -p {{destdir}} + echo bar > {{destdir}}/bar diff --git a/examples/binutils.yml b/examples/binutils.yml index c05d965..422a108 100644 --- a/examples/binutils.yml +++ b/examples/binutils.yml @@ -4,23 +4,20 @@ tasks: - name: 'binutils-2.37.tar.xz' sha256: '820d9724f020a3e69cb337893a0b63c2db161dadcb0e06fc11dc29eb1e84a32c' run: | - tar xf "${DLDIR}/binutils-2.37.tar.xz" + tar xf {{dldir}}/binutils-2.37.tar.xz configure: inherit: task: 'binutils:unpack' run: | - BUILD=x86_64-linux-gnu - HOST=x86_64-linux-gnu - TARGET=aarch64-linux-gnu mkdir binutils-build cd binutils-build ../binutils-2.37/configure \ - --build="${BUILD}" \ - --host="${HOST}" \ - --target="${TARGET}" \ - --prefix="${BUILD_PREFIX}" \ - --with-sysroot="${SYSROOT}" \ + --build={{build.target_triple}} \ + --host={{host.target_triple}} \ + --target={{target.target_triple}} \ + --prefix={{host.prefix}} \ + --with-sysroot={{sysroot}} \ --enable-libssp \ --enable-deterministic-archives \ --enable-plugins \ @@ -45,4 +42,4 @@ tasks: default: {} run: | cd binutils-build - make DESTDIR="$DESTDIR" install + make DESTDIR={{destdir}} install diff --git a/examples/foo.yml b/examples/foo.yml index 6b82812..2982795 100644 --- a/examples/foo.yml +++ b/examples/foo.yml @@ -5,8 +5,8 @@ tasks: run: | ls -lha ls -lha /proc/self/fd - mkdir -p "${DESTDIR}/foo" - cd "${DESTDIR}" + mkdir -p {{destdir}}/foo + cd {{destdir}} echo foobar > foo/bar touch foo/bla touch foo/x diff --git a/examples/gcc-libs.yml b/examples/gcc-libs.yml index 14969db..b9e9e52 100644 --- a/examples/gcc-libs.yml +++ b/examples/gcc-libs.yml @@ -23,11 +23,9 @@ tasks: target: path: 'target' run: | - TARGET=aarch64-linux-gnu - cd gcc-build - make DESTDIR="${DESTDIR}/host" install-target-libgcc install-target-libstdc++-v3 install-target-libgomp install-target-libquadmath install-target-libatomic + make DESTDIR={{destdir}}/host install-target-libgcc install-target-libstdc++-v3 install-target-libgomp install-target-libquadmath install-target-libatomic - mkdir -p "${DESTDIR}/target/${PREFIX}" - mv "${DESTDIR}/host${BUILD_PREFIX}/${TARGET}"/* "${DESTDIR}/target/${PREFIX}" - rmdir "${DESTDIR}/host${BUILD_PREFIX}/${TARGET}" + mkdir -p {{destdir}}/target/{{target.prefix}} + mv {{destdir}}/host{{host.prefix}}/{{target.target_triple}}/* {{destdir}}/target/{{target.prefix}} + rmdir {{destdir}}/host{{host.prefix}}/{{target.target_triple}} diff --git a/examples/gcc.yml b/examples/gcc.yml index fd62466..8dac894 100644 --- a/examples/gcc.yml +++ b/examples/gcc.yml @@ -4,7 +4,7 @@ tasks: - name: 'gcc-11.2.0.tar.xz' sha256: 'd08edc536b54c372a1010ff6619dd274c0f1603aa49212ba20f7aa2cda36fa8b' run: | - tar xf "${DLDIR}/gcc-11.2.0.tar.xz" + tar xf {{dldir}}/gcc-11.2.0.tar.xz sed -i -e 's@^MULTILIB_OSDIRNAMES@# &@' gcc-11.2.0/gcc/config/*/t-* @@ -12,8 +12,8 @@ tasks: output: default: {} run: | - mkdir -p "${DESTDIR}${PREFIX}/include" - touch "${DESTDIR}${PREFIX}/include/limits.h" + mkdir -p {{destdir}}{{target.prefix}}/include + touch {{destdir}}{{target.prefix}}/include/limits.h configure: inherit: @@ -23,22 +23,18 @@ tasks: depends: - task: 'gcc:header-stubs' run: | - BUILD=x86_64-linux-gnu - HOST=x86_64-linux-gnu - TARGET=aarch64-linux-gnu - mkdir gcc-build cd gcc-build ../gcc-11.2.0/configure \ - --build="${BUILD}" \ - --host="${HOST}" \ - --target=${TARGET} \ - --prefix="${BUILD_PREFIX}" \ - --libdir=${BUILD_PREFIX}/lib \ - --libexecdir=${BUILD_PREFIX}/lib \ - --with-sysroot="${SYSROOT}" \ - --with-native-system-header-dir=${PREFIX}/include \ - --with-build-time-tools="${BUILD_PREFIX}/${TARGET}/bin" \ + --build={{build.target_triple}} \ + --host={{host.target_triple}} \ + --target={{target.target_triple}} \ + --prefix={{host.prefix}} \ + --libdir={{host.prefix}}/lib \ + --libexecdir={{host.prefix}}/lib \ + --with-sysroot={{sysroot}} \ + --with-native-system-header-dir={{target.prefix}}/include \ + --with-build-time-tools={{host.prefix}}/{{target.target_triple}}/bin \ --with-gnu-ld \ --enable-shared \ --enable-languages=c,c++ \ @@ -88,5 +84,5 @@ tasks: - task: 'binutils:install' run: | cd gcc-build - make DESTDIR="$DESTDIR" install-host + make DESTDIR={{destdir}} install-host diff --git a/examples/glibc.yml b/examples/glibc.yml index 8e1cf5a..6c2ca8d 100644 --- a/examples/glibc.yml +++ b/examples/glibc.yml @@ -4,7 +4,7 @@ tasks: - name: 'glibc-2.34.tar.xz' sha256: '44d26a1fe20b8853a48f470ead01e4279e869ac149b195dda4e44a195d981ab2' run: | - tar xf "${DLDIR}/glibc-2.34.tar.xz" + tar xf {{dldir}}/glibc-2.34.tar.xz configure: inherit: @@ -15,10 +15,7 @@ tasks: depends: - task: 'linux-uapi-headers:install' run: | - BUILD=x86_64-linux-gnu - HOST=aarch64-linux-gnu - - CROSS_COMPILE=${BUILD_PREFIX}/bin/${HOST}- + CROSS_COMPILE={{build.prefix}}/bin/{{host.target_triple}}- export BUILD_CC=gcc export CC=${CROSS_COMPILE}gcc @@ -31,12 +28,12 @@ tasks: mkdir glibc-build cd glibc-build ../glibc-2.34/configure \ - --build="${BUILD}" \ - --host=${HOST} \ - --prefix="${PREFIX}" \ - --includedir="${PREFIX}/include" \ - --libdir="${PREFIX}/lib" \ - --libexecdir="${PREFIX}/lib" \ + --build={{build.target_triple}} \ + --host={{host.target_triple}} \ + --prefix={{host.prefix}} \ + --includedir={{host.prefix}}/include \ + --libdir={{host.prefix}}/lib \ + --libexecdir={{host.prefix}}/lib \ --enable-add-ons \ --enable-obsolete-rpc \ --enable-kernel=2.6.32 \ @@ -46,9 +43,9 @@ tasks: --enable-lock-elision \ --disable-werror - echo 'slibdir=/usr/lib' >> configparms - echo 'sbindir=/usr/bin' >> configparms - echo 'rootsbindir=/usr/bin' >> configparms + echo slibdir={{host.prefix}}/lib >> configparms + echo sbindir={{host.prefix}}/bin >> configparms + echo rootsbindir={{host.prefix}}/bin >> configparms compile: inherit: @@ -76,4 +73,4 @@ tasks: - task: 'linux-uapi-headers:install' run: | cd glibc-build - make install_root="${DESTDIR}" install + make install_root={{destdir}} install diff --git a/examples/libgcc-initial.yml b/examples/libgcc-initial.yml index f34ea34..cf26211 100644 --- a/examples/libgcc-initial.yml +++ b/examples/libgcc-initial.yml @@ -7,8 +7,6 @@ tasks: depends: - task: 'gcc:header-stubs' run: | - TARGET=aarch64-linux-gnu - cd gcc-build make configure-target-libgcc @@ -21,7 +19,7 @@ tasks: sed -i -r \ -e 's@^(thread_header =).*@\1 gthr-single.h@' \ -e 's@^(enable_shared =).*@\1 no@' \ - "${TARGET}/libgcc/Makefile" + {{target.target_triple}}/libgcc/Makefile compile: inherit: @@ -43,5 +41,5 @@ tasks: default: {} run: | cd gcc-build - make DESTDIR="$DESTDIR" install-target-libgcc - ln -s libgcc.a "${DESTDIR}${BUILD_PREFIX}/lib/gcc/aarch64-linux-gnu/11.2.0/libgcc_eh.a" + make DESTDIR={{destdir}} install-target-libgcc + ln -s libgcc.a {{destdir}}{{host.prefix}}/lib/gcc/{{target.target_triple}}/11.2.0/libgcc_eh.a diff --git a/examples/linux-uapi-headers.yml b/examples/linux-uapi-headers.yml index 95f6d11..9c6f284 100644 --- a/examples/linux-uapi-headers.yml +++ b/examples/linux-uapi-headers.yml @@ -4,7 +4,7 @@ tasks: - name: 'linux-5.10.66.tar.xz' sha256: '5dfa06bbbbd164b9ea669ec637b1e6d05fb5fea8ef3aeb6729f2cbcd0dfcc8a7' run: | - tar xf "${DLDIR}/linux-5.10.66.tar.xz" + tar xf {{dldir}}/linux-5.10.66.tar.xz install: inherit: @@ -12,8 +12,6 @@ tasks: output: default: {} run: | - TARGET_ARCH=arm64 - cd linux-5.10.66 - make INSTALL_HDR_PATH="${DESTDIR}${PREFIX}" ARCH="${TARGET_ARCH}" headers_install + make INSTALL_HDR_PATH={{destdir}}{{host.prefix}} ARCH={{host.karch}} headers_install diff --git a/examples/ls.yml b/examples/ls.yml index b576266..b9cdb28 100644 --- a/examples/ls.yml +++ b/examples/ls.yml @@ -7,4 +7,4 @@ tasks: default: {} run: | ls -lh / - ls -lhR "${SYSROOT}" + ls -lhR {{sysroot}} diff --git a/src/args.rs b/src/args.rs new file mode 100644 index 0000000..cc1ca38 --- /dev/null +++ b/src/args.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, hash, rc::Rc}; + +#[derive(Debug, Serialize)] +pub struct Platform<'a> { + pub target_triple: &'a str, + pub karch: &'a str, + pub prefix: &'a str, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct TaskArgs(pub HashMap<String, Rc<serde_json::Value>>); + +impl hash::Hash for TaskArgs { + fn hash<H: hash::Hasher>(&self, _state: &mut H) { + // Don't do anything: Properly hashing the task args is likely to cost + // much more performance than the hash collisions caused by TaskRefs + // that only differ by the args + } +} + +pub fn arg<T: Serialize>(key: &str, value: T) -> (String, Rc<serde_json::Value>) { + ( + key.to_string(), + Rc::new(serde_json::to_value(value).unwrap()), + ) +} diff --git a/src/context.rs b/src/context.rs index 9ebd549..4706945 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,13 +1,19 @@ use std::{ + borrow::Cow, collections::{HashMap, HashSet}, fmt::Display, hash::Hash, ops::Index, + rc::Rc, }; use serde::Serialize; -use crate::task::*; +use crate::{ + args::{self, arg, TaskArgs}, + task::*, + types::TaskID, +}; #[derive(Clone, Debug, Serialize, PartialEq, Eq, Hash)] pub struct TaskRef<'ctx> { @@ -27,43 +33,76 @@ pub struct OutputRef<'ctx> { pub output: &'ctx str, } +const BUILD_PLATFORM: args::Platform = args::Platform { + target_triple: "x86_64-linux-gnu", + karch: "x86", + prefix: "/opt/toolchain", +}; +const HOST_PLATFORM: args::Platform = args::Platform { + target_triple: "aarch64-linux-gnu", + karch: "arm64", + prefix: "/usr", +}; + #[derive(Debug)] pub struct Context { tasks: HashMap<TaskID, TaskDef>, + default_args: HashMap<String, Rc<serde_json::Value>>, } impl Context { pub fn new(tasks: HashMap<TaskID, TaskDef>) -> Self { - Context { tasks } + let default_args = [arg("build", BUILD_PLATFORM), arg("host", HOST_PLATFORM)] + .iter() + .cloned() + .collect(); + + Context { + tasks, + default_args, + } } pub fn get(&self, id: &TaskID) -> Option<&TaskDef> { self.tasks.get(id) } - fn task_ref<'a>(&'a self, id: &'a TaskID, args: &TaskArgs) -> TaskRef { + fn task_ref<'a>(&'a self, id: &'a TaskID, args: Cow<'_, TaskArgs>) -> TaskRef { TaskRef { id, - args: args.clone(), + args: args.into_owned(), } } - pub fn make_ref(&self, id: &TaskID, args: &TaskArgs) -> Option<TaskRef> { - let (map_id, _) = self.tasks.get_key_value(id)?; - Some(self.task_ref(map_id, args)) + pub fn parse(&self, task: &str) -> Option<TaskRef> { + let (id, _) = self.tasks.get_key_value(task)?; + Some(self.task_ref(id, Cow::Owned(TaskArgs(self.default_args.clone())))) } fn inherit_ref<'a>(&'a self, dep: &'a InheritDep, args: &TaskArgs) -> TaskRef { - self.task_ref(&dep.task, args) + self.task_ref(&dep.task, Cow::Borrowed(args)) } pub fn output_ref<'a>(&'a self, dep: &'a OutputDep, args: &TaskArgs) -> OutputRef<'a> { OutputRef { - task: self.task_ref(&dep.task, args), + task: self.task_ref(&dep.task, Cow::Borrowed(args)), output: &dep.output, } } + fn build_depend_args(args: &TaskArgs) -> TaskArgs { + let mut ret = args.clone(); + + ret.0.insert("host".to_string(), args.0["build"].clone()); + + // TODO: Hack for correct target of toolchain build depends (gcc -> binutils) + if !ret.0.contains_key("target") { + ret.0.insert("target".to_string(), args.0["host"].clone()); + } + + ret + } + pub fn get_inherit_depend(&self, task_ref: &TaskRef) -> Option<TaskRef> { let task = &self[task_ref.id]; Some(self.inherit_ref(task.inherit.as_ref()?, &task_ref.args)) @@ -73,7 +112,7 @@ impl Context { let task = &self[task_ref.id]; task.build_depends .iter() - .map(|dep| self.output_ref(dep, &task_ref.args)) + .map(|dep| self.output_ref(dep, &Context::build_depend_args(&task_ref.args))) .collect() } @@ -85,26 +124,21 @@ impl Context { .collect() } - pub fn get_dependent_tasks(&self, task_ref: &TaskRef) -> HashSet<TaskRef> { - let task = &self[task_ref.id]; - task.inherit - .iter() - .map(|dep| self.inherit_ref(dep, &task_ref.args)) - .chain( - task.build_depends - .iter() - .chain(task.depends.iter()) - .map(|dep| self.task_ref(&dep.task, &task_ref.args)), - ) + pub fn get_dependent_outputs(&self, task_ref: &TaskRef) -> HashSet<OutputRef> { + self.get_build_depends(task_ref) + .into_iter() + .chain(self.get_host_depends(task_ref).into_iter()) .collect() } - pub fn get_dependent_outputs(&self, task_ref: &TaskRef) -> HashSet<OutputRef> { - let task = &self[task_ref.id]; - task.build_depends - .iter() - .chain(task.depends.iter()) - .map(|dep| self.output_ref(dep, &task_ref.args)) + pub fn get_dependent_tasks(&self, task_ref: &TaskRef) -> HashSet<TaskRef> { + self.get_inherit_depend(task_ref) + .into_iter() + .chain( + self.get_dependent_outputs(task_ref) + .into_iter() + .map(|dep| dep.task), + ) .collect() } } diff --git a/src/executor.rs b/src/executor.rs index b736062..cbefeb4 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; use crate::{ + args::{self, TaskArgs}, context::*, paths, resolve, runner, task::*, @@ -11,16 +12,16 @@ use crate::{ util::{cjson, error::*, fs}, }; -// TODO: Generate dynamically from paths module +const BASE_ARGS: &[(&str, &str)] = &[ + ("workdir", "/build/work"), + ("dldir", "/build/downloads"), + ("sysroot", "/build/sysroot"), + ("destdir", "/build/dest"), +]; + const TASK_ENV: &[(&str, &str)] = &[ ("PATH", "/usr/sbin:/usr/bin:/sbin:/bin"), ("HOME", "/build"), - ("DESTDIR", "/build/dest"), - ("DLDIR", "/build/downloads"), - ("SYSROOT", "/build/sysroot"), - ("WORKDIR", "/build/work"), - ("BUILD_PREFIX", "/opt/toolchain"), - ("PREFIX", "/usr"), ]; #[derive(Clone, Debug, Serialize)] @@ -80,6 +81,7 @@ pub struct Executor<'ctx> { tasks_done: HashMap<TaskRef<'ctx>, TaskMeta>, rdeps: HashMap<TaskRef<'ctx>, Vec<TaskRef<'ctx>>>, env: HashMap<String, String>, + base_args: TaskArgs, tpl: TemplateEngine, } @@ -90,6 +92,8 @@ impl<'ctx> Executor<'ctx> { .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); + let base_args = TaskArgs(BASE_ARGS.iter().map(|(k, v)| args::arg(k, v)).collect()); + let mut exc = Executor { ctx, tasks_blocked: HashSet::new(), @@ -97,6 +101,7 @@ impl<'ctx> Executor<'ctx> { tasks_done: HashMap::new(), rdeps: HashMap::new(), env, + base_args, tpl: TemplateEngine::new(), }; @@ -155,6 +160,12 @@ impl<'ctx> Executor<'ctx> { .collect() } + fn eval_command(&self, task_def: &TaskDef, task_args: &TaskArgs) -> Result<String> { + let mut args = self.base_args.clone(); + args.0.extend(task_args.0.clone()); + self.tpl.eval(&task_def, &args) + } + fn run_one(&self, task_ref: &TaskRef, runner: &impl runner::Runner) -> Result<TaskMeta> { let task_def = &self.ctx[task_ref.id]; let task_deps = self.task_deps(task_ref); @@ -180,9 +191,11 @@ impl<'ctx> Executor<'ctx> { (Vec::new(), None) }; - let command = self.tpl.eval(&task_def, &task_ref.args).with_context(|| { - format!("Failed to evaluate command template for task {}", task_ref) - })?; + let command = self + .eval_command(&task_def, &task_ref.args) + .with_context(|| { + format!("Failed to evaluate command template for task {}", task_ref) + })?; let input_hash = TaskInput { inherit: inherit_hash, diff --git a/src/main.rs b/src/main.rs index fbff23d..3a92566 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +mod args; mod context; mod executor; mod paths; @@ -10,8 +11,6 @@ mod types; mod unshare; mod util; -use types::*; - use clap::Clap; #[derive(Clap)] @@ -31,9 +30,7 @@ fn main() { let mut rsv = resolve::Resolver::new(&ctx); for task in opts.tasks { - let task_ref = ctx - .make_ref(&TaskID(task), &TaskArgs(Default::default())) - .expect("Unknown task"); + let task_ref = ctx.parse(&task).expect("Unknown task"); let errors = rsv.add_goal(&task_ref); if !errors.is_empty() { for error in errors { diff --git a/src/recipe.rs b/src/recipe.rs index 446ca63..15f4c3b 100644 --- a/src/recipe.rs +++ b/src/recipe.rs @@ -1,8 +1,9 @@ -use serde::Deserialize; use std::{collections::HashMap, fmt, fs::File, io, path::Path}; + +use serde::Deserialize; use walkdir::WalkDir; -use crate::task::*; +use crate::{task::TaskDef, types::TaskID}; #[derive(Debug, Deserialize)] struct Recipe { diff --git a/src/task.rs b/src/task.rs index a421538..19502c3 100644 --- a/src/task.rs +++ b/src/task.rs @@ -1,9 +1,7 @@ use serde::Deserialize; use std::collections::{HashMap, HashSet}; -use crate::types::StringHash; - -pub use crate::types::{TaskArgs, TaskID}; +use crate::types::{StringHash, TaskID}; #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)] pub struct Fetch { diff --git a/src/template.rs b/src/template.rs index 035e09a..2873f00 100644 --- a/src/template.rs +++ b/src/template.rs @@ -1,6 +1,6 @@ use handlebars::Handlebars; -use crate::{task::*, util::error::*}; +use crate::{args::TaskArgs, task::TaskDef, util::error::*}; fn escape(s: &str) -> String { format!("'{}'", s.replace("'", "'\\''")) diff --git a/src/types.rs b/src/types.rs index 25f21d3..d9a7b28 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,6 +1,7 @@ +use std::{borrow::Borrow, fmt::Display}; + use serde::{Deserialize, Serialize}; use sha2::Sha256; -use std::{collections::HashMap, fmt::Display, hash, rc::Rc}; use crate::util::cjson; @@ -67,14 +68,9 @@ impl Display for TaskID { } } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct TaskArgs(pub HashMap<String, Rc<serde_json::Value>>); - -impl hash::Hash for TaskArgs { - fn hash<H: hash::Hasher>(&self, _state: &mut H) { - // Don't do anything: Properly hashing the task args is likely to cost - // much more performance than the hash collisions caused by TaskRefs - // that only differ by the args +impl Borrow<str> for TaskID { + fn borrow(&self) -> &str { + &self.0 } } |