From e462674c78a7c7dea7bac0eaf9e177cff0280df9 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sun, 31 Oct 2021 12:27:00 +0100 Subject: runner: unpack dependencies only once Reuse unpacked dependencies across multiple tasks by mounting them into a single task's build directory. We lose support for file conflict detection for now. --- crates/runner/src/paths.rs | 18 +++++-- crates/runner/src/task.rs | 118 ++++++++++++++++++++++++++++----------------- 2 files changed, 90 insertions(+), 46 deletions(-) diff --git a/crates/runner/src/paths.rs b/crates/runner/src/paths.rs index 5e183cb..2e174df 100644 --- a/crates/runner/src/paths.rs +++ b/crates/runner/src/paths.rs @@ -28,14 +28,14 @@ //! └── tmp/ # temporary files (cleaned on start) //!    ├── dev/ # container /dev //!    ├── rootfs/ # unpacked rootfs.tar +//!    ├── depends/ # unpacked dependencies //!    └── task/ //!    └── / //! ├── build/ # mount point for /build directory //! │ ├── downloads/ # downloaded sources //! │ ├── task/ # internal runner files //! │ └── work/ # build overlay mountpoint -//! ├── rootfs/ # rootfs overlay mountpoint -//! └── depends/ # overlayed on rootfs in container +//! └── rootfs/ # rootfs overlay mountpoint //! ``` use common::string_hash::*; @@ -47,10 +47,10 @@ pub const DOWNLOADS_DIR: &str = "build/downloads"; pub const TMP_DIR: &str = "build/tmp"; pub const DEV_DIR: &str = "build/tmp/dev"; pub const ROOTFS_DIR: &str = "build/tmp/rootfs"; +pub const DEPENDS_TMP_DIR: &str = "build/tmp/depends"; pub const TASK_TMP_DIR: &str = "build/tmp/task"; pub const TASK_TMP_ROOTFS_SUBDIR: &str = "rootfs"; -pub const TASK_TMP_DEPENDS_SUBDIR: &str = "depends"; pub const LOCKFILE: &str = "build/build.lock"; pub const OUTPUT_STATE_DIR: &str = "build/state/output"; @@ -97,6 +97,18 @@ pub fn layer_dir(hash: &LayerHash) -> String { join(&[LAYER_STATE_DIR, &hash.to_string()]) } +pub fn depend_tmp_dir(hash: &ArchiveHash) -> String { + join(&[DEPENDS_TMP_DIR, &hash.to_string()]) +} + +pub fn depend_dir(hash: &ArchiveHash) -> String { + join(&[DEPENDS_TMP_DIR, &hash.to_string()]) +} + +pub fn depend_lock_filename(hash: &ArchiveHash) -> String { + join(&[DEPENDS_TMP_DIR, &format!("{}.lock", hash.to_string())]) +} + pub fn archive_tmp_filename(hash: &InputHash) -> String { join(&[OUTPUT_STATE_DIR, &format!("{}.tar.tmp", hash)]) } diff --git a/crates/runner/src/task.rs b/crates/runner/src/task.rs index 5d3f664..941e5f1 100644 --- a/crates/runner/src/task.rs +++ b/crates/runner/src/task.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, io::{self, BufWriter}, os::unix::prelude::CommandExt, path::Path, @@ -24,7 +24,10 @@ use super::{ ns, tar, util::{checkable::Checkable, cjson, fs}, }; -use crate::{paths, util::unix}; +use crate::{ + paths, + util::{stack::Stack, unix}, +}; const BUILD_UID: Uid = Uid::from_raw(1000); const BUILD_GID: Gid = Gid::from_raw(1000); @@ -34,6 +37,8 @@ type DependencyHasher = blake3::Hasher; type LayerHasher = blake3::Hasher; type ArchiveHasher = blake3::Hasher; +type DependMap = BTreeMap>; + fn dependency_hash(dep: &Dependency) -> DependencyHash { DependencyHash(StringHash( cjson::digest::(dep).unwrap().into(), @@ -121,27 +126,55 @@ fn init_task(input_hash: &InputHash, task: &Task) -> Result { Ok(mount) } -fn init_task_rootfs(input_hash: &InputHash) -> Result { +fn init_task_rootfs(input_hash: &InputHash, depends: &DependMap) -> Result> { let task_tmp_dir = paths::task_tmp_dir(input_hash); - let depends_dir = paths::join(&[&task_tmp_dir, paths::TASK_TMP_DEPENDS_SUBDIR]); let mount_target = paths::join(&[&task_tmp_dir, paths::TASK_TMP_ROOTFS_SUBDIR]); - let lower = [&depends_dir, paths::ROOTFS_DIR].join(":"); - let options = format!( - "xino=off,index=off,metacopy=off,lowerdir={lower}", - lower = lower, + let mut mounts = Stack::new(); + + mounts.push( + fs::mount( + paths::ROOTFS_DIR, + &mount_target, + None, + MsFlags::MS_BIND, + None, + ) + .with_context(|| format!("Failed to bind mount rootfs to {:?}", mount_target))?, ); - let mount = fs::mount( - "overlay", - &mount_target, - Some("overlay"), - MsFlags::empty(), - Some(&options), - ) - .with_context(|| format!("Failed to mount rootfs overlay to {:?}", mount_target))?; + for (path, dep_hashes) in depends { + assert!(!dep_hashes.is_empty()); - Ok(mount) + if !path.is_empty() && !path.starts_with('/') { + return Err(Error::new(format!( + "Dependency path {:?} must be absolute", + path + ))); + } + + let dep_target = mount_target.clone() + path; + let dep_paths: Box<[_]> = dep_hashes.iter().map(paths::depend_dir).collect(); + + let options = format!( + "xino=off,index=off,metacopy=off,lowerdir={lower}:{base}", + lower = dep_paths.join(":"), + base = dep_target, + ); + + mounts.push( + fs::mount( + "overlay", + dep_target.as_str(), + Some("overlay"), + MsFlags::MS_RDONLY, + Some(&options), + ) + .with_context(|| format!("Failed to mount overlay to {:?}", dep_target))?, + ); + } + + Ok(mounts) } fn cleanup_task(input_hash: &InputHash) -> Result<()> { @@ -156,18 +189,24 @@ fn cleanup_task(input_hash: &InputHash) -> Result<()> { Ok(()) } -fn unpack_dependency, P2: AsRef>( - filename: P1, - dest: P2, - expected_hash: &ArchiveHash, -) -> Result<()> { +fn unpack_dependency>(filename: P, hash: &ArchiveHash) -> Result<()> { + let _lock = unix::lock(paths::depend_lock_filename(hash), true, true); + + let dest = paths::depend_dir(hash); + if Path::new(&dest).is_dir() { + return Ok(()); + } + (|| -> Result<()> { + let tmp_dest = paths::depend_tmp_dir(hash); + fs::ensure_removed(&tmp_dest)?; + let file = fs::open(filename.as_ref())?; let hasher = ArchiveHasher::new(); let buffered_hasher = BufWriter::with_capacity(16 * 1024 * 1024, hasher); let mut reader = TeeReader::new(file, buffered_hasher, false); - tar::unpack(&mut reader, dest.as_ref())?; + tar::unpack(&mut reader, &tmp_dest)?; // Read file to end to get the correct hash io::copy(&mut reader, &mut io::sink())?; @@ -177,31 +216,26 @@ fn unpack_dependency, P2: AsRef>( let actual_hash = ArchiveHash(StringHash(hasher.finalize().into())); - if &actual_hash != expected_hash { + if &actual_hash != hash { return Err(Error::new(format!( "Incorrect file hash for {:?} (expected: {}, actual: {})", filename.as_ref(), - expected_hash, + hash, actual_hash ))); } + fs::rename(&tmp_dest, &dest)?; + Ok(()) })() - .with_context(|| { - format!( - "Failed to unpack {:?} to {:?}", - filename.as_ref(), - dest.as_ref() - ) - }) + .with_context(|| format!("Failed to unpack {:?}", filename.as_ref(),)) } -fn unpack_dependencies(input_hash: &InputHash, task: &Task) -> Result<()> { +fn unpack_dependencies(input_hash: &InputHash, task: &Task) -> Result { let task_tmp_dir = paths::task_tmp_dir(input_hash); - let depends_dir = paths::join(&[&task_tmp_dir, paths::TASK_TMP_DEPENDS_SUBDIR]); - fs::mkdir(&depends_dir)?; + let mut ret = DependMap::new(); for dep in &task.depends { match dep { @@ -216,16 +250,13 @@ fn unpack_dependencies(input_hash: &InputHash, task: &Task) -> Result<()> { )?; } Dependency::Task { output, path } => { - unpack_dependency( - paths::archive_filename(output), - paths::join(&[&depends_dir, path]), - output, - )?; + unpack_dependency(paths::archive_filename(output), output)?; + ret.entry(path.clone()).or_default().push(*output); } } } - Ok(()) + Ok(ret) } fn collect_output(input_hash: &InputHash, path: &str) -> Result> { @@ -272,8 +303,9 @@ fn collect_outputs(input_hash: &InputHash, task: &Task) -> Result Result<()> { let _workdir_mount = init_task(input_hash, task).context("Failed to initialize task")?; - unpack_dependencies(input_hash, task).context("Failed to unpack dependencies")?; - let _rootfs_mount = init_task_rootfs(input_hash).context("Failed to initialize task rootfs")?; + let depends = unpack_dependencies(input_hash, task).context("Failed to unpack dependencies")?; + let _rootfs_mounts = + init_task_rootfs(input_hash, &depends).context("Failed to initialize task rootfs")?; let task_tmp_dir = paths::task_tmp_dir(input_hash); let rootfs = paths::join(&[&task_tmp_dir, paths::TASK_TMP_ROOTFS_SUBDIR]); -- cgit v1.2.3