runner: unpack dependencies only once

Reuse unpacked dependencies across multiple tasks by mounting them into
a single task's build directory. We lose support for file conflict
detection for now.

2 files changed, 90 insertions, 46 deletions

diff --git a/crates/runner/src/paths.rs b/crates/runner/src/paths.rs
index 5e183cb..2e174df 100644
--- a/crates/runner/src/paths.rs
+++ b/crates/runner/src/paths.rs
@@ -28,14 +28,14 @@
 //! └── tmp/                          # temporary files (cleaned on start)
 //!     ├── dev/                      # container /dev
 //!     ├── rootfs/                   # unpacked rootfs.tar
+//!     ├── depends/                  # unpacked dependencies
 //!     └── task/
 //!         └── <input hash>/
 //!             ├── build/            # mount point for /build directory
 //!             │   ├── downloads/    # downloaded sources
 //!             │   ├── task/         # internal runner files
 //!             │   └── work/         # build overlay mountpoint
-//!             ├── rootfs/           # rootfs overlay mountpoint
-//!             └── depends/          # overlayed on rootfs in container
+//!             └── rootfs/           # rootfs overlay mountpoint
 //! ```
 
 use common::string_hash::*;
@@ -47,10 +47,10 @@ pub const DOWNLOADS_DIR: &str = "build/downloads";
 pub const TMP_DIR: &str = "build/tmp";
 pub const DEV_DIR: &str = "build/tmp/dev";
 pub const ROOTFS_DIR: &str = "build/tmp/rootfs";
+pub const DEPENDS_TMP_DIR: &str = "build/tmp/depends";
 pub const TASK_TMP_DIR: &str = "build/tmp/task";
 
 pub const TASK_TMP_ROOTFS_SUBDIR: &str = "rootfs";
-pub const TASK_TMP_DEPENDS_SUBDIR: &str = "depends";
 
 pub const LOCKFILE: &str = "build/build.lock";
 pub const OUTPUT_STATE_DIR: &str = "build/state/output";
@@ -97,6 +97,18 @@ pub fn layer_dir(hash: &LayerHash) -> String {
 	join(&[LAYER_STATE_DIR, &hash.to_string()])
 }
 
+pub fn depend_tmp_dir(hash: &ArchiveHash) -> String {
+	join(&[DEPENDS_TMP_DIR, &hash.to_string()])
+}
+
+pub fn depend_dir(hash: &ArchiveHash) -> String {
+	join(&[DEPENDS_TMP_DIR, &hash.to_string()])
+}
+
+pub fn depend_lock_filename(hash: &ArchiveHash) -> String {
+	join(&[DEPENDS_TMP_DIR, &format!("{}.lock", hash.to_string())])
+}
+
 pub fn archive_tmp_filename(hash: &InputHash) -> String {
 	join(&[OUTPUT_STATE_DIR, &format!("{}.tar.tmp", hash)])
 }
diff --git a/crates/runner/src/task.rs b/crates/runner/src/task.rs
index 5d3f664..941e5f1 100644
--- a/crates/runner/src/task.rs
+++ b/crates/runner/src/task.rs
@@ -1,5 +1,5 @@
 use std::{
-	collections::HashMap,
+	collections::{BTreeMap, HashMap},
 	io::{self, BufWriter},
 	os::unix::prelude::CommandExt,
 	path::Path,
@@ -24,7 +24,10 @@ use super::{
 	ns, tar,
 	util::{checkable::Checkable, cjson, fs},
 };
-use crate::{paths, util::unix};
+use crate::{
+	paths,
+	util::{stack::Stack, unix},
+};
 
 const BUILD_UID: Uid = Uid::from_raw(1000);
 const BUILD_GID: Gid = Gid::from_raw(1000);
@@ -34,6 +37,8 @@ type DependencyHasher = blake3::Hasher;
 type LayerHasher = blake3::Hasher;
 type ArchiveHasher = blake3::Hasher;
 
+type DependMap = BTreeMap<String, Vec<ArchiveHash>>;
+
 fn dependency_hash(dep: &Dependency) -> DependencyHash {
 	DependencyHash(StringHash(
 		cjson::digest::<DependencyHasher, _>(dep).unwrap().into(),
@@ -121,27 +126,55 @@ fn init_task(input_hash: &InputHash, task: &Task) -> Result<fs::Mount> {
 	Ok(mount)
 }
 
-fn init_task_rootfs(input_hash: &InputHash) -> Result<fs::Mount> {
+fn init_task_rootfs(input_hash: &InputHash, depends: &DependMap) -> Result<Stack<fs::Mount>> {
 	let task_tmp_dir = paths::task_tmp_dir(input_hash);
-	let depends_dir = paths::join(&[&task_tmp_dir, paths::TASK_TMP_DEPENDS_SUBDIR]);
 	let mount_target = paths::join(&[&task_tmp_dir, paths::TASK_TMP_ROOTFS_SUBDIR]);
 
-	let lower = [&depends_dir, paths::ROOTFS_DIR].join(":");
-	let options = format!(
-		"xino=off,index=off,metacopy=off,lowerdir={lower}",
-		lower = lower,
+	let mut mounts = Stack::new();
+
+	mounts.push(
+		fs::mount(
+			paths::ROOTFS_DIR,
+			&mount_target,
+			None,
+			MsFlags::MS_BIND,
+			None,
+		)
+		.with_context(|| format!("Failed to bind mount rootfs to {:?}", mount_target))?,
 	);
 
-	let mount = fs::mount(
-		"overlay",
-		&mount_target,
-		Some("overlay"),
-		MsFlags::empty(),
-		Some(&options),
-	)
-	.with_context(|| format!("Failed to mount rootfs overlay to {:?}", mount_target))?;
+	for (path, dep_hashes) in depends {
+		assert!(!dep_hashes.is_empty());
 
-	Ok(mount)
+		if !path.is_empty() && !path.starts_with('/') {
+			return Err(Error::new(format!(
+				"Dependency path {:?} must be absolute",
+				path
+			)));
+		}
+
+		let dep_target = mount_target.clone() + path;
+		let dep_paths: Box<[_]> = dep_hashes.iter().map(paths::depend_dir).collect();
+
+		let options = format!(
+			"xino=off,index=off,metacopy=off,lowerdir={lower}:{base}",
+			lower = dep_paths.join(":"),
+			base = dep_target,
+		);
+
+		mounts.push(
+			fs::mount(
+				"overlay",
+				dep_target.as_str(),
+				Some("overlay"),
+				MsFlags::MS_RDONLY,
+				Some(&options),
+			)
+			.with_context(|| format!("Failed to mount overlay to {:?}", dep_target))?,
+		);
+	}
+
+	Ok(mounts)
 }
 
 fn cleanup_task(input_hash: &InputHash) -> Result<()> {
@@ -156,18 +189,24 @@ fn cleanup_task(input_hash: &InputHash) -> Result<()> {
 	Ok(())
 }
 
-fn unpack_dependency<P1: AsRef<Path>, P2: AsRef<Path>>(
-	filename: P1,
-	dest: P2,
-	expected_hash: &ArchiveHash,
-) -> Result<()> {
+fn unpack_dependency<P: AsRef<Path>>(filename: P, hash: &ArchiveHash) -> Result<()> {
+	let _lock = unix::lock(paths::depend_lock_filename(hash), true, true);
+
+	let dest = paths::depend_dir(hash);
+	if Path::new(&dest).is_dir() {
+		return Ok(());
+	}
+
 	(|| -> Result<()> {
+		let tmp_dest = paths::depend_tmp_dir(hash);
+		fs::ensure_removed(&tmp_dest)?;
+
 		let file = fs::open(filename.as_ref())?;
 		let hasher = ArchiveHasher::new();
 		let buffered_hasher = BufWriter::with_capacity(16 * 1024 * 1024, hasher);
 		let mut reader = TeeReader::new(file, buffered_hasher, false);
 
-		tar::unpack(&mut reader, dest.as_ref())?;
+		tar::unpack(&mut reader, &tmp_dest)?;
 
 		// Read file to end to get the correct hash
 		io::copy(&mut reader, &mut io::sink())?;
@@ -177,31 +216,26 @@ fn unpack_dependency<P1: AsRef<Path>, P2: AsRef<Path>>(
 
 		let actual_hash = ArchiveHash(StringHash(hasher.finalize().into()));
 
-		if &actual_hash != expected_hash {
+		if &actual_hash != hash {
 			return Err(Error::new(format!(
 				"Incorrect file hash for {:?} (expected: {}, actual: {})",
 				filename.as_ref(),
-				expected_hash,
+				hash,
 				actual_hash
 			)));
 		}
 
+		fs::rename(&tmp_dest, &dest)?;
+
 		Ok(())
 	})()
-	.with_context(|| {
-		format!(
-			"Failed to unpack {:?} to {:?}",
-			filename.as_ref(),
-			dest.as_ref()
-		)
-	})
+	.with_context(|| format!("Failed to unpack {:?}", filename.as_ref(),))
 }
 
-fn unpack_dependencies(input_hash: &InputHash, task: &Task) -> Result<()> {
+fn unpack_dependencies(input_hash: &InputHash, task: &Task) -> Result<DependMap> {
 	let task_tmp_dir = paths::task_tmp_dir(input_hash);
-	let depends_dir = paths::join(&[&task_tmp_dir, paths::TASK_TMP_DEPENDS_SUBDIR]);
 
-	fs::mkdir(&depends_dir)?;
+	let mut ret = DependMap::new();
 
 	for dep in &task.depends {
 		match dep {
@@ -216,16 +250,13 @@ fn unpack_dependencies(input_hash: &InputHash, task: &Task) -> Result<()> {
 				)?;
 			}
 			Dependency::Task { output, path } => {
-				unpack_dependency(
-					paths::archive_filename(output),
-					paths::join(&[&depends_dir, path]),
-					output,
-				)?;
+				unpack_dependency(paths::archive_filename(output), output)?;
+				ret.entry(path.clone()).or_default().push(*output);
 			}
 		}
 	}
 
-	Ok(())
+	Ok(ret)
 }
 
 fn collect_output(input_hash: &InputHash, path: &str) -> Result<Option<ArchiveHash>> {
@@ -272,8 +303,9 @@ fn collect_outputs(input_hash: &InputHash, task: &Task) -> Result<HashMap<String
 
 fn run_task(input_hash: &InputHash, task: &Task, jobserver: &mut Jobserver) -> Result<()> {
 	let _workdir_mount = init_task(input_hash, task).context("Failed to initialize task")?;
-	unpack_dependencies(input_hash, task).context("Failed to unpack dependencies")?;
-	let _rootfs_mount = init_task_rootfs(input_hash).context("Failed to initialize task rootfs")?;
+	let depends = unpack_dependencies(input_hash, task).context("Failed to unpack dependencies")?;
+	let _rootfs_mounts =
+		init_task_rootfs(input_hash, &depends).context("Failed to initialize task rootfs")?;
 
 	let task_tmp_dir = paths::task_tmp_dir(input_hash);
 	let rootfs = paths::join(&[&task_tmp_dir, paths::TASK_TMP_ROOTFS_SUBDIR]);