container: replace crun with our own minimal container runtime

Our own setup is more flexible, faster, and allows for better process
management.

7 files changed, 128 insertions, 306 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 880dc50..8a74226 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -104,6 +104,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
+name = "capctl"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eea0d91a34c56f0a0779e1cc2ec7040fa7f672819c4d3fe7d9dd4af3d2e78aca"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
+[[package]]
 name = "cc"
 version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -190,72 +200,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "darling"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c"
-dependencies = [
- "darling_core",
- "darling_macro",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36"
-dependencies = [
- "fnv",
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
-dependencies = [
- "darling_core",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "derive_builder"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d13202debe11181040ae9063d739fa32cfcaaebe2275fe387703460ae2365b30"
-dependencies = [
- "derive_builder_macro",
-]
-
-[[package]]
-name = "derive_builder_core"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5"
-dependencies = [
- "darling",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "derive_builder_macro"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73"
-dependencies = [
- "derive_builder_core",
- "syn",
-]
-
-[[package]]
 name = "digest"
 version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -372,18 +316,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "getset"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24b328c01a4d71d2d8173daa93562a73ab0fe85616876f02500f53d82948c504"
-dependencies = [
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
 name = "handlebars"
 version = "4.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -431,12 +363,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
-[[package]]
 name = "indexmap"
 version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -608,19 +534,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "oci-spec"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63f0f82a50257e72a6f13616dc093f3d0874294629bf8b7d0bc2a098e3db324f"
-dependencies = [
- "derive_builder",
- "getset",
- "serde",
- "serde_json",
- "thiserror",
-]
-
-[[package]]
 name = "olpc-cjson"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -829,6 +742,7 @@ name = "rebel"
 version = "0.1.0"
 dependencies = [
  "blake3",
+ "capctl",
  "clap",
  "digest 0.9.0",
  "enum-kinds",
@@ -839,7 +753,6 @@ dependencies = [
  "lazy_static",
  "libc",
  "nix",
- "oci-spec",
  "olpc-cjson",
  "regex",
  "scoped-tls-hkt",
@@ -1041,26 +954,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "thiserror"
-version = "1.0.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
 name = "tinyvec"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index e3bfa0b..25b2105 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ edition = "2018"
 
 [dependencies]
 blake3 = { version = "1.0.0", features = ["traits-preview"] }
+capctl = "0.2.0"
 clap = "3.0.0-beta.2"
 digest = "0.9.0"
 enum-kinds = "0.5.1"
@@ -19,7 +20,6 @@ ipc-channel = { git = "https://github.com/servo/ipc-channel.git" }
 lazy_static = "1.4.0"
 libc = "0.2.84"
 nix = "0.23.0"
-oci-spec = "0.5.1"
 olpc-cjson = "0.1.0"
 regex = "1.5.4"
 scoped-tls-hkt = "0.1.2"
diff --git a/src/paths.rs b/src/paths.rs
index a5e1577..ffbc3a5 100644
--- a/src/paths.rs
+++ b/src/paths.rs
@@ -50,8 +50,6 @@ pub const TASK_TMP_DIR: &str = "build/tmp/task";
 pub const TASK_TMP_ROOTFS_SUBDIR: &str = "rootfs";
 pub const TASK_TMP_DEPENDS_SUBDIR: &str = "depends";
 
-pub const TASK_TMP_CONTAINERS_ROOT_SUBDIR: &str = "../../containers";
-
 pub const OUTPUT_STATE_DIR: &str = "build/state/output";
 pub const TASK_STATE_DIR: &str = "build/state/task";
 pub const LAYER_STATE_DIR: &str = "build/state/layer";
diff --git a/src/runner/container/mod.rs b/src/runner/container/mod.rs
index 1c6c021..c9032e4 100644
--- a/src/runner/container/mod.rs
+++ b/src/runner/container/mod.rs
@@ -1,6 +1,5 @@
 mod init;
 mod ns;
-mod spec;
 mod tar;
 mod task;
 
diff --git a/src/runner/container/ns.rs b/src/runner/container/ns.rs
index 4143ea9..da53853 100644
--- a/src/runner/container/ns.rs
+++ b/src/runner/container/ns.rs
@@ -51,3 +51,32 @@ pub fn pivot_root(path: &str) {
 	})()
 	.expect("Failed to pivot root");
 }
+
+pub fn container_mounts() -> Result<()> {
+	mount::mount(
+		Some("tmp"),
+		"/tmp",
+		Some("tmpfs"),
+		MsFlags::MS_NODEV | MsFlags::MS_NOSUID,
+		Some("mode=1777,size=1048576k"),
+	)
+	.context("Failed to mount /tmp")?;
+	mount::mount(
+		Some("devpts"),
+		"/dev/pts",
+		Some("devpts"),
+		MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC,
+		Some("newinstance,ptmxmode=0666,mode=0620"),
+	)
+	.context("Failed to mount /dev/pts")?;
+	mount::mount(
+		Some("shm"),
+		"/dev/shm",
+		Some("tmpfs"),
+		MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC | MsFlags::MS_NODEV,
+		Some("mode=1777,size=65536k"),
+	)
+	.context("Failed to mount /dev/shm")?;
+
+	Ok(())
+}
diff --git a/src/runner/container/spec.rs b/src/runner/container/spec.rs
deleted file mode 100644
index f122da7..0000000
--- a/src/runner/container/spec.rs
+++ /dev/null
@@ -1,147 +0,0 @@
-use oci_spec::runtime::{self, Capability};
-use serde_json::json;
-
-use crate::paths;
-
-pub fn generate_spec(command: &[&str]) -> runtime::Spec {
-	let capabilities: runtime::Capabilities =
-		IntoIterator::into_iter([Capability::DacReadSearch, Capability::Setfcap]).collect();
-
-	serde_json::from_value(json!({
-		"ociVersion": "1.0.2",
-		"process": {
-			"terminal": false,
-			"user": {
-				"uid": 0,
-				"gid": 0,
-			},
-			"args": command,
-			"env": [
-				"PATH=/usr/sbin:/usr/bin:/sbin:/bin",
-				"HOME=/build",
-			],
-			"cwd": paths::abs(paths::TASK_WORKDIR),
-			"noNewPrivileges": true,
-			"capabilities": {
-				"bounding": capabilities,
-				"permitted": capabilities,
-				"inheritable": capabilities,
-				"effective": capabilities,
-			},
-		},
-		"root": {
-			"path": paths::TASK_TMP_ROOTFS_SUBDIR,
-			"readonly": true
-		},
-		"hostname": "rebel-builder",
-		"mounts": [
-			{
-				"destination": paths::abs(paths::TASK_BUILDDIR),
-				"type": "none",
-				"source": paths::TASK_BUILDDIR,
-				"options": [
-					"rbind"
-				]
-			},
-			{
-				"destination": "/tmp",
-				"type": "tmpfs",
-				"source": "tmp",
-				"options": [
-					"nodev",
-					"nosuid",
-					"mode=1777",
-					"size=1048576k"
-				]
-			},
-			{
-				"destination": "/proc",
-				"type": "proc",
-				"source": "proc"
-			},
-			{
-				"destination": "/dev",
-				"type": "tmpfs",
-				"source": "tmpfs",
-				"options": [
-					"nosuid",
-					"strictatime",
-					"mode=755",
-					"size=65536k"
-				]
-			},
-			{
-				"destination": "/dev/pts",
-				"type": "devpts",
-				"source": "devpts",
-				"options": [
-					"nosuid",
-					"noexec",
-					"newinstance",
-					"ptmxmode=0666",
-					"mode=0620"
-				]
-			},
-			{
-				"destination": "/dev/shm",
-				"type": "tmpfs",
-				"source": "shm",
-				"options": [
-					"nosuid",
-					"noexec",
-					"nodev",
-					"mode=1777",
-					"size=65536k"
-				]
-			},
-			{
-				"destination": "/dev/mqueue",
-				"type": "mqueue",
-				"source": "mqueue",
-				"options": [
-					"nosuid",
-					"noexec",
-					"nodev"
-				]
-			},
-		],
-		"linux": {
-			"namespaces": [
-				{
-					"type": "pid"
-				},
-				{
-					"type": "network"
-				},
-				{
-					"type": "ipc"
-				},
-				{
-					"type": "uts"
-				},
-				{
-					"type": "mount"
-				},
-			],
-			"maskedPaths": [
-				"/proc/acpi",
-				"/proc/asound",
-				"/proc/kcore",
-				"/proc/keys",
-				"/proc/latency_stats",
-				"/proc/timer_list",
-				"/proc/timer_stats",
-				"/proc/sched_debug",
-				"/proc/scsi"
-			],
-			"readonlyPaths": [
-				"/proc/bus",
-				"/proc/fs",
-				"/proc/irq",
-				"/proc/sys",
-				"/proc/sysrq-trigger"
-			]
-		}
-	}))
-	.unwrap()
-}
diff --git a/src/runner/container/task.rs b/src/runner/container/task.rs
index 73d0d64..923f15b 100644
--- a/src/runner/container/task.rs
+++ b/src/runner/container/task.rs
@@ -1,22 +1,30 @@
 use std::{
 	collections::HashMap,
+	fs::File,
 	io::BufWriter,
+	os::unix::prelude::CommandExt,
 	path::{Path, PathBuf},
-	process,
+	process::{self, Command, Stdio},
 	time::Instant,
 };
 
+use capctl::prctl;
 use indoc::formatdoc;
-use nix::mount::MsFlags;
+use nix::{
+	mount::{self, MsFlags},
+	sched::CloneFlags,
+	sys::wait,
+	unistd,
+};
 use tee_readwrite::TeeWriter;
 
 use crate::{
 	paths, runner,
 	types::*,
-	util::{cjson, error::*, fs},
+	util::{cjson, error::*, fs, Checkable},
 };
 
-use super::{spec, tar};
+use super::{ns, tar};
 
 pub const BUILD_UID: u32 = 1000;
 pub const BUILD_GID: u32 = 1000;
@@ -204,6 +212,10 @@ fn run_task(input_hash: &InputHash, task: &runner::Task) -> Result<()> {
 	let _rootfs_mount = init_task_rootfs(input_hash).context("Failed to initialize task rootfs")?;
 
 	let task_tmp_dir = paths::task_tmp_dir(input_hash);
+	let rootfs = paths::join(&[&task_tmp_dir, paths::TASK_TMP_ROOTFS_SUBDIR]);
+
+	let builddir_source = paths::join(&[&task_tmp_dir, paths::TASK_BUILDDIR]);
+	let builddir_target = paths::join(&[&rootfs, paths::TASK_BUILDDIR]);
 
 	let command = formatdoc! {"
 		INPUT_HASH={input_hash}
@@ -213,46 +225,84 @@ fn run_task(input_hash: &InputHash, task: &runner::Task) -> Result<()> {
 		command = task.input.command,
 	};
 
-	spec::generate_spec(&[
-		"unshare",
-		"--user",
-		&format!("--map-user={}", BUILD_UID),
-		&format!("--map-group={}", BUILD_GID),
-		"sh",
-		"-exc",
-		&command,
-	])
-	.save(paths::join(&[&task_tmp_dir, "config.json"]))
-	.map_err(Error::new)
-	.context("Failed to save runtime config")?;
-
 	let log_filename = paths::task_log_filename(input_hash);
 	let log = fs::create(&log_filename)?;
-	let log_stdout = log
-		.try_clone()
-		.context("Failed to duplicate log file descriptor")?;
-	let log_stderr = log
-		.try_clone()
-		.context("Failed to duplicate log file descriptor")?;
-
-	let status = process::Command::new("crun")
-		.arg("--root")
-		.arg(paths::TASK_TMP_CONTAINERS_ROOT_SUBDIR)
-		.arg("run")
-		.arg(input_hash.to_string())
-		.current_dir(task_tmp_dir)
-		.stdin(process::Stdio::null())
-		.stdout(log_stdout)
-		.stderr(log_stderr)
-		.status()
-		.context("Failed to start container runtime")?;
+
+	let exec_cmd = |log: File| -> Result<()> {
+		mount::mount::<_, _, str, str>(
+			Some(paths::join(&[paths::ROOTFS_DIR, "dev"]).as_str()),
+			paths::join(&[&rootfs, "dev"]).as_str(),
+			None,
+			MsFlags::MS_BIND | MsFlags::MS_REC,
+			None,
+		)
+		.expect("Failed to bind mount /dev directory");
+		mount::mount::<_, _, str, str>(
+			Some(builddir_source.as_str()),
+			builddir_target.as_str(),
+			None,
+			MsFlags::MS_BIND | MsFlags::MS_REC,
+			None,
+		)
+		.expect("Failed to bind mount build directory");
+
+		ns::pivot_root(&rootfs);
+		ns::container_mounts().context("Failed to set up container mounts")?;
+
+		unistd::sethostname("rebel-builder").context("Failed to set hostname")?;
+
+		prctl::set_no_new_privs().context("set_no_new_privs()")?;
+
+		let log_stdout = log
+			.try_clone()
+			.context("Failed to duplicate log file descriptor")?;
+		let log_stderr = log
+			.try_clone()
+			.context("Failed to duplicate log file descriptor")?;
+
+		let mut cmd = Command::new("unshare");
+		let err = cmd
+			.args(&[
+				"--user",
+				&format!("--map-user={}", BUILD_UID),
+				&format!("--map-group={}", BUILD_GID),
+				"sh",
+				"-exc",
+				&command,
+			])
+			.stdin(Stdio::null())
+			.stdout(log_stdout)
+			.stderr(log_stderr)
+			.current_dir(paths::TASK_WORKDIR)
+			.env_clear()
+			.env("PATH", "/usr/sbin:/usr/bin:/sbin:/bin")
+			.env("HOME", "/build")
+			.exec();
+		eprintln!("{}", err);
+		process::exit(127);
+	};
+
+	let (pid, log) = unsafe {
+		ns::spawn(
+			CloneFlags::CLONE_NEWNS
+				| CloneFlags::CLONE_NEWPID
+				| CloneFlags::CLONE_NEWIPC
+				| CloneFlags::CLONE_NEWNET
+				| CloneFlags::CLONE_NEWUTS,
+			log,
+			|log| exec_cmd(log).unwrap(),
+		)
+	}
+	.context("Failed to run task container")?;
+
+	let status = wait::waitpid(pid, None)?;
 
 	log.sync_all().context("Failed to write log output")?;
 
-	if !status.success() {
+	if let Err(err) = status.check() {
 		return Err(Error::new(format!(
 			"Task failed: {}\nOutput: {}",
-			status, log_filename
+			err, log_filename
 		)));
 	}