minedmap: add support for parallel processing

For now, only RegionProcessor and TileMipmapper are run in parallel.

Cargo.lock

@@ -212,6 +212,49 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "crossbeam-utils",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "either"
 version = "1.9.0"
@@ -297,6 +340,43 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "futures-core"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "futures-task"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+
+[[package]]
+name = "futures-util"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+dependencies = [
+ "futures-core",
+ "futures-macro",
+ "futures-task",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
 [[package]]
 name = "glam"
 version = "0.24.1"
@@ -416,6 +496,15 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "minedmap"
 version = "0.1.0"
@@ -427,12 +516,15 @@ dependencies = [
  "enumflags2",
  "fastnbt",
  "flate2",
+ "futures-util",
  "glam",
  "image",
  "indexmap",
  "itertools",
  "lru",
  "num-integer",
+ "num_cpus",
+ "rayon",
  "rustc-hash",
  "serde",
  "serde_json",
@@ -479,12 +571,34 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
+[[package]]
+name = "pin-project-lite"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
 [[package]]
 name = "pkg-config"
 version = "0.3.27"
@@ -522,6 +636,28 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "num_cpus",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -547,6 +683,12 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "serde"
 version = "1.0.183"
@@ -593,6 +735,15 @@ version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
 
+[[package]]
+name = "slab"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "strsim"
 version = "0.10.0"

Cargo.toml

@@ -15,12 +15,15 @@ clap = { version = "4.1.4", features = ["derive"] }
 enumflags2 = "0.7.5"
 fastnbt = "2.3.2"
 flate2 = "1.0.25"
+futures-util = "0.3.28"
 glam = "0.24.0"
 image = { version = "0.24.5", default-features = false, features = ["png"] }
 indexmap = { version = "2.0.0", features = ["serde"] }
 itertools = "0.11.0"
 lru = "0.11.0"
 num-integer = "0.1.45"
+num_cpus = "1.16.0"
+rayon = "1.7.0"
 rustc-hash = "1.1.0"
 serde = { version = "1.0.152", features = ["rc"] }
 serde_json = "1.0.99"

src/bin/minedmap/common.rs

@@ -52,6 +52,7 @@ pub struct ProcessedRegion {
 }
 
 pub struct Config {
+	pub num_threads: usize,
 	pub region_dir: PathBuf,
 	pub processed_dir: PathBuf,
 	pub output_dir: PathBuf,
@@ -70,16 +71,23 @@ pub enum TileKind {
 }
 
 impl Config {
-	pub fn new(args: super::Args) -> Self {
+	pub fn new(args: &super::Args) -> Self {
+		let num_threads = match args.jobs {
+			Some(0) => num_cpus::get(),
+			Some(threads) => threads,
+			None => 1,
+		};
+
 		let region_dir = [&args.input_dir, Path::new("region")].iter().collect();
 		let processed_dir = [&args.output_dir, Path::new("processed")].iter().collect();
 		let level_dat_path = [&args.input_dir, Path::new("level.dat")].iter().collect();
 		let metadata_path = [&args.output_dir, Path::new("info.json")].iter().collect();
 
 		Config {
+			num_threads,
 			region_dir,
 			processed_dir,
-			output_dir: args.output_dir,
+			output_dir: args.output_dir.clone(),
 			level_dat_path,
 			metadata_path,
 		}

src/bin/minedmap/main.rs

@@ -7,7 +7,7 @@ mod tile_renderer;
 
 use std::path::PathBuf;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use clap::Parser;
 
 use common::Config;
@@ -18,15 +18,30 @@ use tile_renderer::TileRenderer;
 
 #[derive(Debug, Parser)]
 pub struct Args {
+	/// Number of parallel threads to use for processing
+	///
+	/// If not given, only a single thread is used. Pass 0 to
+	/// use one thread per logical CPU core.
+	#[arg(short, long)]
+	pub jobs: Option<usize>,
 	/// Minecraft save directory
 	pub input_dir: PathBuf,
 	/// MinedMap data directory
 	pub output_dir: PathBuf,
 }
 
+fn setup_threads(num_threads: usize) -> Result<()> {
+	rayon::ThreadPoolBuilder::new()
+		.num_threads(num_threads)
+		.build_global()
+		.context("Failed to configure thread pool")
+}
+
 fn main() -> Result<()> {
 	let args = Args::parse();
-	let config = Config::new(args);
+	let config = Config::new(&args);
+
+	setup_threads(config.num_threads)?;
 
 	let regions = RegionProcessor::new(&config).run()?;
 	TileRenderer::new(&config).run(&regions)?;

src/bin/minedmap/region_processor.rs

@@ -1,8 +1,9 @@
 use std::{ffi::OsStr, path::Path, time::SystemTime};
 
 use anyhow::{Context, Result};
-
 use indexmap::IndexSet;
+use rayon::prelude::*;
+
 use minedmap::{
 	io::{fs, storage},
 	resource::{self, Biome},
@@ -183,11 +184,11 @@ impl<'a> RegionProcessor<'a> {
 		fs::create_dir_all(&self.config.processed_dir)?;
 		fs::create_dir_all(&self.config.tile_dir(TileKind::Lightmap, 0))?;
 
-		for &coords in &regions {
+		regions.par_iter().for_each(|&coords| {
 			if let Err(err) = self.process_region(coords) {
 				eprintln!("Failed to process region {:?}: {:?}", coords, err);
 			}
-		}
+		});
 
 		Ok(regions)
 	}

src/bin/minedmap/tile_mipmapper.rs

@@ -1,4 +1,5 @@
 use anyhow::{Context, Result};
+use rayon::prelude::*;
 
 use minedmap::{io::fs, types::*};
 
@@ -151,8 +152,8 @@ impl<'a> TileMipmapper<'a> {
 
 			let next = Self::map_coords(prev);
 
-			for (&z, xs) in &next.0 {
+			next.0.par_iter().try_for_each(|(&z, xs)| {
-				for &x in xs {
+				xs.par_iter().try_for_each(|&x| {
 					let coords = TileCoords { x, z };
 					self.render_mipmap::<image::Rgba<u8>>(TileKind::Map, level, coords, prev)?;
 					self.render_mipmap::<image::LumaA<u8>>(
@@ -161,8 +162,9 @@ impl<'a> TileMipmapper<'a> {
 						coords,
 						prev,
 					)?;
-				}
+					anyhow::Ok(())
-			}
+				})
+			})?;
 
 			tile_stack.push(next);
 		}