From 78fe1ec50efc3c4f912e5e28f1590e1c2b84b4e2 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Mon, 14 Aug 2023 15:48:05 +0200 Subject: [PATCH] minedmap: add support for parallel processing For now, only RegionProcessor and TileMipmapper are run in parallel. --- Cargo.lock | 151 +++++++++++++++++++++++++++ Cargo.toml | 3 + src/bin/minedmap/common.rs | 12 ++- src/bin/minedmap/main.rs | 19 +++- src/bin/minedmap/region_processor.rs | 7 +- src/bin/minedmap/tile_mipmapper.rs | 10 +- 6 files changed, 191 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e73977..c6cae9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,49 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "either" version = "1.9.0" @@ -297,6 +340,43 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "glam" version = "0.24.1" @@ -416,6 +496,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "minedmap" version = "0.1.0" @@ -427,12 +516,15 @@ dependencies = [ "enumflags2", "fastnbt", "flate2", + "futures-util", "glam", "image", "indexmap", "itertools", "lru", "num-integer", + "num_cpus", + "rayon", "rustc-hash", "serde", "serde_json", @@ -479,12 +571,34 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "pin-project-lite" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.27" @@ -522,6 +636,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -547,6 +683,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.183" @@ -593,6 +735,15 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "strsim" version = "0.10.0" diff --git a/Cargo.toml b/Cargo.toml index 5d74ce8..c6013ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,12 +15,15 @@ clap = { version = "4.1.4", features = ["derive"] } enumflags2 = "0.7.5" fastnbt = "2.3.2" flate2 = "1.0.25" +futures-util = "0.3.28" glam = "0.24.0" image = { version = "0.24.5", default-features = false, features = ["png"] } indexmap = { version = "2.0.0", features = ["serde"] } itertools = "0.11.0" lru = "0.11.0" num-integer = "0.1.45" +num_cpus = "1.16.0" +rayon = "1.7.0" rustc-hash = "1.1.0" serde = { version = "1.0.152", features = ["rc"] } serde_json = "1.0.99" diff --git a/src/bin/minedmap/common.rs b/src/bin/minedmap/common.rs index a063bf8..c0c4c8c 100644 --- a/src/bin/minedmap/common.rs +++ b/src/bin/minedmap/common.rs @@ -52,6 +52,7 @@ pub struct ProcessedRegion { } pub struct Config { + pub num_threads: usize, pub region_dir: PathBuf, pub processed_dir: PathBuf, pub output_dir: PathBuf, @@ -70,16 +71,23 @@ pub enum TileKind { } impl Config { - pub fn new(args: super::Args) -> Self { + pub fn new(args: &super::Args) -> Self { + let num_threads = match args.jobs { + Some(0) => num_cpus::get(), + Some(threads) => threads, + None => 1, + }; + let region_dir = [&args.input_dir, Path::new("region")].iter().collect(); let processed_dir = [&args.output_dir, Path::new("processed")].iter().collect(); let level_dat_path = [&args.input_dir, Path::new("level.dat")].iter().collect(); let metadata_path = [&args.output_dir, Path::new("info.json")].iter().collect(); Config { + num_threads, region_dir, processed_dir, - output_dir: args.output_dir, + output_dir: args.output_dir.clone(), level_dat_path, metadata_path, } diff --git a/src/bin/minedmap/main.rs b/src/bin/minedmap/main.rs index 8d3bbea..55158ed 100644 --- a/src/bin/minedmap/main.rs +++ b/src/bin/minedmap/main.rs @@ -7,7 +7,7 @@ mod tile_renderer; use std::path::PathBuf; -use anyhow::Result; +use anyhow::{Context, Result}; use clap::Parser; use common::Config; @@ -18,15 +18,30 @@ use tile_renderer::TileRenderer; #[derive(Debug, Parser)] pub struct Args { + /// Number of parallel threads to use for processing + /// + /// If not given, only a single thread is used. Pass 0 to + /// use one thread per logical CPU core. + #[arg(short, long)] + pub jobs: Option, /// Minecraft save directory pub input_dir: PathBuf, /// MinedMap data directory pub output_dir: PathBuf, } +fn setup_threads(num_threads: usize) -> Result<()> { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global() + .context("Failed to configure thread pool") +} + fn main() -> Result<()> { let args = Args::parse(); - let config = Config::new(args); + let config = Config::new(&args); + + setup_threads(config.num_threads)?; let regions = RegionProcessor::new(&config).run()?; TileRenderer::new(&config).run(®ions)?; diff --git a/src/bin/minedmap/region_processor.rs b/src/bin/minedmap/region_processor.rs index 639dac7..6aa2cb4 100644 --- a/src/bin/minedmap/region_processor.rs +++ b/src/bin/minedmap/region_processor.rs @@ -1,8 +1,9 @@ use std::{ffi::OsStr, path::Path, time::SystemTime}; use anyhow::{Context, Result}; - use indexmap::IndexSet; +use rayon::prelude::*; + use minedmap::{ io::{fs, storage}, resource::{self, Biome}, @@ -183,11 +184,11 @@ impl<'a> RegionProcessor<'a> { fs::create_dir_all(&self.config.processed_dir)?; fs::create_dir_all(&self.config.tile_dir(TileKind::Lightmap, 0))?; - for &coords in ®ions { + regions.par_iter().for_each(|&coords| { if let Err(err) = self.process_region(coords) { eprintln!("Failed to process region {:?}: {:?}", coords, err); } - } + }); Ok(regions) } diff --git a/src/bin/minedmap/tile_mipmapper.rs b/src/bin/minedmap/tile_mipmapper.rs index 40add3a..2c1e9d6 100644 --- a/src/bin/minedmap/tile_mipmapper.rs +++ b/src/bin/minedmap/tile_mipmapper.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use rayon::prelude::*; use minedmap::{io::fs, types::*}; @@ -151,8 +152,8 @@ impl<'a> TileMipmapper<'a> { let next = Self::map_coords(prev); - for (&z, xs) in &next.0 { - for &x in xs { + next.0.par_iter().try_for_each(|(&z, xs)| { + xs.par_iter().try_for_each(|&x| { let coords = TileCoords { x, z }; self.render_mipmap::>(TileKind::Map, level, coords, prev)?; self.render_mipmap::>( @@ -161,8 +162,9 @@ impl<'a> TileMipmapper<'a> { coords, prev, )?; - } - } + anyhow::Ok(()) + }) + })?; tile_stack.push(next); }