mirror of
https://github.com/neocturne/MinedMap.git
synced 2025-03-04 17:23:33 +01:00
minedmap: add support for parallel processing
For now, only RegionProcessor and TileMipmapper are run in parallel.
This commit is contained in:
parent
c1260a63b5
commit
78fe1ec50e
GPG key ID:
16EF3F64CB201D9C
6 changed files with 191 additions and 11 deletions
151
Cargo.lock
generated
151
Cargo.lock
generated
|
|
@ -212,6 +212,49 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.9.0"
|
||||
|
|
@ -297,6 +340,43 @@ dependencies = [
|
|||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-macro",
|
||||
"futures-task",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glam"
|
||||
version = "0.24.1"
|
||||
|
|
@ -416,6 +496,15 @@ dependencies = [
|
|||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minedmap"
|
||||
version = "0.1.0"
|
||||
|
|
@ -427,12 +516,15 @@ dependencies = [
|
|||
"enumflags2",
|
||||
"fastnbt",
|
||||
"flate2",
|
||||
"futures-util",
|
||||
"glam",
|
||||
"image",
|
||||
"indexmap",
|
||||
"itertools",
|
||||
"lru",
|
||||
"num-integer",
|
||||
"num_cpus",
|
||||
"rayon",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -479,12 +571,34 @@ dependencies = [
|
|||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.27"
|
||||
|
|
@ -522,6 +636,28 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
|
|
@ -547,6 +683,12 @@ version = "1.0.15"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.183"
|
||||
|
|
@ -593,6 +735,15 @@ version = "0.3.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
|
|
|
|||
|
|
@ -15,12 +15,15 @@ clap = { version = "4.1.4", features = ["derive"] }
|
|||
enumflags2 = "0.7.5"
|
||||
fastnbt = "2.3.2"
|
||||
flate2 = "1.0.25"
|
||||
futures-util = "0.3.28"
|
||||
glam = "0.24.0"
|
||||
image = { version = "0.24.5", default-features = false, features = ["png"] }
|
||||
indexmap = { version = "2.0.0", features = ["serde"] }
|
||||
itertools = "0.11.0"
|
||||
lru = "0.11.0"
|
||||
num-integer = "0.1.45"
|
||||
num_cpus = "1.16.0"
|
||||
rayon = "1.7.0"
|
||||
rustc-hash = "1.1.0"
|
||||
serde = { version = "1.0.152", features = ["rc"] }
|
||||
serde_json = "1.0.99"
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ pub struct ProcessedRegion {
|
|||
}
|
||||
|
||||
pub struct Config {
|
||||
pub num_threads: usize,
|
||||
pub region_dir: PathBuf,
|
||||
pub processed_dir: PathBuf,
|
||||
pub output_dir: PathBuf,
|
||||
|
|
@ -70,16 +71,23 @@ pub enum TileKind {
|
|||
}
|
||||
|
||||
impl Config {
|
||||
pub fn new(args: super::Args) -> Self {
|
||||
pub fn new(args: &super::Args) -> Self {
|
||||
let num_threads = match args.jobs {
|
||||
Some(0) => num_cpus::get(),
|
||||
Some(threads) => threads,
|
||||
None => 1,
|
||||
};
|
||||
|
||||
let region_dir = [&args.input_dir, Path::new("region")].iter().collect();
|
||||
let processed_dir = [&args.output_dir, Path::new("processed")].iter().collect();
|
||||
let level_dat_path = [&args.input_dir, Path::new("level.dat")].iter().collect();
|
||||
let metadata_path = [&args.output_dir, Path::new("info.json")].iter().collect();
|
||||
|
||||
Config {
|
||||
num_threads,
|
||||
region_dir,
|
||||
processed_dir,
|
||||
output_dir: args.output_dir,
|
||||
output_dir: args.output_dir.clone(),
|
||||
level_dat_path,
|
||||
metadata_path,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ mod tile_renderer;
|
|||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
|
||||
use common::Config;
|
||||
|
|
@ -18,15 +18,30 @@ use tile_renderer::TileRenderer;
|
|||
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Args {
|
||||
/// Number of parallel threads to use for processing
|
||||
///
|
||||
/// If not given, only a single thread is used. Pass 0 to
|
||||
/// use one thread per logical CPU core.
|
||||
#[arg(short, long)]
|
||||
pub jobs: Option<usize>,
|
||||
/// Minecraft save directory
|
||||
pub input_dir: PathBuf,
|
||||
/// MinedMap data directory
|
||||
pub output_dir: PathBuf,
|
||||
}
|
||||
|
||||
fn setup_threads(num_threads: usize) -> Result<()> {
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(num_threads)
|
||||
.build_global()
|
||||
.context("Failed to configure thread pool")
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
let config = Config::new(args);
|
||||
let config = Config::new(&args);
|
||||
|
||||
setup_threads(config.num_threads)?;
|
||||
|
||||
let regions = RegionProcessor::new(&config).run()?;
|
||||
TileRenderer::new(&config).run(®ions)?;
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use std::{ffi::OsStr, path::Path, time::SystemTime};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use indexmap::IndexSet;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use minedmap::{
|
||||
io::{fs, storage},
|
||||
resource::{self, Biome},
|
||||
|
|
@ -183,11 +184,11 @@ impl<'a> RegionProcessor<'a> {
|
|||
fs::create_dir_all(&self.config.processed_dir)?;
|
||||
fs::create_dir_all(&self.config.tile_dir(TileKind::Lightmap, 0))?;
|
||||
|
||||
for &coords in ®ions {
|
||||
regions.par_iter().for_each(|&coords| {
|
||||
if let Err(err) = self.process_region(coords) {
|
||||
eprintln!("Failed to process region {:?}: {:?}", coords, err);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(regions)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use anyhow::{Context, Result};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use minedmap::{io::fs, types::*};
|
||||
|
||||
|
|
@ -151,8 +152,8 @@ impl<'a> TileMipmapper<'a> {
|
|||
|
||||
let next = Self::map_coords(prev);
|
||||
|
||||
for (&z, xs) in &next.0 {
|
||||
for &x in xs {
|
||||
next.0.par_iter().try_for_each(|(&z, xs)| {
|
||||
xs.par_iter().try_for_each(|&x| {
|
||||
let coords = TileCoords { x, z };
|
||||
self.render_mipmap::<image::Rgba<u8>>(TileKind::Map, level, coords, prev)?;
|
||||
self.render_mipmap::<image::LumaA<u8>>(
|
||||
|
|
@ -161,8 +162,9 @@ impl<'a> TileMipmapper<'a> {
|
|||
coords,
|
||||
prev,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
anyhow::Ok(())
|
||||
})
|
||||
})?;
|
||||
|
||||
tile_stack.push(next);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue