From 30e5aee09e1e3387ae7c652039946832410021d9 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 18:27:49 +0200 Subject: [PATCH 01/11] Update dependencies --- Cargo.lock | 64 ++++++++++++++++++++++++++---------------------------- Cargo.toml | 4 ++-- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4687cf0..5cdfec3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,9 +132,9 @@ checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" @@ -189,7 +189,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -285,7 +285,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -340,7 +340,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -461,9 +461,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -476,9 +476,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libz-ng-sys" @@ -508,18 +508,18 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" +checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" dependencies = [ "hashbrown", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -632,9 +632,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -732,9 +732,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -827,7 +827,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -843,9 +843,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b21f559e07218024e7e9f90f96f601825397de0e25420135f7f952453fed0b" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -890,9 +890,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -911,9 +911,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "parking_lot", @@ -940,7 +940,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1092,30 +1092,28 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index fcef957..f9b1f0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ git-version = "0.3.5" glam = "0.24.0" image = { version = "0.24.5", default-features = false, features = ["png"] } indexmap = { version = "2.0.0", features = ["serde"] } -lru = "0.11.0" +lru = "0.12.0" minedmap-nbt = { version = "0.1.0", path = "crates/nbt", default-features = false } minedmap-resource = { version = "0.1.0", path = "crates/resource" } minedmap-types = { version = "0.1.0", path = "crates/types" } @@ -59,7 +59,7 @@ serde_json = "1.0.99" tokio = { version = "1.31.0", features = ["rt", "parking_lot", "sync"] } tracing = "0.1.37" tracing-subscriber = "0.3.17" -zstd = "0.12.3" +zstd = "0.13.0" [features] default = ["zlib-ng"] From 920547f64a57648aabef0b298ebc0ea3ada1bc2b Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 19:14:13 +0200 Subject: [PATCH 02/11] core/tile_mipmapper: use mpsc channels for counters Make the code a bit easier to understand. --- src/core/tile_mipmapper.rs | 73 ++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/src/core/tile_mipmapper.rs b/src/core/tile_mipmapper.rs index d2b1ac5..cc1a8d4 100644 --- a/src/core/tile_mipmapper.rs +++ b/src/core/tile_mipmapper.rs @@ -1,5 +1,7 @@ //! The [TileMipmapper] +use std::sync::mpsc; + use anyhow::{Context, Result}; use rayon::prelude::*; use tracing::{debug, info, warn}; @@ -58,7 +60,9 @@ impl<'a> TileMipmapper<'a> { level: usize, coords: TileCoords, prev: &TileCoordMap, - ) -> Result<(bool, bool)> + count_total: &mpsc::Sender<()>, + count_processed: &mpsc::Sender<()>, + ) -> Result<()> where [P::Subpixel]: image::EncodableLayout, image::ImageBuffer>: Into, @@ -93,9 +97,11 @@ impl<'a> TileMipmapper<'a> { .collect(); let Some(input_timestamp) = sources.iter().map(|(_, _, ts)| *ts).max() else { - return Ok((false, false)); + return Ok(()); }; + count_total.send(()).unwrap(); + if Some(input_timestamp) <= output_timestamp { debug!( "Skipping unchanged mipmap tile {}", @@ -104,7 +110,7 @@ impl<'a> TileMipmapper<'a> { .expect("tile path must be in output directory") .display(), ); - return Ok((true, false)); + return Ok(()); } debug!( @@ -145,7 +151,8 @@ impl<'a> TileMipmapper<'a> { .context("Failed to save image") })?; - Ok((true, true)) + count_processed.send(()).unwrap(); + Ok(()) } /// Runs the mipmap generation @@ -174,34 +181,38 @@ impl<'a> TileMipmapper<'a> { let next = Self::map_coords(prev); - let (total, processed) = next - .0 + let (total_send, total_recv) = mpsc::channel(); + let (processed_send, processed_recv) = mpsc::channel(); + + next.0 .par_iter() - .flat_map(|(&z, xs)| { - let mipmapper = &self; - xs.par_iter().map(move |&x| { - let coords = TileCoords { x, z }; - let (found_map, processed_map) = mipmapper - .render_mipmap::>(TileKind::Map, level, coords, prev)?; - let (found_light, processed_light) = mipmapper - .render_mipmap::>( - TileKind::Lightmap, - level, - coords, - prev, - )?; - anyhow::Ok(( - usize::from(found_map) + usize::from(found_light), - usize::from(processed_map) + usize::from(processed_light), - )) - }) - }) - .try_reduce( - || (0, 0), - |(found_a, processed_a), (found_b, processed_b)| { - Ok((found_a + found_b, processed_a + processed_b)) - }, - )?; + .flat_map(|(&z, xs)| xs.par_iter().map(move |&x| TileCoords { x, z })) + .try_for_each(|coords| { + self.render_mipmap::>( + TileKind::Map, + level, + coords, + prev, + &total_send, + &processed_send, + )?; + self.render_mipmap::>( + TileKind::Lightmap, + level, + coords, + prev, + &total_send, + &processed_send, + )?; + + anyhow::Ok(()) + })?; + + drop(total_send); + let total = total_recv.into_iter().count(); + + drop(processed_send); + let processed = processed_recv.into_iter().count(); info!( "Generated level {} mipmaps ({} processed, {} unchanged)", From 38da1616d5af2edf4c16e204073a93bbd0d3ba73 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 18:30:28 +0200 Subject: [PATCH 03/11] core/region_processor: rename path to input_path in process_region() Make the variable names more consistent. --- src/core/region_processor.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index ac6b42e..55f7227 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -139,8 +139,8 @@ impl<'a> RegionProcessor<'a> { let mut processed_region = ProcessedRegion::default(); let mut lightmap = image::GrayAlphaImage::new(N, N); - let path = self.config.region_path(coords); - let input_timestamp = fs::modified_timestamp(&path)?; + let input_path = self.config.region_path(coords); + let input_timestamp = fs::modified_timestamp(&input_path)?; let output_path = self.config.processed_path(coords); let output_timestamp = fs::read_timestamp(&output_path, FILE_META_VERSION); @@ -155,7 +155,7 @@ impl<'a> RegionProcessor<'a> { debug!("Processing region r.{}.{}.mca", coords.x, coords.z); - crate::nbt::region::from_file(path)?.foreach_chunk( + crate::nbt::region::from_file(input_path)?.foreach_chunk( |chunk_coords, data: world::de::Chunk| { let Some(layer::LayerData { blocks, From 46c04e632fb2e0e545cb1f8f47e21c5a7f7e8f44 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 19:31:02 +0200 Subject: [PATCH 04/11] core/region_processor: temporarily disable stat output --- src/core/region_processor.rs | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index 55f7227..6e5adbe 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -205,29 +205,19 @@ impl<'a> RegionProcessor<'a> { info!("Processing region files..."); - let mut results = vec![]; - regions - .par_iter() - .map(|&coords| { - let result = self.process_region(coords); - if let Err(err) = &result { - error!("Failed to process region {:?}: {:?}", coords, err); - } - result - }) - .collect_into_vec(&mut results); + regions.par_iter().for_each(|&coords| { + let result = self.process_region(coords); + if let Err(err) = &result { + error!("Failed to process region {:?}: {:?}", coords, err); + } + }); - let processed = results - .iter() - .filter(|result| matches!(result, Ok(true))) - .count(); - let errors = results.iter().filter(|result| result.is_err()).count(); - info!( - "Processed region files ({} processed, {} unchanged, {} errors)", - processed, - results.len() - processed, - errors, - ); + // info!( + // "Processed region files ({} processed, {} unchanged, {} errors)", + // processed, + // results.len() - processed, + // errors, + // ); Ok(regions) } From 284892cea6a15cc16b68557a99d3c1f519d10abe Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 19:36:18 +0200 Subject: [PATCH 05/11] core/region_processor: make all process_region() error fatal Less severe errors will be changed not to be passed up to run(). --- src/core/region_processor.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index 6e5adbe..db25fc2 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -5,7 +5,7 @@ use std::{ffi::OsStr, path::Path, time::SystemTime}; use anyhow::{Context, Result}; use indexmap::IndexSet; use rayon::prelude::*; -use tracing::{debug, error, info}; +use tracing::{debug, info}; use super::common::*; use crate::{ @@ -205,12 +205,12 @@ impl<'a> RegionProcessor<'a> { info!("Processing region files..."); - regions.par_iter().for_each(|&coords| { - let result = self.process_region(coords); - if let Err(err) = &result { - error!("Failed to process region {:?}: {:?}", coords, err); - } - }); + regions.par_iter().try_for_each(|&coords| { + let _ = self + .process_region(coords) + .with_context(|| format!("Failed to process region {:?}", coords))?; + anyhow::Ok(()) + })?; // info!( // "Processed region files ({} processed, {} unchanged, {} errors)", From 09374d755e25a5dec400177ab7a58f40424e40c1 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 19:44:14 +0200 Subject: [PATCH 06/11] core/region_processor: introduce RegionProcessorStatus enum --- src/core/region_processor.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index db25fc2..fff0b25 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -31,6 +31,19 @@ fn parse_region_filename(file_name: &OsStr) -> Option { }) } +/// [RegionProcessor::process_region] return values +#[derive(Debug, Clone, Copy)] +enum RegionProcessorStatus { + /// Region was processed + Ok, + /// Region was unchanged and skipped + Skipped, + /// Reading the region failed, previous processed data is reused + ErrorOk, + /// Reading the region failed, no previous data available + ErrorMissing, +} + /// Type with methods for processing the regions of a Minecraft save directory /// /// The RegionProcessor builds lightmap tiles as well as processed region data @@ -132,7 +145,7 @@ impl<'a> RegionProcessor<'a> { } /// Processes a single region file - fn process_region(&self, coords: TileCoords) -> Result { + fn process_region(&self, coords: TileCoords) -> Result { /// Width/height of the region data const N: u32 = (BLOCKS_PER_CHUNK * CHUNKS_PER_REGION) as u32; @@ -150,7 +163,7 @@ impl<'a> RegionProcessor<'a> { if Some(input_timestamp) <= output_timestamp && Some(input_timestamp) <= lightmap_timestamp { debug!("Skipping unchanged region r.{}.{}.mca", coords.x, coords.z); - return Ok(false); + return Ok(RegionProcessorStatus::Skipped); } debug!("Processing region r.{}.{}.mca", coords.x, coords.z); @@ -188,7 +201,7 @@ impl<'a> RegionProcessor<'a> { Self::save_lightmap(&lightmap_path, &lightmap, input_timestamp)?; } - Ok(true) + Ok(RegionProcessorStatus::Ok) } /// Iterates over all region files of a Minecraft save directory From fd48f94f16b1028227bd26bee82d1597ae311352 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 19:49:26 +0200 Subject: [PATCH 07/11] core/region_processor: make regular processing errors non-fatal again Never fail because of invalid save files. --- src/core/region_processor.rs | 64 ++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index fff0b25..c1193b6 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -5,7 +5,7 @@ use std::{ffi::OsStr, path::Path, time::SystemTime}; use anyhow::{Context, Result}; use indexmap::IndexSet; use rayon::prelude::*; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use super::common::*; use crate::{ @@ -168,31 +168,47 @@ impl<'a> RegionProcessor<'a> { debug!("Processing region r.{}.{}.mca", coords.x, coords.z); - crate::nbt::region::from_file(input_path)?.foreach_chunk( - |chunk_coords, data: world::de::Chunk| { - let Some(layer::LayerData { - blocks, - biomes, - block_light, - depths, - }) = self - .process_chunk(&mut processed_region.biome_list, data) - .with_context(|| format!("Failed to process chunk {:?}", chunk_coords))? - else { - return Ok(()); - }; - processed_region.chunks[chunk_coords] = Some(Box::new(ProcessedChunk { - blocks, - biomes, - depths, - })); + if let Err(err) = (|| -> Result<()> { + crate::nbt::region::from_file(input_path)?.foreach_chunk( + |chunk_coords, data: world::de::Chunk| { + let Some(layer::LayerData { + blocks, + biomes, + block_light, + depths, + }) = self + .process_chunk(&mut processed_region.biome_list, data) + .with_context(|| format!("Failed to process chunk {:?}", chunk_coords))? + else { + return Ok(()); + }; + processed_region.chunks[chunk_coords] = Some(Box::new(ProcessedChunk { + blocks, + biomes, + depths, + })); - let chunk_lightmap = Self::render_chunk_lightmap(block_light); - overlay_chunk(&mut lightmap, &chunk_lightmap, chunk_coords); + let chunk_lightmap = Self::render_chunk_lightmap(block_light); + overlay_chunk(&mut lightmap, &chunk_lightmap, chunk_coords); - Ok(()) - }, - )?; + Ok(()) + }, + ) + })() { + if output_timestamp.is_some() && lightmap_timestamp.is_some() { + warn!( + "Failed to process region {:?}, using old data: {:?}", + coords, err + ); + return Ok(RegionProcessorStatus::ErrorOk); + } else { + warn!( + "Failed to process region {:?}, no old data available: {:?}", + coords, err + ); + return Ok(RegionProcessorStatus::ErrorMissing); + } + } if Some(input_timestamp) > output_timestamp { Self::save_region(&output_path, &processed_region, input_timestamp)?; From 506631a18f9e20875e7bde703700851b61e15b24 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 19:59:14 +0200 Subject: [PATCH 08/11] core/region_processor: only return available regions Ignore regions that failed to process and have no old processed data. --- src/core/region_processor.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index c1193b6..caaf8ec 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -1,6 +1,6 @@ //! The [RegionProcessor] and related functions -use std::{ffi::OsStr, path::Path, time::SystemTime}; +use std::{ffi::OsStr, path::Path, sync::mpsc, time::SystemTime}; use anyhow::{Context, Result}; use indexmap::IndexSet; @@ -32,7 +32,7 @@ fn parse_region_filename(file_name: &OsStr) -> Option { } /// [RegionProcessor::process_region] return values -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum RegionProcessorStatus { /// Region was processed Ok, @@ -224,23 +224,28 @@ impl<'a> RegionProcessor<'a> { /// /// Returns a list of the coordinates of all processed regions pub fn run(self) -> Result> { - let mut regions = self.collect_regions()?; - - // Sort regions in a zig-zag pattern to optimize cache usage - regions.sort_unstable_by_key(|&TileCoords { x, z }| (x, if x % 2 == 0 { z } else { -z })); - fs::create_dir_all(&self.config.processed_dir)?; fs::create_dir_all(&self.config.tile_dir(TileKind::Lightmap, 0))?; info!("Processing region files..."); - regions.par_iter().try_for_each(|&coords| { - let _ = self + let (region_send, region_recv) = mpsc::channel(); + + self.collect_regions()?.par_iter().try_for_each(|&coords| { + let ret = self .process_region(coords) .with_context(|| format!("Failed to process region {:?}", coords))?; + + if ret != RegionProcessorStatus::ErrorMissing { + region_send.send(coords).unwrap(); + } + anyhow::Ok(()) })?; + drop(region_send); + let mut regions: Vec<_> = region_recv.into_iter().collect(); + // info!( // "Processed region files ({} processed, {} unchanged, {} errors)", // processed, @@ -248,6 +253,9 @@ impl<'a> RegionProcessor<'a> { // errors, // ); + // Sort regions in a zig-zag pattern to optimize cache usage + regions.sort_unstable_by_key(|&TileCoords { x, z }| (x, if x % 2 == 0 { z } else { -z })); + Ok(regions) } } From fba9b6cb553c2760d7399abca196f858f572f6a7 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 12 Oct 2023 20:08:02 +0200 Subject: [PATCH 09/11] core/region_processor: reenable stat output --- src/core/region_processor.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index caaf8ec..6442da0 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -230,6 +230,8 @@ impl<'a> RegionProcessor<'a> { info!("Processing region files..."); let (region_send, region_recv) = mpsc::channel(); + let (processed_send, processed_recv) = mpsc::channel(); + let (error_send, error_recv) = mpsc::channel(); self.collect_regions()?.par_iter().try_for_each(|&coords| { let ret = self @@ -240,18 +242,31 @@ impl<'a> RegionProcessor<'a> { region_send.send(coords).unwrap(); } + match ret { + RegionProcessorStatus::Ok => processed_send.send(()).unwrap(), + RegionProcessorStatus::Skipped => {} + RegionProcessorStatus::ErrorOk | RegionProcessorStatus::ErrorMissing => { + error_send.send(()).unwrap() + } + } + anyhow::Ok(()) })?; drop(region_send); let mut regions: Vec<_> = region_recv.into_iter().collect(); - // info!( - // "Processed region files ({} processed, {} unchanged, {} errors)", - // processed, - // results.len() - processed, - // errors, - // ); + drop(processed_send); + let processed = processed_recv.into_iter().count(); + drop(error_send); + let errors = error_recv.into_iter().count(); + + info!( + "Processed region files ({} processed, {} unchanged, {} errors)", + processed, + regions.len() - processed - errors, + errors, + ); // Sort regions in a zig-zag pattern to optimize cache usage regions.sort_unstable_by_key(|&TileCoords { x, z }| (x, if x % 2 == 0 { z } else { -z })); From 2c1c8c17ef7a7b156d8f4813289a8f5c0c5f515c Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sat, 18 Nov 2023 18:49:25 +0100 Subject: [PATCH 10/11] Update dependencies --- Cargo.lock | 179 ++++++++++++++++++++-------------------- crates/types/Cargo.toml | 2 +- 2 files changed, 91 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5cdfec3..fc57421 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,13 +19,14 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -160,9 +161,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.6" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -170,9 +171,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -182,21 +183,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.38", + "syn", ] [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cmake" @@ -285,7 +286,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn", ] [[package]] @@ -308,18 +309,18 @@ dependencies = [ [[package]] name = "fdeflate" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10" +checksum = "64d6dafc854908ff5da46ff3f8f473c6984119a2876a383a860246dd7841a868" dependencies = [ "simd-adler32", ] [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "libz-ng-sys", @@ -328,32 +329,32 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn", ] [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", "futures-macro", @@ -371,24 +372,22 @@ checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "git-version" -version = "0.3.5" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b0decc02f4636b9ccad390dcbe77b722a77efedfa393caf8379a51d5c61899" +checksum = "13ad01ffa8221f7fe8b936d6ffb2a3e7ad428885a04fad51866a5f33eafda57c" dependencies = [ "git-version-macro", - "proc-macro-hack", ] [[package]] name = "git-version-macro" -version = "0.3.5" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe69f1cbdb6e28af2bac214e943b99ce8a0a06b447d15d3e61161b0423139f3f" +checksum = "84488ccbdb24ad6f56dc1863b4a8154a7856cd3c6c7610401634fab3cb588dae" dependencies = [ - "proc-macro-hack", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -399,9 +398,9 @@ checksum = "b5418c17512bdf42730f9032c74e1ae39afc408745ebb2acf72fbc4691c17945" [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash", "allocator-api2", @@ -435,9 +434,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown", @@ -446,9 +445,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" dependencies = [ "either", ] @@ -476,9 +475,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libz-ng-sys" @@ -492,9 +491,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -682,9 +681,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", @@ -724,12 +723,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.69" @@ -770,9 +763,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags", ] @@ -803,9 +796,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] @@ -821,20 +814,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -867,9 +860,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "strsim" @@ -879,20 +872,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -911,9 +893,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "parking_lot", @@ -922,11 +904,10 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -934,20 +915,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -955,20 +936,20 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -1090,6 +1071,26 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zstd" version = "0.13.0" diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 4097eea..3e8f63d 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -8,5 +8,5 @@ readme.workspace = true repository.workspace = true [dependencies] -itertools = "0.11.0" +itertools = "0.12.0" serde = { version = "1.0.183", features = ["derive"] } From 412568603e1c4b11dced9a2859f51152954fcd23 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sat, 18 Nov 2023 18:51:11 +0100 Subject: [PATCH 11/11] README.md: minor updates --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4dcd106..75877b4 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ MinedMap consists of two components: a map renderer generating map tiles from Minecraft save games, and a viewer for displaying and navigating maps in a browser based on [Leaflet](https://leafletjs.com/). The map renderer is heavily inspired by -[MapRend](https://github.com/YSelfTool/MapRend), but it has been implemented in C++ -from scratch for highest performance. +[MapRend](https://github.com/YSelfTool/MapRend), but has been reimplemented from scratch +(first in C++, now in Rust) for highest performance. The viewer expects the the map data in a directory named `data`. To generate a new map, create this empty directory inside the viewer directory. Next, to generate the @@ -48,8 +48,8 @@ the generated map files to public webspace to make the map available to others. ## Installation -Building the MinedMap map generator requires a recent Rust toolchain. The -following command can be used to build the current development version from source: +Building the MinedMap map generator from source requires a recent Rust toolchain. The +following command can be used to build the current development version: ```shell cargo install --git 'https://github.com/NeoRaider/MinedMap.git' ```