From fbdd5ed457f2e5ca12bc7a7d6f6992f47494e062 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 21:11:59 +0100 Subject: [PATCH 1/9] Dockerfile: switch back to docker.io/library/rust:alpine image bincode 2 required Rust 1.85, so our options are to switch to Alpine edge or to use the rust image. While using the rust image results in a statically linked binary, this does not actually increase the size of the image, as we were already using jemalloc, so almost nothing of libc is actually used. --- Dockerfile | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 818971d..0f0eecd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM docker.io/library/alpine:latest AS builder +FROM docker.io/library/rust:alpine AS builder WORKDIR /build -RUN apk add --no-cache build-base cargo +RUN apk add --no-cache build-base tini-static ARG MINEDMAP_VERSION @@ -9,14 +9,9 @@ COPY . . RUN cargo build -r RUN strip target/release/minedmap -FROM docker.io/library/alpine:latest +FROM scratch -RUN addgroup -g 1000 -S minedmap \ - && adduser -S -D -H -u 1000 -h /output -s /sbin/nologin -G minedmap -g minedmap minedmap +COPY --from=builder /sbin/tini-static /build/target/release/minedmap /bin/ +ENTRYPOINT [ "/bin/tini-static", "--", "/bin/minedmap" ] -RUN apk add --no-cache libgcc tini - -COPY --from=builder /build/target/release/minedmap /bin/minedmap -ENTRYPOINT [ "/sbin/tini", "--", "/bin/minedmap" ] - -USER minedmap:minedmap +USER 1000:1000 From deb232ddf356efd80a4658b0b0f8c329e420b244 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 20:34:16 +0100 Subject: [PATCH 2/9] Update dependencies --- Cargo.lock | 118 ++++++++++++++++++++++++++--------------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bd83ee..0142276 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.96" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "autocfg" @@ -126,15 +126,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "bytemuck" -version = "1.21.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" +checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" [[package]] name = "byteorder" @@ -173,9 +173,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.31" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" dependencies = [ "clap_builder", "clap_derive", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.31" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" dependencies = [ "anstream", "anstyle", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.28" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck", "proc-macro2", @@ -254,9 +254,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "either" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "enum-map" @@ -468,9 +468,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "image" @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown", @@ -512,7 +512,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "inotify-sys", "libc", ] @@ -543,9 +543,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" @@ -584,9 +584,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.170" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libredox" @@ -594,7 +594,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "libc", "redox_syscall", ] @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.15" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9" [[package]] name = "lock_api" @@ -745,7 +745,7 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "filetime", "fsevent-sys", "inotify", @@ -813,9 +813,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" [[package]] name = "overload" @@ -902,9 +902,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "png" @@ -921,9 +921,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -936,9 +936,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -980,11 +980,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -1030,11 +1030,11 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "0.38.44" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys", @@ -1043,9 +1043,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -1064,27 +1064,27 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.15" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", @@ -1093,9 +1093,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.139" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -1153,9 +1153,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.98" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -1164,9 +1164,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" dependencies = [ "rustix", "windows-sys 0.59.0", @@ -1204,9 +1204,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.43.0" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" dependencies = [ "backtrace", "parking_lot", @@ -1272,9 +1272,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "utf8parse" From 404ad74235cc69935204ea50497a80f9acce6aad Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Wed, 12 Mar 2025 20:54:04 +0100 Subject: [PATCH 3/9] core: deserialize biome list into Vec Only use IndexSet for deduplication while processing the biome; when deserializing, no deduplication is required, so using a Vec is faster (besides IndexSet missing non-serde support for bincode 2). --- src/core/common.rs | 3 +-- src/core/region_processor.rs | 12 ++++++++++-- src/core/tile_renderer.rs | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/core/common.rs b/src/core/common.rs index 8230a79..b25d7b8 100644 --- a/src/core/common.rs +++ b/src/core/common.rs @@ -8,7 +8,6 @@ use std::{ use anyhow::{Context, Result}; use clap::ValueEnum; -use indexmap::IndexSet; use regex::{Regex, RegexSet}; use serde::{Deserialize, Serialize}; @@ -102,7 +101,7 @@ pub struct ProcessedRegion { /// List of biomes used in the region /// /// Indexed by [ProcessedChunk] biome data - pub biome_list: IndexSet, + pub biome_list: Vec, /// Processed chunk data pub chunks: ChunkArray>>, } diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index e448f5e..1dc50fb 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -1,9 +1,11 @@ //! The [RegionProcessor] and related functions -use std::{ffi::OsStr, path::PathBuf, sync::mpsc, time::SystemTime}; +use std::{ffi::OsStr, mem, path::PathBuf, sync::mpsc, time::SystemTime}; use anyhow::{Context, Result}; use enum_map::{Enum, EnumMap}; +use indexmap::IndexSet; +use minedmap_resource::Biome; use rayon::prelude::*; use tracing::{debug, info, warn}; @@ -73,6 +75,8 @@ struct SingleRegionProcessor<'a> { lightmap_needed: bool, /// True if entity output file needs to be updated entities_needed: bool, + /// [IndexSet] of biomes used by the processed region + biome_list: IndexSet, /// Processed region intermediate data processed_region: ProcessedRegion, /// Lightmap intermediate data @@ -108,6 +112,7 @@ impl<'a> SingleRegionProcessor<'a> { let entities_needed = Some(input_timestamp) > entities_timestamp; let processed_region = ProcessedRegion::default(); + let biome_list = IndexSet::default(); let lightmap = image::GrayAlphaImage::new(N, N); let entities = ProcessedEntities::default(); @@ -127,6 +132,7 @@ impl<'a> SingleRegionProcessor<'a> { lightmap_needed, entities_needed, processed_region, + biome_list, lightmap, entities, image_format: processor.config.tile_image_format(), @@ -220,7 +226,7 @@ impl<'a> SingleRegionProcessor<'a> { biomes, block_light, depths, - }) = world::layer::top_layer(&mut self.processed_region.biome_list, &chunk) + }) = world::layer::top_layer(&mut self.biome_list, &chunk) .with_context(|| format!("Failed to process chunk {:?}", chunk_coords))? { if self.output_needed { @@ -291,6 +297,8 @@ impl<'a> SingleRegionProcessor<'a> { } } + self.processed_region.biome_list = mem::take(&mut self.biome_list).into_iter().collect(); + self.save_region()?; self.save_lightmap()?; self.save_entities()?; diff --git a/src/core/tile_renderer.rs b/src/core/tile_renderer.rs index 24af234..e47e20e 100644 --- a/src/core/tile_renderer.rs +++ b/src/core/tile_renderer.rs @@ -187,7 +187,7 @@ impl<'a> TileRenderer<'a> { for ((region_x, region_z, index), w) in weights.into_values() { let region = region_group.get(region_x, region_z)?; - let biome = region.biome_list.get_index(index.into())?; + let biome = region.biome_list.get(usize::from(index))?; total += w; color += w * block_color(block, Some(biome), depth.0 as f32); From 53a0f2460006b89413feb80858a03673f2746b68 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Wed, 12 Mar 2025 20:34:26 +0100 Subject: [PATCH 4/9] treewide: update to bincode 2 Consistently use bincode's Encode/Decode to avoid issues with incompatible serde features. Support for storing some temporary files as JSON is removed. The size of the "processed" directory is reduced by ~8% with the new default encoding of bincode 2. Performance is more or less unaffected. --- Cargo.lock | 31 ++++++++++++++++++++--- Cargo.toml | 2 +- crates/resource/Cargo.toml | 2 +- crates/resource/src/lib.rs | 49 +++++++++++++++++++++++++++++++----- crates/types/Cargo.toml | 2 +- crates/types/src/lib.rs | 6 ++--- src/core/common.rs | 14 ++++++----- src/core/entity_collector.rs | 25 +++++++++--------- src/core/metadata_writer.rs | 5 ++-- src/core/region_processor.rs | 2 -- src/core/tile_renderer.rs | 3 +-- src/io/storage.rs | 46 +++++++++++++-------------------- src/world/block_entity.rs | 11 ++++---- src/world/json_text.rs | 7 ++++-- src/world/layer.rs | 4 +-- src/world/sign.rs | 5 ++-- 16 files changed, 133 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0142276..5c5d477 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,11 +111,22 @@ dependencies = [ [[package]] name = "bincode" -version = "1.3.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" dependencies = [ + "bincode_derive", "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", ] [[package]] @@ -704,17 +715,17 @@ dependencies = [ name = "minedmap-resource" version = "0.6.0" dependencies = [ + "bincode", "enumflags2", "glam", - "serde", ] [[package]] name = "minedmap-types" version = "0.1.4" dependencies = [ + "bincode", "itertools", - "serde", ] [[package]] @@ -1276,6 +1287,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1288,6 +1305,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 812ad33..4109bfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ pre-release-replacements = [ [dependencies] anyhow = "1.0.68" -bincode = "1.3.3" +bincode = "2.0.1" clap = { version = "4.1.4", features = ["derive", "wrap_help"] } enum-map = "2.7.3" fastnbt = "2.3.2" diff --git a/crates/resource/Cargo.toml b/crates/resource/Cargo.toml index 07b6d1f..c654f40 100644 --- a/crates/resource/Cargo.toml +++ b/crates/resource/Cargo.toml @@ -8,6 +8,6 @@ readme.workspace = true repository.workspace = true [dependencies] +bincode = "2.0.1" enumflags2 = { version = "0.7.7", features = ["serde"] } glam = "0.30.0" -serde = { version = "1.0.183", features = ["derive"] } diff --git a/crates/resource/src/lib.rs b/crates/resource/src/lib.rs index a633d06..01b460b 100644 --- a/crates/resource/src/lib.rs +++ b/crates/resource/src/lib.rs @@ -10,13 +10,13 @@ mod legacy_block_types; use std::collections::HashMap; +use bincode::{BorrowDecode, Decode, Encode}; use enumflags2::{bitflags, BitFlags}; -use serde::{Deserialize, Serialize}; /// Flags describing special properties of [BlockType]s #[bitflags] #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum BlockFlag { /// The block type is opaque Opaque, @@ -38,14 +38,14 @@ pub enum BlockFlag { } /// An RGB color with u8 components -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)] pub struct Color(pub [u8; 3]); /// An RGB color with f32 components pub type Colorf = glam::Vec3; /// A block type specification -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy)] pub struct BlockColor { /// Bit set of [BlockFlag]s describing special properties of the block type pub flags: BitFlags, @@ -61,6 +61,43 @@ impl BlockColor { } } +impl Encode for BlockColor { + fn encode( + &self, + encoder: &mut E, + ) -> Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.flags.bits(), encoder)?; + bincode::Encode::encode(&self.color, encoder)?; + Ok(()) + } +} + +impl Decode for BlockColor { + fn decode>( + decoder: &mut D, + ) -> Result { + Ok(BlockColor { + flags: BitFlags::from_bits(bincode::Decode::decode(decoder)?).or(Err( + bincode::error::DecodeError::Other("invalid block flags"), + ))?, + color: bincode::Decode::decode(decoder)?, + }) + } +} + +impl<'de, Context> BorrowDecode<'de, Context> for BlockColor { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + Ok(BlockColor { + flags: BitFlags::from_bits(bincode::BorrowDecode::borrow_decode(decoder)?).or(Err( + bincode::error::DecodeError::Other("invalid block flags"), + ))?, + color: bincode::BorrowDecode::borrow_decode(decoder)?, + }) + } +} + /// A block type specification (for use in constants) #[derive(Debug, Clone)] struct ConstBlockType { @@ -137,7 +174,7 @@ impl BlockTypes { pub use block_color::{block_color, needs_biome}; /// Grass color modifier used by a biome -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Encode, Decode)] pub enum BiomeGrassColorModifier { /// Grass color modifier used by the dark forest biome DarkForest, @@ -149,7 +186,7 @@ pub enum BiomeGrassColorModifier { /// /// A Biome contains all information about a biome necessary to compute a block /// color given a block type and depth -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Encode, Decode)] pub struct Biome { /// Temperature value /// diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index c5bd47e..2b83c39 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -8,5 +8,5 @@ readme.workspace = true repository.workspace = true [dependencies] +bincode = "2.0.1" itertools = "0.14.0" -serde = { version = "1.0.183", features = ["derive"] } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index b4f12c2..e770f6a 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -8,8 +8,8 @@ use std::{ ops::{Index, IndexMut}, }; +use bincode::{Decode, Encode}; use itertools::iproduct; -use serde::{Deserialize, Serialize}; /// Const generic AXIS arguments for coordinate types pub mod axis { @@ -110,7 +110,7 @@ impl LayerBlockCoords { /// Generic array for data stored per block of a chunk layer /// /// Includes various convenient iteration functions. -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, Encode, Decode)] pub struct LayerBlockArray(pub [[T; BLOCKS_PER_CHUNK]; BLOCKS_PER_CHUNK]); impl Index for LayerBlockArray { @@ -196,7 +196,7 @@ impl Debug for ChunkCoords { /// Generic array for data stored per chunk of a region /// /// Includes various convenient iteration functions. -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, Encode, Decode)] pub struct ChunkArray(pub [[T; CHUNKS_PER_REGION]; CHUNKS_PER_REGION]); impl ChunkArray { diff --git a/src/core/common.rs b/src/core/common.rs index b25d7b8..06a3277 100644 --- a/src/core/common.rs +++ b/src/core/common.rs @@ -3,13 +3,15 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, + hash::Hash, path::{Path, PathBuf}, }; use anyhow::{Context, Result}; +use bincode::{Decode, Encode}; use clap::ValueEnum; use regex::{Regex, RegexSet}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use crate::{ io::fs::FileMetaVersion, @@ -24,7 +26,7 @@ use crate::{ /// /// Increase when the generation of processed regions from region data changes /// (usually because of updated resource data) -pub const REGION_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(5); +pub const REGION_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(6); /// MinedMap map tile data version number /// @@ -46,7 +48,7 @@ pub const MIPMAP_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(0); /// MinedMap processed entity data version number /// /// Increase when entity collection changes bacause of code changes. -pub const ENTITIES_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(1); +pub const ENTITIES_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(2); /// Coordinate pair of a generated tile /// @@ -85,7 +87,7 @@ impl TileCoordMap { } /// Data structure for storing chunk data between processing and rendering steps -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Encode, Decode)] pub struct ProcessedChunk { /// Block type data pub blocks: Box, @@ -96,7 +98,7 @@ pub struct ProcessedChunk { } /// Data structure for storing region data between processing and rendering steps -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Default, Encode, Decode)] pub struct ProcessedRegion { /// List of biomes used in the region /// @@ -107,7 +109,7 @@ pub struct ProcessedRegion { } /// Data structure for storing entity data between processing and collection steps -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Default, Encode, Decode)] pub struct ProcessedEntities { /// List of block entities pub block_entities: Vec, diff --git a/src/core/entity_collector.rs b/src/core/entity_collector.rs index 30b3a86..0d18090 100644 --- a/src/core/entity_collector.rs +++ b/src/core/entity_collector.rs @@ -78,23 +78,22 @@ impl<'a> EntityCollector<'a> { let mut output = ProcessedEntities::default(); for source_path in sources { - let mut source: ProcessedEntities = - match storage::read_file(source_path.as_ref(), storage::Format::Json) { - Ok(source) => source, - Err(err) => { - warn!( - "Failed to read entity data file {}: {:?}", - source_path.as_ref().display(), - err, - ); - continue; - } - }; + let mut source: ProcessedEntities = match storage::read_file(source_path.as_ref()) { + Ok(source) => source, + Err(err) => { + warn!( + "Failed to read entity data file {}: {:?}", + source_path.as_ref().display(), + err, + ); + continue; + } + }; output.block_entities.append(&mut source.block_entities); } - storage::write(file, &output, storage::Format::Json).context("Failed to write entity data") + storage::write(file, &output).context("Failed to write entity data") } /// Runs the mipmap generation diff --git a/src/core/metadata_writer.rs b/src/core/metadata_writer.rs index eb5f59f..40c5796 100644 --- a/src/core/metadata_writer.rs +++ b/src/core/metadata_writer.rs @@ -179,9 +179,8 @@ impl<'a> MetadataWriter<'a> { /// Generates [Entities] data from collected entity lists fn entities(&self) -> Result { - let data: ProcessedEntities = - storage::read_file(&self.config.entities_path_final, storage::Format::Json) - .context("Failed to read entity data file")?; + let data: ProcessedEntities = storage::read_file(&self.config.entities_path_final) + .context("Failed to read entity data file")?; let ret = Entities { signs: data diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index 1dc50fb..56f54ed 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -168,7 +168,6 @@ impl<'a> SingleRegionProcessor<'a> { storage::write_file( &self.output_path, &self.processed_region, - storage::Format::Bincode, REGION_FILE_META_VERSION, self.input_timestamp, ) @@ -207,7 +206,6 @@ impl<'a> SingleRegionProcessor<'a> { storage::write_file( &self.entities_path, &self.entities, - storage::Format::Json, ENTITIES_FILE_META_VERSION, self.input_timestamp, ) diff --git a/src/core/tile_renderer.rs b/src/core/tile_renderer.rs index e47e20e..e990a63 100644 --- a/src/core/tile_renderer.rs +++ b/src/core/tile_renderer.rs @@ -105,8 +105,7 @@ impl<'a> TileRenderer<'a> { region_loader .get_or_try_init(|| async { - storage::read_file(&processed_path, storage::Format::Bincode) - .context("Failed to load processed region data") + storage::read_file(&processed_path).context("Failed to load processed region data") }) .await .cloned() diff --git a/src/io/storage.rs b/src/io/storage.rs index 9296166..ae311de 100644 --- a/src/io/storage.rs +++ b/src/io/storage.rs @@ -10,28 +10,16 @@ use std::{ }; use anyhow::{Context, Result}; -use serde::{de::DeserializeOwned, Serialize}; +use bincode::{Decode, Encode}; use super::fs; -/// Storage format -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum Format { - /// Encode as Bincode - /// - /// Bincode is more efficient than JSON, but cannot handle many of - /// serde's features like flatten, conditional skipping, ... - Bincode, - /// Encode as JSON - Json, -} +/// Bincode configuration +const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard(); /// Serializes data and writes it to a writer -pub fn write(writer: &mut W, value: &T, format: Format) -> Result<()> { - let data = match format { - Format::Bincode => bincode::serialize(value)?, - Format::Json => serde_json::to_vec(value)?, - }; +pub fn write(writer: &mut W, value: &T) -> Result<()> { + let data = bincode::encode_to_vec(value, BINCODE_CONFIG)?; let len = u32::try_from(data.len())?; let compressed = zstd::bulk::compress(&data, 1)?; drop(data); @@ -45,18 +33,21 @@ pub fn write(writer: &mut W, value: &T, format: Format) /// Serializes data and stores it in a file /// /// A timestamp is stored in an assiciated metadata file. -pub fn write_file( +pub fn write_file( path: &Path, value: &T, - format: Format, version: fs::FileMetaVersion, timestamp: SystemTime, ) -> Result<()> { - fs::create_with_timestamp(path, version, timestamp, |file| write(file, value, format)) + fs::create_with_timestamp(path, version, timestamp, |file| write(file, value)) } /// Reads data from a reader and deserializes it -pub fn read(reader: &mut R, format: Format) -> Result { +pub fn read(reader: &mut R) -> Result +where + R: Read, + T: Decode<()>, +{ let mut len_buf = [0u8; 4]; reader.read_exact(&mut len_buf)?; let len = usize::try_from(u32::from_be_bytes(len_buf))?; @@ -66,18 +57,17 @@ pub fn read(reader: &mut R, format: Format) -> Res let data = zstd::bulk::decompress(&compressed, len)?; drop(compressed); - let value = match format { - Format::Bincode => bincode::deserialize(&data)?, - Format::Json => serde_json::from_slice(&data)?, - }; - Ok(value) + Ok(bincode::decode_from_slice(&data, BINCODE_CONFIG)?.0) } /// Reads data from a file and deserializes it -pub fn read_file(path: &Path, format: Format) -> Result { +pub fn read_file(path: &Path) -> Result +where + T: Decode<()>, +{ (|| -> Result { let mut file = File::open(path)?; - read(&mut file, format) + read(&mut file) })() .with_context(|| format!("Failed to read file {}", path.display())) } diff --git a/src/world/block_entity.rs b/src/world/block_entity.rs index 182ad50..6ad58a1 100644 --- a/src/world/block_entity.rs +++ b/src/world/block_entity.rs @@ -1,7 +1,8 @@ //! Processing of block entity data +use bincode::{Decode, Encode}; use minedmap_resource::{BlockFlag, BlockType}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::{ de, @@ -9,7 +10,7 @@ use super::{ }; /// Kind of sign block -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] #[serde(rename_all = "snake_case")] pub enum SignKind { /// Standing sign @@ -23,7 +24,7 @@ pub enum SignKind { } /// Processed sign data -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] pub struct Sign { /// The kind of the sign pub kind: SignKind, @@ -54,7 +55,7 @@ impl Sign { } /// Data for different kinds of [BlockEntity] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum BlockEntityData { /// A sign block @@ -62,7 +63,7 @@ pub enum BlockEntityData { } /// A processed block entity -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] pub struct BlockEntity { /// Global X coordinate pub x: i32, diff --git a/src/world/json_text.rs b/src/world/json_text.rs index fa18527..6a2d8ba 100644 --- a/src/world/json_text.rs +++ b/src/world/json_text.rs @@ -2,6 +2,7 @@ use std::{collections::VecDeque, fmt::Display}; +use bincode::{Decode, Encode}; use minedmap_resource::Color; use serde::{Deserialize, Serialize}; @@ -12,7 +13,9 @@ use serde::{Deserialize, Serialize}; /// is handled by [DeserializedText]. /// /// Formatting that is not set in a node is inherited from the parent. -#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] +#[derive( + Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Encode, Decode, +)] pub struct FormattedText { #[serde(default)] /// Text content @@ -84,7 +87,7 @@ impl From for FormattedTextTree { } /// List of [FormattedText] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Encode, Decode)] pub struct FormattedTextList(pub Vec); impl FormattedTextList { diff --git a/src/world/layer.rs b/src/world/layer.rs index e59593c..deb4b48 100644 --- a/src/world/layer.rs +++ b/src/world/layer.rs @@ -3,8 +3,8 @@ use std::num::NonZeroU16; use anyhow::{Context, Result}; +use bincode::{Decode, Encode}; use indexmap::IndexSet; -use serde::{Deserialize, Serialize}; use super::chunk::{Chunk, SectionIterItem}; use crate::{ @@ -13,7 +13,7 @@ use crate::{ }; /// Height (Y coordinate) of a block -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] pub struct BlockHeight(pub i32); impl BlockHeight { diff --git a/src/world/sign.rs b/src/world/sign.rs index eff319f..579f5b3 100644 --- a/src/world/sign.rs +++ b/src/world/sign.rs @@ -2,8 +2,9 @@ use std::fmt::Display; +use bincode::{Decode, Encode}; use minedmap_resource::Color; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::{ de, @@ -104,7 +105,7 @@ impl BlockEntitySignExt for de::BlockEntitySign { } } -#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Default, Serialize, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)] /// Deserialized and linearized sign text pub struct SignText(pub Vec); From 708fb9645dc89f6e454f3564b81f8e9a265457d7 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 20:27:51 +0100 Subject: [PATCH 5/9] core/region_processor: refactor Separate configuration and mutable state, also allowing to avoid a few mem::take(). --- src/core/region_processor.rs | 115 ++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 48 deletions(-) diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index 56f54ed..e638342 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -1,6 +1,6 @@ //! The [RegionProcessor] and related functions -use std::{ffi::OsStr, mem, path::PathBuf, sync::mpsc, time::SystemTime}; +use std::{ffi::OsStr, path::PathBuf, sync::mpsc, time::SystemTime}; use anyhow::{Context, Result}; use enum_map::{Enum, EnumMap}; @@ -45,6 +45,37 @@ enum RegionProcessorStatus { ErrorMissing, } +/// Data of a region being processed by a [SingleRegionProcessor] +#[derive(Debug)] +struct SingleRegionData { + /// [IndexSet] of biomes used by the processed region + biome_list: IndexSet, + /// Processed region chunk intermediate data + chunks: ChunkArray>>, + /// Lightmap intermediate data + lightmap: image::GrayAlphaImage, + /// Processed entity intermediate data + entities: ProcessedEntities, + /// True if any unknown block or biome types were encountered during processing + has_unknown: bool, +} + +impl Default for SingleRegionData { + fn default() -> Self { + /// Width/height of the region data + const N: u32 = (BLOCKS_PER_CHUNK * CHUNKS_PER_REGION) as u32; + + let lightmap = image::GrayAlphaImage::new(N, N); + Self { + biome_list: Default::default(), + chunks: Default::default(), + lightmap, + entities: Default::default(), + has_unknown: false, + } + } +} + /// Handles processing for a single region struct SingleRegionProcessor<'a> { /// Registry of known block types @@ -75,26 +106,13 @@ struct SingleRegionProcessor<'a> { lightmap_needed: bool, /// True if entity output file needs to be updated entities_needed: bool, - /// [IndexSet] of biomes used by the processed region - biome_list: IndexSet, - /// Processed region intermediate data - processed_region: ProcessedRegion, - /// Lightmap intermediate data - lightmap: image::GrayAlphaImage, - /// Processed entity intermediate data - entities: ProcessedEntities, /// Format of generated map tiles image_format: image::ImageFormat, - /// True if any unknown block or biome types were encountered during processing - has_unknown: bool, } impl<'a> SingleRegionProcessor<'a> { /// Initializes a [SingleRegionProcessor] fn new(processor: &'a RegionProcessor<'a>, coords: TileCoords) -> Result { - /// Width/height of the region data - const N: u32 = (BLOCKS_PER_CHUNK * CHUNKS_PER_REGION) as u32; - let input_path = processor.config.region_path(coords); let input_timestamp = fs::modified_timestamp(&input_path)?; @@ -111,11 +129,6 @@ impl<'a> SingleRegionProcessor<'a> { let lightmap_needed = Some(input_timestamp) > lightmap_timestamp; let entities_needed = Some(input_timestamp) > entities_timestamp; - let processed_region = ProcessedRegion::default(); - let biome_list = IndexSet::default(); - let lightmap = image::GrayAlphaImage::new(N, N); - let entities = ProcessedEntities::default(); - Ok(SingleRegionProcessor { block_types: &processor.block_types, biome_types: &processor.biome_types, @@ -131,12 +144,7 @@ impl<'a> SingleRegionProcessor<'a> { output_needed, lightmap_needed, entities_needed, - processed_region, - biome_list, - lightmap, - entities, image_format: processor.config.tile_image_format(), - has_unknown: false, }) } @@ -160,14 +168,14 @@ impl<'a> SingleRegionProcessor<'a> { /// Saves processed region data /// /// The timestamp is the time of the last modification of the input region data. - fn save_region(&self) -> Result<()> { + fn save_region(&self, processed_region: &ProcessedRegion) -> Result<()> { if !self.output_needed { return Ok(()); } storage::write_file( &self.output_path, - &self.processed_region, + processed_region, REGION_FILE_META_VERSION, self.input_timestamp, ) @@ -176,7 +184,7 @@ impl<'a> SingleRegionProcessor<'a> { /// Saves a lightmap tile /// /// The timestamp is the time of the last modification of the input region data. - fn save_lightmap(&self) -> Result<()> { + fn save_lightmap(&self, lightmap: &image::GrayAlphaImage) -> Result<()> { if !self.lightmap_needed { return Ok(()); } @@ -186,7 +194,7 @@ impl<'a> SingleRegionProcessor<'a> { LIGHTMAP_FILE_META_VERSION, self.input_timestamp, |file| { - self.lightmap + lightmap .write_to(file, self.image_format) .context("Failed to save image") }, @@ -196,27 +204,32 @@ impl<'a> SingleRegionProcessor<'a> { /// Saves processed entity data /// /// The timestamp is the time of the last modification of the input region data. - fn save_entities(&mut self) -> Result<()> { + fn save_entities(&self, entities: &mut ProcessedEntities) -> Result<()> { if !self.entities_needed { return Ok(()); } - self.entities.block_entities.sort_unstable(); + entities.block_entities.sort_unstable(); storage::write_file( &self.entities_path, - &self.entities, + entities, ENTITIES_FILE_META_VERSION, self.input_timestamp, ) } /// Processes a single chunk - fn process_chunk(&mut self, chunk_coords: ChunkCoords, data: world::de::Chunk) -> Result<()> { + fn process_chunk( + &self, + data: &mut SingleRegionData, + chunk_coords: ChunkCoords, + chunk_data: world::de::Chunk, + ) -> Result<()> { let (chunk, has_unknown) = - world::chunk::Chunk::new(&data, self.block_types, self.biome_types) + world::chunk::Chunk::new(&chunk_data, self.block_types, self.biome_types) .with_context(|| format!("Failed to decode chunk {:?}", chunk_coords))?; - self.has_unknown |= has_unknown; + data.has_unknown |= has_unknown; if self.output_needed || self.lightmap_needed { if let Some(layer::LayerData { @@ -224,11 +237,11 @@ impl<'a> SingleRegionProcessor<'a> { biomes, block_light, depths, - }) = world::layer::top_layer(&mut self.biome_list, &chunk) + }) = world::layer::top_layer(&mut data.biome_list, &chunk) .with_context(|| format!("Failed to process chunk {:?}", chunk_coords))? { if self.output_needed { - self.processed_region.chunks[chunk_coords] = Some(Box::new(ProcessedChunk { + data.chunks[chunk_coords] = Some(Box::new(ProcessedChunk { blocks, biomes, depths, @@ -237,7 +250,7 @@ impl<'a> SingleRegionProcessor<'a> { if self.lightmap_needed { let chunk_lightmap = Self::render_chunk_lightmap(block_light); - overlay_chunk(&mut self.lightmap, &chunk_lightmap, chunk_coords); + overlay_chunk(&mut data.lightmap, &chunk_lightmap, chunk_coords); } } } @@ -249,20 +262,21 @@ impl<'a> SingleRegionProcessor<'a> { chunk_coords, ) })?; - self.entities.block_entities.append(&mut block_entities); + data.entities.block_entities.append(&mut block_entities); } Ok(()) } /// Processes the chunks of the region - fn process_chunks(&mut self) -> Result<()> { - crate::nbt::region::from_file(&self.input_path)? - .foreach_chunk(|chunk_coords, data| self.process_chunk(chunk_coords, data)) + fn process_chunks(&self, data: &mut SingleRegionData) -> Result<()> { + crate::nbt::region::from_file(&self.input_path)?.foreach_chunk( + |chunk_coords, chunk_data| self.process_chunk(data, chunk_coords, chunk_data), + ) } /// Processes the region - fn run(mut self) -> Result { + fn run(&self) -> Result { if !self.output_needed && !self.lightmap_needed && !self.entities_needed { debug!( "Skipping unchanged region r.{}.{}.mca", @@ -276,7 +290,9 @@ impl<'a> SingleRegionProcessor<'a> { self.coords.x, self.coords.z ); - if let Err(err) = self.process_chunks() { + let mut data = SingleRegionData::default(); + + if let Err(err) = self.process_chunks(&mut data) { if self.output_timestamp.is_some() && self.lightmap_timestamp.is_some() && self.entities_timestamp.is_some() @@ -295,13 +311,16 @@ impl<'a> SingleRegionProcessor<'a> { } } - self.processed_region.biome_list = mem::take(&mut self.biome_list).into_iter().collect(); + let processed_region = ProcessedRegion { + biome_list: data.biome_list.into_iter().collect(), + chunks: data.chunks, + }; - self.save_region()?; - self.save_lightmap()?; - self.save_entities()?; + self.save_region(&processed_region)?; + self.save_lightmap(&data.lightmap)?; + self.save_entities(&mut data.entities)?; - Ok(if self.has_unknown { + Ok(if data.has_unknown { RegionProcessorStatus::OkWithUnknown } else { RegionProcessorStatus::Ok From 5c8568755441c8e91fc8bf2ea598a021a0bd5477 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 20:40:23 +0100 Subject: [PATCH 6/9] Clean up dependency features --- Cargo.lock | 2 -- Cargo.toml | 4 ++-- crates/resource/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c5d477..4f5bfa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -296,7 +296,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" dependencies = [ "enumflags2_derive", - "serde", ] [[package]] @@ -514,7 +513,6 @@ checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown", - "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4109bfd..da9f2ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ futures-util = "0.3.28" git-version = "0.3.5" humantime = "2.1.0" image = { version = "0.25.1", default-features = false, features = ["png", "webp"] } -indexmap = { version = "2.0.0", features = ["serde"] } +indexmap = "2.0.0" lru = "0.13.0" minedmap-default-alloc = { version = "0.1.0", path = "crates/default-alloc", optional = true } minedmap-nbt = { version = "0.1.1", path = "crates/nbt", default-features = false } @@ -60,7 +60,7 @@ phf = { version = "0.11.2", features = ["macros"] } rayon = "1.7.0" regex = "1.10.2" rustc-hash = "2.0.0" -serde = { version = "1.0.152", features = ["rc", "derive"] } +serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.99" tokio = { version = "1.31.0", features = ["rt", "parking_lot", "sync"] } tracing = "0.1.37" diff --git a/crates/resource/Cargo.toml b/crates/resource/Cargo.toml index c654f40..9fc157c 100644 --- a/crates/resource/Cargo.toml +++ b/crates/resource/Cargo.toml @@ -9,5 +9,5 @@ repository.workspace = true [dependencies] bincode = "2.0.1" -enumflags2 = { version = "0.7.7", features = ["serde"] } +enumflags2 = "0.7.7" glam = "0.30.0" From 7bba5bae558780a30e0c3218fc41ff770230b4cc Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 21:24:44 +0100 Subject: [PATCH 7/9] Update to Rust 2024 With bincode 2, we require rust 1.85 anyways, so we might as well upgrade, too. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index da9f2ca..0051203 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ members = ["crates/*"] [workspace.package] -edition = "2021" +edition = "2024" license = "MIT" readme = "README.md" repository = "https://github.com/neocturne/MinedMap" From 775fcb2d1b6adcd8aca3196ebdd1b2b0f16818cb Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 21:44:10 +0100 Subject: [PATCH 8/9] treewide: cargo fmt with 1.85 --- crates/nbt/src/region.rs | 4 ++-- crates/resource/src/lib.rs | 2 +- crates/types/src/lib.rs | 8 ++++---- src/core/mod.rs | 8 +++++--- src/core/tile_renderer.rs | 2 +- src/world/chunk.rs | 4 ++-- src/world/json_text.rs | 2 +- src/world/section.rs | 8 ++------ 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/crates/nbt/src/region.rs b/crates/nbt/src/region.rs index 8a52b9d..1325919 100644 --- a/crates/nbt/src/region.rs +++ b/crates/nbt/src/region.rs @@ -2,11 +2,11 @@ use std::{ fs::File, - io::{prelude::*, SeekFrom}, + io::{SeekFrom, prelude::*}, path::Path, }; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use flate2::read::ZlibDecoder; use serde::de::DeserializeOwned; diff --git a/crates/resource/src/lib.rs b/crates/resource/src/lib.rs index 01b460b..86cfb0f 100644 --- a/crates/resource/src/lib.rs +++ b/crates/resource/src/lib.rs @@ -11,7 +11,7 @@ mod legacy_block_types; use std::collections::HashMap; use bincode::{BorrowDecode, Decode, Encode}; -use enumflags2::{bitflags, BitFlags}; +use enumflags2::{BitFlags, bitflags}; /// Flags describing special properties of [BlockType]s #[bitflags] diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index e770f6a..f2dc0e1 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -48,10 +48,10 @@ macro_rules! coord_type { /// Returns an iterator over all possible values of the type #[inline] pub fn iter() -> impl DoubleEndedIterator> - + ExactSizeIterator - + FusedIterator - + Clone - + Debug { + + ExactSizeIterator + + FusedIterator + + Clone + + Debug { (0..Self::MAX as u8).map($t) } } diff --git a/src/core/mod.rs b/src/core/mod.rs index a16f620..fce2cb5 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -205,8 +205,10 @@ pub fn cli() -> Result<()> { if config.num_threads != config.num_threads_initial { pool = setup_threads(config.num_threads)?; } - pool.install(move || loop { - wait_watcher(&args, &watch_channel)?; - generate(&config, &rt)?; + pool.install(move || { + loop { + wait_watcher(&args, &watch_channel)?; + generate(&config, &rt)?; + } }) } diff --git a/src/core/tile_renderer.rs b/src/core/tile_renderer.rs index e990a63..9d7e188 100644 --- a/src/core/tile_renderer.rs +++ b/src/core/tile_renderer.rs @@ -16,7 +16,7 @@ use tracing::{debug, info}; use super::{common::*, region_group::RegionGroup}; use crate::{ io::{fs, storage}, - resource::{block_color, needs_biome, Colorf}, + resource::{Colorf, block_color, needs_biome}, types::*, util::coord_offset, }; diff --git a/src/world/chunk.rs b/src/world/chunk.rs index daee023..c4744d0 100644 --- a/src/world/chunk.rs +++ b/src/world/chunk.rs @@ -4,11 +4,11 @@ //! over different data versions as much as possible. use std::{ - collections::{btree_map, BTreeMap}, + collections::{BTreeMap, btree_map}, iter::{self, FusedIterator}, }; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use super::{block_entity::BlockEntity, de, section::*}; use crate::{ diff --git a/src/world/json_text.rs b/src/world/json_text.rs index 6a2d8ba..7d3ff3a 100644 --- a/src/world/json_text.rs +++ b/src/world/json_text.rs @@ -185,9 +185,9 @@ mod json_color { use minedmap_resource::Color; use serde::{ + Deserializer, Serializer, de::{self, Visitor}, ser::Error as _, - Deserializer, Serializer, }; /// Named JSON text colors diff --git a/src/world/section.rs b/src/world/section.rs index 845ddae..dc5c9a6 100644 --- a/src/world/section.rs +++ b/src/world/section.rs @@ -5,7 +5,7 @@ use std::fmt::Debug; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use num_integer::div_rem; use tracing::debug; @@ -400,10 +400,6 @@ impl<'a> BlockLight<'a> { let (offset, nibble) = div_rem(coords.offset(), 2); let byte = block_light[offset] as u8; - if nibble == 1 { - byte >> 4 - } else { - byte & 0xf - } + if nibble == 1 { byte >> 4 } else { byte & 0xf } } } From 5ee826a11b78905eeefdacfc78fced7592ec8962 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Mar 2025 21:52:35 +0100 Subject: [PATCH 9/9] ci: set fixed Rust version 1.85 Avoid CI failure due to new warnings or fmt changes in new Rust version. --- .github/workflows/MinedMap.yml | 11 +++++------ Dockerfile | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/MinedMap.yml b/.github/workflows/MinedMap.yml index 24895c9..16a3047 100644 --- a/.github/workflows/MinedMap.yml +++ b/.github/workflows/MinedMap.yml @@ -48,7 +48,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: stable + toolchain: '1.85' components: rustfmt - run: cargo fmt --all -- --check @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: stable + toolchain: '1.85' components: clippy - uses: swatinem/rust-cache@v2 - uses: actions-rs/clippy-check@v1 @@ -72,7 +72,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: stable + toolchain: '1.85' components: rust-docs - uses: swatinem/rust-cache@v2 - run: cargo doc --workspace --no-deps --document-private-items @@ -83,12 +83,11 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - rust: [stable] steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ matrix.rust }} + toolchain: '1.85' - uses: swatinem/rust-cache@v2 - run: cargo test --workspace - run: cargo test --workspace --no-default-features @@ -128,7 +127,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: - toolchain: stable + toolchain: '1.85' targets: '${{ matrix.target }}' - uses: swatinem/rust-cache@v2 diff --git a/Dockerfile b/Dockerfile index 0f0eecd..41d4c6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/rust:alpine AS builder +FROM docker.io/library/rust:1.85-alpine AS builder WORKDIR /build RUN apk add --no-cache build-base tini-static