From 53a0f2460006b89413feb80858a03673f2746b68 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Wed, 12 Mar 2025 20:34:26 +0100 Subject: [PATCH] treewide: update to bincode 2 Consistently use bincode's Encode/Decode to avoid issues with incompatible serde features. Support for storing some temporary files as JSON is removed. The size of the "processed" directory is reduced by ~8% with the new default encoding of bincode 2. Performance is more or less unaffected. --- Cargo.lock | 31 ++++++++++++++++++++--- Cargo.toml | 2 +- crates/resource/Cargo.toml | 2 +- crates/resource/src/lib.rs | 49 +++++++++++++++++++++++++++++++----- crates/types/Cargo.toml | 2 +- crates/types/src/lib.rs | 6 ++--- src/core/common.rs | 14 ++++++----- src/core/entity_collector.rs | 25 +++++++++--------- src/core/metadata_writer.rs | 5 ++-- src/core/region_processor.rs | 2 -- src/core/tile_renderer.rs | 3 +-- src/io/storage.rs | 46 +++++++++++++-------------------- src/world/block_entity.rs | 11 ++++---- src/world/json_text.rs | 7 ++++-- src/world/layer.rs | 4 +-- src/world/sign.rs | 5 ++-- 16 files changed, 133 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0142276..5c5d477 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,11 +111,22 @@ dependencies = [ [[package]] name = "bincode" -version = "1.3.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" dependencies = [ + "bincode_derive", "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", ] [[package]] @@ -704,17 +715,17 @@ dependencies = [ name = "minedmap-resource" version = "0.6.0" dependencies = [ + "bincode", "enumflags2", "glam", - "serde", ] [[package]] name = "minedmap-types" version = "0.1.4" dependencies = [ + "bincode", "itertools", - "serde", ] [[package]] @@ -1276,6 +1287,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1288,6 +1305,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 812ad33..4109bfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ pre-release-replacements = [ [dependencies] anyhow = "1.0.68" -bincode = "1.3.3" +bincode = "2.0.1" clap = { version = "4.1.4", features = ["derive", "wrap_help"] } enum-map = "2.7.3" fastnbt = "2.3.2" diff --git a/crates/resource/Cargo.toml b/crates/resource/Cargo.toml index 07b6d1f..c654f40 100644 --- a/crates/resource/Cargo.toml +++ b/crates/resource/Cargo.toml @@ -8,6 +8,6 @@ readme.workspace = true repository.workspace = true [dependencies] +bincode = "2.0.1" enumflags2 = { version = "0.7.7", features = ["serde"] } glam = "0.30.0" -serde = { version = "1.0.183", features = ["derive"] } diff --git a/crates/resource/src/lib.rs b/crates/resource/src/lib.rs index a633d06..01b460b 100644 --- a/crates/resource/src/lib.rs +++ b/crates/resource/src/lib.rs @@ -10,13 +10,13 @@ mod legacy_block_types; use std::collections::HashMap; +use bincode::{BorrowDecode, Decode, Encode}; use enumflags2::{bitflags, BitFlags}; -use serde::{Deserialize, Serialize}; /// Flags describing special properties of [BlockType]s #[bitflags] #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum BlockFlag { /// The block type is opaque Opaque, @@ -38,14 +38,14 @@ pub enum BlockFlag { } /// An RGB color with u8 components -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)] pub struct Color(pub [u8; 3]); /// An RGB color with f32 components pub type Colorf = glam::Vec3; /// A block type specification -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy)] pub struct BlockColor { /// Bit set of [BlockFlag]s describing special properties of the block type pub flags: BitFlags, @@ -61,6 +61,43 @@ impl BlockColor { } } +impl Encode for BlockColor { + fn encode( + &self, + encoder: &mut E, + ) -> Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.flags.bits(), encoder)?; + bincode::Encode::encode(&self.color, encoder)?; + Ok(()) + } +} + +impl Decode for BlockColor { + fn decode>( + decoder: &mut D, + ) -> Result { + Ok(BlockColor { + flags: BitFlags::from_bits(bincode::Decode::decode(decoder)?).or(Err( + bincode::error::DecodeError::Other("invalid block flags"), + ))?, + color: bincode::Decode::decode(decoder)?, + }) + } +} + +impl<'de, Context> BorrowDecode<'de, Context> for BlockColor { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + Ok(BlockColor { + flags: BitFlags::from_bits(bincode::BorrowDecode::borrow_decode(decoder)?).or(Err( + bincode::error::DecodeError::Other("invalid block flags"), + ))?, + color: bincode::BorrowDecode::borrow_decode(decoder)?, + }) + } +} + /// A block type specification (for use in constants) #[derive(Debug, Clone)] struct ConstBlockType { @@ -137,7 +174,7 @@ impl BlockTypes { pub use block_color::{block_color, needs_biome}; /// Grass color modifier used by a biome -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Encode, Decode)] pub enum BiomeGrassColorModifier { /// Grass color modifier used by the dark forest biome DarkForest, @@ -149,7 +186,7 @@ pub enum BiomeGrassColorModifier { /// /// A Biome contains all information about a biome necessary to compute a block /// color given a block type and depth -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Encode, Decode)] pub struct Biome { /// Temperature value /// diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index c5bd47e..2b83c39 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -8,5 +8,5 @@ readme.workspace = true repository.workspace = true [dependencies] +bincode = "2.0.1" itertools = "0.14.0" -serde = { version = "1.0.183", features = ["derive"] } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index b4f12c2..e770f6a 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -8,8 +8,8 @@ use std::{ ops::{Index, IndexMut}, }; +use bincode::{Decode, Encode}; use itertools::iproduct; -use serde::{Deserialize, Serialize}; /// Const generic AXIS arguments for coordinate types pub mod axis { @@ -110,7 +110,7 @@ impl LayerBlockCoords { /// Generic array for data stored per block of a chunk layer /// /// Includes various convenient iteration functions. -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, Encode, Decode)] pub struct LayerBlockArray(pub [[T; BLOCKS_PER_CHUNK]; BLOCKS_PER_CHUNK]); impl Index for LayerBlockArray { @@ -196,7 +196,7 @@ impl Debug for ChunkCoords { /// Generic array for data stored per chunk of a region /// /// Includes various convenient iteration functions. -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, Encode, Decode)] pub struct ChunkArray(pub [[T; CHUNKS_PER_REGION]; CHUNKS_PER_REGION]); impl ChunkArray { diff --git a/src/core/common.rs b/src/core/common.rs index b25d7b8..06a3277 100644 --- a/src/core/common.rs +++ b/src/core/common.rs @@ -3,13 +3,15 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, + hash::Hash, path::{Path, PathBuf}, }; use anyhow::{Context, Result}; +use bincode::{Decode, Encode}; use clap::ValueEnum; use regex::{Regex, RegexSet}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use crate::{ io::fs::FileMetaVersion, @@ -24,7 +26,7 @@ use crate::{ /// /// Increase when the generation of processed regions from region data changes /// (usually because of updated resource data) -pub const REGION_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(5); +pub const REGION_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(6); /// MinedMap map tile data version number /// @@ -46,7 +48,7 @@ pub const MIPMAP_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(0); /// MinedMap processed entity data version number /// /// Increase when entity collection changes bacause of code changes. -pub const ENTITIES_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(1); +pub const ENTITIES_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(2); /// Coordinate pair of a generated tile /// @@ -85,7 +87,7 @@ impl TileCoordMap { } /// Data structure for storing chunk data between processing and rendering steps -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Encode, Decode)] pub struct ProcessedChunk { /// Block type data pub blocks: Box, @@ -96,7 +98,7 @@ pub struct ProcessedChunk { } /// Data structure for storing region data between processing and rendering steps -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Default, Encode, Decode)] pub struct ProcessedRegion { /// List of biomes used in the region /// @@ -107,7 +109,7 @@ pub struct ProcessedRegion { } /// Data structure for storing entity data between processing and collection steps -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Debug, Default, Encode, Decode)] pub struct ProcessedEntities { /// List of block entities pub block_entities: Vec, diff --git a/src/core/entity_collector.rs b/src/core/entity_collector.rs index 30b3a86..0d18090 100644 --- a/src/core/entity_collector.rs +++ b/src/core/entity_collector.rs @@ -78,23 +78,22 @@ impl<'a> EntityCollector<'a> { let mut output = ProcessedEntities::default(); for source_path in sources { - let mut source: ProcessedEntities = - match storage::read_file(source_path.as_ref(), storage::Format::Json) { - Ok(source) => source, - Err(err) => { - warn!( - "Failed to read entity data file {}: {:?}", - source_path.as_ref().display(), - err, - ); - continue; - } - }; + let mut source: ProcessedEntities = match storage::read_file(source_path.as_ref()) { + Ok(source) => source, + Err(err) => { + warn!( + "Failed to read entity data file {}: {:?}", + source_path.as_ref().display(), + err, + ); + continue; + } + }; output.block_entities.append(&mut source.block_entities); } - storage::write(file, &output, storage::Format::Json).context("Failed to write entity data") + storage::write(file, &output).context("Failed to write entity data") } /// Runs the mipmap generation diff --git a/src/core/metadata_writer.rs b/src/core/metadata_writer.rs index eb5f59f..40c5796 100644 --- a/src/core/metadata_writer.rs +++ b/src/core/metadata_writer.rs @@ -179,9 +179,8 @@ impl<'a> MetadataWriter<'a> { /// Generates [Entities] data from collected entity lists fn entities(&self) -> Result { - let data: ProcessedEntities = - storage::read_file(&self.config.entities_path_final, storage::Format::Json) - .context("Failed to read entity data file")?; + let data: ProcessedEntities = storage::read_file(&self.config.entities_path_final) + .context("Failed to read entity data file")?; let ret = Entities { signs: data diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index 1dc50fb..56f54ed 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -168,7 +168,6 @@ impl<'a> SingleRegionProcessor<'a> { storage::write_file( &self.output_path, &self.processed_region, - storage::Format::Bincode, REGION_FILE_META_VERSION, self.input_timestamp, ) @@ -207,7 +206,6 @@ impl<'a> SingleRegionProcessor<'a> { storage::write_file( &self.entities_path, &self.entities, - storage::Format::Json, ENTITIES_FILE_META_VERSION, self.input_timestamp, ) diff --git a/src/core/tile_renderer.rs b/src/core/tile_renderer.rs index e47e20e..e990a63 100644 --- a/src/core/tile_renderer.rs +++ b/src/core/tile_renderer.rs @@ -105,8 +105,7 @@ impl<'a> TileRenderer<'a> { region_loader .get_or_try_init(|| async { - storage::read_file(&processed_path, storage::Format::Bincode) - .context("Failed to load processed region data") + storage::read_file(&processed_path).context("Failed to load processed region data") }) .await .cloned() diff --git a/src/io/storage.rs b/src/io/storage.rs index 9296166..ae311de 100644 --- a/src/io/storage.rs +++ b/src/io/storage.rs @@ -10,28 +10,16 @@ use std::{ }; use anyhow::{Context, Result}; -use serde::{de::DeserializeOwned, Serialize}; +use bincode::{Decode, Encode}; use super::fs; -/// Storage format -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum Format { - /// Encode as Bincode - /// - /// Bincode is more efficient than JSON, but cannot handle many of - /// serde's features like flatten, conditional skipping, ... - Bincode, - /// Encode as JSON - Json, -} +/// Bincode configuration +const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard(); /// Serializes data and writes it to a writer -pub fn write(writer: &mut W, value: &T, format: Format) -> Result<()> { - let data = match format { - Format::Bincode => bincode::serialize(value)?, - Format::Json => serde_json::to_vec(value)?, - }; +pub fn write(writer: &mut W, value: &T) -> Result<()> { + let data = bincode::encode_to_vec(value, BINCODE_CONFIG)?; let len = u32::try_from(data.len())?; let compressed = zstd::bulk::compress(&data, 1)?; drop(data); @@ -45,18 +33,21 @@ pub fn write(writer: &mut W, value: &T, format: Format) /// Serializes data and stores it in a file /// /// A timestamp is stored in an assiciated metadata file. -pub fn write_file( +pub fn write_file( path: &Path, value: &T, - format: Format, version: fs::FileMetaVersion, timestamp: SystemTime, ) -> Result<()> { - fs::create_with_timestamp(path, version, timestamp, |file| write(file, value, format)) + fs::create_with_timestamp(path, version, timestamp, |file| write(file, value)) } /// Reads data from a reader and deserializes it -pub fn read(reader: &mut R, format: Format) -> Result { +pub fn read(reader: &mut R) -> Result +where + R: Read, + T: Decode<()>, +{ let mut len_buf = [0u8; 4]; reader.read_exact(&mut len_buf)?; let len = usize::try_from(u32::from_be_bytes(len_buf))?; @@ -66,18 +57,17 @@ pub fn read(reader: &mut R, format: Format) -> Res let data = zstd::bulk::decompress(&compressed, len)?; drop(compressed); - let value = match format { - Format::Bincode => bincode::deserialize(&data)?, - Format::Json => serde_json::from_slice(&data)?, - }; - Ok(value) + Ok(bincode::decode_from_slice(&data, BINCODE_CONFIG)?.0) } /// Reads data from a file and deserializes it -pub fn read_file(path: &Path, format: Format) -> Result { +pub fn read_file(path: &Path) -> Result +where + T: Decode<()>, +{ (|| -> Result { let mut file = File::open(path)?; - read(&mut file, format) + read(&mut file) })() .with_context(|| format!("Failed to read file {}", path.display())) } diff --git a/src/world/block_entity.rs b/src/world/block_entity.rs index 182ad50..6ad58a1 100644 --- a/src/world/block_entity.rs +++ b/src/world/block_entity.rs @@ -1,7 +1,8 @@ //! Processing of block entity data +use bincode::{Decode, Encode}; use minedmap_resource::{BlockFlag, BlockType}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::{ de, @@ -9,7 +10,7 @@ use super::{ }; /// Kind of sign block -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] #[serde(rename_all = "snake_case")] pub enum SignKind { /// Standing sign @@ -23,7 +24,7 @@ pub enum SignKind { } /// Processed sign data -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] pub struct Sign { /// The kind of the sign pub kind: SignKind, @@ -54,7 +55,7 @@ impl Sign { } /// Data for different kinds of [BlockEntity] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum BlockEntityData { /// A sign block @@ -62,7 +63,7 @@ pub enum BlockEntityData { } /// A processed block entity -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)] pub struct BlockEntity { /// Global X coordinate pub x: i32, diff --git a/src/world/json_text.rs b/src/world/json_text.rs index fa18527..6a2d8ba 100644 --- a/src/world/json_text.rs +++ b/src/world/json_text.rs @@ -2,6 +2,7 @@ use std::{collections::VecDeque, fmt::Display}; +use bincode::{Decode, Encode}; use minedmap_resource::Color; use serde::{Deserialize, Serialize}; @@ -12,7 +13,9 @@ use serde::{Deserialize, Serialize}; /// is handled by [DeserializedText]. /// /// Formatting that is not set in a node is inherited from the parent. -#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] +#[derive( + Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Encode, Decode, +)] pub struct FormattedText { #[serde(default)] /// Text content @@ -84,7 +87,7 @@ impl From for FormattedTextTree { } /// List of [FormattedText] -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Encode, Decode)] pub struct FormattedTextList(pub Vec); impl FormattedTextList { diff --git a/src/world/layer.rs b/src/world/layer.rs index e59593c..deb4b48 100644 --- a/src/world/layer.rs +++ b/src/world/layer.rs @@ -3,8 +3,8 @@ use std::num::NonZeroU16; use anyhow::{Context, Result}; +use bincode::{Decode, Encode}; use indexmap::IndexSet; -use serde::{Deserialize, Serialize}; use super::chunk::{Chunk, SectionIterItem}; use crate::{ @@ -13,7 +13,7 @@ use crate::{ }; /// Height (Y coordinate) of a block -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] pub struct BlockHeight(pub i32); impl BlockHeight { diff --git a/src/world/sign.rs b/src/world/sign.rs index eff319f..579f5b3 100644 --- a/src/world/sign.rs +++ b/src/world/sign.rs @@ -2,8 +2,9 @@ use std::fmt::Display; +use bincode::{Decode, Encode}; use minedmap_resource::Color; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::{ de, @@ -104,7 +105,7 @@ impl BlockEntitySignExt for de::BlockEntitySign { } } -#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Default, Serialize, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)] /// Deserialized and linearized sign text pub struct SignText(pub Vec);