neocturne/MinedMap
1
0
Fork 0
mirror of https://github.com/neocturne/MinedMap.git synced 2025-04-12 00:05:07 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

treewide: update to bincode 2

Browse source 
Consistently use bincode's Encode/Decode to avoid issues with
incompatible serde features. Support for storing some temporary files as
JSON is removed.

The size of the "processed" directory is reduced by ~8% with the new
default encoding of bincode 2. Performance is more or less unaffected.
This commit is contained in:
Matthias Schiffer 2025-03-12 20:34:26 +01:00
parent 404ad74235
commit 53a0f24600
Signed by: neocturne
GPG key ID: 16EF3F64CB201D9C
16 changed files with 133 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
Cargo.lock generated
View file

@ -111,11 +111,22 @@ dependencies = [
[[package]]
name = "bincode"
version = "1.3.3"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]]
@ -704,17 +715,17 @@ dependencies = [
name = "minedmap-resource"
version = "0.6.0"
dependencies = [
"bincode",
"enumflags2",
"glam",
"serde",
]
[[package]]
name = "minedmap-types"
version = "0.1.4"
dependencies = [
"bincode",
"itertools",
"serde",
]
[[package]]
@ -1276,6 +1287,12 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "utf8parse"
version = "0.2.2"
@ -1288,6 +1305,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "walkdir"
version = "2.5.0"

2
Cargo.toml
View file

@ -38,7 +38,7 @@ pre-release-replacements = [
[dependencies]
anyhow = "1.0.68"
bincode = "1.3.3"
bincode = "2.0.1"
clap = { version = "4.1.4", features = ["derive", "wrap_help"] }
enum-map = "2.7.3"
fastnbt = "2.3.2"

2
crates/resource/Cargo.toml
View file

@ -8,6 +8,6 @@ readme.workspace = true
repository.workspace = true
[dependencies]
bincode = "2.0.1"
enumflags2 = { version = "0.7.7", features = ["serde"] }
glam = "0.30.0"
serde = { version = "1.0.183", features = ["derive"] }

49
crates/resource/src/lib.rs
View file

@ -10,13 +10,13 @@ mod legacy_block_types;
use std::collections::HashMap;
use bincode::{BorrowDecode, Decode, Encode};
use enumflags2::{bitflags, BitFlags};
use serde::{Deserialize, Serialize};
/// Flags describing special properties of [BlockType]s
#[bitflags]
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum BlockFlag {
/// The block type is opaque
Opaque,
@ -38,14 +38,14 @@ pub enum BlockFlag {
}
/// An RGB color with u8 components
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)]
pub struct Color(pub [u8; 3]);
/// An RGB color with f32 components
pub type Colorf = glam::Vec3;
/// A block type specification
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy)]
pub struct BlockColor {
/// Bit set of [BlockFlag]s describing special properties of the block type
pub flags: BitFlags<BlockFlag>,
@ -61,6 +61,43 @@ impl BlockColor {
}
}
impl Encode for BlockColor {
fn encode<E: bincode::enc::Encoder>(
&self,
encoder: &mut E,
) -> Result<(), bincode::error::EncodeError> {
bincode::Encode::encode(&self.flags.bits(), encoder)?;
bincode::Encode::encode(&self.color, encoder)?;
Ok(())
}
}
impl<Context> Decode<Context> for BlockColor {
fn decode<D: bincode::de::Decoder<Context = Context>>(
decoder: &mut D,
) -> Result<Self, bincode::error::DecodeError> {
Ok(BlockColor {
flags: BitFlags::from_bits(bincode::Decode::decode(decoder)?).or(Err(
bincode::error::DecodeError::Other("invalid block flags"),
))?,
color: bincode::Decode::decode(decoder)?,
})
}
}
impl<'de, Context> BorrowDecode<'de, Context> for BlockColor {
fn borrow_decode<D: bincode::de::BorrowDecoder<'de, Context = Context>>(
decoder: &mut D,
) -> Result<Self, bincode::error::DecodeError> {
Ok(BlockColor {
flags: BitFlags::from_bits(bincode::BorrowDecode::borrow_decode(decoder)?).or(Err(
bincode::error::DecodeError::Other("invalid block flags"),
))?,
color: bincode::BorrowDecode::borrow_decode(decoder)?,
})
}
}
/// A block type specification (for use in constants)
#[derive(Debug, Clone)]
struct ConstBlockType {
@ -137,7 +174,7 @@ impl BlockTypes {
pub use block_color::{block_color, needs_biome};
/// Grass color modifier used by a biome
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Encode, Decode)]
pub enum BiomeGrassColorModifier {
/// Grass color modifier used by the dark forest biome
DarkForest,
@ -149,7 +186,7 @@ pub enum BiomeGrassColorModifier {
///
/// A Biome contains all information about a biome necessary to compute a block
/// color given a block type and depth
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Encode, Decode)]
pub struct Biome {
/// Temperature value
///

2
crates/types/Cargo.toml
View file

@ -8,5 +8,5 @@ readme.workspace = true
repository.workspace = true
[dependencies]
bincode = "2.0.1"
itertools = "0.14.0"
serde = { version = "1.0.183", features = ["derive"] }

6
crates/types/src/lib.rs
View file

@ -8,8 +8,8 @@ use std::{
ops::{Index, IndexMut},
};
use bincode::{Decode, Encode};
use itertools::iproduct;
use serde::{Deserialize, Serialize};
/// Const generic AXIS arguments for coordinate types
pub mod axis {
@ -110,7 +110,7 @@ impl LayerBlockCoords {
/// Generic array for data stored per block of a chunk layer
///
/// Includes various convenient iteration functions.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, Default, Encode, Decode)]
pub struct LayerBlockArray<T>(pub [[T; BLOCKS_PER_CHUNK]; BLOCKS_PER_CHUNK]);
impl<T> Index<LayerBlockCoords> for LayerBlockArray<T> {
@ -196,7 +196,7 @@ impl Debug for ChunkCoords {
/// Generic array for data stored per chunk of a region
///
/// Includes various convenient iteration functions.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, Default, Encode, Decode)]
pub struct ChunkArray<T>(pub [[T; CHUNKS_PER_REGION]; CHUNKS_PER_REGION]);
impl<T> ChunkArray<T> {

14
src/core/common.rs
View file

@ -3,13 +3,15 @@
use std::{
collections::{BTreeMap, BTreeSet},
fmt::Debug,
hash::Hash,
path::{Path, PathBuf},
};
use anyhow::{Context, Result};
use bincode::{Decode, Encode};
use clap::ValueEnum;
use regex::{Regex, RegexSet};
use serde::{Deserialize, Serialize};
use serde::Serialize;
use crate::{
io::fs::FileMetaVersion,
@ -24,7 +26,7 @@ use crate::{
///
/// Increase when the generation of processed regions from region data changes
/// (usually because of updated resource data)
pub const REGION_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(5);
pub const REGION_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(6);
/// MinedMap map tile data version number
///
@ -46,7 +48,7 @@ pub const MIPMAP_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(0);
/// MinedMap processed entity data version number
///
/// Increase when entity collection changes bacause of code changes.
pub const ENTITIES_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(1);
pub const ENTITIES_FILE_META_VERSION: FileMetaVersion = FileMetaVersion(2);
/// Coordinate pair of a generated tile
///
@ -85,7 +87,7 @@ impl TileCoordMap {
}
/// Data structure for storing chunk data between processing and rendering steps
#[derive(Debug, Serialize, Deserialize)]
#[derive(Debug, Encode, Decode)]
pub struct ProcessedChunk {
/// Block type data
pub blocks: Box<layer::BlockArray>,
@ -96,7 +98,7 @@ pub struct ProcessedChunk {
}
/// Data structure for storing region data between processing and rendering steps
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Debug, Default, Encode, Decode)]
pub struct ProcessedRegion {
/// List of biomes used in the region
///
@ -107,7 +109,7 @@ pub struct ProcessedRegion {
}
/// Data structure for storing entity data between processing and collection steps
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Debug, Default, Encode, Decode)]
pub struct ProcessedEntities {
/// List of block entities
pub block_entities: Vec<BlockEntity>,

25
src/core/entity_collector.rs
View file

@ -78,23 +78,22 @@ impl<'a> EntityCollector<'a> {
let mut output = ProcessedEntities::default();
for source_path in sources {
let mut source: ProcessedEntities =
match storage::read_file(source_path.as_ref(), storage::Format::Json) {
Ok(source) => source,
Err(err) => {
warn!(
"Failed to read entity data file {}: {:?}",
source_path.as_ref().display(),
err,
);
continue;
}
};
let mut source: ProcessedEntities = match storage::read_file(source_path.as_ref()) {
Ok(source) => source,
Err(err) => {
warn!(
"Failed to read entity data file {}: {:?}",
source_path.as_ref().display(),
err,
);
continue;
}
};
output.block_entities.append(&mut source.block_entities);
}
storage::write(file, &output, storage::Format::Json).context("Failed to write entity data")
storage::write(file, &output).context("Failed to write entity data")
}
/// Runs the mipmap generation

5
src/core/metadata_writer.rs
View file

@ -179,9 +179,8 @@ impl<'a> MetadataWriter<'a> {
/// Generates [Entities] data from collected entity lists
fn entities(&self) -> Result<Entities> {
let data: ProcessedEntities =
storage::read_file(&self.config.entities_path_final, storage::Format::Json)
.context("Failed to read entity data file")?;
let data: ProcessedEntities = storage::read_file(&self.config.entities_path_final)
.context("Failed to read entity data file")?;
let ret = Entities {
signs: data

2
src/core/region_processor.rs
View file

@ -168,7 +168,6 @@ impl<'a> SingleRegionProcessor<'a> {
storage::write_file(
&self.output_path,
&self.processed_region,
storage::Format::Bincode,
REGION_FILE_META_VERSION,
self.input_timestamp,
)
@ -207,7 +206,6 @@ impl<'a> SingleRegionProcessor<'a> {
storage::write_file(
&self.entities_path,
&self.entities,
storage::Format::Json,
ENTITIES_FILE_META_VERSION,
self.input_timestamp,
)

3
src/core/tile_renderer.rs
View file

@ -105,8 +105,7 @@ impl<'a> TileRenderer<'a> {
region_loader
.get_or_try_init(|| async {
storage::read_file(&processed_path, storage::Format::Bincode)
.context("Failed to load processed region data")
storage::read_file(&processed_path).context("Failed to load processed region data")
})
.await
.cloned()

46
src/io/storage.rs
View file

@ -10,28 +10,16 @@ use std::{
};
use anyhow::{Context, Result};
use serde::{de::DeserializeOwned, Serialize};
use bincode::{Decode, Encode};
use super::fs;
/// Storage format
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Format {
/// Encode as Bincode
///
/// Bincode is more efficient than JSON, but cannot handle many of
/// serde's features like flatten, conditional skipping, ...
Bincode,
/// Encode as JSON
Json,
}
/// Bincode configuration
const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
/// Serializes data and writes it to a writer
pub fn write<W: Write, T: Serialize>(writer: &mut W, value: &T, format: Format) -> Result<()> {
let data = match format {
Format::Bincode => bincode::serialize(value)?,
Format::Json => serde_json::to_vec(value)?,
};
pub fn write<W: Write, T: Encode>(writer: &mut W, value: &T) -> Result<()> {
let data = bincode::encode_to_vec(value, BINCODE_CONFIG)?;
let len = u32::try_from(data.len())?;
let compressed = zstd::bulk::compress(&data, 1)?;
drop(data);
@ -45,18 +33,21 @@ pub fn write<W: Write, T: Serialize>(writer: &mut W, value: &T, format: Format)
/// Serializes data and stores it in a file
///
/// A timestamp is stored in an assiciated metadata file.
pub fn write_file<T: Serialize>(
pub fn write_file<T: Encode>(
path: &Path,
value: &T,
format: Format,
version: fs::FileMetaVersion,
timestamp: SystemTime,
) -> Result<()> {
fs::create_with_timestamp(path, version, timestamp, |file| write(file, value, format))
fs::create_with_timestamp(path, version, timestamp, |file| write(file, value))
}
/// Reads data from a reader and deserializes it
pub fn read<R: Read, T: DeserializeOwned>(reader: &mut R, format: Format) -> Result<T> {
pub fn read<R, T>(reader: &mut R) -> Result<T>
where
R: Read,
T: Decode<()>,
{
let mut len_buf = [0u8; 4];
reader.read_exact(&mut len_buf)?;
let len = usize::try_from(u32::from_be_bytes(len_buf))?;
@ -66,18 +57,17 @@ pub fn read<R: Read, T: DeserializeOwned>(reader: &mut R, format: Format) -> Res
let data = zstd::bulk::decompress(&compressed, len)?;
drop(compressed);
let value = match format {
Format::Bincode => bincode::deserialize(&data)?,
Format::Json => serde_json::from_slice(&data)?,
};
Ok(value)
Ok(bincode::decode_from_slice(&data, BINCODE_CONFIG)?.0)
}
/// Reads data from a file and deserializes it
pub fn read_file<T: DeserializeOwned>(path: &Path, format: Format) -> Result<T> {
pub fn read_file<T>(path: &Path) -> Result<T>
where
T: Decode<()>,
{
(|| -> Result<T> {
let mut file = File::open(path)?;
read(&mut file, format)
read(&mut file)
})()
.with_context(|| format!("Failed to read file {}", path.display()))
}

11
src/world/block_entity.rs
View file

@ -1,7 +1,8 @@
//! Processing of block entity data
use bincode::{Decode, Encode};
use minedmap_resource::{BlockFlag, BlockType};
use serde::{Deserialize, Serialize};
use serde::Serialize;
use super::{
de,
@ -9,7 +10,7 @@ use super::{
};
/// Kind of sign block
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SignKind {
/// Standing sign
@ -23,7 +24,7 @@ pub enum SignKind {
}
/// Processed sign data
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)]
pub struct Sign {
/// The kind of the sign
pub kind: SignKind,
@ -54,7 +55,7 @@ impl Sign {
}
/// Data for different kinds of [BlockEntity]
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BlockEntityData {
/// A sign block
@ -62,7 +63,7 @@ pub enum BlockEntityData {
}
/// A processed block entity
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Serialize)]
pub struct BlockEntity {
/// Global X coordinate
pub x: i32,

7
src/world/json_text.rs
View file

@ -2,6 +2,7 @@
use std::{collections::VecDeque, fmt::Display};
use bincode::{Decode, Encode};
use minedmap_resource::Color;
use serde::{Deserialize, Serialize};
@ -12,7 +13,9 @@ use serde::{Deserialize, Serialize};
/// is handled by [DeserializedText].
///
/// Formatting that is not set in a node is inherited from the parent.
#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)]
#[derive(
Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Encode, Decode,
)]
pub struct FormattedText {
#[serde(default)]
/// Text content
@ -84,7 +87,7 @@ impl From<String> for FormattedTextTree {
}
/// List of [FormattedText]
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Encode, Decode)]
pub struct FormattedTextList(pub Vec<FormattedText>);
impl FormattedTextList {

4
src/world/layer.rs
View file

@ -3,8 +3,8 @@
use std::num::NonZeroU16;
use anyhow::{Context, Result};
use bincode::{Decode, Encode};
use indexmap::IndexSet;
use serde::{Deserialize, Serialize};
use super::chunk::{Chunk, SectionIterItem};
use crate::{
@ -13,7 +13,7 @@ use crate::{
};
/// Height (Y coordinate) of a block
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)]
pub struct BlockHeight(pub i32);
impl BlockHeight {

5
src/world/sign.rs
View file

@ -2,8 +2,9 @@
use std::fmt::Display;
use bincode::{Decode, Encode};
use minedmap_resource::Color;
use serde::{Deserialize, Serialize};
use serde::Serialize;
use super::{
de,
@ -104,7 +105,7 @@ impl BlockEntitySignExt for de::BlockEntitySign {
}
}
#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, Default, Serialize, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
/// Deserialized and linearized sign text
pub struct SignText(pub Vec<FormattedTextList>);