treewide: update to bincode 2

Consistently use bincode's Encode/Decode to avoid issues with
incompatible serde features. Support for storing some temporary files as
JSON is removed.

The size of the "processed" directory is reduced by ~8% with the new
default encoding of bincode 2. Performance is more or less unaffected.
This commit is contained in:
Matthias Schiffer 2025-03-12 20:34:26 +01:00
parent 404ad74235
commit 53a0f24600
Signed by: neocturne
GPG key ID: 16EF3F64CB201D9C
16 changed files with 133 additions and 81 deletions

View file

@ -10,28 +10,16 @@ use std::{
};
use anyhow::{Context, Result};
use serde::{de::DeserializeOwned, Serialize};
use bincode::{Decode, Encode};
use super::fs;
/// Storage format
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Format {
/// Encode as Bincode
///
/// Bincode is more efficient than JSON, but cannot handle many of
/// serde's features like flatten, conditional skipping, ...
Bincode,
/// Encode as JSON
Json,
}
/// Bincode configuration
const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
/// Serializes data and writes it to a writer
pub fn write<W: Write, T: Serialize>(writer: &mut W, value: &T, format: Format) -> Result<()> {
let data = match format {
Format::Bincode => bincode::serialize(value)?,
Format::Json => serde_json::to_vec(value)?,
};
pub fn write<W: Write, T: Encode>(writer: &mut W, value: &T) -> Result<()> {
let data = bincode::encode_to_vec(value, BINCODE_CONFIG)?;
let len = u32::try_from(data.len())?;
let compressed = zstd::bulk::compress(&data, 1)?;
drop(data);
@ -45,18 +33,21 @@ pub fn write<W: Write, T: Serialize>(writer: &mut W, value: &T, format: Format)
/// Serializes data and stores it in a file
///
/// A timestamp is stored in an assiciated metadata file.
pub fn write_file<T: Serialize>(
pub fn write_file<T: Encode>(
path: &Path,
value: &T,
format: Format,
version: fs::FileMetaVersion,
timestamp: SystemTime,
) -> Result<()> {
fs::create_with_timestamp(path, version, timestamp, |file| write(file, value, format))
fs::create_with_timestamp(path, version, timestamp, |file| write(file, value))
}
/// Reads data from a reader and deserializes it
pub fn read<R: Read, T: DeserializeOwned>(reader: &mut R, format: Format) -> Result<T> {
pub fn read<R, T>(reader: &mut R) -> Result<T>
where
R: Read,
T: Decode<()>,
{
let mut len_buf = [0u8; 4];
reader.read_exact(&mut len_buf)?;
let len = usize::try_from(u32::from_be_bytes(len_buf))?;
@ -66,18 +57,17 @@ pub fn read<R: Read, T: DeserializeOwned>(reader: &mut R, format: Format) -> Res
let data = zstd::bulk::decompress(&compressed, len)?;
drop(compressed);
let value = match format {
Format::Bincode => bincode::deserialize(&data)?,
Format::Json => serde_json::from_slice(&data)?,
};
Ok(value)
Ok(bincode::decode_from_slice(&data, BINCODE_CONFIG)?.0)
}
/// Reads data from a file and deserializes it
pub fn read_file<T: DeserializeOwned>(path: &Path, format: Format) -> Result<T> {
pub fn read_file<T>(path: &Path) -> Result<T>
where
T: Decode<()>,
{
(|| -> Result<T> {
let mut file = File::open(path)?;
read(&mut file, format)
read(&mut file)
})()
.with_context(|| format!("Failed to read file {}", path.display()))
}