diff --git a/src/core/region_processor.rs b/src/core/region_processor.rs index 04699d3..52cb165 100644 --- a/src/core/region_processor.rs +++ b/src/core/region_processor.rs @@ -148,6 +148,7 @@ impl<'a> SingleRegionProcessor<'a> { storage::write_file( &self.output_path, &self.processed_region, + storage::Format::Bincode, REGION_FILE_META_VERSION, self.input_timestamp, ) diff --git a/src/core/tile_renderer.rs b/src/core/tile_renderer.rs index ce9fed3..09ad8a1 100644 --- a/src/core/tile_renderer.rs +++ b/src/core/tile_renderer.rs @@ -105,7 +105,8 @@ impl<'a> TileRenderer<'a> { region_loader .get_or_try_init(|| async { - storage::read_file(&processed_path).context("Failed to load processed region data") + storage::read_file(&processed_path, storage::Format::Bincode) + .context("Failed to load processed region data") }) .await .cloned() diff --git a/src/io/storage.rs b/src/io/storage.rs index 634f082..9296166 100644 --- a/src/io/storage.rs +++ b/src/io/storage.rs @@ -14,9 +14,24 @@ use serde::{de::DeserializeOwned, Serialize}; use super::fs; +/// Storage format +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Format { + /// Encode as Bincode + /// + /// Bincode is more efficient than JSON, but cannot handle many of + /// serde's features like flatten, conditional skipping, ... + Bincode, + /// Encode as JSON + Json, +} + /// Serializes data and writes it to a writer -pub fn write(writer: &mut W, value: &T) -> Result<()> { - let data = bincode::serialize(value)?; +pub fn write(writer: &mut W, value: &T, format: Format) -> Result<()> { + let data = match format { + Format::Bincode => bincode::serialize(value)?, + Format::Json => serde_json::to_vec(value)?, + }; let len = u32::try_from(data.len())?; let compressed = zstd::bulk::compress(&data, 1)?; drop(data); @@ -33,14 +48,15 @@ pub fn write(writer: &mut W, value: &T) -> Result<()> { pub fn write_file( path: &Path, value: &T, + format: Format, version: fs::FileMetaVersion, timestamp: SystemTime, ) -> Result<()> { - fs::create_with_timestamp(path, version, timestamp, |file| write(file, value)) + fs::create_with_timestamp(path, version, timestamp, |file| write(file, value, format)) } /// Reads data from a reader and deserializes it -pub fn read(reader: &mut R) -> Result { +pub fn read(reader: &mut R, format: Format) -> Result { let mut len_buf = [0u8; 4]; reader.read_exact(&mut len_buf)?; let len = usize::try_from(u32::from_be_bytes(len_buf))?; @@ -50,14 +66,18 @@ pub fn read(reader: &mut R) -> Result { let data = zstd::bulk::decompress(&compressed, len)?; drop(compressed); - Ok(bincode::deserialize(&data)?) + let value = match format { + Format::Bincode => bincode::deserialize(&data)?, + Format::Json => serde_json::from_slice(&data)?, + }; + Ok(value) } /// Reads data from a file and deserializes it -pub fn read_file(path: &Path) -> Result { +pub fn read_file(path: &Path, format: Format) -> Result { (|| -> Result { let mut file = File::open(path)?; - read(&mut file) + read(&mut file, format) })() .with_context(|| format!("Failed to read file {}", path.display())) }