io/storage: split out inner part of read/write

Allow reusing the read/write logic without requiring control over the file
open.
This commit is contained in:
Matthias Schiffer 2023-11-26 12:47:31 +01:00
parent f0e0db63d3
commit 5d40d061a4
Signed by: neocturne
GPG key ID: 16EF3F64CB201D9C
3 changed files with 31 additions and 24 deletions

View file

@ -145,7 +145,7 @@ impl<'a> SingleRegionProcessor<'a> {
return Ok(());
}
storage::write(
storage::write_file(
&self.output_path,
&self.processed_region,
REGION_FILE_META_VERSION,

View file

@ -105,7 +105,7 @@ impl<'a> TileRenderer<'a> {
region_loader
.get_or_try_init(|| async {
storage::read(&processed_path).context("Failed to load processed region data")
storage::read_file(&processed_path).context("Failed to load processed region data")
})
.await
.cloned()

View file

@ -14,43 +14,50 @@ use serde::{de::DeserializeOwned, Serialize};
use super::fs;
/// Serializes data and writes it to a writer
pub fn write<W: Write, T: Serialize>(writer: &mut W, value: &T) -> Result<()> {
let data = bincode::serialize(value)?;
let len = u32::try_from(data.len())?;
let compressed = zstd::bulk::compress(&data, 1)?;
drop(data);
writer.write_all(&len.to_be_bytes())?;
writer.write_all(&compressed)?;
Ok(())
}
/// Serializes data and stores it in a file
///
/// A timestamp is stored in an assiciated metadata file.
pub fn write<T: Serialize>(
pub fn write_file<T: Serialize>(
path: &Path,
value: &T,
version: fs::FileMetaVersion,
timestamp: SystemTime,
) -> Result<()> {
fs::create_with_timestamp(path, version, timestamp, |file| {
let data = bincode::serialize(value)?;
let len = u32::try_from(data.len())?;
let compressed = zstd::bulk::compress(&data, 1)?;
drop(data);
fs::create_with_timestamp(path, version, timestamp, |file| write(file, value))
}
file.write_all(&len.to_be_bytes())?;
file.write_all(&compressed)?;
/// Reads data from a reader and deserializes it
pub fn read<R: Read, T: DeserializeOwned>(reader: &mut R) -> Result<T> {
let mut len_buf = [0u8; 4];
reader.read_exact(&mut len_buf)?;
let len = usize::try_from(u32::from_be_bytes(len_buf))?;
Ok(())
})
let mut compressed = vec![];
reader.read_to_end(&mut compressed)?;
let data = zstd::bulk::decompress(&compressed, len)?;
drop(compressed);
Ok(bincode::deserialize(&data)?)
}
/// Reads data from a file and deserializes it
pub fn read<T: DeserializeOwned>(path: &Path) -> Result<T> {
pub fn read_file<T: DeserializeOwned>(path: &Path) -> Result<T> {
(|| -> Result<T> {
let mut file = File::open(path)?;
let mut len_buf = [0u8; 4];
file.read_exact(&mut len_buf)?;
let len = usize::try_from(u32::from_be_bytes(len_buf))?;
let mut compressed = vec![];
file.read_to_end(&mut compressed)?;
let data = zstd::bulk::decompress(&compressed, len)?;
drop(compressed);
Ok(bincode::deserialize(&data)?)
read(&mut file)
})()
.with_context(|| format!("Failed to read file {}", path.display()))
}