neocturne/MinedMap
1
0
Fork 0
mirror of https://github.com/neocturne/MinedMap.git synced 2025-03-04 17:23:33 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

io/storage: compress/decompress data in bulk

Browse source 
The bulk API is signifiantly faster than the stream API.
This commit is contained in:
Matthias Schiffer 2023-03-03 20:58:35 +01:00
parent fa04e3e94a
commit a4b726992a
Signed by: neocturne
GPG key ID: 16EF3F64CB201D9C
1 changed files with 22 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/io/storage.rs
View file

@ -1,6 +1,6 @@
use std::{
fs::File,
io::{BufReader, BufWriter, Write},
io::{Read, Write},
path::Path,
};
@ -9,13 +9,16 @@ use serde::{de::DeserializeOwned, Serialize};
pub fn write<T: Serialize>(path: &Path, value: &T) -> Result<()> {
(|| -> Result<()> {
let file = File::create(path)?;
let writer = BufWriter::new(file);
let mut compressor = zstd::Encoder::new(writer, 1)?;
bincode::serialize_into(&mut compressor, value)?;
let writer = compressor.finish()?;
let mut file = writer.into_inner()?;
let data = bincode::serialize(value)?;
let len = u32::try_from(data.len())?;
let compressed = zstd::bulk::compress(&data, 1)?;
drop(data);
let mut file = File::create(path)?;
file.write_all(&len.to_be_bytes())?;
file.write_all(&compressed)?;
file.flush()?;
Ok(())
})()
.with_context(|| format!("Failed to write file {}", path.display()))
@ -23,10 +26,18 @@ pub fn write<T: Serialize>(path: &Path, value: &T) -> Result<()> {
pub fn read<T: DeserializeOwned>(path: &Path) -> Result<T> {
(|| -> Result<T> {
let file = File::open(path)?;
let reader = BufReader::new(file);
let decompressor = zstd::Decoder::new(reader)?;
Ok(bincode::deserialize_from(decompressor)?)
let mut file = File::open(path)?;
let mut len_buf = [0u8; 4];
file.read_exact(&mut len_buf)?;
let len = usize::try_from(u32::from_be_bytes(len_buf))?;
let mut compressed = vec![];
file.read_to_end(&mut compressed)?;
let data = zstd::bulk::decompress(&compressed, len)?;
drop(compressed);
Ok(bincode::deserialize(&data)?)
})()
.with_context(|| format!("Failed to read file {}", path.display()))
}