From 21fd3a42ca29f7d6b1124799db1b79c34993a8c9 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Tue, 28 Feb 2023 00:44:49 +0100 Subject: [PATCH] io: add storage helper Use bincode+zstd to serialize data structures for temporary storage. --- Cargo.lock | 59 +++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/io/mod.rs | 1 + src/io/storage.rs | 22 ++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 src/io/storage.rs diff --git a/Cargo.lock b/Cargo.lock index 7d96fbc..5adcd72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -43,6 +52,9 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] [[package]] name = "cesu8" @@ -214,6 +226,15 @@ dependencies = [ "either", ] +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + [[package]] name = "libc" version = "0.2.139" @@ -231,6 +252,7 @@ name = "minedmap" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "bytemuck", "clap", "enumflags2", @@ -239,6 +261,7 @@ dependencies = [ "itertools", "num-integer", "serde", + "zstd", ] [[package]] @@ -281,6 +304,12 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -500,3 +529,33 @@ name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.4+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.7+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index b5ef027..6f40d32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ default-run = "minedmap" [dependencies] anyhow = "1.0.68" +bincode = "1.3.3" bytemuck = "1.13.0" clap = { version = "4.1.4", features = ["derive"] } enumflags2 = "0.7.5" @@ -17,3 +18,4 @@ flate2 = "1.0.25" itertools = "0.10.5" num-integer = "0.1.45" serde = "1.0.152" +zstd = "0.12.3" diff --git a/src/io/mod.rs b/src/io/mod.rs index f109d15..4098f3b 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -1,2 +1,3 @@ pub mod data; pub mod region; +pub mod storage; diff --git a/src/io/storage.rs b/src/io/storage.rs new file mode 100644 index 0000000..86430b3 --- /dev/null +++ b/src/io/storage.rs @@ -0,0 +1,22 @@ +use std::{ + fs::File, + io::{BufWriter, Write}, + path::Path, +}; + +use anyhow::{Context, Result}; +use serde::Serialize; + +pub fn write(path: &Path, value: &T) -> Result<()> { + (|| -> Result<()> { + let file = File::create(path)?; + let writer = BufWriter::new(file); + let mut compressor = zstd::Encoder::new(writer, 1)?; + bincode::serialize_into(&mut compressor, value)?; + let writer = compressor.finish()?; + let mut file = writer.into_inner()?; + file.flush()?; + Ok(()) + })() + .with_context(|| format!("Failed to write file {}", path.display())) +}