From 84bee6d6d99baca742675613e04e2bf9ccd6e3c9 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sat, 5 Aug 2023 20:13:35 +0200 Subject: [PATCH] minedmap: add region cache Cache the last loaded processed regions. --- Cargo.lock | 37 ++++++++++++++++++++++++++ Cargo.toml | 3 ++- src/bin/minedmap/main.rs | 8 ++++-- src/bin/minedmap/tile_renderer.rs | 43 +++++++++++++++++++++++-------- 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1bfe6e..839c9ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,23 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "anstream" version = "0.3.2" @@ -291,6 +308,10 @@ name = "hashbrown" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -386,6 +407,15 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +[[package]] +name = "lru" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eedb2bdbad7e0634f83989bf596f497b070130daaa398ab22d84c39e266deec5" +dependencies = [ + "hashbrown", +] + [[package]] name = "minedmap" version = "0.1.0" @@ -401,6 +431,7 @@ dependencies = [ "image", "indexmap", "itertools", + "lru", "num-integer", "rustc-hash", "serde", @@ -591,6 +622,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index c74b56a..5d74ce8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,10 @@ glam = "0.24.0" image = { version = "0.24.5", default-features = false, features = ["png"] } indexmap = { version = "2.0.0", features = ["serde"] } itertools = "0.11.0" +lru = "0.11.0" num-integer = "0.1.45" rustc-hash = "1.1.0" -serde = "1.0.152" +serde = { version = "1.0.152", features = ["rc"] } serde_json = "1.0.99" zstd = "0.12.3" diff --git a/src/bin/minedmap/main.rs b/src/bin/minedmap/main.rs index 8d3bbea..838728c 100644 --- a/src/bin/minedmap/main.rs +++ b/src/bin/minedmap/main.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; use anyhow::Result; use clap::Parser; -use common::Config; +use common::{Config, TileCoords}; use metadata_writer::MetadataWriter; use region_processor::RegionProcessor; use tile_mipmapper::TileMipmapper; @@ -28,7 +28,11 @@ fn main() -> Result<()> { let args = Args::parse(); let config = Config::new(args); - let regions = RegionProcessor::new(&config).run()?; + let mut regions = RegionProcessor::new(&config).run()?; + + // Sort regions in a zig-zag pattern to optimize cache usage + regions.sort_unstable_by_key(|&TileCoords { x, z }| (x, if x % 2 == 0 { z } else { -z })); + TileRenderer::new(&config).run(®ions)?; let tiles = TileMipmapper::new(&config).run(®ions)?; MetadataWriter::new(&config).run(tiles)?; diff --git a/src/bin/minedmap/tile_renderer.rs b/src/bin/minedmap/tile_renderer.rs index 646e098..abacffd 100644 --- a/src/bin/minedmap/tile_renderer.rs +++ b/src/bin/minedmap/tile_renderer.rs @@ -1,10 +1,13 @@ use std::{ + num::NonZeroUsize, path::{Path, PathBuf}, + rc::Rc, time::SystemTime, }; use anyhow::{Context, Result}; use glam::Vec3; +use lru::LruCache; use num_integer::div_mod_floor; use minedmap::{ @@ -39,7 +42,7 @@ fn coord_offset( } fn biome_at( - region_group: &RegionGroup>, + region_group: &RegionGroup>, chunk: ChunkCoords, block: LayerBlockCoords, dx: i32, @@ -72,18 +75,30 @@ impl<'a> TileRenderer<'a> { TileRenderer { config } } - fn load_region(processed_path: &Path) -> Result> { - storage::read(processed_path).context("Failed to load processed region data") + fn load_region( + region_cache: &mut LruCache>, + processed_path: &Path, + ) -> Result> { + if let Some(region) = region_cache.get(processed_path) { + return Ok(region.clone()); + } + + let region: Rc = + storage::read(processed_path).context("Failed to load processed region data")?; + region_cache.put(processed_path.to_owned(), region.clone()); + + Ok(region) } fn load_region_group( + region_cache: &mut LruCache>, processed_paths: RegionGroup, - ) -> Result>> { - processed_paths.try_map(|path| Self::load_region(&path)) + ) -> Result>> { + processed_paths.try_map(|path| Self::load_region(region_cache, &path)) } fn block_color_at( - region_group: &RegionGroup>, + region_group: &RegionGroup>, chunk: &ProcessedChunk, chunk_coords: ChunkCoords, block_coords: LayerBlockCoords, @@ -141,7 +156,7 @@ impl<'a> TileRenderer<'a> { fn render_chunk( image: &mut image::RgbaImage, - region_group: &RegionGroup>, + region_group: &RegionGroup>, chunk: &ProcessedChunk, chunk_coords: ChunkCoords, ) { @@ -164,7 +179,7 @@ impl<'a> TileRenderer<'a> { fn render_region( image: &mut image::RgbaImage, - region_group: &RegionGroup>, + region_group: &RegionGroup>, ) { for (coords, chunk) in region_group.center().chunks.iter() { let Some(chunk) = chunk else { @@ -200,7 +215,11 @@ impl<'a> TileRenderer<'a> { Ok((paths, max_timestamp)) } - fn render_tile(&self, coords: TileCoords) -> Result<()> { + fn render_tile( + &self, + region_cache: &mut LruCache>, + coords: TileCoords, + ) -> Result<()> { const N: u32 = (BLOCKS_PER_CHUNK * CHUNKS_PER_REGION) as u32; let (processed_paths, processed_timestamp) = self.processed_sources(coords)?; @@ -227,7 +246,7 @@ impl<'a> TileRenderer<'a> { .display(), ); - let region_group = Self::load_region_group(processed_paths) + let region_group = Self::load_region_group(region_cache, processed_paths) .with_context(|| format!("Region {:?} from previous step must be loadable", coords))?; let mut image = image::RgbaImage::new(N, N); Self::render_region(&mut image, ®ion_group); @@ -247,8 +266,10 @@ impl<'a> TileRenderer<'a> { pub fn run(self, regions: &[TileCoords]) -> Result<()> { fs::create_dir_all(&self.config.tile_dir(TileKind::Map, 0))?; + let mut region_cache = LruCache::new(NonZeroUsize::new(12).unwrap()); + for &coords in regions { - if let Err(err) = self.render_tile(coords) { + if let Err(err) = self.render_tile(&mut region_cache, coords) { eprintln!("Failed to render tile {:?}: {:?}", coords, err); } }