mirror of
https://github.com/neocturne/MinedMap.git
synced 2025-03-04 17:23:33 +01:00
minedmap/tile_renderer: async region cache
Prepare for sharing the region cache between multiple threads by making lookup/load async.
This commit is contained in:
parent
dcc9e6e794
commit
b5980b82af
GPG key ID:
16EF3F64CB201D9C
4 changed files with 165 additions and 39 deletions
111
Cargo.lock
generated
111
Cargo.lock
generated
|
|
@ -2,6 +2,15 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler"
|
||||
version = "1.0.2"
|
||||
|
|
@ -86,6 +95,21 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
|
|
@ -377,6 +401,12 @@ dependencies = [
|
|||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.27.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
|
||||
|
||||
[[package]]
|
||||
name = "glam"
|
||||
version = "0.24.1"
|
||||
|
|
@ -487,6 +517,16 @@ version = "0.4.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.11.0"
|
||||
|
|
@ -496,6 +536,12 @@ dependencies = [
|
|||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
|
|
@ -528,6 +574,7 @@ dependencies = [
|
|||
"rustc-hash",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
|
|
@ -581,12 +628,44 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.31.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.12"
|
||||
|
|
@ -658,6 +737,21 @@ dependencies = [
|
|||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
|
|
@ -744,6 +838,12 @@ dependencies = [
|
|||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
|
|
@ -761,6 +861,17 @@ dependencies = [
|
|||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40de3a2ba249dcb097e01be5e67a5ff53cf250397715a071a81543e8a832a920"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.11"
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ rayon = "1.7.0"
|
|||
rustc-hash = "1.1.0"
|
||||
serde = { version = "1.0.152", features = ["rc"] }
|
||||
serde_json = "1.0.99"
|
||||
tokio = { version = "1.31.0", features = ["rt", "parking_lot", "sync"] }
|
||||
zstd = "0.12.3"
|
||||
|
||||
[features]
|
||||
|
|
|
|||
|
|
@ -43,8 +43,12 @@ fn main() -> Result<()> {
|
|||
|
||||
setup_threads(config.num_threads)?;
|
||||
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let regions = RegionProcessor::new(&config).run()?;
|
||||
TileRenderer::new(&config).run(®ions)?;
|
||||
TileRenderer::new(&config, &rt).run(®ions)?;
|
||||
let tiles = TileMipmapper::new(&config).run(®ions)?;
|
||||
MetadataWriter::new(&config).run(tiles)?;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
use std::{
|
||||
num::NonZeroUsize,
|
||||
path::{Path, PathBuf},
|
||||
rc::Rc,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use glam::Vec3;
|
||||
use lru::LruCache;
|
||||
use tokio::sync::OnceCell;
|
||||
|
||||
use minedmap::{
|
||||
io::{fs, storage},
|
||||
|
|
@ -18,8 +19,10 @@ use minedmap::{
|
|||
|
||||
use super::{common::*, region_group::RegionGroup};
|
||||
|
||||
type RegionRef = Arc<ProcessedRegion>;
|
||||
|
||||
fn biome_at(
|
||||
region_group: &RegionGroup<Rc<ProcessedRegion>>,
|
||||
region_group: &RegionGroup<RegionRef>,
|
||||
chunk: ChunkCoords,
|
||||
block: LayerBlockCoords,
|
||||
dx: i32,
|
||||
|
|
@ -45,37 +48,51 @@ fn biome_at(
|
|||
|
||||
pub struct TileRenderer<'a> {
|
||||
config: &'a Config,
|
||||
rt: &'a tokio::runtime::Runtime,
|
||||
region_cache: Mutex<LruCache<PathBuf, Arc<OnceCell<RegionRef>>>>,
|
||||
}
|
||||
|
||||
impl<'a> TileRenderer<'a> {
|
||||
pub fn new(config: &'a Config) -> Self {
|
||||
TileRenderer { config }
|
||||
pub fn new(config: &'a Config, rt: &'a tokio::runtime::Runtime) -> Self {
|
||||
let region_cache = Mutex::new(LruCache::new(NonZeroUsize::new(12).unwrap()));
|
||||
TileRenderer {
|
||||
config,
|
||||
rt,
|
||||
region_cache,
|
||||
}
|
||||
}
|
||||
|
||||
fn load_region(
|
||||
region_cache: &mut LruCache<PathBuf, Rc<ProcessedRegion>>,
|
||||
processed_path: &Path,
|
||||
) -> Result<Rc<ProcessedRegion>> {
|
||||
if let Some(region) = region_cache.get(processed_path) {
|
||||
return Ok(region.clone());
|
||||
async fn load_region(&self, processed_path: PathBuf) -> Result<RegionRef> {
|
||||
let region_loader = {
|
||||
let mut region_cache = self.region_cache.lock().unwrap();
|
||||
if let Some(region_loader) = region_cache.get(&processed_path) {
|
||||
Arc::clone(region_loader)
|
||||
} else {
|
||||
let region_loader = Default::default();
|
||||
region_cache.put(processed_path.clone(), Arc::clone(®ion_loader));
|
||||
region_loader
|
||||
}
|
||||
};
|
||||
|
||||
region_loader
|
||||
.get_or_try_init(|| async {
|
||||
storage::read(&processed_path).context("Failed to load processed region data")
|
||||
})
|
||||
.await
|
||||
.cloned()
|
||||
}
|
||||
|
||||
let region: Rc<ProcessedRegion> =
|
||||
storage::read(processed_path).context("Failed to load processed region data")?;
|
||||
region_cache.put(processed_path.to_owned(), region.clone());
|
||||
|
||||
Ok(region)
|
||||
}
|
||||
|
||||
fn load_region_group(
|
||||
region_cache: &mut LruCache<PathBuf, Rc<ProcessedRegion>>,
|
||||
async fn load_region_group(
|
||||
&self,
|
||||
processed_paths: RegionGroup<PathBuf>,
|
||||
) -> Result<RegionGroup<Rc<ProcessedRegion>>> {
|
||||
processed_paths.try_map(|path| Self::load_region(region_cache, &path))
|
||||
) -> Result<RegionGroup<RegionRef>> {
|
||||
processed_paths
|
||||
.async_try_map(move |path| self.load_region(path))
|
||||
.await
|
||||
}
|
||||
|
||||
fn block_color_at(
|
||||
region_group: &RegionGroup<Rc<ProcessedRegion>>,
|
||||
region_group: &RegionGroup<RegionRef>,
|
||||
chunk: &ProcessedChunk,
|
||||
chunk_coords: ChunkCoords,
|
||||
block_coords: LayerBlockCoords,
|
||||
|
|
@ -139,7 +156,7 @@ impl<'a> TileRenderer<'a> {
|
|||
|
||||
fn render_chunk(
|
||||
image: &mut image::RgbaImage,
|
||||
region_group: &RegionGroup<Rc<ProcessedRegion>>,
|
||||
region_group: &RegionGroup<RegionRef>,
|
||||
chunk: &ProcessedChunk,
|
||||
chunk_coords: ChunkCoords,
|
||||
) {
|
||||
|
|
@ -160,10 +177,7 @@ impl<'a> TileRenderer<'a> {
|
|||
overlay_chunk(image, &chunk_image, chunk_coords);
|
||||
}
|
||||
|
||||
fn render_region(
|
||||
image: &mut image::RgbaImage,
|
||||
region_group: &RegionGroup<Rc<ProcessedRegion>>,
|
||||
) {
|
||||
fn render_region(image: &mut image::RgbaImage, region_group: &RegionGroup<RegionRef>) {
|
||||
for (coords, chunk) in region_group.center().chunks.iter() {
|
||||
let Some(chunk) = chunk else {
|
||||
continue;
|
||||
|
|
@ -198,11 +212,7 @@ impl<'a> TileRenderer<'a> {
|
|||
Ok((paths, max_timestamp))
|
||||
}
|
||||
|
||||
fn render_tile(
|
||||
&self,
|
||||
region_cache: &mut LruCache<PathBuf, Rc<ProcessedRegion>>,
|
||||
coords: TileCoords,
|
||||
) -> Result<()> {
|
||||
fn render_tile(&self, coords: TileCoords) -> Result<()> {
|
||||
const N: u32 = (BLOCKS_PER_CHUNK * CHUNKS_PER_REGION) as u32;
|
||||
|
||||
let (processed_paths, processed_timestamp) = self.processed_sources(coords)?;
|
||||
|
|
@ -229,7 +239,9 @@ impl<'a> TileRenderer<'a> {
|
|||
.display(),
|
||||
);
|
||||
|
||||
let region_group = Self::load_region_group(region_cache, processed_paths)
|
||||
let region_group = self
|
||||
.rt
|
||||
.block_on(self.load_region_group(processed_paths))
|
||||
.with_context(|| format!("Region {:?} from previous step must be loadable", coords))?;
|
||||
let mut image = image::RgbaImage::new(N, N);
|
||||
Self::render_region(&mut image, ®ion_group);
|
||||
|
|
@ -249,10 +261,8 @@ impl<'a> TileRenderer<'a> {
|
|||
pub fn run(self, regions: &[TileCoords]) -> Result<()> {
|
||||
fs::create_dir_all(&self.config.tile_dir(TileKind::Map, 0))?;
|
||||
|
||||
let mut region_cache = LruCache::new(NonZeroUsize::new(12).unwrap());
|
||||
|
||||
for &coords in regions {
|
||||
if let Err(err) = self.render_tile(&mut region_cache, coords) {
|
||||
if let Err(err) = self.render_tile(coords) {
|
||||
eprintln!("Failed to render tile {:?}: {:?}", coords, err);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue