From fd5e69922a5a7d050b2e2df6e5fcf5ffd53cfe13 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Thu, 12 Nov 2026 14:09:19 +0800 Subject: [PATCH 1/2] fix: bound Nydus cache backfill under ChunkDB contention Cap queued and in-flight decompressed chunks across mounts sharing a ChunkDB. Drop expired backfill and pause the worker after MapFull while preserving synchronous GC and retries for explicit batch writes. Cover cross-process reservations, MapFull, and Nydus read and backfill behavior with regression tests. --- src/backend/cache_write_budget.rs | 273 ++++++++++++++++++++++++++++++ src/backend/chunkdb.rs | 136 ++++++++++----- src/backend/chunkdb/tests.rs | 97 +++++++++++ src/backend/mod.rs | 2 + src/image/nydus.rs | 211 ++++++++++++++++------- tests/mount.rs | 20 ++- 6 files changed, 635 insertions(+), 104 deletions(-) create mode 100644 src/backend/cache_write_budget.rs diff --git a/src/backend/cache_write_budget.rs b/src/backend/cache_write_budget.rs new file mode 100644 index 0000000..7c22ab3 --- /dev/null +++ b/src/backend/cache_write_budget.rs @@ -0,0 +1,273 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::{Duration, Instant}; + +const LOCK_FILE_NAME: &str = ".cache-write-budget"; +const SLOT_RETRY_DELAY: Duration = Duration::from_millis(50); + +/// All daemons sharing a ChunkDB use the same number of equal-size slots. +/// Each daemon holds a slot only while it has pending cache writes. +pub(crate) struct CacheWriteBudget { + path: PathBuf, + slot_count: usize, + slot_bytes: usize, + state: Mutex, +} + +#[derive(Default)] +struct SlotState { + slot: Weak, + retry_after: Option, +} + +struct Slot { + // Closing this open file description releases the slot, including on OOM. + _file: File, + used_bytes: AtomicUsize, +} + +pub(crate) struct CacheWritePermit { + slot: Arc, + bytes: usize, +} + +impl Drop for CacheWritePermit { + fn drop(&mut self) { + self.slot + .used_bytes + .fetch_sub(self.bytes, Ordering::Relaxed); + } +} + +impl CacheWriteBudget { + pub(crate) fn new( + chunk_db_dir: &Path, + slot_count: usize, + slot_bytes: usize, + ) -> io::Result { + if slot_count == 0 + || slot_bytes == 0 + || libc::off_t::try_from(slot_count).is_err() + || slot_count.checked_mul(slot_bytes).is_none() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid cache write budget", + )); + } + + let path = chunk_db_dir.join(LOCK_FILE_NAME); + // Keep the file at a stable path so all daemons lock the same inode. + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&path)?; + + Ok(Self { + path, + slot_count, + slot_bytes, + state: Mutex::default(), + }) + } + + pub(crate) fn try_reserve(&self, bytes: usize) -> Option { + if bytes == 0 || bytes > self.slot_bytes { + return None; + } + + let mut state = self.state.try_lock().ok()?; + let slot = if let Some(slot) = state.slot.upgrade() { + slot + } else { + if state + .retry_after + .is_some_and(|retry| Instant::now() < retry) + { + return None; + } + // Avoid scanning all slots on every cache miss while they are busy. + let Some(slot) = self.try_lock_slot() else { + state.retry_after = Some(Instant::now() + SLOT_RETRY_DELAY); + return None; + }; + let slot = Arc::new(slot); + state.slot = Arc::downgrade(&slot); + state.retry_after = None; + slot + }; + + slot.used_bytes + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + used.checked_add(bytes) + .filter(|total| *total <= self.slot_bytes) + }) + .ok()?; + Some(CacheWritePermit { slot, bytes }) + } + + fn try_lock_slot(&self) -> Option { + // A distinct open file description is required for each slot holder. + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&self.path) + .ok()?; + let start = std::process::id() as usize % self.slot_count; + // The fields are integer values, and zero is required for l_pid. + let mut lock: libc::flock = unsafe { std::mem::zeroed() }; + lock.l_type = libc::F_WRLCK as _; + lock.l_whence = libc::SEEK_SET as _; + lock.l_len = 1; + for index in (start..self.slot_count).chain(0..start) { + lock.l_start = index as libc::off_t; + // file and lock remain valid for the duration of the call. + let result = + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_OFD_SETLK, &lock as *const _) }; + if result == 0 { + return Some(Slot { + _file: file, + used_bytes: AtomicUsize::new(0), + }); + } + if io::Error::last_os_error().raw_os_error() != Some(libc::EAGAIN) { + break; + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::{Child, Command, Stdio}; + use std::thread; + + fn wait_for_slot(budget: &CacheWriteBudget, bytes: usize) -> CacheWritePermit { + let deadline = Instant::now() + Duration::from_secs(1); + loop { + if let Some(permit) = budget.try_reserve(bytes) { + return permit; + } + assert!(Instant::now() < deadline, "slot did not become available"); + thread::sleep(Duration::from_millis(10)); + } + } + + #[test] + fn reservations_hold_the_slot_until_the_last_drop() { + let dir = tempfile::tempdir().unwrap(); + let budget = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + assert!(budget.try_reserve(9).is_none()); + + let first = budget.try_reserve(3).unwrap(); + let second = budget.try_reserve(5).unwrap(); + assert!(budget.try_reserve(1).is_none()); + assert!(other.try_reserve(1).is_none()); + + drop(first); + assert!(budget.try_reserve(4).is_none()); + drop(budget); + assert!(other.try_reserve(1).is_none()); + drop(second); + let _permit = wait_for_slot(&other, 8); + } + + #[test] + fn defers_rescanning_after_slot_acquisition_fails() { + let dir = tempfile::tempdir().unwrap(); + let budget = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + let permit = budget.try_reserve(8).unwrap(); + assert!(other.try_reserve(8).is_none()); + let mut state = other.state.lock().unwrap(); + assert!(state.retry_after.is_some()); + state.retry_after = Some(Instant::now() + Duration::from_secs(10)); + drop(state); + + drop(permit); + assert!(other.try_reserve(8).is_none()); + other.state.lock().unwrap().retry_after = Some(Instant::now()); + assert!(other.try_reserve(8).is_some()); + } + + struct ChildGuard(Child); + + impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[test] + fn shares_budget_across_processes_and_recovers_after_kill() { + const CHILD_DIR: &str = "DISTILL_FS_CACHE_WRITE_BUDGET_TEST_CHILD_DIR"; + if let Some(dir) = std::env::var_os(CHILD_DIR) { + let dir = PathBuf::from(dir); + let budget = CacheWriteBudget::new(&dir, 2, 8).unwrap(); + let _permit = budget.try_reserve(8).unwrap(); + File::create(dir.join("ready")).unwrap(); + // Hold the slot until the parent kills this process or exits. + let _ = io::stdin().read_line(&mut String::new()); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let mut child = ChildGuard( + Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "backend::cache_write_budget::tests::shares_budget_across_processes_and_recovers_after_kill", + ]) + .env(CHILD_DIR, dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(), + ); + + let deadline = Instant::now() + Duration::from_secs(10); + while !dir.path().join("ready").is_file() { + assert!(child.0.try_wait().unwrap().is_none(), "child exited early"); + assert!(Instant::now() < deadline, "child did not reserve a slot"); + thread::sleep(Duration::from_millis(10)); + } + + let budget = CacheWriteBudget::new(dir.path(), 2, 8).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 2, 8).unwrap(); + let _permit = budget.try_reserve(8).unwrap(); + assert!(budget.try_reserve(1).is_none()); + assert!(other.try_reserve(1).is_none()); + + child.0.kill().unwrap(); + child.0.wait().unwrap(); + let _recovered = wait_for_slot(&other, 8); + } +} diff --git a/src/backend/chunkdb.rs b/src/backend/chunkdb.rs index 32d4099..5fc33ea 100644 --- a/src/backend/chunkdb.rs +++ b/src/backend/chunkdb.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(target_os = "linux")] +use super::cache_write_budget::CacheWritePermit; use super::slow_write_txn; use crate::utils::now_epoch_secs; use bytemuck::{Pod, Zeroable}; @@ -424,6 +426,14 @@ impl std::fmt::Debug for ChunkDB { use std::collections::VecDeque; use std::sync::mpsc; +struct ChunkBatch { + chunks: Vec<(CheckSum, Vec)>, + deadline: Option, + #[cfg(target_os = "linux")] + // Drop the payloads before returning their reservations. + _permits: Vec, +} + enum WriteRequest { AddChunk { cs: CheckSum, @@ -431,7 +441,7 @@ enum WriteRequest { reply: mpsc::SyncSender>, }, AddChunksBatch { - chunks: Vec<(CheckSum, Vec)>, + batch: ChunkBatch, reply: mpsc::SyncSender>, }, DeleteKeys { @@ -557,8 +567,9 @@ impl WriterThread { let result = self.do_add_chunk(&cs, &data); let _ = reply.send(result); } - WriteRequest::AddChunksBatch { chunks, reply } => { - let result = self.do_add_chunks_batch(&chunks); + WriteRequest::AddChunksBatch { batch, reply } => { + let result = self.do_add_chunks_batch(&batch); + drop(batch); let _ = reply.send(result); } _ => unreachable!(), @@ -658,53 +669,64 @@ impl WriterThread { } } - fn do_add_chunks_batch(&self, chunks: &[(CheckSum, Vec)]) -> anyhow::Result<()> { - for batch in chunks.chunks(ADD_CHUNKS_BATCH_SIZE) { - self.do_add_chunks_sub_batch(batch)?; + fn do_add_chunks_batch(&self, batch: &ChunkBatch) -> anyhow::Result<()> { + for chunks in batch.chunks.chunks(ADD_CHUNKS_BATCH_SIZE) { + self.do_add_chunks_sub_batch(chunks, batch.deadline)?; } Ok(()) } - fn do_add_chunks_sub_batch(&self, batch: &[(CheckSum, Vec)]) -> anyhow::Result<()> { + fn do_add_chunks_sub_batch( + &self, + batch: &[(CheckSum, Vec)], + deadline: Option, + ) -> anyhow::Result<()> { + let check_deadline = || { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Err(io::Error::new( + ErrorKind::TimedOut, + "Cache backfill expired", + )); + } + Ok(()) + }; loop { + check_deadline()?; let mut wtxn = slow_write_txn(&self.env, "WriterThread::add_chunks")?; - let mut map_full = false; - for (cs, data) in batch { + check_deadline()?; + let result = batch.iter().try_for_each(|(cs, data)| { let cs_on_disk = CheckSumOnDisk::from(*cs); - match self.data_db.get_or_put_reserved( - &mut wtxn, - &cs_on_disk, - data.len(), - |reserved| reserved.write_all(data), - ) { - Ok(None) | Ok(Some(_)) => {} - Err(heed::Error::Mdb(MdbError::MapFull)) => { - map_full = true; - break; - } - Err(e) => return Err(e.into()), - } - } - if !map_full { - wtxn.commit()?; - // Register immediately after commit so that even if a later - // sub-batch fails, these already-durable chunks are visible - // in the chunk index. - if let Some(ctl) = &self.index_ctl { - let checksums: Vec = batch.iter().map(|(cs, _)| *cs).collect(); - let _ = ctl.register_chunks(&checksums); + self.data_db + .get_or_put_reserved(&mut wtxn, &cs_on_disk, data.len(), |reserved| { + reserved.write_all(data) + }) + .map(drop) + }); + match result { + Ok(()) => { + wtxn.commit()?; + break; } - self.channel - .enqueue_access_many(batch.iter().map(|(cs, _)| CheckSumOnDisk::from(*cs))); - return Ok(()); - } else { - drop(wtxn); - let removed = self.gc_lru_direct(GC_BATCH)?; - if removed == 0 { - return Err(heed::Error::Mdb(MdbError::MapFull).into()); + Err(heed::Error::Mdb(MdbError::MapFull)) if deadline.is_none() => { + // Explicit batch writes retain synchronous reclamation. + drop(wtxn); + if self.gc_lru_direct(GC_BATCH)? == 0 { + return Err(heed::Error::Mdb(MdbError::MapFull).into()); + } } + // Optional backfill leaves reclamation to periodic GC. + Err(e) => return Err(e.into()), } } + + // A later sub-batch may fail. Publish each committed sub-batch immediately. + if let Some(ctl) = &self.index_ctl { + let checksums: Vec = batch.iter().map(|(cs, _)| *cs).collect(); + let _ = ctl.register_chunks(&checksums); + } + self.channel + .enqueue_access_many(batch.iter().map(|(cs, _)| CheckSumOnDisk::from(*cs))); + Ok(()) } fn do_delete_keys(&self, keys: Vec) -> anyhow::Result { @@ -910,15 +932,43 @@ impl ChunkDB { } pub fn add_chunks(&self, chunks: Vec<(CheckSum, Vec)>) -> anyhow::Result<()> { - if chunks.is_empty() { + self.submit_chunks(ChunkBatch { + chunks, + deadline: None, + #[cfg(target_os = "linux")] + _permits: Vec::new(), + }) + } + + #[cfg(target_os = "linux")] + pub(crate) fn path(&self) -> &Path { + self.env.path() + } + + #[cfg(target_os = "linux")] + pub(crate) fn add_cache_chunks( + &self, + chunks: Vec<(CheckSum, Vec)>, + permits: Vec, + deadline: Instant, + ) -> anyhow::Result<()> { + self.submit_chunks(ChunkBatch { + chunks, + deadline: Some(deadline), + _permits: permits, + }) + } + + fn submit_chunks(&self, batch: ChunkBatch) -> anyhow::Result<()> { + if batch.chunks.is_empty() { return Ok(()); } - let total_bytes: u64 = chunks.iter().map(|(_, data)| data.len() as u64).sum(); - let chunk_count = chunks.len() as u64; + let total_bytes: u64 = batch.chunks.iter().map(|(_, data)| data.len() as u64).sum(); + let chunk_count = batch.chunks.len() as u64; let (tx, rx) = mpsc::sync_channel(1); self.writer_channel - .submit_high(WriteRequest::AddChunksBatch { chunks, reply: tx }); + .submit_high(WriteRequest::AddChunksBatch { batch, reply: tx }); let result: anyhow::Result<()> = rx .recv() .map_err(|_| anyhow::anyhow!("Writer thread dropped"))?; diff --git a/src/backend/chunkdb/tests.rs b/src/backend/chunkdb/tests.rs index 090312a..ade6602 100644 --- a/src/backend/chunkdb/tests.rs +++ b/src/backend/chunkdb/tests.rs @@ -38,6 +38,32 @@ fn wait_for_access_time(db: &ChunkDB, cs: &CheckSumOnDisk) -> Option { } } +#[cfg(target_os = "linux")] +#[test] +fn expired_cache_backfill_releases_its_reservation() { + use crate::backend::cache_write_budget::CacheWriteBudget; + + let dir = TempDir::new().unwrap(); + let db = ChunkDB::new(dir.path()).unwrap(); + let data = vec![1; 1024]; + let checksum = CheckSum::from_data(&data, CheckSumMethod::Blake3); + let bytes = data.capacity(); + let budget = CacheWriteBudget::new(dir.path(), 1, bytes).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 1, bytes).unwrap(); + let permit = budget.try_reserve(bytes).unwrap(); + assert!(other.try_reserve(1).is_none()); + + let err = db + .add_cache_chunks(vec![(checksum, data)], vec![permit], Instant::now()) + .unwrap_err(); + assert_eq!( + err.downcast_ref::().unwrap().kind(), + ErrorKind::TimedOut + ); + assert!(!db.has_chunk(&checksum).unwrap()); + assert!(budget.try_reserve(bytes).is_some()); +} + #[derive(Debug, Default)] struct RecordingChunkIndex { registered: Arc>>, @@ -584,6 +610,77 @@ fn test_add_chunks_registers_per_sub_batch() { assert_eq!(registered, expected_sorted); } +#[test] +fn batch_map_full_skips_gc_and_preserves_committed_chunks() { + let temp = TempDir::new().unwrap(); + let control = Arc::new(RecordingChunkIndex::default()); + let db = + ChunkDB::new_with_size_and_index_ctl(temp.path(), Some(8 << 20), Some(control.clone())) + .unwrap(); + let cached = CheckSum::from_data(b"cached", CheckSumMethod::Blake3); + db.add_chunk(&cached, b"cached".to_vec()).unwrap(); + assert!(wait_for_access_time(&db, &CheckSumOnDisk::from(cached)).is_some()); + + let mut chunks: Vec<_> = (0..ADD_CHUNKS_BATCH_SIZE) + .map(|i| { + let data = format!("committed-{i}").into_bytes(); + (CheckSum::from_data(&data, CheckSumMethod::Blake3), data) + }) + .collect(); + let mut registered = vec![cached]; + registered.extend(chunks.iter().map(|(cs, _)| *cs)); + + let pending = b"uncommitted".to_vec(); + let pending_cs = CheckSum::from_data(&pending, CheckSumMethod::Blake3); + let oversized = vec![42; 9 << 20]; + let oversized_cs = CheckSum::from_data(&oversized, CheckSumMethod::Blake3); + chunks.extend([(pending_cs, pending), (oversized_cs, oversized)]); + + let bytes = chunks.iter().map(|(_, data)| data.capacity()).sum(); + let budget = + crate::backend::cache_write_budget::CacheWriteBudget::new(temp.path(), 1, bytes).unwrap(); + let permit = budget.try_reserve(bytes).unwrap(); + let err = db + .add_cache_chunks( + chunks, + vec![permit], + Instant::now() + Duration::from_secs(10), + ) + .unwrap_err(); + assert!(matches!( + err.downcast_ref::(), + Some(heed::Error::Mdb(MdbError::MapFull)) + )); + assert_eq!(control.registered(), registered); + assert!(control.unregistered().is_empty()); + for cs in registered { + assert!(db.has_chunk(&cs).unwrap()); + } + assert!(!db.has_chunk(&pending_cs).unwrap()); + assert!(!db.has_chunk(&oversized_cs).unwrap()); + assert!(budget.try_reserve(bytes).is_some()); +} + +#[test] +fn explicit_batch_write_runs_gc_on_map_full() { + let temp = TempDir::new().unwrap(); + let db = ChunkDB::new_with_size_and_index_ctl(temp.path(), Some(8 << 20), None).unwrap(); + let cached = b"cached"; + let cached_cs = CheckSum::from_data(cached, CheckSumMethod::Blake3); + db.add_chunk(&cached_cs, cached.to_vec()).unwrap(); + assert!(wait_for_access_time(&db, &CheckSumOnDisk::from(cached_cs)).is_some()); + + let oversized = vec![42; 9 << 20]; + let oversized_cs = CheckSum::from_data(&oversized, CheckSumMethod::Blake3); + let err = db.add_chunks(vec![(oversized_cs, oversized)]).unwrap_err(); + assert!(matches!( + err.downcast_ref::(), + Some(heed::Error::Mdb(MdbError::MapFull)) + )); + assert!(!db.has_chunk(&cached_cs).unwrap()); + assert!(!db.has_chunk(&oversized_cs).unwrap()); +} + #[test] fn test_gc_expired_large_batch_batched_deletion() { let temp = TempDir::new().unwrap(); diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 23da673..b81fd80 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -20,6 +20,8 @@ use std::fmt::Debug; use std::sync::Arc; use std::time::{Duration, Instant}; pub mod cache; +#[cfg(target_os = "linux")] +pub(crate) mod cache_write_budget; pub mod chunkdb; pub mod dedup; pub mod general; diff --git a/src/image/nydus.rs b/src/image/nydus.rs index 0d4b38d..a4fa536 100644 --- a/src/image/nydus.rs +++ b/src/image/nydus.rs @@ -15,6 +15,8 @@ #[cfg(target_os = "linux")] use crate::backend::cache::Cache; #[cfg(target_os = "linux")] +use crate::backend::cache_write_budget::{CacheWriteBudget, CacheWritePermit}; +#[cfg(target_os = "linux")] use crate::backend::chunkdb::CheckSumMethod; use crate::backend::chunkdb::{CheckSum, ChunkDB}; #[cfg(target_os = "linux")] @@ -28,6 +30,7 @@ use crate::image::FsReadMetrics; use crate::rate_limited_log; #[cfg(target_os = "linux")] use crate::utils::new_std_io_error; +#[cfg(target_os = "linux")] use crate::utils::RateLimitedLog; #[cfg(target_os = "linux")] use fuse_backend_rs::abi::fuse_abi::statvfs64; @@ -57,12 +60,13 @@ use std::io::{self, ErrorKind}; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; use std::path::Path; -use std::sync::atomic::AtomicU64; -#[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; use std::sync::atomic::Ordering::Relaxed; +#[cfg(target_os = "linux")] +use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(target_os = "linux")] use std::sync::mpsc::{self, Receiver, SyncSender}; use std::sync::{Arc, Mutex}; +#[cfg(target_os = "linux")] use std::thread; #[cfg(target_os = "linux")] use std::time::Duration; @@ -72,21 +76,37 @@ use std::time::{SystemTime, UNIX_EPOCH}; #[cfg(target_os = "linux")] use tracing::error; use tracing::info; +#[cfg(target_os = "linux")] use tracing::warn; +#[cfg(target_os = "linux")] const CHUNK_STORE_REQ_BUF_SIZE: usize = 4096; +#[cfg(target_os = "linux")] const CHUNK_STORE_BATCH_SIZE: usize = 64; +#[cfg(target_os = "linux")] +const CACHE_WRITE_SLOTS: usize = 64; +#[cfg(target_os = "linux")] +const CACHE_WRITE_SLOT_BYTES: usize = 16 * 1024 * 1024; +#[cfg(target_os = "linux")] +const CACHE_WRITE_MAX_AGE: Duration = Duration::from_secs(2); +#[cfg(target_os = "linux")] +const MAP_FULL_RETRY_DELAY: Duration = Duration::from_secs(5); +#[cfg(target_os = "linux")] struct ChunkStoreRequest { checksum: CheckSum, data: Vec, + deadline: Instant, + permit: CacheWritePermit, } +#[cfg(target_os = "linux")] struct ChunkStoreWorker { chunk_db: Arc, rx: Receiver, } +#[cfg(target_os = "linux")] impl ChunkStoreWorker { fn new(chunk_db: Arc, rx: Receiver) -> Self { Self { chunk_db, rx } @@ -94,40 +114,34 @@ impl ChunkStoreWorker { fn run(self) { let mut batch = Vec::with_capacity(CHUNK_STORE_BATCH_SIZE); - let mut batch_data: Vec> = Vec::with_capacity(CHUNK_STORE_BATCH_SIZE); + let warn_limiter = RateLimitedLog::new(5); + let mut retry_after = None; while let Ok(req) = self.rx.recv() { - batch.push(req.checksum); - batch_data.push(req.data); - - // Try to collect more requests without blocking - while batch.len() < CHUNK_STORE_BATCH_SIZE { - match self.rx.try_recv() { - Ok(req) => { - batch.push(req.checksum); - batch_data.push(req.data); - } - Err(mpsc::TryRecvError::Empty) => break, - Err(mpsc::TryRecvError::Disconnected) => break, - } + if retry_after.is_some_and(|deadline| Instant::now() < deadline) { + continue; } - - // Process the batch - let chunks: Vec<(CheckSum, Vec)> = - batch.drain(..).zip(batch_data.drain(..)).collect(); - - if let Err(e) = self.chunk_db.add_chunks(chunks) { - warn!(err = debug(e), "Failed to store chunk batch into chunkdb."); - } - } - - // Process any remaining requests in the batch - if !batch.is_empty() { - let chunks: Vec<(CheckSum, Vec)> = - batch.drain(..).zip(batch_data.drain(..)).collect(); - - if let Err(e) = self.chunk_db.add_chunks(chunks) { - warn!(err = debug(e), "Failed to store chunk batch into chunkdb."); + batch.push(req); + batch.extend(self.rx.try_iter().take(CHUNK_STORE_BATCH_SIZE - 1)); + let now = Instant::now(); + batch.retain(|req| req.deadline > now); + let Some(deadline) = batch.iter().map(|req| req.deadline).min() else { + continue; + }; + let (chunks, permits) = batch + .drain(..) + .map(|req| ((req.checksum, req.data), req.permit)) + .unzip(); + if let Err(e) = self.chunk_db.add_cache_chunks(chunks, permits, deadline) { + if matches!( + e.downcast_ref::(), + Some(heed::Error::Mdb(heed::MdbError::MapFull)) + ) { + retry_after = Some(Instant::now() + MAP_FULL_RETRY_DELAY); + } + rate_limited_log!(warn_limiter, { + warn!(err = debug(e), "Failed to store chunk batch into chunkdb."); + }); } } } @@ -146,13 +160,15 @@ pub struct NydusImage { dedup_db: Option<(Arc, Arc)>, #[cfg_attr(not(target_os = "linux"), allow(dead_code))] local_chunk_client: Option>, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] + store_budget: Option, + #[cfg(target_os = "linux")] store_tx: Option>, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] store_worker: Mutex>>, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] store_dropped_count: AtomicU64, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] store_warn_limiter: RateLimitedLog, #[cfg_attr(not(target_os = "linux"), allow(dead_code))] cache_dir: String, @@ -204,14 +220,31 @@ impl NydusImage { let i_uid = unsafe { libc::geteuid() } as u32; let i_gid = unsafe { libc::getegid() } as u32; - let (store_tx, store_worker) = if let Some((chunk_db, _)) = &dedup_db { - let (tx, rx) = mpsc::sync_channel(CHUNK_STORE_REQ_BUF_SIZE); - let worker = ChunkStoreWorker::new(chunk_db.clone(), rx); - let handle = thread::spawn(move || worker.run()); - (Some(tx), Mutex::new(Some(handle))) - } else { - (None, Mutex::new(None)) - }; + #[cfg(target_os = "linux")] + let store_budget = + dedup_db.as_ref().and_then(|(chunk_db, _)| { + match CacheWriteBudget::new( + chunk_db.path(), + CACHE_WRITE_SLOTS, + CACHE_WRITE_SLOT_BYTES, + ) { + Ok(budget) => Some(budget), + Err(e) => { + warn!(err = debug(e), "Chunk backfill budget unavailable."); + None + } + } + }); + #[cfg(target_os = "linux")] + let (store_tx, store_worker) = + if let (Some((chunk_db, _)), Some(_)) = (&dedup_db, &store_budget) { + let (tx, rx) = mpsc::sync_channel(CHUNK_STORE_REQ_BUF_SIZE); + let worker = ChunkStoreWorker::new(chunk_db.clone(), rx); + let handle = thread::spawn(move || worker.run()); + (Some(tx), Mutex::new(Some(handle))) + } else { + (None, Mutex::new(None)) + }; Ok(Self { sb: Arc::new(sb), @@ -220,9 +253,15 @@ impl NydusImage { backend, dedup_db, local_chunk_client, + #[cfg(target_os = "linux")] + store_budget, + #[cfg(target_os = "linux")] store_tx, + #[cfg(target_os = "linux")] store_worker, + #[cfg(target_os = "linux")] store_dropped_count: AtomicU64::new(0), + #[cfg(target_os = "linux")] store_warn_limiter: RateLimitedLog::new(5), cache_dir: cache_dir.to_string(), i_uid, @@ -377,20 +416,33 @@ impl NydusImage { #[cfg(target_os = "linux")] fn store_chunk_bg(&self, cs: CheckSum, data: Vec) { - if let Some(tx) = &self.store_tx { - let req = ChunkStoreRequest { checksum: cs, data }; - if let Err(_e) = tx.try_send(req) { - let dropped = self.store_dropped_count.fetch_add(1, Ordering::Relaxed) + 1; - rate_limited_log!(self.store_warn_limiter, { - warn!( - dropped_count = dropped, - queue_size = CHUNK_STORE_REQ_BUF_SIZE, - "Chunk store queue full, dropping requests." - ); - self.store_dropped_count.store(0, Ordering::Relaxed); - }); + let (Some(tx), Some(budget)) = (&self.store_tx, &self.store_budget) else { + return; + }; + let message = if let Some(permit) = budget.try_reserve(data.capacity()) { + let req = ChunkStoreRequest { + checksum: cs, + data, + deadline: Instant::now() + CACHE_WRITE_MAX_AGE, + permit, + }; + if tx.try_send(req).is_ok() { + return; } - } + "Chunk store queue full, dropping requests." + } else { + drop(data); + "Chunk store byte budget unavailable, dropping requests." + }; + self.store_dropped_count.fetch_add(1, Ordering::Relaxed); + rate_limited_log!(self.store_warn_limiter, { + let dropped = self.store_dropped_count.swap(0, Ordering::Relaxed); + warn!( + dropped_count = dropped, + queue_size = CHUNK_STORE_REQ_BUF_SIZE, + "{message}" + ); + }); } #[cfg(target_os = "linux")] @@ -798,6 +850,7 @@ impl fuse_backend_rs::api::filesystem::FileSystem for NydusImage { } } +#[cfg(target_os = "linux")] impl Drop for NydusImage { fn drop(&mut self) { // Drop the sender to signal the worker thread to exit @@ -811,3 +864,43 @@ impl Drop for NydusImage { } } } + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + #[test] + fn map_full_pauses_later_backfill_batches() { + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new( + ChunkDB::new_with_size_and_index_ctl(dir.path(), Some(8 << 20), None).unwrap(), + ); + let slot_bytes = 16 << 20; + let budget = CacheWriteBudget::new(dir.path(), 1, slot_bytes).unwrap(); + let (tx, rx) = mpsc::sync_channel(CHUNK_STORE_BATCH_SIZE + 1); + let deadline = Instant::now() + Duration::from_secs(30); + let enqueue = |data: Vec| { + let checksum = CheckSum::from_data(&data, CheckSumMethod::Blake3); + let permit = budget.try_reserve(data.capacity()).unwrap(); + tx.send(ChunkStoreRequest { + checksum, + data, + deadline, + permit, + }) + .unwrap(); + checksum + }; + + enqueue(vec![0; 9 << 20]); + for _ in 1..CHUNK_STORE_BATCH_SIZE { + enqueue(vec![1]); + } + // This chunk would fit if the next batch immediately retried LMDB. + let next = enqueue(vec![2]); + drop(tx); + ChunkStoreWorker::new(db.clone(), rx).run(); + assert!(!db.has_chunk(&next).unwrap()); + assert!(budget.try_reserve(slot_bytes).is_some()); + } +} diff --git a/tests/mount.rs b/tests/mount.rs index 2a2c161..39339ac 100644 --- a/tests/mount.rs +++ b/tests/mount.rs @@ -18,9 +18,12 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; +use distill_fs::backend::chunkdb::{CheckSum, CheckSumMethod, ChunkDB}; +use distill_fs::backend::indexdb::IndexDB; use fuse_backend_rs::api::filesystem::{Context, FileSystem, ZeroCopyWriter}; fn init_tracing() { @@ -204,8 +207,18 @@ fn nydus_direct_reads_file() { let temp = tempfile::TempDir::new().unwrap(); let expected = b"distill-fs nydus mount\n"; let (bootstrap, config) = build_nydus_fixture(&nydus_image, &temp, expected); - let image = - distill_fs::image::nydus::NydusImage::new(&bootstrap, &config, "", None, None).unwrap(); + let chunk_dir = tempfile::TempDir::new_in(temp.path()).unwrap(); + let index_dir = tempfile::TempDir::new_in(temp.path()).unwrap(); + let chunk_db = Arc::new(ChunkDB::new(chunk_dir.path()).unwrap()); + let index_db = Arc::new(IndexDB::open(index_dir.path()).unwrap()); + let image = distill_fs::image::nydus::NydusImage::new( + &bootstrap, + &config, + "", + Some((chunk_db.clone(), index_db)), + None, + ) + .unwrap(); let ctx = Context::new(); let name = std::ffi::CString::new("hello.txt").unwrap(); @@ -227,6 +240,9 @@ fn nydus_direct_reads_file() { .unwrap(); assert_eq!(n, expected.len()); assert_eq!(writer.data, expected); + drop(image); + let checksum = CheckSum::from_data(expected, CheckSumMethod::Blake3); + assert!(chunk_db.has_chunk(&checksum).unwrap()); } #[test] From 38ecca607291e3e89c3817ebea53a2b140318f3d Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Thu, 12 Nov 2026 14:57:58 +0800 Subject: [PATCH 2/2] test: cover backfill expiry while waiting for LMDB Hold the LMDB writer lock past the batch deadline. Verify the writer returns TimedOut after acquiring the lock, leaves the cache unchanged, and releases the reservation. --- src/backend/chunkdb.rs | 8 ++++++ src/backend/chunkdb/tests.rs | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/backend/chunkdb.rs b/src/backend/chunkdb.rs index 5fc33ea..6962e08 100644 --- a/src/backend/chunkdb.rs +++ b/src/backend/chunkdb.rs @@ -464,6 +464,8 @@ struct WriteQueue { struct WriterChannel { inner: Mutex, cond: Condvar, + #[cfg(test)] + before_batch_write_txn: Mutex>>, } impl Default for WriterChannel { @@ -471,6 +473,8 @@ impl Default for WriterChannel { Self { inner: Mutex::new(WriteQueue::default()), cond: Condvar::new(), + #[cfg(test)] + before_batch_write_txn: Mutex::new(None), } } } @@ -692,6 +696,10 @@ impl WriterThread { }; loop { check_deadline()?; + #[cfg(test)] + if let Some(ready) = self.channel.before_batch_write_txn.lock().unwrap().take() { + let _ = ready.send(()); + } let mut wtxn = slow_write_txn(&self.env, "WriterThread::add_chunks")?; check_deadline()?; let result = batch.iter().try_for_each(|(cs, data)| { diff --git a/src/backend/chunkdb/tests.rs b/src/backend/chunkdb/tests.rs index ade6602..615eed0 100644 --- a/src/backend/chunkdb/tests.rs +++ b/src/backend/chunkdb/tests.rs @@ -64,6 +64,56 @@ fn expired_cache_backfill_releases_its_reservation() { assert!(budget.try_reserve(bytes).is_some()); } +#[test] +fn cache_backfill_expires_while_waiting_for_writer_lock() { + use crate::backend::cache_write_budget::CacheWriteBudget; + + let dir = TempDir::new().unwrap(); + let db = ChunkDB::new(dir.path()).unwrap(); + let data = vec![1; 1024]; + let checksum = CheckSum::from_data(&data, CheckSumMethod::Blake3); + let bytes = data.capacity(); + let budget = CacheWriteBudget::new(dir.path(), 1, bytes).unwrap(); + let permit = budget.try_reserve(bytes).unwrap(); + + let held_txn = db.env.write_txn().unwrap(); + let (ready, started) = mpsc::channel(); + *db.writer_channel.before_batch_write_txn.lock().unwrap() = Some(ready); + let (reply, result) = mpsc::sync_channel(1); + let deadline = Instant::now() + Duration::from_secs(1); + db.writer_channel.submit_high(WriteRequest::AddChunksBatch { + batch: ChunkBatch { + chunks: vec![(checksum, data)], + deadline: Some(deadline), + _permits: vec![permit], + }, + reply, + }); + started + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("writer did not reach the lock attempt before expiry"); + + // The writer must remain blocked past the deadline with the permit charged. + let wait = deadline.saturating_duration_since(Instant::now()) + Duration::from_millis(50); + assert_eq!( + result.recv_timeout(wait).unwrap_err(), + mpsc::RecvTimeoutError::Timeout + ); + assert!(budget.try_reserve(1).is_none()); + + drop(held_txn); + let err = result + .recv_timeout(Duration::from_secs(5)) + .expect("writer did not finish after lock release") + .expect_err("expired backfill succeeded after lock release"); + assert_eq!( + err.downcast_ref::().unwrap().kind(), + ErrorKind::TimedOut + ); + assert!(!db.has_chunk(&checksum).unwrap()); + assert!(budget.try_reserve(bytes).is_some()); +} + #[derive(Debug, Default)] struct RecordingChunkIndex { registered: Arc>>,