diff --git a/src/backend/cache_write_budget.rs b/src/backend/cache_write_budget.rs new file mode 100644 index 0000000..7c22ab3 --- /dev/null +++ b/src/backend/cache_write_budget.rs @@ -0,0 +1,273 @@ +// Copyright (c) 2026 Ant Group Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::{Duration, Instant}; + +const LOCK_FILE_NAME: &str = ".cache-write-budget"; +const SLOT_RETRY_DELAY: Duration = Duration::from_millis(50); + +/// All daemons sharing a ChunkDB use the same number of equal-size slots. +/// Each daemon holds a slot only while it has pending cache writes. +pub(crate) struct CacheWriteBudget { + path: PathBuf, + slot_count: usize, + slot_bytes: usize, + state: Mutex, +} + +#[derive(Default)] +struct SlotState { + slot: Weak, + retry_after: Option, +} + +struct Slot { + // Closing this open file description releases the slot, including on OOM. + _file: File, + used_bytes: AtomicUsize, +} + +pub(crate) struct CacheWritePermit { + slot: Arc, + bytes: usize, +} + +impl Drop for CacheWritePermit { + fn drop(&mut self) { + self.slot + .used_bytes + .fetch_sub(self.bytes, Ordering::Relaxed); + } +} + +impl CacheWriteBudget { + pub(crate) fn new( + chunk_db_dir: &Path, + slot_count: usize, + slot_bytes: usize, + ) -> io::Result { + if slot_count == 0 + || slot_bytes == 0 + || libc::off_t::try_from(slot_count).is_err() + || slot_count.checked_mul(slot_bytes).is_none() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid cache write budget", + )); + } + + let path = chunk_db_dir.join(LOCK_FILE_NAME); + // Keep the file at a stable path so all daemons lock the same inode. + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&path)?; + + Ok(Self { + path, + slot_count, + slot_bytes, + state: Mutex::default(), + }) + } + + pub(crate) fn try_reserve(&self, bytes: usize) -> Option { + if bytes == 0 || bytes > self.slot_bytes { + return None; + } + + let mut state = self.state.try_lock().ok()?; + let slot = if let Some(slot) = state.slot.upgrade() { + slot + } else { + if state + .retry_after + .is_some_and(|retry| Instant::now() < retry) + { + return None; + } + // Avoid scanning all slots on every cache miss while they are busy. + let Some(slot) = self.try_lock_slot() else { + state.retry_after = Some(Instant::now() + SLOT_RETRY_DELAY); + return None; + }; + let slot = Arc::new(slot); + state.slot = Arc::downgrade(&slot); + state.retry_after = None; + slot + }; + + slot.used_bytes + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + used.checked_add(bytes) + .filter(|total| *total <= self.slot_bytes) + }) + .ok()?; + Some(CacheWritePermit { slot, bytes }) + } + + fn try_lock_slot(&self) -> Option { + // A distinct open file description is required for each slot holder. + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&self.path) + .ok()?; + let start = std::process::id() as usize % self.slot_count; + // The fields are integer values, and zero is required for l_pid. + let mut lock: libc::flock = unsafe { std::mem::zeroed() }; + lock.l_type = libc::F_WRLCK as _; + lock.l_whence = libc::SEEK_SET as _; + lock.l_len = 1; + for index in (start..self.slot_count).chain(0..start) { + lock.l_start = index as libc::off_t; + // file and lock remain valid for the duration of the call. + let result = + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_OFD_SETLK, &lock as *const _) }; + if result == 0 { + return Some(Slot { + _file: file, + used_bytes: AtomicUsize::new(0), + }); + } + if io::Error::last_os_error().raw_os_error() != Some(libc::EAGAIN) { + break; + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::{Child, Command, Stdio}; + use std::thread; + + fn wait_for_slot(budget: &CacheWriteBudget, bytes: usize) -> CacheWritePermit { + let deadline = Instant::now() + Duration::from_secs(1); + loop { + if let Some(permit) = budget.try_reserve(bytes) { + return permit; + } + assert!(Instant::now() < deadline, "slot did not become available"); + thread::sleep(Duration::from_millis(10)); + } + } + + #[test] + fn reservations_hold_the_slot_until_the_last_drop() { + let dir = tempfile::tempdir().unwrap(); + let budget = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + assert!(budget.try_reserve(9).is_none()); + + let first = budget.try_reserve(3).unwrap(); + let second = budget.try_reserve(5).unwrap(); + assert!(budget.try_reserve(1).is_none()); + assert!(other.try_reserve(1).is_none()); + + drop(first); + assert!(budget.try_reserve(4).is_none()); + drop(budget); + assert!(other.try_reserve(1).is_none()); + drop(second); + let _permit = wait_for_slot(&other, 8); + } + + #[test] + fn defers_rescanning_after_slot_acquisition_fails() { + let dir = tempfile::tempdir().unwrap(); + let budget = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 1, 8).unwrap(); + let permit = budget.try_reserve(8).unwrap(); + assert!(other.try_reserve(8).is_none()); + let mut state = other.state.lock().unwrap(); + assert!(state.retry_after.is_some()); + state.retry_after = Some(Instant::now() + Duration::from_secs(10)); + drop(state); + + drop(permit); + assert!(other.try_reserve(8).is_none()); + other.state.lock().unwrap().retry_after = Some(Instant::now()); + assert!(other.try_reserve(8).is_some()); + } + + struct ChildGuard(Child); + + impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[test] + fn shares_budget_across_processes_and_recovers_after_kill() { + const CHILD_DIR: &str = "DISTILL_FS_CACHE_WRITE_BUDGET_TEST_CHILD_DIR"; + if let Some(dir) = std::env::var_os(CHILD_DIR) { + let dir = PathBuf::from(dir); + let budget = CacheWriteBudget::new(&dir, 2, 8).unwrap(); + let _permit = budget.try_reserve(8).unwrap(); + File::create(dir.join("ready")).unwrap(); + // Hold the slot until the parent kills this process or exits. + let _ = io::stdin().read_line(&mut String::new()); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let mut child = ChildGuard( + Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "backend::cache_write_budget::tests::shares_budget_across_processes_and_recovers_after_kill", + ]) + .env(CHILD_DIR, dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(), + ); + + let deadline = Instant::now() + Duration::from_secs(10); + while !dir.path().join("ready").is_file() { + assert!(child.0.try_wait().unwrap().is_none(), "child exited early"); + assert!(Instant::now() < deadline, "child did not reserve a slot"); + thread::sleep(Duration::from_millis(10)); + } + + let budget = CacheWriteBudget::new(dir.path(), 2, 8).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 2, 8).unwrap(); + let _permit = budget.try_reserve(8).unwrap(); + assert!(budget.try_reserve(1).is_none()); + assert!(other.try_reserve(1).is_none()); + + child.0.kill().unwrap(); + child.0.wait().unwrap(); + let _recovered = wait_for_slot(&other, 8); + } +} diff --git a/src/backend/chunkdb.rs b/src/backend/chunkdb.rs index 32d4099..6962e08 100644 --- a/src/backend/chunkdb.rs +++ b/src/backend/chunkdb.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(target_os = "linux")] +use super::cache_write_budget::CacheWritePermit; use super::slow_write_txn; use crate::utils::now_epoch_secs; use bytemuck::{Pod, Zeroable}; @@ -424,6 +426,14 @@ impl std::fmt::Debug for ChunkDB { use std::collections::VecDeque; use std::sync::mpsc; +struct ChunkBatch { + chunks: Vec<(CheckSum, Vec)>, + deadline: Option, + #[cfg(target_os = "linux")] + // Drop the payloads before returning their reservations. + _permits: Vec, +} + enum WriteRequest { AddChunk { cs: CheckSum, @@ -431,7 +441,7 @@ enum WriteRequest { reply: mpsc::SyncSender>, }, AddChunksBatch { - chunks: Vec<(CheckSum, Vec)>, + batch: ChunkBatch, reply: mpsc::SyncSender>, }, DeleteKeys { @@ -454,6 +464,8 @@ struct WriteQueue { struct WriterChannel { inner: Mutex, cond: Condvar, + #[cfg(test)] + before_batch_write_txn: Mutex>>, } impl Default for WriterChannel { @@ -461,6 +473,8 @@ impl Default for WriterChannel { Self { inner: Mutex::new(WriteQueue::default()), cond: Condvar::new(), + #[cfg(test)] + before_batch_write_txn: Mutex::new(None), } } } @@ -557,8 +571,9 @@ impl WriterThread { let result = self.do_add_chunk(&cs, &data); let _ = reply.send(result); } - WriteRequest::AddChunksBatch { chunks, reply } => { - let result = self.do_add_chunks_batch(&chunks); + WriteRequest::AddChunksBatch { batch, reply } => { + let result = self.do_add_chunks_batch(&batch); + drop(batch); let _ = reply.send(result); } _ => unreachable!(), @@ -658,53 +673,68 @@ impl WriterThread { } } - fn do_add_chunks_batch(&self, chunks: &[(CheckSum, Vec)]) -> anyhow::Result<()> { - for batch in chunks.chunks(ADD_CHUNKS_BATCH_SIZE) { - self.do_add_chunks_sub_batch(batch)?; + fn do_add_chunks_batch(&self, batch: &ChunkBatch) -> anyhow::Result<()> { + for chunks in batch.chunks.chunks(ADD_CHUNKS_BATCH_SIZE) { + self.do_add_chunks_sub_batch(chunks, batch.deadline)?; } Ok(()) } - fn do_add_chunks_sub_batch(&self, batch: &[(CheckSum, Vec)]) -> anyhow::Result<()> { + fn do_add_chunks_sub_batch( + &self, + batch: &[(CheckSum, Vec)], + deadline: Option, + ) -> anyhow::Result<()> { + let check_deadline = || { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Err(io::Error::new( + ErrorKind::TimedOut, + "Cache backfill expired", + )); + } + Ok(()) + }; loop { + check_deadline()?; + #[cfg(test)] + if let Some(ready) = self.channel.before_batch_write_txn.lock().unwrap().take() { + let _ = ready.send(()); + } let mut wtxn = slow_write_txn(&self.env, "WriterThread::add_chunks")?; - let mut map_full = false; - for (cs, data) in batch { + check_deadline()?; + let result = batch.iter().try_for_each(|(cs, data)| { let cs_on_disk = CheckSumOnDisk::from(*cs); - match self.data_db.get_or_put_reserved( - &mut wtxn, - &cs_on_disk, - data.len(), - |reserved| reserved.write_all(data), - ) { - Ok(None) | Ok(Some(_)) => {} - Err(heed::Error::Mdb(MdbError::MapFull)) => { - map_full = true; - break; - } - Err(e) => return Err(e.into()), - } - } - if !map_full { - wtxn.commit()?; - // Register immediately after commit so that even if a later - // sub-batch fails, these already-durable chunks are visible - // in the chunk index. - if let Some(ctl) = &self.index_ctl { - let checksums: Vec = batch.iter().map(|(cs, _)| *cs).collect(); - let _ = ctl.register_chunks(&checksums); + self.data_db + .get_or_put_reserved(&mut wtxn, &cs_on_disk, data.len(), |reserved| { + reserved.write_all(data) + }) + .map(drop) + }); + match result { + Ok(()) => { + wtxn.commit()?; + break; } - self.channel - .enqueue_access_many(batch.iter().map(|(cs, _)| CheckSumOnDisk::from(*cs))); - return Ok(()); - } else { - drop(wtxn); - let removed = self.gc_lru_direct(GC_BATCH)?; - if removed == 0 { - return Err(heed::Error::Mdb(MdbError::MapFull).into()); + Err(heed::Error::Mdb(MdbError::MapFull)) if deadline.is_none() => { + // Explicit batch writes retain synchronous reclamation. + drop(wtxn); + if self.gc_lru_direct(GC_BATCH)? == 0 { + return Err(heed::Error::Mdb(MdbError::MapFull).into()); + } } + // Optional backfill leaves reclamation to periodic GC. + Err(e) => return Err(e.into()), } } + + // A later sub-batch may fail. Publish each committed sub-batch immediately. + if let Some(ctl) = &self.index_ctl { + let checksums: Vec = batch.iter().map(|(cs, _)| *cs).collect(); + let _ = ctl.register_chunks(&checksums); + } + self.channel + .enqueue_access_many(batch.iter().map(|(cs, _)| CheckSumOnDisk::from(*cs))); + Ok(()) } fn do_delete_keys(&self, keys: Vec) -> anyhow::Result { @@ -910,15 +940,43 @@ impl ChunkDB { } pub fn add_chunks(&self, chunks: Vec<(CheckSum, Vec)>) -> anyhow::Result<()> { - if chunks.is_empty() { + self.submit_chunks(ChunkBatch { + chunks, + deadline: None, + #[cfg(target_os = "linux")] + _permits: Vec::new(), + }) + } + + #[cfg(target_os = "linux")] + pub(crate) fn path(&self) -> &Path { + self.env.path() + } + + #[cfg(target_os = "linux")] + pub(crate) fn add_cache_chunks( + &self, + chunks: Vec<(CheckSum, Vec)>, + permits: Vec, + deadline: Instant, + ) -> anyhow::Result<()> { + self.submit_chunks(ChunkBatch { + chunks, + deadline: Some(deadline), + _permits: permits, + }) + } + + fn submit_chunks(&self, batch: ChunkBatch) -> anyhow::Result<()> { + if batch.chunks.is_empty() { return Ok(()); } - let total_bytes: u64 = chunks.iter().map(|(_, data)| data.len() as u64).sum(); - let chunk_count = chunks.len() as u64; + let total_bytes: u64 = batch.chunks.iter().map(|(_, data)| data.len() as u64).sum(); + let chunk_count = batch.chunks.len() as u64; let (tx, rx) = mpsc::sync_channel(1); self.writer_channel - .submit_high(WriteRequest::AddChunksBatch { chunks, reply: tx }); + .submit_high(WriteRequest::AddChunksBatch { batch, reply: tx }); let result: anyhow::Result<()> = rx .recv() .map_err(|_| anyhow::anyhow!("Writer thread dropped"))?; diff --git a/src/backend/chunkdb/tests.rs b/src/backend/chunkdb/tests.rs index 090312a..615eed0 100644 --- a/src/backend/chunkdb/tests.rs +++ b/src/backend/chunkdb/tests.rs @@ -38,6 +38,82 @@ fn wait_for_access_time(db: &ChunkDB, cs: &CheckSumOnDisk) -> Option { } } +#[cfg(target_os = "linux")] +#[test] +fn expired_cache_backfill_releases_its_reservation() { + use crate::backend::cache_write_budget::CacheWriteBudget; + + let dir = TempDir::new().unwrap(); + let db = ChunkDB::new(dir.path()).unwrap(); + let data = vec![1; 1024]; + let checksum = CheckSum::from_data(&data, CheckSumMethod::Blake3); + let bytes = data.capacity(); + let budget = CacheWriteBudget::new(dir.path(), 1, bytes).unwrap(); + let other = CacheWriteBudget::new(dir.path(), 1, bytes).unwrap(); + let permit = budget.try_reserve(bytes).unwrap(); + assert!(other.try_reserve(1).is_none()); + + let err = db + .add_cache_chunks(vec![(checksum, data)], vec![permit], Instant::now()) + .unwrap_err(); + assert_eq!( + err.downcast_ref::().unwrap().kind(), + ErrorKind::TimedOut + ); + assert!(!db.has_chunk(&checksum).unwrap()); + assert!(budget.try_reserve(bytes).is_some()); +} + +#[test] +fn cache_backfill_expires_while_waiting_for_writer_lock() { + use crate::backend::cache_write_budget::CacheWriteBudget; + + let dir = TempDir::new().unwrap(); + let db = ChunkDB::new(dir.path()).unwrap(); + let data = vec![1; 1024]; + let checksum = CheckSum::from_data(&data, CheckSumMethod::Blake3); + let bytes = data.capacity(); + let budget = CacheWriteBudget::new(dir.path(), 1, bytes).unwrap(); + let permit = budget.try_reserve(bytes).unwrap(); + + let held_txn = db.env.write_txn().unwrap(); + let (ready, started) = mpsc::channel(); + *db.writer_channel.before_batch_write_txn.lock().unwrap() = Some(ready); + let (reply, result) = mpsc::sync_channel(1); + let deadline = Instant::now() + Duration::from_secs(1); + db.writer_channel.submit_high(WriteRequest::AddChunksBatch { + batch: ChunkBatch { + chunks: vec![(checksum, data)], + deadline: Some(deadline), + _permits: vec![permit], + }, + reply, + }); + started + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + .expect("writer did not reach the lock attempt before expiry"); + + // The writer must remain blocked past the deadline with the permit charged. + let wait = deadline.saturating_duration_since(Instant::now()) + Duration::from_millis(50); + assert_eq!( + result.recv_timeout(wait).unwrap_err(), + mpsc::RecvTimeoutError::Timeout + ); + assert!(budget.try_reserve(1).is_none()); + + drop(held_txn); + let err = result + .recv_timeout(Duration::from_secs(5)) + .expect("writer did not finish after lock release") + .expect_err("expired backfill succeeded after lock release"); + assert_eq!( + err.downcast_ref::().unwrap().kind(), + ErrorKind::TimedOut + ); + assert!(!db.has_chunk(&checksum).unwrap()); + assert!(budget.try_reserve(bytes).is_some()); +} + #[derive(Debug, Default)] struct RecordingChunkIndex { registered: Arc>>, @@ -584,6 +660,77 @@ fn test_add_chunks_registers_per_sub_batch() { assert_eq!(registered, expected_sorted); } +#[test] +fn batch_map_full_skips_gc_and_preserves_committed_chunks() { + let temp = TempDir::new().unwrap(); + let control = Arc::new(RecordingChunkIndex::default()); + let db = + ChunkDB::new_with_size_and_index_ctl(temp.path(), Some(8 << 20), Some(control.clone())) + .unwrap(); + let cached = CheckSum::from_data(b"cached", CheckSumMethod::Blake3); + db.add_chunk(&cached, b"cached".to_vec()).unwrap(); + assert!(wait_for_access_time(&db, &CheckSumOnDisk::from(cached)).is_some()); + + let mut chunks: Vec<_> = (0..ADD_CHUNKS_BATCH_SIZE) + .map(|i| { + let data = format!("committed-{i}").into_bytes(); + (CheckSum::from_data(&data, CheckSumMethod::Blake3), data) + }) + .collect(); + let mut registered = vec![cached]; + registered.extend(chunks.iter().map(|(cs, _)| *cs)); + + let pending = b"uncommitted".to_vec(); + let pending_cs = CheckSum::from_data(&pending, CheckSumMethod::Blake3); + let oversized = vec![42; 9 << 20]; + let oversized_cs = CheckSum::from_data(&oversized, CheckSumMethod::Blake3); + chunks.extend([(pending_cs, pending), (oversized_cs, oversized)]); + + let bytes = chunks.iter().map(|(_, data)| data.capacity()).sum(); + let budget = + crate::backend::cache_write_budget::CacheWriteBudget::new(temp.path(), 1, bytes).unwrap(); + let permit = budget.try_reserve(bytes).unwrap(); + let err = db + .add_cache_chunks( + chunks, + vec![permit], + Instant::now() + Duration::from_secs(10), + ) + .unwrap_err(); + assert!(matches!( + err.downcast_ref::(), + Some(heed::Error::Mdb(MdbError::MapFull)) + )); + assert_eq!(control.registered(), registered); + assert!(control.unregistered().is_empty()); + for cs in registered { + assert!(db.has_chunk(&cs).unwrap()); + } + assert!(!db.has_chunk(&pending_cs).unwrap()); + assert!(!db.has_chunk(&oversized_cs).unwrap()); + assert!(budget.try_reserve(bytes).is_some()); +} + +#[test] +fn explicit_batch_write_runs_gc_on_map_full() { + let temp = TempDir::new().unwrap(); + let db = ChunkDB::new_with_size_and_index_ctl(temp.path(), Some(8 << 20), None).unwrap(); + let cached = b"cached"; + let cached_cs = CheckSum::from_data(cached, CheckSumMethod::Blake3); + db.add_chunk(&cached_cs, cached.to_vec()).unwrap(); + assert!(wait_for_access_time(&db, &CheckSumOnDisk::from(cached_cs)).is_some()); + + let oversized = vec![42; 9 << 20]; + let oversized_cs = CheckSum::from_data(&oversized, CheckSumMethod::Blake3); + let err = db.add_chunks(vec![(oversized_cs, oversized)]).unwrap_err(); + assert!(matches!( + err.downcast_ref::(), + Some(heed::Error::Mdb(MdbError::MapFull)) + )); + assert!(!db.has_chunk(&cached_cs).unwrap()); + assert!(!db.has_chunk(&oversized_cs).unwrap()); +} + #[test] fn test_gc_expired_large_batch_batched_deletion() { let temp = TempDir::new().unwrap(); diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 23da673..b81fd80 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -20,6 +20,8 @@ use std::fmt::Debug; use std::sync::Arc; use std::time::{Duration, Instant}; pub mod cache; +#[cfg(target_os = "linux")] +pub(crate) mod cache_write_budget; pub mod chunkdb; pub mod dedup; pub mod general; diff --git a/src/image/nydus.rs b/src/image/nydus.rs index 0d4b38d..a4fa536 100644 --- a/src/image/nydus.rs +++ b/src/image/nydus.rs @@ -15,6 +15,8 @@ #[cfg(target_os = "linux")] use crate::backend::cache::Cache; #[cfg(target_os = "linux")] +use crate::backend::cache_write_budget::{CacheWriteBudget, CacheWritePermit}; +#[cfg(target_os = "linux")] use crate::backend::chunkdb::CheckSumMethod; use crate::backend::chunkdb::{CheckSum, ChunkDB}; #[cfg(target_os = "linux")] @@ -28,6 +30,7 @@ use crate::image::FsReadMetrics; use crate::rate_limited_log; #[cfg(target_os = "linux")] use crate::utils::new_std_io_error; +#[cfg(target_os = "linux")] use crate::utils::RateLimitedLog; #[cfg(target_os = "linux")] use fuse_backend_rs::abi::fuse_abi::statvfs64; @@ -57,12 +60,13 @@ use std::io::{self, ErrorKind}; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; use std::path::Path; -use std::sync::atomic::AtomicU64; -#[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; use std::sync::atomic::Ordering::Relaxed; +#[cfg(target_os = "linux")] +use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(target_os = "linux")] use std::sync::mpsc::{self, Receiver, SyncSender}; use std::sync::{Arc, Mutex}; +#[cfg(target_os = "linux")] use std::thread; #[cfg(target_os = "linux")] use std::time::Duration; @@ -72,21 +76,37 @@ use std::time::{SystemTime, UNIX_EPOCH}; #[cfg(target_os = "linux")] use tracing::error; use tracing::info; +#[cfg(target_os = "linux")] use tracing::warn; +#[cfg(target_os = "linux")] const CHUNK_STORE_REQ_BUF_SIZE: usize = 4096; +#[cfg(target_os = "linux")] const CHUNK_STORE_BATCH_SIZE: usize = 64; +#[cfg(target_os = "linux")] +const CACHE_WRITE_SLOTS: usize = 64; +#[cfg(target_os = "linux")] +const CACHE_WRITE_SLOT_BYTES: usize = 16 * 1024 * 1024; +#[cfg(target_os = "linux")] +const CACHE_WRITE_MAX_AGE: Duration = Duration::from_secs(2); +#[cfg(target_os = "linux")] +const MAP_FULL_RETRY_DELAY: Duration = Duration::from_secs(5); +#[cfg(target_os = "linux")] struct ChunkStoreRequest { checksum: CheckSum, data: Vec, + deadline: Instant, + permit: CacheWritePermit, } +#[cfg(target_os = "linux")] struct ChunkStoreWorker { chunk_db: Arc, rx: Receiver, } +#[cfg(target_os = "linux")] impl ChunkStoreWorker { fn new(chunk_db: Arc, rx: Receiver) -> Self { Self { chunk_db, rx } @@ -94,40 +114,34 @@ impl ChunkStoreWorker { fn run(self) { let mut batch = Vec::with_capacity(CHUNK_STORE_BATCH_SIZE); - let mut batch_data: Vec> = Vec::with_capacity(CHUNK_STORE_BATCH_SIZE); + let warn_limiter = RateLimitedLog::new(5); + let mut retry_after = None; while let Ok(req) = self.rx.recv() { - batch.push(req.checksum); - batch_data.push(req.data); - - // Try to collect more requests without blocking - while batch.len() < CHUNK_STORE_BATCH_SIZE { - match self.rx.try_recv() { - Ok(req) => { - batch.push(req.checksum); - batch_data.push(req.data); - } - Err(mpsc::TryRecvError::Empty) => break, - Err(mpsc::TryRecvError::Disconnected) => break, - } + if retry_after.is_some_and(|deadline| Instant::now() < deadline) { + continue; } - - // Process the batch - let chunks: Vec<(CheckSum, Vec)> = - batch.drain(..).zip(batch_data.drain(..)).collect(); - - if let Err(e) = self.chunk_db.add_chunks(chunks) { - warn!(err = debug(e), "Failed to store chunk batch into chunkdb."); - } - } - - // Process any remaining requests in the batch - if !batch.is_empty() { - let chunks: Vec<(CheckSum, Vec)> = - batch.drain(..).zip(batch_data.drain(..)).collect(); - - if let Err(e) = self.chunk_db.add_chunks(chunks) { - warn!(err = debug(e), "Failed to store chunk batch into chunkdb."); + batch.push(req); + batch.extend(self.rx.try_iter().take(CHUNK_STORE_BATCH_SIZE - 1)); + let now = Instant::now(); + batch.retain(|req| req.deadline > now); + let Some(deadline) = batch.iter().map(|req| req.deadline).min() else { + continue; + }; + let (chunks, permits) = batch + .drain(..) + .map(|req| ((req.checksum, req.data), req.permit)) + .unzip(); + if let Err(e) = self.chunk_db.add_cache_chunks(chunks, permits, deadline) { + if matches!( + e.downcast_ref::(), + Some(heed::Error::Mdb(heed::MdbError::MapFull)) + ) { + retry_after = Some(Instant::now() + MAP_FULL_RETRY_DELAY); + } + rate_limited_log!(warn_limiter, { + warn!(err = debug(e), "Failed to store chunk batch into chunkdb."); + }); } } } @@ -146,13 +160,15 @@ pub struct NydusImage { dedup_db: Option<(Arc, Arc)>, #[cfg_attr(not(target_os = "linux"), allow(dead_code))] local_chunk_client: Option>, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] + store_budget: Option, + #[cfg(target_os = "linux")] store_tx: Option>, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] store_worker: Mutex>>, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] store_dropped_count: AtomicU64, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + #[cfg(target_os = "linux")] store_warn_limiter: RateLimitedLog, #[cfg_attr(not(target_os = "linux"), allow(dead_code))] cache_dir: String, @@ -204,14 +220,31 @@ impl NydusImage { let i_uid = unsafe { libc::geteuid() } as u32; let i_gid = unsafe { libc::getegid() } as u32; - let (store_tx, store_worker) = if let Some((chunk_db, _)) = &dedup_db { - let (tx, rx) = mpsc::sync_channel(CHUNK_STORE_REQ_BUF_SIZE); - let worker = ChunkStoreWorker::new(chunk_db.clone(), rx); - let handle = thread::spawn(move || worker.run()); - (Some(tx), Mutex::new(Some(handle))) - } else { - (None, Mutex::new(None)) - }; + #[cfg(target_os = "linux")] + let store_budget = + dedup_db.as_ref().and_then(|(chunk_db, _)| { + match CacheWriteBudget::new( + chunk_db.path(), + CACHE_WRITE_SLOTS, + CACHE_WRITE_SLOT_BYTES, + ) { + Ok(budget) => Some(budget), + Err(e) => { + warn!(err = debug(e), "Chunk backfill budget unavailable."); + None + } + } + }); + #[cfg(target_os = "linux")] + let (store_tx, store_worker) = + if let (Some((chunk_db, _)), Some(_)) = (&dedup_db, &store_budget) { + let (tx, rx) = mpsc::sync_channel(CHUNK_STORE_REQ_BUF_SIZE); + let worker = ChunkStoreWorker::new(chunk_db.clone(), rx); + let handle = thread::spawn(move || worker.run()); + (Some(tx), Mutex::new(Some(handle))) + } else { + (None, Mutex::new(None)) + }; Ok(Self { sb: Arc::new(sb), @@ -220,9 +253,15 @@ impl NydusImage { backend, dedup_db, local_chunk_client, + #[cfg(target_os = "linux")] + store_budget, + #[cfg(target_os = "linux")] store_tx, + #[cfg(target_os = "linux")] store_worker, + #[cfg(target_os = "linux")] store_dropped_count: AtomicU64::new(0), + #[cfg(target_os = "linux")] store_warn_limiter: RateLimitedLog::new(5), cache_dir: cache_dir.to_string(), i_uid, @@ -377,20 +416,33 @@ impl NydusImage { #[cfg(target_os = "linux")] fn store_chunk_bg(&self, cs: CheckSum, data: Vec) { - if let Some(tx) = &self.store_tx { - let req = ChunkStoreRequest { checksum: cs, data }; - if let Err(_e) = tx.try_send(req) { - let dropped = self.store_dropped_count.fetch_add(1, Ordering::Relaxed) + 1; - rate_limited_log!(self.store_warn_limiter, { - warn!( - dropped_count = dropped, - queue_size = CHUNK_STORE_REQ_BUF_SIZE, - "Chunk store queue full, dropping requests." - ); - self.store_dropped_count.store(0, Ordering::Relaxed); - }); + let (Some(tx), Some(budget)) = (&self.store_tx, &self.store_budget) else { + return; + }; + let message = if let Some(permit) = budget.try_reserve(data.capacity()) { + let req = ChunkStoreRequest { + checksum: cs, + data, + deadline: Instant::now() + CACHE_WRITE_MAX_AGE, + permit, + }; + if tx.try_send(req).is_ok() { + return; } - } + "Chunk store queue full, dropping requests." + } else { + drop(data); + "Chunk store byte budget unavailable, dropping requests." + }; + self.store_dropped_count.fetch_add(1, Ordering::Relaxed); + rate_limited_log!(self.store_warn_limiter, { + let dropped = self.store_dropped_count.swap(0, Ordering::Relaxed); + warn!( + dropped_count = dropped, + queue_size = CHUNK_STORE_REQ_BUF_SIZE, + "{message}" + ); + }); } #[cfg(target_os = "linux")] @@ -798,6 +850,7 @@ impl fuse_backend_rs::api::filesystem::FileSystem for NydusImage { } } +#[cfg(target_os = "linux")] impl Drop for NydusImage { fn drop(&mut self) { // Drop the sender to signal the worker thread to exit @@ -811,3 +864,43 @@ impl Drop for NydusImage { } } } + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + #[test] + fn map_full_pauses_later_backfill_batches() { + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new( + ChunkDB::new_with_size_and_index_ctl(dir.path(), Some(8 << 20), None).unwrap(), + ); + let slot_bytes = 16 << 20; + let budget = CacheWriteBudget::new(dir.path(), 1, slot_bytes).unwrap(); + let (tx, rx) = mpsc::sync_channel(CHUNK_STORE_BATCH_SIZE + 1); + let deadline = Instant::now() + Duration::from_secs(30); + let enqueue = |data: Vec| { + let checksum = CheckSum::from_data(&data, CheckSumMethod::Blake3); + let permit = budget.try_reserve(data.capacity()).unwrap(); + tx.send(ChunkStoreRequest { + checksum, + data, + deadline, + permit, + }) + .unwrap(); + checksum + }; + + enqueue(vec![0; 9 << 20]); + for _ in 1..CHUNK_STORE_BATCH_SIZE { + enqueue(vec![1]); + } + // This chunk would fit if the next batch immediately retried LMDB. + let next = enqueue(vec![2]); + drop(tx); + ChunkStoreWorker::new(db.clone(), rx).run(); + assert!(!db.has_chunk(&next).unwrap()); + assert!(budget.try_reserve(slot_bytes).is_some()); + } +} diff --git a/tests/mount.rs b/tests/mount.rs index 2a2c161..39339ac 100644 --- a/tests/mount.rs +++ b/tests/mount.rs @@ -18,9 +18,12 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; +use distill_fs::backend::chunkdb::{CheckSum, CheckSumMethod, ChunkDB}; +use distill_fs::backend::indexdb::IndexDB; use fuse_backend_rs::api::filesystem::{Context, FileSystem, ZeroCopyWriter}; fn init_tracing() { @@ -204,8 +207,18 @@ fn nydus_direct_reads_file() { let temp = tempfile::TempDir::new().unwrap(); let expected = b"distill-fs nydus mount\n"; let (bootstrap, config) = build_nydus_fixture(&nydus_image, &temp, expected); - let image = - distill_fs::image::nydus::NydusImage::new(&bootstrap, &config, "", None, None).unwrap(); + let chunk_dir = tempfile::TempDir::new_in(temp.path()).unwrap(); + let index_dir = tempfile::TempDir::new_in(temp.path()).unwrap(); + let chunk_db = Arc::new(ChunkDB::new(chunk_dir.path()).unwrap()); + let index_db = Arc::new(IndexDB::open(index_dir.path()).unwrap()); + let image = distill_fs::image::nydus::NydusImage::new( + &bootstrap, + &config, + "", + Some((chunk_db.clone(), index_db)), + None, + ) + .unwrap(); let ctx = Context::new(); let name = std::ffi::CString::new("hello.txt").unwrap(); @@ -227,6 +240,9 @@ fn nydus_direct_reads_file() { .unwrap(); assert_eq!(n, expected.len()); assert_eq!(writer.data, expected); + drop(image); + let checksum = CheckSum::from_data(expected, CheckSumMethod::Blake3); + assert!(chunk_db.has_chunk(&checksum).unwrap()); } #[test]