diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebc47625..0c59af23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,14 @@ jobs: timeout-minutes: 10 run: cargo test --release --test fsx -- --test-threads=1 --nocapture + - name: fsx paranoid (CAS round-trip per mutation, --sparse-writes) + timeout-minutes: 15 + run: cargo test --release --test fsx_paranoid -- --test-threads=1 --nocapture + + - name: sparse concurrent real-CAS (multi-worker same-inode, --sparse-writes) + timeout-minutes: 15 + run: cargo test --release --test sparse_concurrent_real -- --test-threads=1 --nocapture + xfstests: name: xfstests (filesystem exerciser) runs-on: diff --git a/Cargo.lock b/Cargo.lock index 1945abe2..c938967c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -372,7 +372,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -572,7 +572,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -632,7 +632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1249,7 +1249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" dependencies = [ "io-lifetimes 3.0.1", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -1586,7 +1586,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1942,7 +1942,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -2226,7 +2226,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2293,7 +2293,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2662,7 +2662,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3334,7 +3334,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3461,7 +3461,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3470,16 +3470,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -3497,31 +3488,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -3539,96 +3513,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.3" @@ -3751,7 +3677,7 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "xet-client" version = "1.5.2" -source = "git+https://github.com/huggingface/xet-core.git?branch=main#40f9530753e48f9517b5ad09e2339df7059d8de4" +source = "git+https://github.com/huggingface/xet-core.git?rev=40f9530753e48f9517b5ad09e2339df7059d8de4#40f9530753e48f9517b5ad09e2339df7059d8de4" dependencies = [ "anyhow", "async-trait", @@ -3788,7 +3714,7 @@ dependencies = [ [[package]] name = "xet-core-structures" version = "1.5.2" -source = "git+https://github.com/huggingface/xet-core.git?branch=main#40f9530753e48f9517b5ad09e2339df7059d8de4" +source = "git+https://github.com/huggingface/xet-core.git?rev=40f9530753e48f9517b5ad09e2339df7059d8de4#40f9530753e48f9517b5ad09e2339df7059d8de4" dependencies = [ "async-trait", "base64", @@ -3824,7 +3750,7 @@ dependencies = [ [[package]] name = "xet-data" version = "1.5.2" -source = "git+https://github.com/huggingface/xet-core.git?branch=main#40f9530753e48f9517b5ad09e2339df7059d8de4" +source = "git+https://github.com/huggingface/xet-core.git?rev=40f9530753e48f9517b5ad09e2339df7059d8de4#40f9530753e48f9517b5ad09e2339df7059d8de4" dependencies = [ "anyhow", "async-trait", @@ -3856,7 +3782,7 @@ dependencies = [ [[package]] name = "xet-runtime" version = "1.5.2" -source = "git+https://github.com/huggingface/xet-core.git?branch=main#40f9530753e48f9517b5ad09e2339df7059d8de4" +source = "git+https://github.com/huggingface/xet-core.git?rev=40f9530753e48f9517b5ad09e2339df7059d8de4#40f9530753e48f9517b5ad09e2339df7059d8de4" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index d3f115f8..f7185aa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,10 +10,10 @@ categories = ["filesystem", "command-line-utilities"] [dependencies] # xet-core crates -xet-client = { git = "https://github.com/huggingface/xet-core.git", branch = "main" } -xet-core-structures = { git = "https://github.com/huggingface/xet-core.git", branch = "main" } -xet-data = { git = "https://github.com/huggingface/xet-core.git", branch = "main" } -xet-runtime = { git = "https://github.com/huggingface/xet-core.git", branch = "main" } +xet-client = { git = "https://github.com/huggingface/xet-core.git", rev = "40f9530753e48f9517b5ad09e2339df7059d8de4" } +xet-core-structures = { git = "https://github.com/huggingface/xet-core.git", rev = "40f9530753e48f9517b5ad09e2339df7059d8de4" } +xet-data = { git = "https://github.com/huggingface/xet-core.git", rev = "40f9530753e48f9517b5ad09e2339df7059d8de4" } +xet-runtime = { git = "https://github.com/huggingface/xet-core.git", rev = "40f9530753e48f9517b5ad09e2339df7059d8de4" } # External crates async-trait = "0.1" diff --git a/src/nfs.rs b/src/nfs.rs index 47ad9496..9e2837a9 100644 --- a/src/nfs.rs +++ b/src/nfs.rs @@ -816,6 +816,15 @@ fn errno_to_nfs(e: i32) -> nfsstat3 { libc::ENOTEMPTY => nfsstat3::NFS3ERR_NOTEMPTY, libc::EBADF => nfsstat3::NFS3ERR_STALE, libc::ENOSPC => nfsstat3::NFS3ERR_NOSPC, + // EAGAIN is used internally as the sparse-write drift sentinel + // (`open_advanced_write` returns it when poll rotates xet_hash + // between snapshot and install). The internal retry loop in + // `open()` normally upgrades a final EAGAIN to EIO so userspace + // never sees it, but map it defensively here to NFS3ERR_JUKEBOX + // (the conventional retryable-busy code) so a future code path + // that leaks EAGAIN doesn't degrade to NFS3ERR_IO on an NFS + // export. + libc::EAGAIN => nfsstat3::NFS3ERR_JUKEBOX, _ => nfsstat3::NFS3ERR_IO, } } diff --git a/src/setup.rs b/src/setup.rs index cfa77535..42e2c170 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -104,6 +104,16 @@ pub struct MountOptions { #[arg(long, default_value_t = false)] pub advanced_writes: bool, + /// Sparse writes: open-for-write punches a hole instead of downloading + /// the original CAS content, and flush composes the new file via + /// `range_upload` (CAS prefix/suffix + re-chunked dirty windows). Reads + /// outside the dirty regions fetch from CAS on demand and populate the + /// local staging file as a cache, so each byte is fetched at most once + /// per mount lifetime. Saves bandwidth on small-edit workloads against + /// large CAS-backed files. Implies `--advanced-writes`. Experimental. + #[arg(long, default_value_t = false)] + pub sparse_writes: bool, + /// Interval in seconds for polling remote changes (0 to disable). #[arg(long, default_value_t = 30)] pub poll_interval_secs: u64, @@ -427,7 +437,13 @@ pub fn build_with_runtime( let upload_config = if remote_read_only { None } else { Some(cas_config) }; let xet_sessions = XetSessions::new(xet_ctx, download_session, upload_config, cached_client, xorb_cache); - let advanced_writes = options.advanced_writes || options.overlay || (is_nfs && !read_only); + // --sparse-writes implies --advanced-writes: the sparse path lives on + // top of the staging substrate that advanced writes provides. Forcing + // the implication here is what makes the implication real — without + // this, VfsConfig would silently downgrade sparse_writes to false and + // the user would see the legacy write path behind their back. + let advanced_writes = options.advanced_writes || options.sparse_writes || options.overlay || (is_nfs && !read_only); + let sparse_writes = options.sparse_writes; // Overlay: open a pre-mount fd to the mount point directory. The fd is // held by OverlayBacking so overlay-local filesystem ops can stay rooted @@ -489,12 +505,16 @@ pub fn build_with_runtime( access_mode, backend_name, ); + if sparse_writes { + warn!("--sparse-writes is experimental; report issues at https://github.com/huggingface/hf-mount/issues"); + } info!( - "Config: advanced_writes={} overlay={} remote_read_only={} direct_io={} poll_interval={}s \ + "Config: advanced_writes={} sparse_writes={} overlay={} remote_read_only={} direct_io={} poll_interval={}s \ poll_listing_concurrency={} metadata_ttl={}ms \ cache_dir={:?} cache_size={} no_disk_cache={} cache_mode={:?} max_staging_size={} max_threads={} \ flush_debounce={}ms flush_max_batch={}ms uid={} gid={} filter_os_files={}", advanced_writes, + sparse_writes, options.overlay, remote_read_only, options.direct_io, @@ -526,6 +546,15 @@ pub fn build_with_runtime( VfsConfig { read_only, advanced_writes, + sparse_writes, + // Sparse-write fallback threshold. Files smaller than this bypass + // the sparse path and use the standard download-then-upload route. + // Default 256 MiB (bench-derived crossover for typical cloud + // networks); override via HF_MOUNT_SPARSE_MIN_BYTES. + sparse_min_size_bytes: std::env::var("HF_MOUNT_SPARSE_MIN_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(256 * 1024 * 1024), uid, gid, poll_interval_secs: options.poll_interval_secs, diff --git a/src/test_mocks.rs b/src/test_mocks.rs index 8d945c38..63d2527e 100644 --- a/src/test_mocks.rs +++ b/src/test_mocks.rs @@ -12,7 +12,7 @@ use xet_data::processing::XetFileInfo; use crate::error::{Error, Result}; use crate::hub_api::{BatchOp, HeadFileInfo, HubOps, SourceKind, TreeEntry}; use crate::overlay::OverlayBacking; -use crate::xet::{DownloadStreamOps, StagingDir, StreamingWriterOps, XetOps}; +use crate::xet::{DownloadStreamOps, RangeSnapshot, StagingDir, StreamingWriterOps, XetOps}; // ── MockHub ─────────────────────────────────────────────────────────── @@ -278,10 +278,14 @@ impl HubOps for MockHub { // ── MockXet ─────────────────────────────────────────────────────────── pub struct MockXet { - files: Mutex>>, + pub files: Mutex>>, pub next_hash: AtomicU64, writer_create_fail: AtomicBool, upload_fail: AtomicBool, + /// Force only the next `upload_files` (Pass B) call to fail, without + /// affecting `range_upload` (Pass A). Used to assert that a Pass B + /// failure doesn't pollute Pass A successes in the same flush batch. + upload_files_fail: AtomicBool, download_fail: AtomicBool, writer_fail_after: AtomicU64, /// Number of range download calls that should fail before succeeding. @@ -313,6 +317,7 @@ impl MockXet { next_hash: AtomicU64::new(1), writer_create_fail: AtomicBool::new(false), upload_fail: AtomicBool::new(false), + upload_files_fail: AtomicBool::new(false), download_fail: AtomicBool::new(false), writer_fail_after: AtomicU64::new(u64::MAX), range_fail_count: AtomicU32::new(0), @@ -348,6 +353,14 @@ impl MockXet { self.upload_fail.store(true, Ordering::SeqCst); } + /// Force ONLY the next `upload_files` (batched Pass B) call to fail. + /// `range_upload` (Pass A) is unaffected. Used to verify that a Pass B + /// abort doesn't misattribute its error to sparse items that already + /// succeeded in Pass A. + pub fn fail_next_upload_files(&self) { + self.upload_files_fail.store(true, Ordering::SeqCst); + } + pub fn fail_writer_after(&self, bytes: u64) { self.writer_fail_after.store(bytes, Ordering::SeqCst); } @@ -407,7 +420,7 @@ impl XetOps for MockXet { gate.release.notified().await; self.uploads_inflight.fetch_sub(1, Ordering::SeqCst); } - if self.upload_fail.swap(false, Ordering::SeqCst) { + if self.upload_fail.swap(false, Ordering::SeqCst) || self.upload_files_fail.swap(false, Ordering::SeqCst) { return Err(Error::Xet("mock upload failure".into())); } let mut results = Vec::new(); @@ -460,6 +473,43 @@ impl XetOps for MockXet { chunk_size: 4096, })) } + + async fn range_upload( + &self, + original_hash: &str, + original_size: u64, + new_file_size: u64, + dirty_snapshots: Vec, + ) -> Result { + if self.upload_fail.swap(false, Ordering::SeqCst) { + return Err(Error::Xet("mock range upload failure".into())); + } + // No-op: matches the real XetSessions short-circuit. Lets tests pin + // the "nothing changed → original hash preserved" invariant. + if dirty_snapshots.is_empty() && new_file_size == original_size { + return Ok(XetFileInfo::new(original_hash.to_string(), original_size)); + } + // Build the new file by overlaying dirty bytes on top of the original + // content. Truncate / extend as needed to match new_file_size. + let original = { + let files = self.files.lock().unwrap(); + files.get(original_hash).cloned().unwrap_or_default() + }; + let new_size_usize = new_file_size as usize; + let mut composed = vec![0u8; new_size_usize]; + let copy_len = original.len().min(new_size_usize); + composed[..copy_len].copy_from_slice(&original[..copy_len]); + for snap in dirty_snapshots { + let start = snap.offset as usize; + let end = (start + snap.data.len()).min(new_size_usize); + if start < new_size_usize { + composed[start..end].copy_from_slice(&snap.data[..end - start]); + } + } + let new_hash = self.next_hash_string(); + self.files.lock().unwrap().insert(new_hash.clone(), composed); + Ok(XetFileInfo::new(new_hash, new_file_size)) + } } // ── MockStreamingWriter ─────────────────────────────────────────────── @@ -522,6 +572,11 @@ impl DownloadStreamOps for MockDownloadStream { pub struct TestOpts { pub read_only: bool, pub advanced_writes: bool, + /// Enables sparse-write staging. The mock-driven test_vfs honors this + /// faithfully even when `advanced_writes` is false: the helper turns the + /// effective advanced_writes flag on so the underlying VfsConfig + /// derivation does not silently downgrade `sparse_writes` to false. + pub sparse_writes: bool, pub overlay: bool, pub serve_lookup_from_cache: bool, pub metadata_ttl: Duration, @@ -534,6 +589,7 @@ impl Default for TestOpts { Self { read_only: false, advanced_writes: false, + sparse_writes: false, overlay: false, serve_lookup_from_cache: false, metadata_ttl: Duration::from_secs(1), @@ -581,7 +637,12 @@ pub fn make_test_vfs( opts: TestOpts, runtime: &tokio::runtime::Runtime, ) -> Arc { - let effective_advanced_writes = opts.advanced_writes || opts.overlay; + // Sparse writes need a staging substrate (advanced or overlay). Force + // advanced_writes on when the test opted into sparse_writes so VfsConfig + // does not silently downgrade. Without this, tests using + // `TestOpts { sparse_writes: true, ..Default::default() }` would get the + // non-sparse path and the feature wouldn't actually be exercised. + let effective_advanced_writes = opts.advanced_writes || opts.overlay || opts.sparse_writes; let overlay_backing = if opts.overlay { let overlay_root = fresh_test_dir("hf_mount_overlay"); @@ -609,7 +670,12 @@ pub fn make_test_vfs( overlay_backing, crate::virtual_fs::VfsConfig { read_only: opts.read_only, - advanced_writes: opts.advanced_writes, + advanced_writes: effective_advanced_writes, + sparse_writes: opts.sparse_writes, + // Mock tests use tiny files (10-50 bytes) and need sparse semantics + // regardless of size — set the threshold to 0 so every file + // engages the sparse path under test. + sparse_min_size_bytes: 0, uid: 1000, gid: 1000, poll_interval_secs: 0, @@ -653,6 +719,8 @@ pub fn make_overlay_test_vfs_with_root( crate::virtual_fs::VfsConfig { read_only: false, advanced_writes: false, + sparse_writes: false, + sparse_min_size_bytes: 0, uid: 1000, gid: 1000, poll_interval_secs: 0, diff --git a/src/virtual_fs/flush.rs b/src/virtual_fs/flush.rs index 5dad4c7c..2747e2b7 100644 --- a/src/virtual_fs/flush.rs +++ b/src/virtual_fs/flush.rs @@ -7,10 +7,13 @@ use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; use crate::hub_api::{BatchOp, HubOps}; -use crate::xet::XetOps; +use crate::xet::{RangeSnapshot, XetOps}; +use bytes::Bytes; -use super::inode::InodeTable; +use super::inode::{InodeTable, SparseWriteState}; use super::staging::StagingCoordinator; +use std::os::unix::fs::FileExt; +use xet_data::processing::XetFileInfo; enum FlushSignal { /// Flush a dirty inode. @@ -270,6 +273,62 @@ struct FlushItem { /// Hash from the last successful commit, used to skip redundant Hub commits /// when the CAS upload produces the same hash (content unchanged). prev_xet_hash: Option, + /// Current file size at snapshot time. Captured under the inode read + /// lock together with the sparse_write Arc, so range_upload composes a + /// file of the right shape even if a concurrent setattr races later. + file_size: u64, + /// Sparse-write snapshot. When Some, the flush composes a new CAS file + /// from this state's dirty ranges via range_upload (no re-upload of the + /// unchanged prefix/suffix). When None, the flush takes the regular path + /// and uploads the entire staging file via upload_files. + sparse_snapshot: Option>, +} + +/// Read each dirty range of `sparse` from the staging file at `staging_path`, +/// producing a Vec of `RangeSnapshot` ready for `range_upload`. Each range is +/// read under the per-inode io_lock so a concurrent pwrite cannot interleave +/// with our reads. +fn snapshot_dirty_bytes( + staging: &StagingCoordinator, + ino: u64, + staging_path: &std::path::Path, + sparse: &SparseWriteState, +) -> std::io::Result> { + if sparse.dirty_ranges.is_empty() { + return Ok(Vec::new()); + } + let file = std::fs::File::open(staging_path)?; + let io_lock = staging.io_lock(ino); + let _io_guard = io_lock.lock().expect("staging io_lock poisoned"); + let mut snapshots = Vec::with_capacity(sparse.dirty_ranges.len()); + for &(start, end) in &sparse.dirty_ranges { + let len = (end - start) as usize; + let mut buf = vec![0u8; len]; + // The staging file MUST hold every byte the dirty range claims: the + // flush holds `staging.lock(ino)` across this snapshot (see flush_batch), + // and `setattr`-shrink takes the same lock before truncating, so it + // cannot shrink staging underneath us. A short read therefore signals a + // real invariant violation, not a benign race; surface it as an error + // (the dirty_generation guard makes the next flush retry) instead of + // silently composing a wrong-length file from a truncated buffer. + // Translate UnexpectedEof into an explicit invariant-violation message + // so the failure mode is greppable in logs (the default Display is + // just "failed to fill whole buffer" which leaks zero context). + file.read_exact_at(&mut buf, start).map_err(|e| { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + std::io::Error::other(format!( + "staging short-read at dirty range [{start}, {end}) — staging file shorter than dirty_ranges claims (invariant violation)" + )) + } else { + e + } + })?; + snapshots.push(RangeSnapshot { + offset: start, + data: Bytes::from(buf), + }); + } + Ok(snapshots) } #[allow(clippy::too_many_arguments)] @@ -334,6 +393,8 @@ async fn flush_batch( pending_deletes: entry.pending_deletes.clone(), dirty_generation: entry.dirty_generation, prev_xet_hash: entry.xet_hash.clone(), + file_size: entry.size, + sparse_snapshot: entry.sparse_write.clone(), }) }) .collect() @@ -350,11 +411,99 @@ async fn flush_batch( // Upload in chunks to bound FD usage (xet-core opens all staging files per // upload session), but accumulate all batch ops for a single Hub commit to // preserve the global adds-before-deletes ordering required by the Hub API. + // + // Sparse items take the per-item range_upload path: dirty bytes are + // snapshotted under the per-inode io_lock, then composed against the + // existing CAS reconstruction without re-uploading the unchanged prefix/ + // suffix. Regular items go through the batched upload_files path. const UPLOAD_CHUNK_SIZE: usize = 500; - let mut upload_results = Vec::with_capacity(to_flush.len()); + let mut upload_results: Vec> = vec![None; to_flush.len()]; + + // Pass A: per-item range_upload for sparse items whose coverage has + // HOLES (the staging file is incomplete and we want the wire savings + // of composing against the old CAS base). A failure on one sparse item + // is recorded per-inode and the loop continues — independent sparse + // items are not head-of-line-blocked by a sibling's sticky failure. + // The failing item leaves its slot in `upload_results` as None: + // downstream commit-apply skips it, clear_dirty_if is never invoked, + // the next flush retries it. + // + // FAST PATH: when sparse_write.coverage spans the full file (typical + // after a long-lived handle that read most of the file plus a few + // dirty writes), the staging file IS the committed content. Take the + // non-sparse upload_files path instead — same wire bytes (xet CDC + // dedup re-uses existing xorbs for unchanged chunks) but skips + // range_upload's per-window CAS re-fetches of bytes we already have + // locally. Bench at 1.2 GB / 12 MB blob: the fast path runs at the + // upload_files speed (~1.37 s) instead of range_upload's ~1.5 s, and + // the saving grows with file size as range_upload's stream_cas_range + // calls download more "around the edit" bytes for the composition. + // Per-item flag set when a sparse item is routed through the fast-path + // upload_files instead of range_upload. The post-upload commit-apply uses + // this to dispatch `apply_commit` (drop sparse_write — staging IS the + // committed content) instead of `apply_commit_sparse` (re-key sparse_write + // — staging is a partial mirror). Vec for O(1) lookup; an aligned + // index avoids the previous O(n²) `fast_path_indices.contains` scan. + let mut is_fast_path = vec![false; to_flush.len()]; + for (idx, item) in to_flush.iter().enumerate() { + let Some(sw) = item.sparse_snapshot.as_ref() else { + continue; + }; + // Fast path requires both: (a) full coverage so the staging file is + // a valid full-file content, and (b) at least one dirty range so + // there's a real change to commit. With no dirty ranges, range_upload + // short-circuits to the original hash unchanged ("no-op flush" + // semantic preserved by `apply_noop_commit`-style retention of the + // hash). Forcing upload_files here would re-hash the staging into a + // new (CDC-equivalent) hash and lose that semantic. + if sw.is_fully_covered(item.file_size) && !sw.dirty_ranges.is_empty() { + is_fast_path[idx] = true; + continue; + } + // Snapshot dirty bytes under io_lock so a concurrent pwrite cannot + // interleave with our reads and produce chimeric content downstream. + let snapshots = match snapshot_dirty_bytes(staging, item.ino, &item.staging_path, sw) { + Ok(s) => s, + Err(e) => { + error!( + "sparse flush: snapshot failed for ino={} path={}: {}", + item.ino, item.full_path, e + ); + flush_errors + .lock() + .expect("flush_errors poisoned") + .insert(item.ino, format!("snapshot failed: {e}")); + continue; + } + }; + match xet_sessions + .range_upload(&sw.original_hash, sw.original_size, item.file_size, snapshots) + .await + { + Ok(info) => upload_results[idx] = Some(info), + Err(e) => { + error!( + "sparse flush: range_upload failed for ino={} path={}: {}", + item.ino, item.full_path, e + ); + flush_errors + .lock() + .expect("flush_errors poisoned") + .insert(item.ino, format!("range_upload failed: {e}")); + } + } + } - for (chunk_idx, chunk) in to_flush.chunks(UPLOAD_CHUNK_SIZE).enumerate() { - let staging_paths: Vec<&std::path::Path> = chunk.iter().map(|item| item.staging_path.as_path()).collect(); + // Pass B: batched upload_files for non-sparse items AND fully-covered + // sparse items that took the fast path above. + let regular_indices: Vec = to_flush + .iter() + .enumerate() + .filter(|(i, it)| it.sparse_snapshot.is_none() || is_fast_path[*i]) + .map(|(i, _)| i) + .collect(); + for chunk in regular_indices.chunks(UPLOAD_CHUNK_SIZE) { + let staging_paths: Vec<&std::path::Path> = chunk.iter().map(|&i| to_flush[i].staging_path.as_path()).collect(); match xet_sessions.upload_files(&staging_paths).await { Ok(results) => { assert_eq!( @@ -364,16 +513,30 @@ async fn flush_batch( results.len(), chunk.len() ); - upload_results.extend(results); + for (slot_idx, info) in chunk.iter().zip(results) { + upload_results[*slot_idx] = Some(info); + } } Err(e) => { - // Abort the entire batch: committing partial results could apply - // deletes without the corresponding adds from this failed chunk. - error!("Batch upload failed (chunk {}), aborting flush: {}", chunk_idx, e); + error!("Batch upload failed, aborting flush: {}", e); let msg = format!("upload failed: {e}"); let mut errs = flush_errors.lock().expect("flush_errors poisoned"); - for item in &to_flush { - errs.insert(item.ino, msg.clone()); + // The whole batch is aborted: no Hub commit is sent for any + // item. Mark only items that have NO upload result yet — the + // ones that actually failed (this chunk or earlier Pass A + // failures) or that the early return prevented from running. + // Items with `upload_results[i] = Some(_)` already pushed + // their CAS xorb successfully (Pass A range_upload or an + // earlier Pass B chunk); the data is durable, only the Hub + // commit is missing. They stay dirty (no apply_commit runs) + // and will retry the Hub commit on the next flush — fsync + // must not surface an "upload failed" error for them. + // `or_insert_with` still preserves the more specific Pass A + // error message on items that did fail there. + for (i, item) in to_flush.iter().enumerate() { + if upload_results[i].is_none() { + errs.entry(item.ino).or_insert_with(|| msg.clone()); + } } return; } @@ -384,7 +547,7 @@ async fn flush_batch( // gc() calls below (which acquire the same per-inode locks) can proceed. drop(_staging_guards); - if upload_results.is_empty() { + if upload_results.iter().all(|r| r.is_none()) { return; } @@ -398,7 +561,12 @@ async fn flush_batch( let mut delete_ops = Vec::new(); let mut unchanged = vec![false; to_flush.len()]; - for (i, (item, file_info)) in to_flush.iter().zip(upload_results.iter()).enumerate() { + for (i, (item, file_info_opt)) in to_flush.iter().zip(upload_results.iter()).enumerate() { + // Skip items whose upload failed in Pass A (sparse range_upload error). + // They stay dirty and are retried on the next flush. + let Some(file_info) = file_info_opt else { + continue; + }; if item.pending_deletes.is_empty() && item.prev_xet_hash.as_deref() == Some(file_info.hash()) { debug!( "flush_batch: unchanged ino={} path={} (hash {})", @@ -409,12 +577,22 @@ async fn flush_batch( unchanged[i] = true; continue; } + // Use `item.file_size` (= entry.size at snapshot time = the actual + // staging file length under staging.lock) instead of + // `file_info.file_size()`. The xet-core upload_files path has been + // observed to return an inflated file_size from deduplication_metrics + // (a 64 MiB file gets reported as ~65.75 MiB), which would propagate + // into entry.size via apply_commit and cause subsequent range_upload + // calls to fail with "caller said original_size=X but reconstruction + // info reports Y". The on-CAS file is correct — only the returned + // XetFileInfo.file_size lies — so the safe authoritative value is + // the size we actually uploaded from staging. info!( "Uploaded file ino={} path={} xet_hash={} size={}", item.ino, item.full_path, file_info.hash(), - file_info.file_size().expect("upload returned XetFileInfo without size") + item.file_size, ); ops.push(BatchOp::AddFile { path: item.full_path.clone(), @@ -430,15 +608,13 @@ async fn flush_batch( // Clear dirty on unchanged files without waiting for the Hub round-trip. { let mut inode_table = inodes.write().expect("inodes poisoned"); - for (i, (item, file_info)) in to_flush.iter().zip(upload_results.iter()).enumerate() { + for (i, item) in to_flush.iter().enumerate() { if unchanged[i] && let Some(entry) = inode_table.get_mut(item.ino) { - entry.apply_commit( - file_info.hash(), - file_info.file_size().expect("upload returned XetFileInfo without size"), - item.dirty_generation, - ); + // Same hash + no pending_deletes: nothing changed remotely. + // Coverage and sparse_write stay intact. + entry.apply_noop_commit(item.dirty_generation); } } } @@ -469,15 +645,41 @@ async fn flush_batch( { let mut inode_table = inodes.write().expect("inodes poisoned"); - for (i, (item, file_info)) in to_flush.iter().zip(upload_results.iter()).enumerate() { + for (i, (item, file_info_opt)) in to_flush.iter().zip(upload_results.iter()).enumerate() { + let Some(file_info) = file_info_opt else { + continue; // Pass A failure — leave dirty for retry. + }; if !unchanged[i] && let Some(entry) = inode_table.get_mut(item.ino) { - entry.apply_commit( - file_info.hash(), - file_info.file_size().expect("upload returned XetFileInfo without size"), - item.dirty_generation, - ); + // Authoritative size = item.file_size (entry.size captured at + // flush snapshot under staging.lock + the staging file's + // on-disk length at upload time, since concurrent O_TRUNC and + // setattr-shrink also take staging.lock). NOT + // file_info.file_size() — xet-core's upload_files has been + // observed to over-count via deduplication_metrics.total_bytes, + // returning ~65.75 MiB for a true 64 MiB file. Trusting that + // value would set entry.size > on-disk staging length and + // poison every subsequent range_upload with a + // caller/reconstruction size mismatch. + let size = item.file_size; + if item.sparse_snapshot.is_some() && !is_fast_path[i] { + // Sparse upload via range_upload (Pass A): staging holds + // covered + dirty bytes that match the new hash, holes + // outside coverage do not. Re-key sparse_write to the + // new hash, clear dirty_ranges, preserve coverage (the + // new CAS file was composed from those staging bytes, so + // staging still matches at those offsets). + entry.apply_commit_sparse(file_info.hash(), size, item.dirty_generation); + } else { + // Non-sparse upload_files OR fast-path upload_files + // (fully-covered sparse item): staging IS the committed + // content byte-for-byte, so drop sparse_write and flag + // staging_is_current. Without this, fast-path items + // would force a full re-download on the next open even + // though staging already matches the committed hash. + entry.apply_commit(file_info.hash(), size, item.dirty_generation); + } } } } diff --git a/src/virtual_fs/inode.rs b/src/virtual_fs/inode.rs index e6238ff8..3d7762ec 100644 --- a/src/virtual_fs/inode.rs +++ b/src/virtual_fs/inode.rs @@ -145,6 +145,223 @@ pub struct InodeEntry { pub last_revalidated: Option, /// Eviction bookkeeping (kernel refcount, LRU recency, pending flag, pinning). pub eviction: EvictionState, + /// Sparse-write tracking state. `Some` iff the inode was prepared for + /// sparse-write modification (open/write/setattr under `--sparse-writes`). + /// `None` for clean inodes, non-sparse mode, and non-file kinds. + /// + /// Wrapped in `Arc` so readers can snapshot it cheaply under the inode + /// read lock then drop the lock before awaiting CAS fetches. Wrapped in + /// `Arc` (not `Arc`) because all writes go through the inode + /// table write lock; readers see a consistent immutable snapshot. + pub sparse_write: Option>, +} + +/// Sparse-write tracking: which parts of the staging file mirror the original +/// CAS content (or have been modified), and which are still sparse holes. +/// +/// Invariants enforced by the helper methods: +/// 1. `dirty_ranges ⊆ coverage`. A byte can only be "modified" if it is "present". +/// 2. `coverage` and `dirty_ranges` are sorted, non-overlapping, with each +/// range satisfying `start < end`. +/// 3. Staging bytes in `coverage \ dirty_ranges` match `original_hash` at the +/// same offset (cached from CAS). +/// 4. Staging bytes in `dirty_ranges` are the user's modifications since the +/// last commit (relative to `original_hash`). +/// 5. Staging bytes outside `coverage` (within `[0, file_size)`) are sparse +/// holes that read paths must fill from CAS on demand. +/// +/// `original_hash` and `original_size` describe the CAS reconstruction this +/// state is keyed to. After `apply_commit_sparse`, they are re-keyed to the +/// new hash and `dirty_ranges` is cleared, but `coverage` is preserved — +/// the bytes in `coverage` were just used to compose the new CAS file, so +/// staging matches the new hash at those offsets. +#[derive(Debug, Clone)] +pub struct SparseWriteState { + /// CAS hash of the file this state is keyed to. + pub original_hash: String, + /// CAS reconstruction size for `original_hash`. Used by `range_upload` + /// to compute original ranges to drop / preserve. + pub original_size: u64, + /// Staging ranges that hold real bytes (either cached from CAS or written + /// by the user). Sorted, non-overlapping, `start < end`. Bytes outside + /// these ranges (within `[0, file_size)`) are holes that need CAS fetch. + pub coverage: Vec<(u64, u64)>, + /// Subset of `coverage` that the user has modified since the last commit. + /// Sorted, non-overlapping, `start < end`. Fed to `range_upload` to + /// compose the new CAS file at flush time. + pub dirty_ranges: Vec<(u64, u64)>, +} + +impl SparseWriteState { + /// Build a state with no coverage and no dirty ranges. Used for opens + /// that punched a sparse hole — reads fill coverage lazily from CAS. + pub fn new(original_hash: String, original_size: u64) -> Self { + Self { + original_hash, + original_size, + coverage: Vec::new(), + dirty_ranges: Vec::new(), + } + } + + /// Build a state with full coverage and no dirty ranges. Used for opens + /// that downloaded the entire CAS file into staging — reads short-circuit + /// because staging already holds every byte. + pub fn new_full(original_hash: String, original_size: u64) -> Self { + let coverage = if original_size > 0 { + vec![(0, original_size)] + } else { + Vec::new() + }; + Self { + original_hash, + original_size, + coverage, + dirty_ranges: Vec::new(), + } + } + + /// Build a fresh state keyed to `original_hash`/`original_size` and project + /// a size change to `new_size`: empty coverage (reads refill from CAS), + /// dirty_ranges populated if `new_size > original_size` (so range_upload + /// composes the zero-fill tail). Used by setattr's sparse-install and + /// drift-rebuild paths. + pub fn new_resized(original_hash: String, original_size: u64, new_size: u64) -> Self { + let mut sw = Self::new(original_hash, original_size); + sw.resize_to(original_size, new_size); + sw + } + + /// Record bytes just fetched from CAS into staging at `[offset, offset+len)`. + /// Adds the range to `coverage` only (not dirty — these bytes match + /// `original_hash`). Subsequent reads of the same range hit staging. + pub fn track_read_fill(&mut self, offset: u64, len: u64) { + if len == 0 { + return; + } + let end = offset.saturating_add(len); + merge_range(&mut self.coverage, offset, end); + } + + /// Record a user write at `[offset, offset+len)`: adds the range to both + /// `coverage` and `dirty_ranges`. A write that lands past the previous EOF + /// (`offset > prev_eof`) also dirties the zero-fill gap `[prev_eof, + /// offset)` so `range_upload` composes those zeros into the new CAS file + /// — without it a write at offset 15 to a 10-byte file would commit a + /// 13-byte file. Idempotent if the range is already tracked. + pub fn track_write(&mut self, offset: u64, len: u64, prev_eof: u64) { + if len == 0 { + return; + } + let start = offset.min(prev_eof); + let end = offset.saturating_add(len); + self.track_read_fill(start, end - start); + merge_range(&mut self.dirty_ranges, start, end); + } + + /// Clip both coverage and dirty_ranges to `[0, new_size)`. Called after + /// a setattr-shrink. Ranges entirely past `new_size` are dropped; ranges + /// straddling `new_size` are capped. + pub fn clip_to_size(&mut self, new_size: u64) { + clip_ranges(&mut self.coverage, new_size); + clip_ranges(&mut self.dirty_ranges, new_size); + } + + /// Apply a `setattr` size change from `prev_size` to `new_size`: a grow + /// tracks the new tail `[prev_size, new_size)` as dirty (so range_upload + /// composes the zero-fill); a shrink clips coverage and dirty_ranges down + /// to `new_size`. Unchanged size is a no-op. + pub fn resize_to(&mut self, prev_size: u64, new_size: u64) { + if new_size > prev_size { + // prev_size == prev_eof on this path (setattr), so the gap + // computation in track_write collapses to start = prev_size and + // end = new_size — exactly the new tail. + self.track_write(prev_size, new_size - prev_size, prev_size); + } else if new_size < prev_size { + self.clip_to_size(new_size); + } + } + + /// Return the sub-ranges of `[offset, offset+len)` that are NOT in + /// `coverage` — i.e. holes that a read must fill from CAS. + pub fn holes_in(&self, offset: u64, len: u64) -> Vec<(u64, u64)> { + if len == 0 { + return Vec::new(); + } + let end = offset.saturating_add(len); + subtract_ranges(offset, end, &self.coverage) + } + + /// Returns true when `coverage` contains `[0, size)` entirely (no holes). + /// Used by the flush path: when the staging file already mirrors the full + /// committed content (via reads + dirty writes), we can upload it whole + /// via `upload_files` (xet-core CDC dedup is more efficient) instead of + /// running `range_upload` (which re-fetches old segments from CAS to + /// compose the new file). At full coverage the two paths produce the + /// same merkle hash, so the swap is safe. + pub fn is_fully_covered(&self, size: u64) -> bool { + if size == 0 { + return true; + } + self.holes_in(0, size).is_empty() + } +} + +/// Merge `[new_start, new_end)` into a sorted, non-overlapping range vec. +/// Coalesces with any overlapping or adjacent ranges. O(log n + k) where k +/// is the number of ranges merged (typically 0–1 for sequential writes). +fn merge_range(ranges: &mut Vec<(u64, u64)>, new_start: u64, new_end: u64) { + if new_start >= new_end { + return; + } + // First range whose end >= new_start — could overlap or abut on the left. + let first = ranges.partition_point(|&(_, e)| e < new_start); + // First range whose start > new_end — past the overlap zone. + let last = ranges[first..].partition_point(|&(s, _)| s <= new_end) + first; + + let (merged_start, merged_end) = if first < last { + (new_start.min(ranges[first].0), new_end.max(ranges[last - 1].1)) + } else { + (new_start, new_end) + }; + ranges.splice(first..last, [(merged_start, merged_end)]); +} + +/// Trim `ranges` to `[0, new_size)`: drop entries fully past `new_size`, +/// cap entries that straddle the boundary, and drop any zero-length leftovers. +fn clip_ranges(ranges: &mut Vec<(u64, u64)>, new_size: u64) { + ranges.retain_mut(|&mut (s, ref mut e)| { + if s >= new_size { + return false; + } + *e = (*e).min(new_size); + s < *e + }); +} + +/// Return the sub-ranges of `[start, end)` not covered by any range in +/// `covered`. `covered` must be sorted and non-overlapping. +fn subtract_ranges(start: u64, end: u64, covered: &[(u64, u64)]) -> Vec<(u64, u64)> { + let mut holes = Vec::new(); + let mut cursor = start; + // Skip ranges entirely before the window. + let first = covered.partition_point(|&(_, e)| e <= start); + for &(cs, ce) in &covered[first..] { + if cs >= end { + break; + } + if cs > cursor { + holes.push((cursor, cs.min(end))); + } + cursor = cursor.max(ce); + if cursor >= end { + break; + } + } + if cursor < end { + holes.push((cursor, end)); + } + holes } impl InodeEntry { @@ -207,8 +424,9 @@ impl InodeEntry { } } - /// Apply a successful commit: update hash, size, timestamps, and - /// conditionally clear dirty + pending_deletes if the generation matches. + /// Apply a successful non-sparse upload (`upload_files`) commit. + /// Clears `sparse_write` because the on-disk staging file IS the just- + /// uploaded content — no holes, no deferred composition needed. pub fn apply_commit(&mut self, hash: &str, size: u64, dirty_generation: u64) { if self.clear_dirty_if(dirty_generation) { // Only update metadata when the generation matches. A concurrent @@ -220,12 +438,80 @@ impl InodeEntry { self.staging_is_current = true; self.size = size; self.pending_deletes.clear(); + self.sparse_write = None; + self.touch_commit_clocks(); } + } + + /// Apply a successful `range_upload` commit. Re-keys `sparse_write` to the + /// new hash with `dirty_ranges` cleared; `coverage` is preserved because + /// the new CAS file was composed from those exact staging bytes, so + /// staging continues to match the new hash at every covered offset. + /// + /// On a generation mismatch (concurrent writer raced past the flush + /// snapshot), nothing is updated — the next flush will re-upload the + /// now-newer content. + pub fn apply_commit_sparse(&mut self, hash: &str, size: u64, dirty_generation: u64) { + if self.clear_dirty_if(dirty_generation) { + self.xet_hash = Some(hash.to_string()); + self.size = size; + self.pending_deletes.clear(); + // staging_is_current: staging holds covered + dirty bytes that + // match the new hash, but holes outside coverage do NOT match + // the new hash (they were filled by composition from old CAS). + // Reads outside coverage must still fetch from CAS, so the + // cache-current flag would lie. Leave it false. + self.staging_is_current = false; + if let Some(sw) = self.sparse_write.as_mut() { + let sw = Arc::make_mut(sw); + sw.original_hash = hash.to_string(); + sw.original_size = size; + sw.dirty_ranges.clear(); + // Coverage preserved: those staging bytes are also the new + // CAS bytes at the same offsets, by construction. + sw.clip_to_size(size); + } + self.touch_commit_clocks(); + } + } + + /// Apply a no-op commit (range_upload returned the original hash + /// unchanged, or the flush detected nothing to do). Clears dirty + + /// pending_deletes if the generation matches. `sparse_write` keeps its + /// coverage (the unchanged hash means staging still matches at covered + /// offsets) but dirty_ranges are cleared since they were just rehashed + /// back to the original. + /// + /// `staging_is_current` is promoted to true ONLY when sparse_write is + /// None: on the non-sparse path the upload hashed the whole staging + /// file and returned the same hash, so staging IS the committed content + /// — a valid cache for the next open. On the sparse path, staging is + /// still a partial mirror (covered + dirty bytes match, holes do not), + /// so the flag must stay false. + pub fn apply_noop_commit(&mut self, dirty_generation: u64) { + if self.clear_dirty_if(dirty_generation) { + self.pending_deletes.clear(); + if let Some(sw) = self.sparse_write.as_mut() { + let sw = Arc::make_mut(sw); + sw.dirty_ranges.clear(); + } else { + self.staging_is_current = true; + } + self.touch_commit_clocks(); + } + } + + /// Bump mtime/ctime to now and mark the inode as freshly revalidated. + /// Called at the end of every commit-apply so subsequent lookups skip HEAD + /// revalidation for metadata_ttl (we just committed this exact state). + /// Must only be called inside the `clear_dirty_if` branch — on a + /// generation mismatch nothing was committed, so stamping + /// `last_revalidated` would let lookups serve stale state from the + /// metadata_ttl cache while the inode is still dirty with newer content. + fn touch_commit_clocks(&mut self) { let now = SystemTime::now(); self.mtime = now; self.ctime = now; - // Mark as recently validated so subsequent lookups skip HEAD revalidation - // for the duration of metadata_ttl (we just committed this exact hash). self.last_revalidated = Some(Instant::now()); } } @@ -293,6 +579,7 @@ impl InodeTable { pending_deletes: Vec::new(), last_revalidated: None, eviction: EvictionState::default(), + sparse_write: None, }; table.inodes.insert(ROOT_INODE, root); table.path_to_inode.insert(root_path, ROOT_INODE); @@ -638,6 +925,7 @@ impl InodeTable { last_touched: AtomicU64::new(touch_seq), ..Default::default() }, + sparse_write: None, }; self.inodes.insert(inode, entry); @@ -714,6 +1002,14 @@ impl InodeTable { // matches xet_hash. An in-flight download observes this under // its post-check and won't re-flag the cache. entry.staging_is_current = false; + // Sparse-write state stays keyed to its `original_hash`, NOT + // the just-rotated `entry.xet_hash`. Readers via fill_sparse_holes + // continue to see the old revision (their frame of reference), + // writers continue to track against the old base. The next open + // recomputes a fresh sparse_write against the new hash. This + // mirrors the equivalent race in the non-sparse path (where a + // writer's flush silently overwrites poll-discovered updates) — + // not made worse by the sparse path. true } else { false @@ -1097,6 +1393,72 @@ mod tests { assert_eq!(entry.pending_deletes.len(), 1, "pending_deletes should be preserved"); } + /// Non-sparse no-op flush (upload_files rehashed staging back to the + /// previous hash): staging IS the committed content, so the cache flag must + /// flip to true — otherwise the next open discards the staging and forces a + /// full re-download even though it matches the committed bytes. + #[test] + fn apply_noop_commit_promotes_staging_is_current_when_non_sparse() { + let mut table = InodeTable::new(false); + let ino = table.insert( + ROOT_INODE, + "test".to_string(), + "test".to_string(), + InodeKind::File, + 10, + UNIX_EPOCH, + Some("h1".to_string()), + 0o644, + 0, + 0, + ); + let entry = table.get_mut(ino).unwrap(); + entry.set_dirty(); + entry.staging_is_current = false; + let snap = entry.dirty_generation; + + entry.apply_noop_commit(snap); + + assert!(!entry.is_dirty(), "noop must clear dirty"); + assert!( + entry.staging_is_current, + "non-sparse noop: staging matches the committed hash, flag must be true" + ); + } + + /// Sparse no-op flush: staging is still a partial mirror (covered + dirty + /// match the hash, holes do not), so the cache flag must stay false even + /// though the hash is unchanged. + #[test] + fn apply_noop_commit_keeps_staging_is_current_false_when_sparse() { + let mut table = InodeTable::new(false); + let ino = table.insert( + ROOT_INODE, + "test".to_string(), + "test".to_string(), + InodeKind::File, + 10, + UNIX_EPOCH, + Some("h1".to_string()), + 0o644, + 0, + 0, + ); + let entry = table.get_mut(ino).unwrap(); + entry.set_dirty(); + entry.staging_is_current = false; + entry.sparse_write = Some(Arc::new(SparseWriteState::new("h1".to_string(), 10))); + let snap = entry.dirty_generation; + + entry.apply_noop_commit(snap); + + assert!(!entry.is_dirty(), "noop must clear dirty"); + assert!( + !entry.staging_is_current, + "sparse noop: staging has unmaterialized holes, flag must remain false" + ); + } + #[test] fn set_dirty_saturates() { let mut table = InodeTable::new(false); @@ -2489,4 +2851,162 @@ mod tests { assert_child_index_consistent(&table); assert_eq!(table.lookup_child(ROOT_INODE, "a.txt").map(|e| e.inode), Some(a2)); } + + // ── SparseWriteState tests ──────────────────────────────────────── + + #[test] + fn merge_range_into_empty() { + let mut v = Vec::new(); + merge_range(&mut v, 10, 20); + assert_eq!(v, vec![(10, 20)]); + } + + #[test] + fn merge_range_disjoint_keeps_separate() { + let mut v = vec![(0, 10)]; + merge_range(&mut v, 20, 30); + assert_eq!(v, vec![(0, 10), (20, 30)]); + } + + #[test] + fn merge_range_coalesces_adjacent() { + let mut v = vec![(0, 10), (20, 30)]; + merge_range(&mut v, 10, 20); + assert_eq!(v, vec![(0, 30)]); + } + + #[test] + fn merge_range_swallows_overlapping_left() { + let mut v = vec![(10, 20)]; + merge_range(&mut v, 5, 15); + assert_eq!(v, vec![(5, 20)]); + } + + #[test] + fn merge_range_swallows_overlapping_right() { + let mut v = vec![(10, 20)]; + merge_range(&mut v, 15, 25); + assert_eq!(v, vec![(10, 25)]); + } + + #[test] + fn merge_range_collapses_multiple() { + let mut v = vec![(0, 5), (10, 15), (20, 25), (30, 35)]; + merge_range(&mut v, 7, 28); + assert_eq!(v, vec![(0, 5), (7, 28), (30, 35)]); + } + + #[test] + fn merge_range_zero_length_is_noop() { + let mut v = vec![(0, 10)]; + merge_range(&mut v, 5, 5); + assert_eq!(v, vec![(0, 10)]); + } + + #[test] + fn clip_ranges_trims_past_size() { + let mut v = vec![(0, 10), (20, 30), (40, 50)]; + clip_ranges(&mut v, 25); + assert_eq!(v, vec![(0, 10), (20, 25)]); + } + + #[test] + fn clip_ranges_to_zero_empties() { + let mut v = vec![(0, 10), (20, 30)]; + clip_ranges(&mut v, 0); + assert!(v.is_empty()); + } + + #[test] + fn subtract_ranges_finds_holes() { + let covered = vec![(10, 20), (30, 40)]; + // Whole window covered by holes. + assert_eq!(subtract_ranges(0, 50, &covered), vec![(0, 10), (20, 30), (40, 50)]); + // Sub-window across one covered range. + assert_eq!(subtract_ranges(15, 35, &covered), vec![(20, 30)]); + // Window inside a hole. + assert_eq!(subtract_ranges(22, 28, &covered), vec![(22, 28)]); + // Window inside coverage. + assert!(subtract_ranges(12, 18, &covered).is_empty()); + } + + #[test] + fn subtract_ranges_handles_window_starting_before_first() { + let covered = vec![(10, 20)]; + assert_eq!(subtract_ranges(0, 30, &covered), vec![(0, 10), (20, 30)]); + } + + #[test] + fn sparse_state_track_write_extends_both_coverage_and_dirty() { + let mut sw = SparseWriteState::new("h".into(), 100); + // prev_eof = offset → no zero-gap, plain in-window write. + sw.track_write(10, 20, 10); + assert_eq!(sw.coverage, vec![(10, 30)]); + assert_eq!(sw.dirty_ranges, vec![(10, 30)]); + } + + #[test] + fn sparse_state_track_write_past_eof_dirties_gap() { + // Write 5 bytes at offset 15 on a 10-byte file: prev_eof=10, the gap + // [10, 15) must also be tracked dirty so range_upload composes those + // zeros instead of producing a 13-byte file. + let mut sw = SparseWriteState::new("h".into(), 10); + sw.track_write(15, 5, 10); + assert_eq!(sw.coverage, vec![(10, 20)]); + assert_eq!(sw.dirty_ranges, vec![(10, 20)]); + } + + #[test] + fn sparse_state_track_read_fill_extends_coverage_only() { + let mut sw = SparseWriteState::new("h".into(), 100); + sw.track_read_fill(0, 40); + assert_eq!(sw.coverage, vec![(0, 40)]); + assert!(sw.dirty_ranges.is_empty()); + } + + #[test] + fn sparse_state_write_after_read_marks_dirty_subset() { + let mut sw = SparseWriteState::new("h".into(), 100); + sw.track_read_fill(0, 100); + sw.track_write(20, 30, 100); + assert_eq!(sw.coverage, vec![(0, 100)]); + assert_eq!(sw.dirty_ranges, vec![(20, 50)]); + } + + #[test] + fn sparse_state_clip_trims_both() { + let mut sw = SparseWriteState::new("h".into(), 100); + sw.track_write(0, 100, 0); + sw.clip_to_size(40); + assert_eq!(sw.coverage, vec![(0, 40)]); + assert_eq!(sw.dirty_ranges, vec![(0, 40)]); + } + + #[test] + fn sparse_state_holes_in_window() { + let mut sw = SparseWriteState::new("h".into(), 100); + sw.track_read_fill(20, 20); // covers [20, 40) + sw.track_write(60, 10, 60); // covers [60, 70) + assert_eq!(sw.holes_in(0, 80), vec![(0, 20), (40, 60), (70, 80)]); + assert!(sw.holes_in(20, 20).is_empty()); // entirely covered + assert_eq!(sw.holes_in(0, 10), vec![(0, 10)]); // entirely hole + } + + #[test] + fn sparse_state_new_full_covers_everything() { + let sw = SparseWriteState::new_full("h".into(), 1000); + assert_eq!(sw.coverage, vec![(0, 1000)]); + assert!(sw.holes_in(0, 1000).is_empty()); + assert!(sw.dirty_ranges.is_empty()); + } + + #[test] + fn sparse_state_new_full_empty_size_is_covered() { + let sw = SparseWriteState::new_full("h".into(), 0); + // An empty file has no coverage ranges and nothing dirty; reads are + // bounded by original_size == 0 so there is never a hole to fill. + assert!(sw.coverage.is_empty()); + assert!(sw.dirty_ranges.is_empty()); + assert!(sw.holes_in(0, 0).is_empty()); + } } diff --git a/src/virtual_fs/mod.rs b/src/virtual_fs/mod.rs index 6517b1e2..ebae670b 100644 --- a/src/virtual_fs/mod.rs +++ b/src/virtual_fs/mod.rs @@ -1,5 +1,6 @@ use std::collections::{HashMap, VecDeque}; use std::fs::{File, OpenOptions}; +use std::os::unix::fs::FileExt; use std::os::unix::io::AsRawFd; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; @@ -72,6 +73,18 @@ fn is_os_junk(name: &str) -> bool { pub struct VfsConfig { pub read_only: bool, pub advanced_writes: bool, + /// Enables sparse-write staging: open-for-write punches a hole instead of + /// downloading the original CAS content, and flush composes the new file + /// via `range_upload` (CAS prefix/suffix + re-chunked dirty windows). Reads + /// outside the dirty regions fetch from CAS on demand and populate the + /// local staging file as a cache. Implies `advanced_writes`. + pub sparse_writes: bool, + /// Minimum file size (bytes) for `sparse_writes` to engage. Files smaller + /// than this transparently fall back to the non-sparse advanced_writes + /// path (download then upload_files). Default 256 MiB at the CLI layer. + /// Setting this to 0 keeps sparse always on (used by the mock-driven + /// tests which exercise sparse semantics on tiny files). + pub sparse_min_size_bytes: u64, pub uid: u32, pub gid: u32, pub poll_interval_secs: u64, @@ -95,20 +108,26 @@ pub struct VfsConfig { /// → inode_table (RwLock, read or write) /// /// staging.lock(ino) (tokio::sync::Mutex, per-inode) -/// → inode_table (RwLock, read or write) +/// → staging.io_lock(ino) (std::sync::Mutex, per-inode I/O critical section) +/// → inode_table (RwLock, read or write) /// → open_files (RwLock, read only — via has_open_handles) /// → negative_cache (RwLock, write — in poll_remote_changes) /// /// StreamingChannel.commit_hook (Mutex) /// → pending_commits (Mutex) /// -/// General discipline: locks are held briefly and never across await points -/// (except the per-inode tokio::sync::Mutex from StagingCoordinator). Most paths acquire a lock, -/// extract data, drop the lock, perform async I/O, then re-acquire to apply. +/// `io_lock` is held by read() (sparse pread + sparse snapshot), write() +/// (pwrite + entry.size update, plus sparse_write track when present), +/// setattr's size-change branch (set_len + entry.size update), and +/// fill_sparse_holes' still_holes recheck + write_at. It serializes the +/// staging file's I/O against concurrent readers/writers/truncates so the +/// on-disk bytes, entry.size, and sparse-coverage view stay consistent. /// -/// Exception: setattr(truncate) holds inode_table.write() across File::create -/// / set_len syscalls (microseconds) to prevent write() from updating -/// inode.size between the file truncation and the metadata update. +/// General discipline: locks are held briefly and never across await points +/// (except the per-inode tokio::sync::Mutex from StagingCoordinator). Most +/// paths acquire a lock, extract data, drop the lock, perform async I/O, then +/// re-acquire to apply. io_lock in particular is never held across awaits — +/// the critical sections it guards are all sync syscalls. pub struct VirtualFs { runtime: tokio::runtime::Handle, hub_client: Arc, @@ -124,6 +143,18 @@ pub struct VirtualFs { overlay_backing: Option>, read_only: bool, advanced_writes: bool, + /// Sparse writes (open punches a hole, flush via range_upload). + /// Derived: requires advanced_writes or overlay — the staging dir is the + /// substrate for both. If `config.sparse_writes` is set without a backing + /// staging dir, this stays false. + sparse_writes: bool, + /// Minimum file size (bytes) below which sparse-writes falls back to the + /// non-sparse advanced_writes path. xet-core's CDC chunk dedup in + /// `upload_files` already wire-trims small files efficiently; the sparse + /// path's per-window compose overhead + reconstruction lookup round-trip + /// only pay off above ~1 GB on typical cloud networks. Configurable via + /// `HF_MOUNT_SPARSE_MIN_BYTES`; default 256 MiB. + sparse_min_size_bytes: u64, inode_table: Arc>, /// Maps file_handle → OpenFile (local fd or lazy remote reference). open_files: Arc>>, @@ -249,6 +280,12 @@ impl VirtualFs { read_only: config.read_only, // Overlay implies advanced_writes (random writes via local backing file). advanced_writes: config.advanced_writes || overlay, + // Sparse writes require a staging substrate (advanced_writes or + // overlay). Without one, downgrade silently rather than failing. + // Callers that need a hard guarantee should validate at the CLI + // layer (see setup.rs: `--sparse-writes` implies `--advanced-writes`). + sparse_writes: config.sparse_writes && (config.advanced_writes || overlay), + sparse_min_size_bytes: config.sparse_min_size_bytes, inode_table: inodes, open_files, next_file_handle: AtomicU64::new(1), @@ -1051,10 +1088,25 @@ impl VirtualFs { Ok((file_handle, channel)) } - /// Open a local file as read-only and return the file handle. - fn open_local_readonly(&self, ino: u64, path: &PathBuf) -> VirtualFsResult { - match File::open(path) { - Ok(file) => self.install_local_handle(ino, Arc::new(file), false), + /// Open a local file for a read FUSE handle and return the file handle. + /// `staging_backed` must be true only when `path` is the per-inode sparse + /// staging file (a dirty file that may carry CAS-fillable holes), and false + /// for complete local files (e.g. the HTTP non-Xet download cache) so the + /// read path keeps its lock-free fast path for them. + fn open_local_readonly(&self, ino: u64, path: &PathBuf, staging_backed: bool) -> VirtualFsResult { + // Staging-backed reads cache fetched holes back into staging via + // fill_sparse_holes' write_at, so the underlying fd needs write + // permission even though this is a read FUSE handle. Without it every + // hole write fails with EBADF, defeating the staging cache (re-fetch + + // warning on each read). Complete cache files are never written here, + // so keep them read-only. + let opened = if staging_backed { + OpenOptions::new().read(true).write(true).open(path) + } else { + File::open(path) + }; + match opened { + Ok(file) => self.install_local_handle(ino, Arc::new(file), false, staging_backed), Err(e) => { error!("Failed to open file {:?}: {}", path, e); Err(libc::EIO) @@ -1065,17 +1117,33 @@ impl VirtualFs { /// Register an already-opened `File` as a read-only `OpenFile::Local` /// handle. Used by the file_cache fast-path so the read fd stays alive /// even if eviction unlinks the on-disk copy after the open. - fn install_local_handle(&self, ino: u64, file: Arc, writable: bool) -> VirtualFsResult { + /// + /// `staging_backed` is true when `file` is the per-inode staging file + /// (sparse, may need CAS hole-fill on read) and false for a complete + /// whole-file cache file (FileCache hit) that must not consult sparse + /// coverage. + fn install_local_handle( + &self, + ino: u64, + file: Arc, + writable: bool, + staging_backed: bool, + ) -> VirtualFsResult { let file_handle = self.alloc_file_handle(); { let inodes = self.inode_table.read().expect("inodes poisoned"); inodes.bump_open_handles(ino); inodes.touch(ino); } - self.open_files - .write() - .expect("open_files poisoned") - .insert(file_handle, OpenFile::Local { ino, file, writable }); + self.open_files.write().expect("open_files poisoned").insert( + file_handle, + OpenFile::Local { + ino, + file, + writable, + staging_backed, + }, + ); Ok(file_handle) } @@ -1549,11 +1617,14 @@ impl VirtualFs { /// Remove any on-disk staging file for `ino`. Safe to call for inodes that /// never had a staging file — NotFound is ignored. Keeps `StagingDir`'s - /// byte budget accurate via `try_remove`. + /// byte budget accurate via `try_remove`. Also drops the per-inode lock + /// map entries so long-running mounts don't accumulate dead Arc + /// entries for inodes that have been evicted. fn drop_staging(&self, ino: u64) { if let Some(sd) = self.staging.dir() { sd.try_remove(ino); } + self.staging.forget_locks(ino); } pub async fn readdir(&self, ino: u64) -> VirtualFsResult> { @@ -1603,15 +1674,47 @@ impl VirtualFs { let staging_path = self.staging.path(ino); if writable && self.advanced_writes { - // Staging file + async flush (supports random writes and seek) - self.open_advanced_write( - ino, - &file_entry.full_path, - &file_entry.xet_hash, - file_entry.size, - truncate, - ) - .await + // Staging file + async flush (supports random writes and seek). + // open_advanced_write re-snapshots xet_hash/size under staging.lock, + // but a narrow window remains between that re-read (under + // inode_table.read()) and the install (under inode_table.write()) + // where apply_commit_sparse can rotate the inode. The drift check + // returns EAGAIN in that case (a dedicated sentinel, NOT EIO, so a + // real disk/IO error surfaces on the first attempt instead of + // being swallowed by the retry budget). Retry a few times + // internally so the FUSE/NFS layer above us never sees a + // transient drift error. After the retry budget is exhausted, + // upgrade the final EAGAIN to EIO — userspace open() returning + // EAGAIN would be surprising for a non-O_NONBLOCK call. + const DRIFT_RETRIES: u32 = 4; + for attempt in 0..DRIFT_RETRIES { + match self + .open_advanced_write( + ino, + &file_entry.full_path, + &file_entry.xet_hash, + file_entry.size, + truncate, + ) + .await + { + Err(libc::EAGAIN) => { + debug!( + "open: ino={} drift from open_advanced_write (attempt {}/{}), retrying", + ino, + attempt + 1, + DRIFT_RETRIES + ); + tokio::time::sleep(Duration::from_millis(5 * (1 << attempt))).await; + } + other => return other, + } + } + error!( + "open: ino={} drift retries exhausted ({}), surfacing as EIO", + ino, DRIFT_RETRIES + ); + Err(libc::EIO) } else if writable && truncate { // Simple streaming write (append-only, synchronous commit on close) self.open_streaming_write(ino, pid).await @@ -1624,26 +1727,41 @@ impl VirtualFs { } /// Advanced writes: prepare a staging file and open it for read-write. + /// + /// `xet_hash` and `size` are the caller's pre-lock snapshot (from `open`'s + /// `get_file_entry`). They are re-read under `staging.lock(ino)` below so + /// any rotation by poll or a just-finished `apply_commit_sparse` is picked + /// up here instead of being detected as drift and returned as EIO — which + /// no FUSE/NFS layer above us is wired to retry. async fn open_advanced_write( &self, ino: u64, full_path: &str, - xet_hash: &str, - size: u64, + _xet_hash: &str, + _size: u64, truncate: bool, ) -> VirtualFsResult { // Serialize staging preparation per inode (prevents concurrent download races) let staging_mutex = self.staging.lock(ino); let _staging_guard = staging_mutex.lock().await; - // Reuse the staging file when either (a) it has pending dirty writes, - // or (b) it's a clean cache flagged as current. In both cases its - // content is the right starting point for this open. - let (is_dirty, staging_is_current) = { + // Re-snapshot the inode state under staging.lock. The pre-lock values + // passed in by `open` can be stale by the time we get here: a flush + // can complete `apply_commit_sparse` (which rotates entry.xet_hash to + // the just-committed hash) between `open`'s snapshot and our lock + // acquisition. Using stale values here was the previous source of + // user-visible EIO on the drift check below. + let (is_dirty, staging_is_current, xet_hash, size) = { let inodes = self.inode_table.read().expect("inodes poisoned"); let entry = inodes.get(ino).ok_or(libc::ENOENT)?; - (entry.is_dirty(), entry.staging_is_current) + ( + entry.is_dirty(), + entry.staging_is_current, + entry.xet_hash.clone().unwrap_or_default(), + entry.size, + ) }; + let xet_hash = xet_hash.as_str(); let local_exists = self.local_backing_exists(ino, full_path).map_err(|e| { error!("Failed to check local backing file for ino={}: {}", ino, e); libc::EIO @@ -1654,6 +1772,22 @@ impl VirtualFs { } let can_reuse_staging = !truncate && (is_dirty || staging_is_current) && local_exists; + // Sparse staging applies only when a remote CAS hash exists to fill + // holes from. truncate skips it: the file is logically empty, no + // original content to preserve. Overlay skips it: overlay writes live + // entirely in user dir, no remote concept. + let has_remote_xet = !self.overlay() && !truncate && !xet_hash.is_empty() && size > 0; + // Small-file threshold: below this size, xet-core's CDC chunk-level + // dedup in `upload_files` already matches what `range_upload` could + // save (only modified chunks go on the wire), AND it avoids the + // reconstruction-info lookup round-trip + per-window compose overhead + // that range_upload pays. Bench shows sparse loses to non-sparse for + // small files (64 MB: ~11× slower; 1 GB: ~par; >1 GB: marginal win). + // Falling back to the non-sparse path here means small files take the + // standard download-then-upload route, which is plenty fast and + // simpler. Threshold is overridable via `HF_MOUNT_SPARSE_MIN_BYTES` + // for benchmarks and tuning. + let want_sparse = self.sparse_writes && has_remote_xet && size >= self.sparse_min_size_bytes; if !can_reuse_staging { // Clear the flag before touching disk so a partial failure (e.g. @@ -1661,12 +1795,49 @@ impl VirtualFs { // reusable. if let Some(entry) = self.inode_table.write().expect("inodes poisoned").get_mut(ino) { entry.staging_is_current = false; + // Drop any stale sparse_write — the staging file is about to + // be recreated. The new sparse_write (if any) is installed + // below against the just-prepared staging. + entry.sparse_write = None; } // GC accounting only matters for non-overlay (overlay files live // in user dir, so file_size returns 0 here on miss). let old_size = self.staging.dir().map(|sd| sd.file_size(ino)).unwrap_or(0); - let needs_download = !self.overlay() && !truncate && !xet_hash.is_empty() && size > 0; - let new_size = if needs_download { + // Sparse mode: punch a hole of `size` bytes via set_len, do NOT + // download. Reads in [0, size) outside the dirty regions fetch + // from CAS lazily via fill_sparse_holes and persist into staging + // as a cache. Flush composes the upload via range_upload. + let new_size = if want_sparse { + let staging_path = self + .staging + .path(ino) + .expect("staging directory required for advanced writes"); + // Hold io_lock(ino) across File::create + set_len: File::create + // is O_TRUNC and shrinks the shared on-disk inode to 0 BEFORE + // set_len extends it back to `size`. A concurrent reader on + // the prior staging file (its Arc still alive from a + // previous open) whose pread lands in that window would read + // 0 bytes and deliver a phantom EOF to FUSE. io_lock + // serializes us against any read()/setattr/fill_sparse_holes + // critical section that touches the staging file. Both + // syscalls are sync, so the lock is never held across await. + let io_lock = self.staging.io_lock(ino); + { + let _io_guard = io_lock.lock().expect("staging io_lock poisoned"); + let file = std::fs::File::create(&staging_path).map_err(|e| { + error!("Failed to create sparse staging file: {}", e); + libc::EIO + })?; + file.set_len(size).map_err(|e| { + error!("Failed to set sparse staging file length: {}", e); + libc::EIO + })?; + } + size + } else if has_remote_xet { + // Non-sparse mode: download the full CAS object into staging. + // Slower for large-file/small-edit workloads but avoids the + // sparse-staging lifecycle edge cases. let staging_path = self .staging .path(ino) @@ -1690,18 +1861,32 @@ impl VirtualFs { if !self.overlay() && let Some(sd) = self.staging.dir() { + // NOTE: a sparse hole is charged at its full logical size here, + // not its (near-zero) on-disk block count. That is conservative + // (a large sparse open can pressure the GC budget before its + // bytes materialize), but it keeps the accounting symmetric with + // try_remove / write / setattr, which all use the logical size: + // charging actual disk usage at only one of those sites skews + // the global bytes_used counter and breaks the budget. Making it + // truly block-based is a separate, whole-model change. sd.resize_bytes(old_size, new_size); } // Flag the cache as current only when the staging actually mirrors - // the remote. Cases to exclude: + // the remote. Cases included: + // - downloaded the full CAS object (staging IS the remote). + // - empty file (xet_hash empty, size == 0): File::create produced + // an empty staging file that trivially matches. + // Cases excluded: + // - sparse staging: staging is a hole + dirty bytes, doesn't match + // the CAS bytes. range_upload composes at flush time, but the + // on-disk file itself is not a clean cache. // - truncated hashed file: empty staging, non-empty xet_hash. - // - non-Xet file with `size > 0` and no xet_hash: File::create - // leaves empty staging, which does not match the remote. // - race with poll: `xet_hash` moved between `open()` reading the - // inode and here, so the downloaded hash is now stale — detected - // by the `entry.xet_hash == xet_hash` post-check. + // inode and here. Detected by the `entry.xet_hash == xet_hash` + // post-check. // Skip in overlay mode: there's no remote materialization concept. - let materializes_remote = !self.overlay() && (needs_download || (xet_hash.is_empty() && size == 0)); + let materializes_remote = + !self.overlay() && !want_sparse && (has_remote_xet || (xet_hash.is_empty() && size == 0)); if materializes_remote && let Some(entry) = self.inode_table.write().expect("inodes poisoned").get_mut(ino) && entry.xet_hash.as_deref().unwrap_or("") == xet_hash @@ -1716,21 +1901,83 @@ impl VirtualFs { libc::EIO })?; - // Re-check inode still exists before committing the open + // Re-check inode still exists before committing the open. Install + // sparse_write here under the same write lock that set_dirty takes, + // so poll can't observe a (dirty, no sparse_write) intermediate + // state and rotate xet_hash from under us. { let mut inodes = self.inode_table.write().expect("inodes poisoned"); let entry = inodes.get_mut(ino).ok_or(libc::ENOENT)?; + // Drift check BEFORE set_dirty: if poll rotated entry.xet_hash + // between `open()`'s snapshot and here, the staging we just + // prepared (sparse hole sized to old `size`, or downloaded old + // hash) no longer matches the inode's reality. Setting dirty + // and proceeding would leave a dirty inode with no sparse_write + // (or a stale one), which the flush would upload as + // hole-zeros via the regular path — silent corruption. + // Surface as EAGAIN (a dedicated sentinel, NOT EIO) so the open() + // retry loop can distinguish drift from a real disk error. + // Nothing has been set dirty yet, so nothing rolls back beyond + // the punched staging file (which will be replaced by the next + // open attempt). + if want_sparse && entry.xet_hash.as_deref() != Some(xet_hash) { + debug!( + "open_advanced_write: ino={} xet_hash drifted between snapshot and install (snap={:?}, now={:?}), signalling EAGAIN", + ino, xet_hash, entry.xet_hash + ); + return Err(libc::EAGAIN); + } entry.set_dirty(); if truncate { entry.size = 0; + entry.sparse_write = None; // POSIX: O_TRUNC must update mtime and ctime let now = SystemTime::now(); entry.mtime = now; entry.ctime = now; + } else if want_sparse { + // Preserve an existing sparse_write when it's already keyed + // to the same hash: the reopen happens after writes to the + // first open that left dirty_ranges/coverage populated, and + // those represent the user's in-flight modifications. + // Replacing the state with a fresh one would drop the dirty + // tracking, make subsequent reads refill user-written bytes + // from CAS, and trigger a no-op flush — silently losing the + // user's pending writes. + let existing_matches = entry + .sparse_write + .as_ref() + .is_some_and(|sw| sw.original_hash == xet_hash); + if !existing_matches { + if !can_reuse_staging { + // We just punched a fresh sparse hole above: staging is + // empty, so empty coverage is correct — reads fill holes + // from CAS lazily. + entry.sparse_write = Some(Arc::new(inode::SparseWriteState::new(xet_hash.to_string(), size))); + } else if staging_is_current { + // Reused a clean cache that mirrors the original in full + // (can_reuse_staging holds here, so this is exactly + // can_reuse_staging && staging_is_current): every byte is + // present, so coverage spans the file. + entry.sparse_write = + Some(Arc::new(inode::SparseWriteState::new_full(xet_hash.to_string(), size))); + } else { + // Reused a DIRTY staging file whose bytes were never + // sparse-tracked — e.g. an O_TRUNC cleared sparse_write, + // then writes landed while it was None. Those bytes are + // real user content, not holes. Installing empty-coverage + // sparse here would make reads overlay original CAS bytes + // over the user's writes and the flush no-op back to the + // original hash, silently losing the writes. Leave + // sparse_write cleared so this inode flushes via the + // regular full-staging upload path instead. + entry.sparse_write = None; + } + } } } - self.install_local_handle(ino, Arc::new(file), true) + self.install_local_handle(ino, Arc::new(file), true, true) } /// Simple streaming write: truncate existing file and set up a new streaming writer. @@ -1800,7 +2047,9 @@ impl VirtualFs { error!("Failed to open local backing file {:?}: {}", fe.full_path, e); libc::EIO })?; - return self.install_local_handle(ino, Arc::new(file), false); + // Overlay backing file: a complete local file, never a sparse + // staging file — not staging-backed. + return self.install_local_handle(ino, Arc::new(file), false, false); } error!("Dirty overlay file ino={} has missing local backing file", ino); return Err(libc::EIO); @@ -1816,8 +2065,9 @@ impl VirtualFs { _ => fe, }; match (fe.is_dirty, &staging_path) { - // Advanced write in progress — read from local staging file. - (true, Some(path)) if path.exists() => self.open_local_readonly(ino, path), + // Advanced write in progress — read from local staging file (which + // may be a sparse staging file with CAS-fillable holes). + (true, Some(path)) if path.exists() => self.open_local_readonly(ino, path, true), // Dirty file but staging file is missing — should not happen. (true, Some(_)) => { @@ -1840,7 +2090,11 @@ impl VirtualFs { _ if !fe.xet_hash.is_empty() => { if let Some(fc) = &self.file_cache { if let Some(file) = fc.try_open(&fe.xet_hash).await { - return self.install_local_handle(ino, file, false); + // Whole-file cache hit: a complete file keyed by hash, + // NOT the sparse staging file. Must not consult sparse + // coverage on read (would EBADF-pwrite into the RO cache + // fd and refetch already-present bytes). + return self.install_local_handle(ino, file, false, false); } self.spawn_populate_file_cache(fe.xet_hash.clone(), fe.size); } @@ -1881,7 +2135,8 @@ impl VirtualFs { libc::EIO })?; } - self.open_local_readonly(ino, &dest) + // Complete HTTP-cached file (non-Xet), never a sparse hole. + self.open_local_readonly(ino, &dest, false) } // Empty file (size=0, no hash). @@ -2032,6 +2287,188 @@ impl VirtualFs { Err(libc::EIO) } + /// Fetch the holes in `[offset, offset + buf.len())` from CAS, overlay them + /// into `buf`, and persist them into the staging file as a cache. + /// + /// `sparse_snapshot` describes which bytes were holes at the moment of the + /// pread that filled `buf`; only those positions get overlaid. The + /// staging-side cache write is gated by re-checking the live coverage + /// under `io_lock` per hole, so we never overwrite bytes a concurrent + /// writer may have just landed at the same offset. + /// + /// Bounded by `original_size`: bytes past the original CAS size aren't + /// in CAS, so they stay as whatever pread returned (zeros from the hole + /// or the user's own extension if a write covered them). + async fn fill_sparse_holes( + &self, + ino: u64, + staging_file: &Arc, + sparse_snapshot: &Arc, + offset: u64, + buf: &mut BytesMut, + ) -> VirtualFsResult<()> { + let buf_len = buf.len() as u64; + if buf_len == 0 { + return Ok(()); + } + // Read window in file coordinates. Clamp to the original CAS size — + // bytes past it are not in CAS. + let window_end = offset.saturating_add(buf_len).min(sparse_snapshot.original_size); + if window_end <= offset { + return Ok(()); + } + let holes = sparse_snapshot.holes_in(offset, window_end - offset); + if holes.is_empty() { + return Ok(()); + } + + // Coalesce the per-hole fetches into a single ranged CAS stream over + // [first_hole_start, last_hole_end). This pulls the already-covered + // bytes that sit between holes too, but turns N reconstruction + // round-trips into one — a large win for fragmented reads, at the cost + // of a little bandwidth. + let fetch_start = holes.first().expect("holes non-empty").0; + let fetch_end = holes.last().expect("holes non-empty").1; + let span = (fetch_end - fetch_start) as usize; + let file_info = XetFileInfo::new(sparse_snapshot.original_hash.clone(), sparse_snapshot.original_size); + + let mut data = Vec::with_capacity(span); + let mut stream = self + .xet_sessions + .download_stream_boxed(&file_info, fetch_start, Some(fetch_end)) + .map_err(|e| { + error!( + "sparse read: open CAS stream {}..{} failed: {}", + fetch_start, fetch_end, e + ); + libc::EIO + })?; + while let Some(bytes) = stream.next().await.map_err(|e| { + error!("sparse read: CAS stream error at {}..{}: {}", fetch_start, fetch_end, e); + libc::EIO + })? { + data.extend_from_slice(&bytes); + if data.len() >= span { + break; + } + } + if data.len() < span { + error!( + "sparse read: CAS stream short ({} of {} bytes) for {}..{}", + data.len(), + span, + fetch_start, + fetch_end, + ); + return Err(libc::EIO); + } + data.truncate(span); + + // Overlay each hole's CAS bytes into buf: the zeros pread'd from those + // hole positions become the real content. Bytes between holes are + // already correct in buf (they came from the staging pread). + for &(hole_start, hole_end) in &holes { + let src_lo = (hole_start - fetch_start) as usize; + let src_hi = (hole_end - fetch_start) as usize; + let buf_lo = (hole_start - offset) as usize; + let buf_hi = (hole_end - offset) as usize; + buf[buf_lo..buf_hi].copy_from_slice(&data[src_lo..src_hi]); + } + + // Cache the fetched bytes into staging, but only where the LIVE coverage + // still shows a hole — a concurrent writer may have landed user bytes + // there since the snapshot, and we must not clobber them. The per-inode + // io_lock serializes against writes (they take the same lock), so + // coverage can't change under us while we hold it. The GLOBAL inode lock + // is taken only briefly to read/update coverage, never across the writes. + let io_lock = self.staging.io_lock(ino); + let _io_guard = io_lock.lock().expect("staging io_lock poisoned"); + + // Still-hole sub-ranges as (file_offset, offset_into_data, len). + // Clamp every hole to the LIVE inode size: setattr's size branch now + // takes io_lock around its set_len + entry.size update, so a shrink + // cannot interleave with this critical section. But the SNAPSHOT + // (taken under io_lock earlier in read()) may have been captured + // before a shrink that completed during the CAS stream await window, + // so we still need to re-read entry.size here and clip to it before + // caching. Without the clamp a write_at past the new EOF would + // re-extend staging with stale CAS bytes and a setattr-grow back + // through the region would expose them instead of POSIX zeros. + let still_holes: Vec<(u64, usize, usize)> = { + let inodes = self.inode_table.read().expect("inodes poisoned"); + let Some(entry) = inodes.get(ino) else { + return Ok(()); // inode vanished; buf overlay is still correct + }; + let live_size = entry.size; + let Some(sw) = entry.sparse_write.as_ref() else { + return Ok(()); // sparse_write cleared (poll/commit) + }; + if sw.original_hash != sparse_snapshot.original_hash { + return Ok(()); // hash rotated; snapshot no longer describes this inode + } + holes + .iter() + .filter_map(|&(hs, he)| { + let clipped_he = he.min(live_size); + if clipped_he <= hs { + return None; + } + Some( + sw.holes_in(hs, clipped_he - hs) + .into_iter() + .map(move |(ss, se)| (ss, (ss - fetch_start) as usize, (se - ss) as usize)), + ) + }) + .flatten() + .collect() + }; + + // Persist each still-hole into staging via the safe positioned write. + // io_lock is held (no concurrent writer), but no await and no global + // inode lock across these syscalls. + let mut filled: Vec<(u64, u64)> = Vec::with_capacity(still_holes.len()); + for (file_off, src, len) in still_holes { + match staging_file.write_at(&data[src..src + len], file_off) { + Ok(n) if n > 0 => filled.push((file_off, n as u64)), + Ok(_) => {} + Err(e) => { + // Don't fail the read — buf is already correct; we just lose + // the staging cache for this range (e.g. a read-only fd). + warn!( + "sparse read: staging cache write at {}..{} failed: {}", + file_off, + file_off + len as u64, + e + ); + } + } + } + + // Record the newly cached ranges as coverage (brief write lock, no + // syscalls under it). Re-check the hash in case a commit re-keyed + // sparse_write while we were writing. + // + // NOTE: the materialized bytes are not separately charged to the GC + // budget here — the staging file was already charged at its full + // logical size when the hole was punched (see open_advanced_write), so + // filling holes adds no new logical bytes. This keeps the accounting + // symmetric with try_remove. + if !filled.is_empty() { + let mut inodes = self.inode_table.write().expect("inodes poisoned"); + if let Some(entry) = inodes.get_mut(ino) + && let Some(sw_arc) = entry.sparse_write.as_mut() + && sw_arc.original_hash == sparse_snapshot.original_hash + { + let sw = Arc::make_mut(sw_arc); + for (off, len) in filled { + sw.track_read_fill(off, len); + } + } + } + + Ok(()) + } + /// Read data from an open file. Returns `(data, eof)`. pub async fn read(&self, file_handle: u64, offset: u64, size: u32) -> VirtualFsResult<(Bytes, bool)> { debug!("read: fh={}, offset={}, size={}", file_handle, offset, size); @@ -2041,7 +2478,16 @@ impl VirtualFs { let read_target = { let files = self.open_files.read().expect("open_files poisoned"); match files.get(&file_handle) { - Some(OpenFile::Local { file, .. }) => ReadTarget::LocalFd(file.clone()), + Some(OpenFile::Local { + file, + ino, + staging_backed, + .. + }) => ReadTarget::LocalFd { + file: file.clone(), + ino: *ino, + staging_backed: *staging_backed, + }, Some(OpenFile::Lazy { prefetch, .. }) => ReadTarget::Remote { prefetch: prefetch.clone(), }, @@ -2052,26 +2498,81 @@ impl VirtualFs { }; match read_target { - ReadTarget::LocalFd(file) => { + ReadTarget::LocalFd { + file, + ino, + staging_backed, + } => { let file_descriptor = file.as_raw_fd(); let mut buf = BytesMut::zeroed(size as usize); - // SAFETY: fd is valid (Arc keeps it alive), buf is correctly sized. - // pread is thread-safe (atomic offset, no shared seek cursor). - let n = unsafe { - libc::pread( - file_descriptor, - buf.as_mut_ptr() as *mut libc::c_void, - size as usize, - offset as i64, - ) - }; - if n < 0 { - Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)) - } else { + + // Fast path: no sparse staging is in play for this read — either + // the mount doesn't use sparse writes, or this handle is a + // complete whole-file cache fd (not the sparse staging file). A + // plain pread is the source of truth; skip the io_lock + the + // coverage snapshot entirely (avoids per-read lock contention on + // the previously lock-free local read path). + if !self.sparse_writes || !staging_backed { + // SAFETY: fd is valid (Arc keeps it alive), buf is + // correctly sized. pread is thread-safe (atomic offset). + let n = unsafe { + libc::pread( + file_descriptor, + buf.as_mut_ptr() as *mut libc::c_void, + size as usize, + offset as i64, + ) + }; + if n < 0 { + return Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)); + } buf.truncate(n as usize); - let eof = (n as u32) < size; - Ok((buf.freeze(), eof)) + return Ok((buf.freeze(), (n as u32) < size)); + } + + // Sparse path: snapshot sparse_write under the io_lock together + // with the pread, so the snapshot describes exactly what staging + // held at the moment buf was filled. A concurrent pwrite cannot + // land between snapshot and pread because both happen under the + // same lock. + let sparse_snapshot; + let n; + { + let io_lock = self.staging.io_lock(ino); + let _io_guard = io_lock.lock().expect("staging io_lock poisoned"); + sparse_snapshot = self + .inode_table + .read() + .expect("inodes poisoned") + .get(ino) + .and_then(|e| e.sparse_write.clone()); + // SAFETY: fd is valid (Arc keeps it alive), buf is + // correctly sized. pread is thread-safe (atomic offset). + n = unsafe { + libc::pread( + file_descriptor, + buf.as_mut_ptr() as *mut libc::c_void, + size as usize, + offset as i64, + ) + }; + } + if n < 0 { + return Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)); } + buf.truncate(n as usize); + + // Sparse staging: bytes pread'd from a hole are zeros. Fill + // them from CAS, overlay into buf, and persist into staging + // (as a cache for subsequent reads of the same range). + if let Some(sw) = sparse_snapshot + && n > 0 + { + self.fill_sparse_holes(ino, &file, &sw, offset, &mut buf).await?; + } + + let eof = (n as u32) < size; + Ok((buf.freeze(), eof)) } ReadTarget::Remote { prefetch } => { let mut prefetch_state = prefetch.lock().await; @@ -2178,6 +2679,7 @@ impl VirtualFs { ino, file, writable: true, + .. }) => WriteTarget::Local { file: file.clone(), ino: *ino, @@ -2194,33 +2696,82 @@ impl VirtualFs { match target { WriteTarget::Local { file, ino: handle_ino } => { let file_descriptor = file.as_raw_fd(); - let n = unsafe { - libc::pwrite( - file_descriptor, - data.as_ptr() as *const libc::c_void, - data.len(), - offset as i64, - ) - }; - - if n < 0 { - Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)) - } else { - let written = n as u32; - let new_end = offset + written as u64; - let mut inodes = self.inode_table.write().expect("inodes poisoned"); - if let Some(entry) = inodes.get_mut(handle_ino) { - if new_end > entry.size { - if let Some(sd) = self.staging.dir() { - sd.resize_bytes(entry.size, new_end); + // pwrite + entry.size update (and sparse_write track when + // present) under the per-inode io_lock so concurrent readers, + // writers, and setattr-shrinks see a consistent view: either + // before this write or after, never half-applied. + let n; + let new_end; + let written; + { + // io_lock serializes this pwrite + entry.size update + // against (a) sparse readers' pread + coverage snapshot + // pair, and (b) setattr's set_len + entry.size update. + // Without io_lock on the non-sparse advanced-writes path, + // a concurrent setattr-shrink can truncate the staging + // file between our pwrite and our entry.size update, + // leaving inode.size larger than the on-disk staging + // length. The non-sparse / non-staging read fast-path + // (FileCache hit, HTTP cache) doesn't take io_lock, but + // those reads never touch the staging file, so taking + // the lock here is harmless for them. + let io_lock = self.staging.io_lock(handle_ino); + let _io_guard = io_lock.lock().expect("staging io_lock poisoned"); + n = unsafe { + libc::pwrite( + file_descriptor, + data.as_ptr() as *const libc::c_void, + data.len(), + offset as i64, + ) + }; + if n < 0 { + return Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)); + } + written = n as u32; + new_end = offset + written as u64; + // A zero-byte write must not extend size, mark dirty, or + // track sparse coverage: pwrite didn't grow the staging + // file, so bumping entry.size past offset would leave + // staging shorter than entry.size and the next flush + // snapshot_dirty_bytes hits UnexpectedEof, stranding the + // inode permanently dirty. Reachable via zero-length + // FUSE writes past EOF (custom clients, NFS re-export, + // test harnesses) since the kernel VFS short-circuit + // doesn't always fire before FUSE. + if written > 0 { + let mut inodes = self.inode_table.write().expect("inodes poisoned"); + if let Some(entry) = inodes.get_mut(handle_ino) { + // Snapshot the previous size BEFORE bumping it: the + // sparse path needs to include the zero-filled gap + // [prev_size, offset) in dirty_ranges when the write + // starts past EOF. Otherwise range_upload would only + // see [offset, new_end) and the resulting CAS file + // would be missing those zeros — a write at offset + // 15 to a 10-byte file would commit a 13-byte file. + let prev_size = entry.size; + if new_end > entry.size { + if let Some(sd) = self.staging.dir() { + sd.resize_bytes(entry.size, new_end); + } + entry.size = new_end; + } + entry.set_dirty(); + // Sparse mode: extend coverage AND dirty_ranges + // for the written window. `track_write` handles + // the zero-fill gap [prev_size, offset) when the + // write lands past the previous EOF. Done under + // both io_lock (vs concurrent readers) and the + // inode write lock (vs flush snapshotting + // sparse_write). + if let Some(sw_arc) = entry.sparse_write.as_mut() { + Arc::make_mut(sw_arc).track_write(offset, written as u64, prev_size); } - entry.size = new_end; } - entry.set_dirty(); + inodes.touch(handle_ino); } - inodes.touch(handle_ino); - Ok(written) } + Ok(written) } WriteTarget::Streaming { ino: handle_ino, @@ -2579,11 +3130,15 @@ impl VirtualFs { return Err(libc::EIO); } + // Authoritative size = bytes the streaming writer actually wrote, NOT + // file_info.file_size() — xet-core's deduplication_metrics-based + // counter has been observed to over-count (see flush.rs for context). + let committed_size = channel.bytes_written.load(Ordering::Relaxed); let mut inodes = self.inode_table.write().expect("inodes poisoned"); if let Some(entry) = inodes.get_mut(ino) { entry.apply_commit( file_info.hash(), - file_info.file_size().expect("upload returned XetFileInfo without size"), + committed_size, channel.dirty_generation_at_open.load(Ordering::Relaxed), ); } @@ -2592,7 +3147,7 @@ impl VirtualFs { "Committed file: {} (hash={}, size={})", full_path, file_info.hash(), - file_info.file_size().expect("upload returned XetFileInfo without size"), + committed_size, ); Ok(()) @@ -2680,6 +3235,9 @@ impl VirtualFs { ino, file: Arc::new(file), writable: true, + // A freshly created file has no CAS hash, so it never + // carries a sparse_write — keep the read fast path. + staging_backed: false, }, ); @@ -3516,6 +4074,35 @@ impl VirtualFs { return Err(libc::EPERM); } + // Snapshot the pre-setattr inode state. Used to decide whether + // the setattr is the first modification of a clean CAS-backed + // file (sparse install path) vs. an extension/clip of an + // already-prepared sparse_write. + let (prev_size, prev_hash, prev_sparse) = { + let inodes = self.inode_table.read().expect("inodes poisoned"); + let entry = inodes.get(ino).ok_or(libc::ENOENT)?; + (entry.size, entry.xet_hash.clone(), entry.sparse_write.is_some()) + }; + + // Sparse-install path: first modification of a clean + // CAS-backed file via setattr (no open happened, no + // sparse_write installed yet). Punch a sparse hole sized to + // `new_size` and install SparseWriteState — the original CAS + // bytes are pulled in lazily by reads, the new tail (if any) + // is tracked as dirty so range_upload knows to compose it. + // Mirrors `open_advanced_write`'s `want_sparse` predicate + // (incl. the small-file threshold) so a setattr-first edit + // takes the same path an open-first edit would for the same + // file — without the threshold check here, a 1 KB file + // shrunk via setattr would install sparse while the same + // file shrunk via open(O_TRUNC) would not. + let want_sparse_install = self.sparse_writes + && !prev_sparse + && !local_exists + && prev_hash.is_some() + && prev_size >= self.sparse_min_size_bytes + && prev_size > 0; + // GC accounting (non-overlay only): snapshot staging bytes before // any mutation so the size delta is applied correctly at the end. let old_staging_size = self @@ -3526,20 +4113,37 @@ impl VirtualFs { .unwrap_or(0); if !local_exists { - if new_size > 0 { + if want_sparse_install { + // Sparse hole, no download. Unlike open_advanced_write's + // equivalent, this does NOT take io_lock around + // File::create + set_len because `!local_exists` + // guarantees no prior staging file existed, so there + // is no live Arc from a previous open that could + // race a pread against File::create's O_TRUNC window. + // staging.lock(ino) (held above) excludes concurrent + // open_advanced_write installs. + let staging_path = self + .staging + .path(ino) + .expect("staging directory required for advanced writes"); + let file = std::fs::File::create(&staging_path).map_err(|e| { + error!("Failed to create sparse staging file for setattr: {}", e); + libc::EIO + })?; + file.set_len(new_size).map_err(|e| { + error!("Failed to set sparse staging file length: {}", e); + libc::EIO + })?; + } else if new_size > 0 { let staging_path = self .staging .path(ino) .expect("staging directory required for advanced writes"); - let (xet_hash, file_size) = { - let inodes = self.inode_table.read().expect("inodes poisoned"); - let entry = inodes.get(ino).ok_or(libc::ENOENT)?; - (entry.xet_hash.clone().unwrap_or_default(), entry.size) - }; - if !xet_hash.is_empty() && file_size > 0 { + let xet_hash = prev_hash.clone().unwrap_or_default(); + if !xet_hash.is_empty() && prev_size > 0 { if let Err(e) = self .xet_sessions - .download_to_file(&xet_hash, file_size, &staging_path) + .download_to_file(&xet_hash, prev_size, &staging_path) .await { error!("Failed to download file for truncate: {}", e); @@ -3555,19 +4159,66 @@ impl VirtualFs { } } - // Apply the size change under the same write lock so write() cannot - // race between the local truncate and inode metadata update. + // Apply the size change under the same write lock so write() + // cannot race between the local truncate and inode metadata + // update. Also hold the per-inode io_lock across set_len + + // entry.size mutation so a concurrent fill_sparse_holes + // cannot interleave its still_holes recheck and write_at + // with this shrink — without the io_lock, the reader can + // re-extend the staging file past the new EOF and cache + // stale CAS bytes there. Lock order matches fill_sparse_holes + // and write(): io_lock first, then inode_table. No awaits + // are held under io_lock (set_len + open_local_backing_file + // are sync, the `?` in the closure only short-circuits). + let io_lock = self.staging.io_lock(ino); + let _io_guard = io_lock.lock().expect("staging io_lock poisoned"); let mut inodes = self.inode_table.write().expect("inodes poisoned"); - let size_result = if new_size == 0 { - self.open_local_backing_file(ino, &full_path, true, true, true, true) - .map(|_| ()) - } else { - self.open_local_backing_file(ino, &full_path, false, true, false, false) - .and_then(|file| file.set_len(new_size)) + // Current revision under the write lock. Poll can rotate + // xet_hash/size on a clean inode between the pre-lock snapshot + // and here (InodeTable::update_remote_file leaves any + // sparse_write keyed to the OLD hash). `sparse_drifted` flags a + // retained sparse_write whose original_hash no longer matches + // the live xet_hash. The staging tail (sized from the stale + // snapshot) must then be rebuilt against the current revision. + // + // Xet-backed files cannot rotate to a non-xet revision on this + // codepath (buckets are uniformly Xet; non-Xet repos are + // read-only so writes never reach here), so cur_hash is + // expected to be Some whenever the sparse path is active. + let (cur_hash, cur_size, sparse_drifted) = match inodes.get(ino) { + Some(entry) => { + let drifted = entry + .sparse_write + .as_ref() + .is_some_and(|sparse| entry.xet_hash.as_deref() != Some(sparse.original_hash.as_str())); + (entry.xet_hash.clone(), entry.size, drifted) + } + None => (None, 0, false), }; - if let Err(e) = size_result { - error!("Failed to set local backing file length: {}", e); - return Err(libc::EIO); + // Skip set_len for the sparse-install path: the staging file + // was just created at exactly `new_size` above. + if !want_sparse_install { + let size_result = if new_size == 0 { + self.open_local_backing_file(ino, &full_path, true, true, true, true) + .map(|_| ()) + } else { + self.open_local_backing_file(ino, &full_path, false, true, false, false) + .and_then(|file| { + // On drift the staging was sized from the stale + // pre-rotation snapshot. Truncate to the current + // revision size before extending so a grow + // zero-fills the extension (POSIX), instead of + // committing stale tail bytes of the old revision. + if sparse_drifted && new_size > cur_size { + file.set_len(cur_size)?; + } + file.set_len(new_size) + }) + }; + if let Err(e) = size_result { + error!("Failed to set local backing file length: {}", e); + return Err(libc::EIO); + } } if !self.overlay() && let Some(sd) = self.staging.dir() @@ -3581,6 +4232,41 @@ impl VirtualFs { entry.set_dirty(); if new_size == 0 { entry.xet_hash = None; + entry.sparse_write = None; + } else { + // Rebuild sparse_write against the current revision + // when: (a) we just punched a sparse-install hole, or + // (b) sparse_write is keyed to a stale hash (poll + // rotated to a NEW xet revision while we held no + // write lock — its compose base would silently + // overwrite the remote update). Otherwise propagate + // the setattr size change in place; any bytes past + // sw.original_size are already in dirty_ranges (Xet + // hashes are content-addressable so size can't grow + // without a hash rotation, and write() under io_lock + // already tracked any user-side extension). + let needs_rebuild = want_sparse_install + || entry + .sparse_write + .as_ref() + .is_some_and(|sw| Some(sw.original_hash.as_str()) != cur_hash.as_deref()); + if needs_rebuild { + // Xet → non-Xet rotation is unreachable on + // writable inodes (buckets are uniformly Xet; + // non-Xet repos are read-only so writes never + // reach here). Keep `expect` rather than + // returning EIO here: entry.size and set_dirty + // have already been applied above, so a mid- + // mutation EIO would leave the inode dirty + // with an empty/stale sparse_write — the next + // flush would commit corrupt content. Failing + // loud is safer than failing silent. + let hash = cur_hash.expect("Xet → non-Xet rotation is unreachable on writable inodes"); + entry.sparse_write = + Some(Arc::new(inode::SparseWriteState::new_resized(hash, cur_size, new_size))); + } else if let Some(sw_arc) = entry.sparse_write.as_mut() { + Arc::make_mut(sw_arc).resize_to(cur_size, new_size); + } } } drop(inodes); @@ -3769,7 +4455,17 @@ struct StreamingChannel { /// An open file handle — either a local fd, lazy remote reference, or streaming writer. enum OpenFile { /// Local file (staging for writes, or dirty reads). - Local { ino: u64, file: Arc, writable: bool }, + /// + /// `staging_backed` is true when `file` is the per-inode sparse staging + /// file (which may carry holes the read path must fill from CAS), and false + /// when it is a complete whole-file cache file (FileCache hit) that holds + /// every byte and must NOT consult the inode's sparse coverage. + Local { + ino: u64, + file: Arc, + writable: bool, + staging_backed: bool, + }, /// Lazy remote — data fetched on-demand with adaptive prefetch buffer. Lazy { ino: u64, @@ -3846,7 +4542,13 @@ async fn streaming_worker( /// What to do in read() after releasing the open_files lock. enum ReadTarget { /// Hold an Arc so the FD stays alive even if release() runs concurrently. - LocalFd(Arc), + /// `ino` is needed to look up `sparse_write` for the sparse-hole fill path; + /// `staging_backed` gates that path off for complete (non-staging) cache fds. + LocalFd { + file: Arc, + ino: u64, + staging_backed: bool, + }, Remote { prefetch: Arc>, }, diff --git a/src/virtual_fs/staging.rs b/src/virtual_fs/staging.rs index cc466a5d..4de85730 100644 --- a/src/virtual_fs/staging.rs +++ b/src/virtual_fs/staging.rs @@ -8,12 +8,24 @@ use crate::xet::StagingDir; use super::inode::InodeTable; -/// Bundles the on-disk staging area with per-inode async locks so subsystems +/// Bundles the on-disk staging area with per-inode locks so subsystems /// outside `VirtualFs` (e.g. the flush-path GC) can take the same lock as /// `open_advanced_write` / `setattr(truncate)` to serialize staging I/O. +/// +/// Two lock maps are tracked per inode: +/// - `locks` (tokio async): held across awaits (download, unlink). Coarse +/// serialization between `open_advanced_write`, `setattr(truncate)`, and +/// the flush-path GC. +/// - `io_locks` (std sync): held briefly around the per-syscall I/O critical +/// sections (`pread` + sparse coverage snapshot, `pwrite` + `track_write`, +/// range_upload's per-chunk reads). Sync because callers include both +/// async tasks (read, flush) and sync code paths (write, called from +/// FUSE/NFS handlers without spawn_blocking). The critical sections hold +/// no `.await`, so a sync mutex is safe everywhere. pub(crate) struct StagingCoordinator { dir: Option, locks: Mutex>>>, + io_locks: Mutex>>>, } impl StagingCoordinator { @@ -21,9 +33,22 @@ impl StagingCoordinator { Self { dir, locks: Mutex::new(HashMap::new()), + io_locks: Mutex::new(HashMap::new()), } } + /// Sync per-inode lock for serializing pread / pwrite / range_upload's + /// per-chunk reads. Held only across non-await operations — never block + /// an async runtime worker. + pub(crate) fn io_lock(&self, ino: u64) -> Arc> { + self.io_locks + .lock() + .expect("staging io_locks poisoned") + .entry(ino) + .or_insert_with(|| Arc::new(std::sync::Mutex::new(()))) + .clone() + } + pub(crate) fn dir(&self) -> Option<&StagingDir> { self.dir.as_ref() } @@ -53,6 +78,25 @@ impl StagingCoordinator { let lock = self.lock(ino); let _guard = lock.lock().await; dir.try_remove(ino); + drop(_guard); + self.forget_locks(ino); + } + + /// Drop the per-inode lock map entries for an inode that is permanently + /// gone (evicted via FUSE forget, unlinked, rename-replaced). Without + /// this, `locks` and `io_locks` grow monotonically — every inode ever + /// touched by read/write/setattr/flush installs a permanent entry, and + /// long-running mounts with high inode churn accumulate dead Arc + /// entries unbounded. + /// + /// Safe even if another task is mid-call holding the Arc: removal + /// from the HashMap only drops THIS reference; the other task continues + /// against its already-cloned Arc. A new caller after removal allocates + /// a fresh Arc, which is correct in practice because the inode is + /// gone — no legitimate caller should be operating on it anymore. + pub(crate) fn forget_locks(&self, ino: u64) { + self.locks.lock().expect("staging locks poisoned").remove(&ino); + self.io_locks.lock().expect("staging io_locks poisoned").remove(&ino); } /// Reclaim a single clean inode's staging file when usage exceeds the diff --git a/src/virtual_fs/tests.rs b/src/virtual_fs/tests.rs index a61e56bf..feefb18b 100644 --- a/src/virtual_fs/tests.rs +++ b/src/virtual_fs/tests.rs @@ -66,6 +66,24 @@ fn vfs_advanced( (rt, vfs) } +/// Build a VFS with sparse writes enabled (implies advanced_writes via TestOpts). +fn vfs_sparse( + hub: &std::sync::Arc, + xet: &std::sync::Arc, +) -> (tokio::runtime::Runtime, std::sync::Arc) { + let rt = new_runtime(); + let vfs = make_test_vfs( + hub.clone(), + xet.clone(), + TestOpts { + sparse_writes: true, + ..Default::default() + }, + &rt, + ); + (rt, vfs) +} + /// Build a read-only VFS. fn vfs_readonly( hub: &std::sync::Arc, @@ -5241,3 +5259,2239 @@ fn overlay_rmdir_remote_dir_eperm() { assert_eq!(err, libc::EPERM); }); } + +// ── Sparse writes ────────────────────────────────────────────────────── +// +// Invariant tests for the sparse-write path. The goal here is to nail down +// observable behaviors (read, write, flush composition), not implementation +// details. The state machine itself (coverage map, dirty tracking) is unit- +// tested in inode.rs. + +/// Poll until `ino` is clean (flushed). Bounded retry; panics on timeout. +async fn wait_for_clean(vfs: &Arc, ino: u64) { + for _ in 0..200 { + { + let inodes = vfs.inode_table.read().unwrap(); + if let Some(entry) = inodes.get(ino) + && !entry.is_dirty() + { + return; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("wait_for_clean: ino={} did not become clean within 4s", ino); +} + +/// Look up the MockXet content for a hash. Helper used by sparse flush tests. +fn xet_content(xet: &Arc, hash: &str) -> Option> { + xet.files.lock().unwrap().get(hash).cloned() +} + +/// Open in sparse mode does not download the full file. Reads outside the +/// dirty window hit CAS lazily via fill_sparse_holes. +#[test] +fn sparse_open_skips_download() { + let hub = MockHub::new(); + hub.add_file("big.bin", 100, Some("hbig"), None); + let xet = MockXet::new(); + xet.add_file("hbig", &(0u8..100).collect::>()); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "big.bin").await.unwrap().ino; + let downloads_before = xet.download_to_file_calls.load(std::sync::atomic::Ordering::SeqCst); + let fh = vfs.open(ino, true, false, None).await.unwrap(); + let downloads_after = xet.download_to_file_calls.load(std::sync::atomic::Ordering::SeqCst); + assert_eq!( + downloads_before, downloads_after, + "sparse open must not call download_to_file" + ); + vfs.release(fh).await.unwrap(); + }); +} + +/// Sparse read fills holes from CAS and returns the original bytes. +#[test] +fn sparse_read_returns_cas_bytes() { + let hub = MockHub::new(); + let content: Vec = (0u8..50).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + let (data, _) = vfs.read(fh, 10, 20).await.unwrap(); + assert_eq!(&data[..], &content[10..30]); + vfs.release(fh).await.unwrap(); + }); +} + +/// Second read of the same region hits the staging cache (no additional CAS +/// stream call) because fill_sparse_holes persisted the bytes on the first read. +#[test] +fn sparse_read_caches_into_staging() { + let hub = MockHub::new(); + let content: Vec = (0u8..50).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // First read populates the cache. + let _ = vfs.read(fh, 0, 30).await.unwrap(); + let calls_after_first = xet.stream_calls.lock().unwrap().len(); + // Second read of the same region should not stream from CAS again. + let (data2, _) = vfs.read(fh, 0, 30).await.unwrap(); + let calls_after_second = xet.stream_calls.lock().unwrap().len(); + assert_eq!( + calls_after_first, calls_after_second, + "second read should hit the staging cache, not CAS" + ); + assert_eq!(&data2[..], &content[..30]); + vfs.release(fh).await.unwrap(); + }); +} + +/// Reads outside the cached window still go to CAS. The cache fills lazily. +#[test] +fn sparse_read_uncached_region_hits_cas() { + let hub = MockHub::new(); + let content: Vec = (0u8..100).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // Cache [0, 20) + let _ = vfs.read(fh, 0, 20).await.unwrap(); + let calls_a = xet.stream_calls.lock().unwrap().len(); + // Read [60, 80) — uncached, should stream. + let (data, _) = vfs.read(fh, 60, 20).await.unwrap(); + let calls_b = xet.stream_calls.lock().unwrap().len(); + assert!(calls_b > calls_a, "uncached read should call CAS"); + assert_eq!(&data[..], &content[60..80]); + vfs.release(fh).await.unwrap(); + }); +} + +/// Write at an offset overlays staging and marks the range dirty. Subsequent +/// read returns the user's bytes (overlaid on holes filled from CAS). +#[test] +fn sparse_write_overlays_on_original() { + let hub = MockHub::new(); + let content: Vec = (0u8..20).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // Overwrite bytes [5, 10) + write_blocking(&vfs, ino, fh, 5, &[0xff; 5]).await.unwrap(); + let (data, _) = vfs.read(fh, 0, 20).await.unwrap(); + let mut expected = content.clone(); + expected[5..10].copy_from_slice(&[0xff; 5]); + assert_eq!(&data[..], &expected[..]); + vfs.release(fh).await.unwrap(); + }); +} + +/// Flush after a sparse write commits a CAS file with the user's modifications +/// applied on top of the original. The new hash differs from the original. +#[test] +fn sparse_flush_commits_composed_file() { + let hub = MockHub::new(); + let content: Vec = (0u8..20).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + write_blocking(&vfs, ino, fh, 10, b"HELLO").await.unwrap(); + vfs.release(fh).await.unwrap(); + // Wait for async flush + wait_for_clean(&vfs, ino).await; + // Verify the new hash exists in MockXet with the expected composed content + let mut expected = content.clone(); + expected[10..15].copy_from_slice(b"HELLO"); + let new_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + assert_ne!(new_hash, "h1", "flush should produce a new hash"); + assert_eq!(xet_content(&xet, &new_hash), Some(expected)); + }); +} + +/// Sparse write past EOF extends the file; the gap [old_size, new_offset) is +/// tracked as dirty so range_upload composes the extension correctly. Without +/// the gap tracking, the real xet-core upload_ranges would commit a truncated +/// file (10 + 3 = 13 bytes instead of 18); MockXet zero-pads to new_file_size +/// so the user-facing read happens to match, but the dirty_ranges check +/// guards against the mock masking the bug. +#[test] +fn sparse_write_past_eof_extends() { + let hub = MockHub::new(); + let content: Vec = (0u8..10).collect(); + hub.add_file("file.bin", 10, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // Write at offset 15 (5 bytes past EOF=10) + write_blocking(&vfs, ino, fh, 15, b"XYZ").await.unwrap(); + let (data, _) = vfs.read(fh, 0, 30).await.unwrap(); + let mut expected = content.clone(); + expected.resize(18, 0); // zeros in [10..15), then "XYZ" + expected[15..18].copy_from_slice(b"XYZ"); + assert_eq!(&data[..], &expected[..]); + + // Dirty range must cover the whole extension [prev_size, new_end) = + // [10, 18), not just [15, 18). Otherwise range_upload omits the + // zero gap and commits a truncated file. + { + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + let sw = entry.sparse_write.as_ref().expect("sparse_write must be set"); + assert_eq!( + sw.dirty_ranges, + vec![(10, 18)], + "past-EOF write must track the zero-gap from old EOF" + ); + } + + vfs.release(fh).await.unwrap(); + }); +} + +/// Setattr shrink trims coverage and dirty_ranges past the new size; the +/// flushed CAS file matches the truncated original. +#[test] +fn sparse_setattr_shrink_truncates_at_flush() { + let hub = MockHub::new(); + let content: Vec = (0u8..50).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + vfs.setattr(ino, Some(20), None, None, None, None, None).await.unwrap(); + wait_for_clean(&vfs, ino).await; + let new_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + assert_eq!(xet_content(&xet, &new_hash), Some(content[..20].to_vec())); + }); +} + +/// Regression: a clean sparse inode whose `xet_hash` is rotated by poll +/// (InodeTable::update_remote_file) keeps its `sparse_write` keyed to the OLD +/// hash. A later setattr must NOT reuse that stale state — doing so makes +/// range_upload compose against the superseded revision and silently overwrite +/// the poll-discovered remote update. The truncate must compose against the +/// current (rotated) revision instead. +#[test] +fn sparse_setattr_after_hash_drift_uses_current_revision() { + let hub = MockHub::new(); + let original: Vec = (0u8..50).collect(); + hub.add_file("file.bin", original.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &original); + // The revision the remote rotates to while the inode is clean. + let rotated: Vec = (100u8..150).collect(); + xet.add_file("h2", &rotated); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + + // Sparse open + write + flush leaves the inode CLEAN with a retained + // sparse_write keyed to the composed hash (the post-apply_commit_sparse + // state). + let fh = vfs.open(ino, true, false, None).await.unwrap(); + write_blocking(&vfs, ino, fh, 10, b"HELLO").await.unwrap(); + vfs.release(fh).await.unwrap(); + wait_for_clean(&vfs, ino).await; + let committed_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + assert_ne!(committed_hash, "h1"); + + // Poll discovers a newer remote revision and rotates xet_hash on the + // clean inode. The retained sparse_write stays keyed to the now-stale + // committed_hash. + { + let mut inodes = vfs.inode_table.write().unwrap(); + assert!( + inodes.update_remote_file(ino, Some("h2".to_string()), None, 50, SystemTime::now()), + "poll must rotate the clean inode" + ); + } + + // Truncate to 30. The flush must compose against h2 (current), not the + // stale committed hash. + vfs.setattr(ino, Some(30), None, None, None, None, None).await.unwrap(); + wait_for_clean(&vfs, ino).await; + + let new_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + assert_eq!( + xet_content(&xet, &new_hash), + Some(rotated[..30].to_vec()), + "truncate after hash drift must compose against the rotated remote (h2), \ + not the stale committed revision" + ); + }); +} + +/// Regression: when poll rotates a clean sparse inode to a SMALLER revision +/// and a later setattr grows the file past the new (smaller) size, the grow +/// tail must be POSIX zero-extension, not stale tail bytes left in the staging +/// file from the larger pre-rotation revision. The drift rebuild re-zeros the +/// staging tail (truncate to cur_size before extending) so range_upload +/// composes zeros over [cur_size, new_size). +#[test] +fn sparse_setattr_grow_after_shrinking_drift_zero_fills_tail() { + let hub = MockHub::new(); + let original: Vec = (0u8..100).collect(); + hub.add_file("file.bin", original.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &original); + // Rotated remote revision is SMALLER (50 bytes) than the original (100). + let rotated: Vec = (200u8..250).collect(); + xet.add_file("h2", &rotated); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + + // Sparse open + write + flush: clean inode, retained sparse_write, and + // a 100-byte staging file persists (no GC pressure in tests). + let fh = vfs.open(ino, true, false, None).await.unwrap(); + write_blocking(&vfs, ino, fh, 60, b"XXXXX").await.unwrap(); + vfs.release(fh).await.unwrap(); + wait_for_clean(&vfs, ino).await; + + // Poll rotates the clean inode to the smaller h2 (50 bytes). The + // retained sparse_write stays keyed to the now-stale committed hash. + { + let mut inodes = vfs.inode_table.write().unwrap(); + assert!( + inodes.update_remote_file(ino, Some("h2".to_string()), None, 50, SystemTime::now()), + "poll must rotate the clean inode" + ); + } + + // Grow to 75 (25 bytes past the current 50-byte revision). + vfs.setattr(ino, Some(75), None, None, None, None, None).await.unwrap(); + wait_for_clean(&vfs, ino).await; + + let new_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + // Expect h2 (50 bytes) followed by 25 zero bytes — NOT stale bytes from + // the 100-byte pre-rotation staging file. + let mut expected = rotated.clone(); + expected.resize(75, 0); + assert_eq!( + xet_content(&xet, &new_hash), + Some(expected), + "grow after shrinking drift must zero-fill the extension, not commit stale tail bytes" + ); + }); +} + +/// A read-only open of a DIRTY sparse file must still cache CAS hole-fills +/// into staging. The read handle's fd is opened writable for exactly this +/// reason; with a read-only fd the fill_sparse_holes write_at fails with EBADF +/// and every read of the same hole re-fetches from CAS. +#[test] +fn sparse_readonly_handle_on_dirty_file_caches_holes() { + let hub = MockHub::new(); + let content: Vec = (0u8..50).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + + // Writable open makes the file dirty with a sparse staging file. + let fh_w = vfs.open(ino, true, false, None).await.unwrap(); + write_blocking(&vfs, ino, fh_w, 5, b"AAAAA").await.unwrap(); + + // Read-only open of the now-dirty file routes through open_readonly -> + // open_local_readonly(staging_backed=true). Hold fh_w so the inode + // stays dirty (no race with the background flush). + let fh_r = vfs.open(ino, false, false, None).await.unwrap(); + + // First read of an uncovered hole streams from CAS and caches it. + let _ = vfs.read(fh_r, 20, 10).await.unwrap(); + let calls_after_first = xet.stream_calls.lock().unwrap().len(); + + // Second read of the same hole must hit the staging cache: the fill + // write_at succeeded (writable fd) and recorded coverage. + let (data, _) = vfs.read(fh_r, 20, 10).await.unwrap(); + let calls_after_second = xet.stream_calls.lock().unwrap().len(); + assert_eq!( + calls_after_first, calls_after_second, + "read-only handle on a dirty sparse file must cache hole-fills, not re-fetch CAS" + ); + assert_eq!(&data[..], &content[20..30]); + + vfs.release(fh_r).await.unwrap(); + vfs.release(fh_w).await.unwrap(); + }); +} + +/// Reopening a dirty sparse inode before its flush completes must preserve +/// the existing sparse_write state (coverage + dirty_ranges). Without this, +/// the second open would replace the state with a fresh one, dropping the +/// dirty tracking — a subsequent flush would see no dirty ranges and commit +/// a no-op even though the user's writes are still in staging. +/// +/// `fh1` is held open across the second open so the inode stays dirty +/// deterministically (no race with the background flush): the reopen path +/// keys on `is_dirty || staging_is_current`, and a held writable handle +/// keeps the inode dirty regardless of debounce timing. +#[test] +fn sparse_reopen_dirty_preserves_state() { + let hub = MockHub::new(); + let content: Vec = (0u8..20).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh1 = vfs.open(ino, true, false, None).await.unwrap(); + write_blocking(&vfs, ino, fh1, 5, b"AAA").await.unwrap(); + + // Reopen while fh1 is still held — the inode is dirty for certain. + let fh2 = vfs.open(ino, true, false, None).await.unwrap(); + + // Dirty state from the first open must survive the reopen. + let dirty_ranges = { + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + entry + .sparse_write + .as_ref() + .expect("sparse_write must persist") + .dirty_ranges + .clone() + }; + assert_eq!( + dirty_ranges, + vec![(5, 8)], + "reopen must preserve dirty_ranges from the first open" + ); + + // The user's bytes are still readable through the reopened handle. + let (data, _) = vfs.read(fh2, 5, 3).await.unwrap(); + assert_eq!(&data[..], b"AAA"); + + vfs.release(fh2).await.unwrap(); + vfs.release(fh1).await.unwrap(); + }); +} + +/// Hash-drift during a sparse open: `open_advanced_write` is robust to a +/// stale `xet_hash` / `size` from the caller's pre-lock snapshot — it +/// re-reads both fields under `staging.lock(ino)` so a poll-induced rotation +/// between `open()` and the install does not return a user-visible EIO. +/// +/// Drives `open_advanced_write` directly with a stale snapshot hash and an +/// off-by-one size: the install must succeed against the live state and +/// install a sparse_write keyed to the CURRENT hash, not the stale one. +#[test] +fn sparse_open_stale_snapshot_uses_live_state() { + let hub = MockHub::new(); + let content: Vec = (0u8..20).collect(); + hub.add_file("file.bin", content.len() as u64, Some("current_hash"), None); + let xet = MockXet::new(); + xet.add_file("current_hash", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + // The inode's live hash is "current_hash" / size 20. Pass STALE values + // to mimic a snapshot taken before a poll rotation completed. + let fh = vfs + .open_advanced_write(ino, "file.bin", "stale_hash", 999, false) + .await + .expect("open must succeed: stale snapshot is re-read under the lock"); + + // sparse_write must be keyed to the LIVE hash, not the stale one. + let entry_hash = { + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + assert!(entry.is_dirty(), "open_advanced_write sets dirty"); + entry + .sparse_write + .as_ref() + .expect("sparse_write must be installed") + .original_hash + .clone() + }; + assert_eq!(entry_hash, "current_hash", "sparse_write must key to the live hash"); + + vfs.release(fh).await.unwrap(); + }); +} + +/// No-op flush (open then release with no writes) keeps the original hash +/// and preserves the staging cache populated by reads. +#[test] +fn sparse_noop_flush_preserves_hash_and_cache() { + let hub = MockHub::new(); + let content: Vec = (0u8..30).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // Read populates the cache but doesn't dirty anything. + let _ = vfs.read(fh, 0, 30).await.unwrap(); + vfs.release(fh).await.unwrap(); + wait_for_clean(&vfs, ino).await; + let final_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + assert_eq!(final_hash, "h1", "no-op flush must keep the original hash"); + }); +} + +/// Regression: O_TRUNC clears sparse_write but keeps xet_hash; a write then +/// lands while sparse_write is None (so it tracks nothing); a reopen WITHOUT +/// O_TRUNC (before the debounced flush) must NOT install an empty-coverage +/// SparseWriteState over the dirty staging. If it did, reads would overlay the +/// ORIGINAL CAS bytes over the user's write and the flush would no-op back to +/// the original hash, silently losing the write entirely. +#[test] +fn sparse_otrunc_then_write_then_reopen_keeps_user_data() { + let hub = MockHub::new(); + let content: Vec = (0u8..20).collect(); + hub.add_file("file.bin", content.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + + // 1. open(O_TRUNC): logically empties the file, clears sparse_write, + // but keeps xet_hash = "h1". + let fh1 = vfs.open(ino, true, true, None).await.unwrap(); + // 2. write fresh bytes while sparse_write is None (untracked). + write_blocking(&vfs, ino, fh1, 0, b"NEWDATA").await.unwrap(); + + // 3. reopen WITHOUT O_TRUNC, before the flush fires (fh1 held open keeps + // the inode dirty deterministically). + let fh2 = vfs.open(ino, true, false, None).await.unwrap(); + + // 4. a read through the reopened handle must return the user's bytes, + // not the resurrected original CAS content. + let (data, _) = vfs.read(fh2, 0, 7).await.unwrap(); + assert_eq!( + &data[..], + b"NEWDATA", + "reopen must not overlay original CAS bytes over the write" + ); + + vfs.release(fh2).await.unwrap(); + vfs.release(fh1).await.unwrap(); + + // 5. the flush must commit the user's content, not no-op back to h1. + wait_for_clean(&vfs, ino).await; + let new_hash = { + let inodes = vfs.inode_table.read().unwrap(); + inodes.get(ino).unwrap().xet_hash.clone().unwrap() + }; + assert_ne!(new_hash, "h1", "flush must not preserve the original hash"); + assert_eq!( + xet_content(&xet, &new_hash), + Some(b"NEWDATA".to_vec()), + "committed CAS content must be exactly the user's write" + ); + }); +} + +// ── Flush batch failure isolation ────────────────────────────────────── +// +// flush_batch processes sparse items in Pass A (per-item range_upload) and +// non-sparse items in Pass B (batched upload_files). A failure in one item +// must not pollute or block sibling items: the failing inode stays dirty and +// retries on the next flush, every other inode commits independently. + +/// Regression for the Pass A fan-out bug: when a sparse range_upload failed, +/// the error was attributed to *every* item in to_flush — including non-sparse +/// items that Pass B had not even attempted yet. After the fix only the failing +/// sparse inode is marked errored; the non-sparse sibling commits cleanly. +#[test] +fn flush_pass_a_failure_does_not_pollute_non_sparse_sibling() { + let hub = MockHub::new(); + let original: Vec = (0u8..40).collect(); + hub.add_file("sparse.bin", original.len() as u64, Some("h_sparse"), None); + let xet = MockXet::new(); + xet.add_file("h_sparse", &original); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + // ino1: existing remote file → sparse path (Pass A). + let ino1 = vfs.lookup(ROOT_INODE, "sparse.bin").await.unwrap().ino; + let fh1 = vfs.open(ino1, true, false, None).await.unwrap(); + write_blocking(&vfs, ino1, fh1, 5, b"ZZZZ").await.unwrap(); + + // ino2: freshly created file with no remote hash → non-sparse (Pass B). + let (attr2, fh2) = vfs + .create(ROOT_INODE, "fresh.bin", 0o644, 1000, 1000, None) + .await + .unwrap(); + let ino2 = attr2.ino; + write_blocking(&vfs, ino2, fh2, 0, b"hello world").await.unwrap(); + + // Arm the one-shot failure: the next sparse range_upload (Pass A on + // ino1) fails; the upload_files in Pass B for ino2 succeeds because + // fail_upload has already been consumed. + xet.fail_upload(); + + vfs.release(fh1).await.unwrap(); + vfs.release(fh2).await.unwrap(); + + // Give the debounce + retries time to settle. ino2 commits on the + // first batch; ino1 stays dirty (will retry on subsequent flushes). + tokio::time::sleep(Duration::from_secs(1)).await; + + let fm = vfs.flush_manager.as_ref().unwrap(); + // ino1 must carry the range_upload failure. + let err1 = fm.check_error(ino1); + assert!( + err1.as_deref().is_some_and(|m| m.contains("range_upload failed")), + "ino1 should be marked range_upload failed, got {err1:?}" + ); + // ino2 must NOT be falsely marked. Before the fix it inherited the + // sparse failure even though Pass B uploaded its content cleanly. + assert!( + fm.check_error(ino2).is_none(), + "ino2 went through a successful upload_files and must not carry the sibling's error" + ); + // And ino2 must actually be clean (committed). + assert!( + !vfs.inode_table.read().unwrap().get(ino2).unwrap().is_dirty(), + "ino2 must be clean after a successful Pass B commit, regardless of Pass A's failure" + ); + }); + + vfs.shutdown(); +} + +/// Regression for head-of-line blocking in Pass A: a failed sparse upload used +/// to abort the entire pass with an early return, starving independent sparse +/// items behind it. After the fix Pass A continues; each sparse item has its +/// own success/failure outcome. +#[test] +fn flush_pass_a_failure_does_not_block_independent_sparse_items() { + let hub = MockHub::new(); + let content_a: Vec = (0u8..30).collect(); + let content_b: Vec = (50u8..90).collect(); + hub.add_file("a.bin", content_a.len() as u64, Some("h_a"), None); + hub.add_file("b.bin", content_b.len() as u64, Some("h_b"), None); + let xet = MockXet::new(); + xet.add_file("h_a", &content_a); + xet.add_file("h_b", &content_b); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino_a = vfs.lookup(ROOT_INODE, "a.bin").await.unwrap().ino; + let ino_b = vfs.lookup(ROOT_INODE, "b.bin").await.unwrap().ino; + + let fh_a = vfs.open(ino_a, true, false, None).await.unwrap(); + write_blocking(&vfs, ino_a, fh_a, 0, b"AAAA").await.unwrap(); + let fh_b = vfs.open(ino_b, true, false, None).await.unwrap(); + write_blocking(&vfs, ino_b, fh_b, 0, b"BBBB").await.unwrap(); + + // Only the first range_upload in the Pass A loop fails. + xet.fail_upload(); + vfs.release(fh_a).await.unwrap(); + vfs.release(fh_b).await.unwrap(); + + // Wait for the batch to settle (the failed inode does not retry within + // this window — the failure is sticky for one debounce cycle). + tokio::time::sleep(Duration::from_secs(1)).await; + + // Exactly one of {a, b} must be clean (the one whose range_upload + // succeeded) and the other must still be dirty. Before the fix neither + // would be clean because the early return killed Pass A entirely. + let (a_dirty, b_dirty) = { + let inodes = vfs.inode_table.read().unwrap(); + ( + inodes.get(ino_a).unwrap().is_dirty(), + inodes.get(ino_b).unwrap().is_dirty(), + ) + }; + assert!( + a_dirty ^ b_dirty, + "exactly one sibling must commit; got a_dirty={a_dirty} b_dirty={b_dirty}" + ); + }); + + vfs.shutdown(); +} + +// ── Sparse smoke / edge cases ─────────────────────────────────────────── + +/// Write that lands EXACTLY at EOF (no zero gap, no overlap) extends the file +/// by the write length. Coverage/dirty must merge with any pre-EOF coverage +/// instead of leaving a phantom seam at the boundary. +#[test] +fn sparse_write_at_eof_boundary_extends_cleanly() { + let hub = MockHub::new(); + let content: Vec = (0u8..10).collect(); + hub.add_file("file.bin", 10, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // Write at offset 10 (current EOF), 5 bytes. No gap, no overlap. + write_blocking(&vfs, ino, fh, 10, b"EDGE!").await.unwrap(); + + // Reading across the boundary must stitch CAS bytes [0..10) and the + // user's bytes [10..15) into one continuous response. + let (data, _) = vfs.read(fh, 7, 8).await.unwrap(); + assert_eq!(&data[..], b"\x07\x08\x09EDGE!", "read across the EOF seam"); + + // Dirty range covers exactly the extension — no spurious zero-gap. + { + let inodes = vfs.inode_table.read().unwrap(); + let sw = inodes.get(ino).unwrap().sparse_write.as_ref().unwrap().clone(); + assert_eq!(sw.dirty_ranges, vec![(10, 15)]); + } + + vfs.release(fh).await.unwrap(); + wait_for_clean(&vfs, ino).await; + let new_hash = vfs + .inode_table + .read() + .unwrap() + .get(ino) + .unwrap() + .xet_hash + .clone() + .unwrap(); + let mut expected = content.clone(); + expected.extend_from_slice(b"EDGE!"); + assert_eq!(xet_content(&xet, &new_hash), Some(expected)); + }); +} + +/// Setattr-shrink to a size INSIDE an existing dirty range must clip the dirty +/// range, not drop it entirely. The flushed CAS file then contains the head of +/// the user's write, truncated at the new size. +#[test] +fn sparse_setattr_shrink_inside_dirty_range_keeps_clipped_writes() { + let hub = MockHub::new(); + let original: Vec = (0u8..100).collect(); + hub.add_file("file.bin", original.len() as u64, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &original); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + // Write [40, 70) = 30 bytes of 'X'. + write_blocking(&vfs, ino, fh, 40, &[b'X'; 30]).await.unwrap(); + vfs.release(fh).await.unwrap(); + + // Shrink to 55 — slices through the middle of the dirty range. The + // committed file must be original[..40] + 'X' * 15. + vfs.setattr(ino, Some(55), None, None, None, None, None).await.unwrap(); + wait_for_clean(&vfs, ino).await; + + let new_hash = vfs + .inode_table + .read() + .unwrap() + .get(ino) + .unwrap() + .xet_hash + .clone() + .unwrap(); + let mut expected = original[..40].to_vec(); + expected.extend(std::iter::repeat_n(b'X', 15)); + assert_eq!( + xet_content(&xet, &new_hash), + Some(expected), + "shrink cutting through a dirty range must keep the clipped prefix" + ); + }); +} + +/// Zero-length write to a sparse file is a no-op: must not mark dirty, must not +/// add to coverage or dirty_ranges, must not extend the file. Defensive smoke +/// test — kernel typically filters these but FUSE/NFS handlers don't always. +#[test] +fn sparse_zero_length_write_is_a_noop() { + let hub = MockHub::new(); + let content: Vec = (0u8..20).collect(); + hub.add_file("file.bin", 20, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + let n = write_blocking(&vfs, ino, fh, 5, b"").await.unwrap(); + assert_eq!(n, 0); + + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + // pwrite of zero bytes still bumps mtime/dirty per POSIX, but the + // tracked dirty range must be empty (no zero-length entries leaking). + if let Some(sw) = entry.sparse_write.as_ref() { + assert!( + sw.dirty_ranges.is_empty(), + "zero-length write must not create a dirty range, got {:?}", + sw.dirty_ranges + ); + } + assert_eq!(entry.size, 20, "zero-length write must not change size"); + }); +} + +/// Regression for the Pass B over-marking bug: when a Pass B (`upload_files`) +/// failure aborts the batch, sparse items whose Pass A `range_upload` already +/// succeeded used to be flagged with the Pass B error in `flush_errors`. A +/// follow-up fsync on the sparse item would return EIO claiming "upload +/// failed" even though its CAS xorb had been uploaded cleanly. After the fix, +/// only items without an upload_result entry (Pass A failures + items the +/// early return skipped) are marked. +#[test] +fn flush_pass_b_failure_preserves_pass_a_sparse_success() { + let hub = MockHub::new(); + let original: Vec = (0u8..40).collect(); + hub.add_file("sparse.bin", original.len() as u64, Some("h_sparse"), None); + let xet = MockXet::new(); + xet.add_file("h_sparse", &original); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + // ino_sparse: existing remote file → Pass A (range_upload succeeds). + let ino_sparse = vfs.lookup(ROOT_INODE, "sparse.bin").await.unwrap().ino; + let fh_sparse = vfs.open(ino_sparse, true, false, None).await.unwrap(); + write_blocking(&vfs, ino_sparse, fh_sparse, 5, b"ZZZZ").await.unwrap(); + + // ino_regular: newly created → Pass B (upload_files fails). + let (attr_regular, fh_regular) = vfs + .create(ROOT_INODE, "fresh.bin", 0o644, 1000, 1000, None) + .await + .unwrap(); + let ino_regular = attr_regular.ino; + write_blocking(&vfs, ino_regular, fh_regular, 0, b"hello world") + .await + .unwrap(); + + // Fail ONLY Pass B (upload_files) — Pass A's range_upload still succeeds. + xet.fail_next_upload_files(); + + vfs.release(fh_sparse).await.unwrap(); + vfs.release(fh_regular).await.unwrap(); + + tokio::time::sleep(Duration::from_secs(1)).await; + + let fm = vfs.flush_manager.as_ref().unwrap(); + // ino_sparse uploaded cleanly in Pass A; its CAS xorb is durable. It + // stayed dirty because the batch's Hub commit was aborted, and the + // next flush will retry the commit, but fsync must NOT surface a + // misleading "upload failed" for it. + let err_sparse = fm.check_error(ino_sparse); + assert!( + err_sparse.is_none(), + "sparse Pass A success must not inherit the Pass B failure, got {err_sparse:?}" + ); + // ino_regular is the one that actually failed in Pass B; its error + // must surface so fsync returns EIO for the caller. + let err_regular = fm.check_error(ino_regular); + assert!( + err_regular.as_deref().is_some_and(|m| m.contains("upload failed")), + "regular Pass B failure must be surfaced, got {err_regular:?}" + ); + }); + + vfs.shutdown(); +} + +/// Regression for `touch_commit_clocks` being stamped on a dirty_generation +/// mismatch: a concurrent writer that bumps the generation between flush +/// snapshot and `apply_commit_sparse` used to still see `last_revalidated = +/// now` written, which lets a subsequent `lookup()` skip HEAD revalidation +/// for metadata_ttl seconds even though nothing was committed for this +/// generation. Mtime/ctime were also overwritten to "now" while the inode +/// was still dirty with the racing writer's bytes. +/// +/// After the fix, all three clock fields (mtime, ctime, last_revalidated) +/// stay untouched on a generation mismatch — only the racing-writer-newer +/// content remains dirty and revalidation is not silenced. +#[test] +fn apply_commit_sparse_mismatch_does_not_stamp_clocks() { + use std::time::Instant; + let hub = MockHub::new(); + let original: Vec = (0u8..20).collect(); + hub.add_file("file.bin", 20, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &original); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + write_blocking(&vfs, ino, fh, 0, b"AAA").await.unwrap(); + + // Snapshot the pre-commit clocks BEFORE simulating the mismatch so we + // can assert nothing moved on the rejected path. + let (mtime_before, ctime_before, lastrev_before, gen_before) = { + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + (entry.mtime, entry.ctime, entry.last_revalidated, entry.dirty_generation) + }; + + // Tiny sleep so any wrongly-bumped `Instant::now()` would differ from + // the snapshot (otherwise the assert is flaky on fast machines). + tokio::time::sleep(Duration::from_millis(10)).await; + + // Simulate a generation mismatch: invoke apply_commit_sparse with a + // generation that is NOT the current one. This mirrors the race where + // the flusher snapshotted gen=N but a concurrent pwrite bumped to N+1 + // before commit. clear_dirty_if returns false → no fields applied. + { + let mut inodes = vfs.inode_table.write().unwrap(); + let entry = inodes.get_mut(ino).unwrap(); + let stale_gen = gen_before.wrapping_sub(1); // any value != gen_before + entry.apply_commit_sparse("hX", 50, stale_gen); + } + + let (mtime_after, ctime_after, lastrev_after) = { + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + (entry.mtime, entry.ctime, entry.last_revalidated) + }; + + assert_eq!(mtime_before, mtime_after, "mtime must not bump on generation mismatch"); + assert_eq!(ctime_before, ctime_after, "ctime must not bump on generation mismatch"); + assert_eq!( + lastrev_before, lastrev_after, + "last_revalidated must not be stamped on a rejected commit, otherwise lookup skips HEAD revalidation while the inode is still dirty" + ); + + // Sanity: the inode should still be dirty (the racing writer's content + // hasn't been committed). Clean up so the test doesn't hang on the + // background flush retrying. + assert!(vfs.inode_table.read().unwrap().get(ino).unwrap().is_dirty()); + let _ = Instant::now(); + + vfs.release(fh).await.unwrap(); + }); + + vfs.shutdown(); +} + +/// Regression for the zero-byte pwrite-past-EOF stuck-dirty bug: a write +/// with `data.len() == 0` (reachable via custom FUSE clients, NFS re-exports, +/// or test harnesses that bypass the kernel's VFS short-circuit) must NOT +/// extend entry.size, mark the inode dirty, or track sparse coverage. Before +/// the fix, the code unconditionally computed `new_end = offset + 0` and +/// bumped entry.size past the on-disk staging length; snapshot_dirty_bytes +/// then hit UnexpectedEof on every flush retry and the inode stayed +/// permanently dirty with fsync returning EIO forever. +#[test] +fn write_zero_bytes_past_eof_is_a_noop() { + let hub = MockHub::new(); + let original: Vec = (0u8..10).collect(); + hub.add_file("file.bin", 10, Some("h1"), None); + let xet = MockXet::new(); + xet.add_file("h1", &original); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, "file.bin").await.unwrap().ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + assert_eq!(vfs.inode_table.read().unwrap().get(ino).unwrap().size, 10); + + // Zero-byte write past EOF (offset > size). Before the fix, the code + // bumped entry.size to offset (50) while the on-disk staging stayed + // at 10 bytes; the next flush snapshot_dirty_bytes would then hit + // UnexpectedEof and the inode would be stuck dirty forever. The fix + // gates the size bump + dirty + sparse-track on `written > 0`. + let written = write_blocking(&vfs, ino, fh, 50, b"").await.unwrap(); + assert_eq!(written, 0, "pwrite of empty slice returns 0"); + + let size_after = vfs.inode_table.read().unwrap().get(ino).unwrap().size; + assert_eq!(size_after, 10, "zero-byte write past EOF must not extend size"); + + // The actual cure: a subsequent flush of any concurrent dirty work + // must succeed (no UnexpectedEof from snapshot_dirty_bytes). Issue a + // real one-byte write to dirty the inode legitimately, then verify + // it commits cleanly — the zero-byte write didn't poison the staging + // file invariant. + write_blocking(&vfs, ino, fh, 0, b"X").await.unwrap(); + vfs.release(fh).await.unwrap(); + wait_for_clean(&vfs, ino).await; + }); + + vfs.shutdown(); +} + +// ── Backports from feat/append-write (PR #41) ─────────────────────────── +// +// These tests target invariants that the rewrite preserves but which were +// not previously exercised end-to-end. Names kept close to the originals +// for cross-referencing during the rewrite review. + +/// Post a sparse flush, the reader must see committed CAS bytes for the +/// holes outside the modified window, NOT the staging zeros. This pins the +/// post-`apply_commit_sparse` invariant: sparse_write stays installed +/// (rekeyed to the new hash), `dirty_ranges` is cleared, `staging_is_current` +/// remains false because the staging file still has unmaterialised holes, +/// and a read fills those holes from the JUST-committed CAS file. +/// +/// Backport of `sparse_post_flush_read_returns_cas_bytes_not_zeros` from +/// PR #41. +#[test] +fn sparse_post_flush_read_returns_cas_bytes_not_zeros() { + let hub = MockHub::new(); + hub.add_file("file.txt", 10, Some("orig_hash"), None); + let xet = MockXet::new(); + xet.add_file("orig_hash", b"0123456789"); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let attr = vfs.lookup(ROOT_INODE, "file.txt").await.unwrap(); + let ino = attr.ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + + // Dirty a small window in the middle of the file. Staging now holds + // zeros everywhere except "XX" at [2..4); the rest are sparse holes. + write_blocking(&vfs, ino, fh, 2, b"XX").await.unwrap(); + + vfs.fsync(ino, fh, None).await.unwrap(); + wait_for_clean(&vfs, ino).await; + + // Post-`apply_commit_sparse` shape: dirty cleared, sparse_write + // preserved (rekeyed to new hash), staging not flagged current + // because holes remain unmaterialised. + { + let inodes = vfs.inode_table.read().unwrap(); + let entry = inodes.get(ino).unwrap(); + assert!( + !entry.staging_is_current, + "staging must not be flagged current after a sparse flush — \ + holes still need CAS fill" + ); + let sw = entry + .sparse_write + .as_ref() + .expect("sparse_write must persist after sparse flush so reads can fill holes"); + assert!(sw.dirty_ranges.is_empty(), "fresh sparse state has no dirty ranges"); + assert_eq!(sw.original_size, 10); + } + + // The actual regression target: bytes the user never wrote must come + // back from the newly-committed CAS file, not zero-filled staging + // holes. Pre-fix this returned "00XX000000" because sparse_write was + // dropped and the reader hit raw staging zeros. + let (data, _) = vfs.read(fh, 0, 10).await.unwrap(); + assert_eq!( + &data[..], + b"01XX456789", + "post-flush read on open handle must return composed CAS bytes, not staging zeros" + ); + + vfs.release(fh).await.unwrap(); + }); + + vfs.shutdown(); +} + +/// A CAS stream that yields zero bytes for a non-empty hole range must +/// surface EIO from `fill_sparse_holes`, NOT panic on an OOB +/// `copy_from_slice` when overlaying empty fetched data onto the read buf. +/// +/// Backport of `repro_fill_sparse_holes_panics_on_short_cas_stream` from +/// PR #41. +#[test] +fn sparse_fill_short_cas_stream_returns_eio_not_panic() { + let hub = MockHub::new(); + let content = b"0123456789"; + hub.add_file("file.txt", content.len() as u64, Some("orig_hash"), None); + let xet = MockXet::new(); + xet.add_file("orig_hash", content); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let attr = vfs.lookup(ROOT_INODE, "file.txt").await.unwrap(); + let ino = attr.ino; + let fh = vfs.open(ino, true, false, None).await.unwrap(); + + // Next CAS download returns an empty stream. fill_sparse_holes + // requests bytes for the full sparse window [0..10) but receives 0 + // bytes — must error out cleanly rather than OOB-slicing. + xet.empty_range_downloads(1); + + let result = vfs.read(fh, 0, 10).await; + assert_eq!( + result.err(), + Some(libc::EIO), + "short CAS stream must surface EIO, not panic or return zeros" + ); + + vfs.release(fh).await.unwrap(); + }); + + vfs.shutdown(); +} + +/// Even when a sparse flush is a no-op (hash unchanged), the inode's mtime +/// must advance — fsync semantics require the modification time reflect the +/// completed write cycle. `apply_noop_commit` calls `touch_commit_clocks` +/// to ensure this. +/// +/// Backport of `repro_noop_flush_does_not_bump_mtime` from PR #41 +/// (test name is awkward but the assert is "mtime DOES advance"). +#[test] +fn sparse_noop_flush_advances_mtime() { + let hub = MockHub::new(); + hub.add_file("file.txt", 10, Some("orig_hash"), None); + let xet = MockXet::new(); + xet.add_file("orig_hash", b"0123456789"); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + rt.block_on(async { + let attr = vfs.lookup(ROOT_INODE, "file.txt").await.unwrap(); + let ino = attr.ino; + let mtime_before = vfs.inode_table.read().unwrap().get(ino).unwrap().mtime; + + // Sleep long enough that any clock bump is observable. SystemTime + // resolution is sub-millisecond but the assertion uses strict >. + tokio::time::sleep(Duration::from_millis(50)).await; + + // Open writable, fsync, release — no actual writes, so the flush is + // a no-op (xet_hash unchanged). The dirty bit is still set by open, + // so the flush manager runs a real flush cycle and ends up in + // `apply_noop_commit`. + let fh = vfs.open(ino, true, false, None).await.unwrap(); + vfs.fsync(ino, fh, None).await.unwrap(); + vfs.release(fh).await.unwrap(); + wait_for_clean(&vfs, ino).await; + + let mtime_after = vfs.inode_table.read().unwrap().get(ino).unwrap().mtime; + assert!( + mtime_after > mtime_before, + "mtime must advance after a dirty open + flush cycle even on a no-op upload \ + (was {mtime_before:?}, still {mtime_after:?})" + ); + }); + + vfs.shutdown(); +} + +// ── Sparse-write stress test ──────────────────────────────────────────── +// +// A multi-worker workload that exercises the sparse-write state machine +// (coverage map, dirty_ranges, dirty_generation, drift handling, io_lock +// discipline) under concurrent ops on multiple files. Each worker owns its +// own file (no cross-worker shadow races), but flushes happen against a +// shared FlushManager → Pass A/B batching, lock map churn, and concurrent +// staging-file lifecycles get exercised for real. +// +// What the random ops cover, by construction: +// - in-file overwrite (offset < EOF, end <= EOF) +// - write spanning EOF (offset < EOF < end) +// - write at EOF (offset == EOF) +// - write past EOF (offset > EOF) → the zero-fill gap +// - setattr-grow (zero-fill tail) +// - setattr-shrink (clip coverage + dirty) +// - setattr-shrink-then-grow (must re-zero, not stale) +// - reopen (sparse_write preservation + drift retry) +// - fsync (force apply_commit_sparse cycle) +// - reads at random offsets (exercises fill_sparse_holes + cache) +// +// Validation: shadow Vec per file mirroring the expected content; after +// every committed state (fsync or release+wait_for_clean), a full-file read +// of the inode must match the shadow byte for byte. A divergence proves +// data loss or corruption. + +const STRESS_FILES: usize = 12; +const STRESS_OPS_PER_FILE: usize = 200; +const STRESS_BASE_SIZE: usize = 4096; +const STRESS_MAX_SIZE: usize = 64 * 1024; + +#[derive(Debug, Clone, Copy)] +enum StressOp { + Write { + offset: usize, + len: usize, + }, + SetattrGrow { + delta: usize, + }, + SetattrShrink { + delta: usize, + }, + Fsync, + Reopen, + VerifyRead, + /// Simulate a poll-induced remote rotation: while the worker's inode is + /// clean, swap the remote content for a freshly-generated blob and call + /// `update_remote_file` so a subsequent reopen sees the new revision. The + /// worker's shadow is updated to match the new remote content so the + /// final CAS-level verify can still catch any data loss along the way. + RogueRotation { + new_size: usize, + }, +} + +struct StressRng(u64); +impl StressRng { + fn new(seed: u64) -> Self { + Self(seed | 1) + } + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn rand_byte(&mut self) -> u8 { + self.next() as u8 + } + fn rand_range(&mut self, hi: usize) -> usize { + if hi == 0 { 0 } else { (self.next() as usize) % hi } + } +} + +fn gen_stress_op(rng: &mut StressRng, current_size: usize) -> StressOp { + let pick = rng.next() % 100; + match pick { + // 50% writes biased toward useful edge cases + 0..=49 => { + let kind = rng.next() % 5; + let (offset, len) = match kind { + // in-file overwrite + 0 if current_size >= 2 => { + let len = 1 + rng.rand_range(current_size.min(1024)); + let offset = rng.rand_range(current_size.saturating_sub(len).max(1)); + (offset, len) + } + // write spanning EOF (offset < size, end > size) + 1 if current_size >= 2 => { + let offset = rng.rand_range(current_size); + let len = (current_size - offset) + 1 + rng.rand_range(1024); + (offset, len) + } + // write at exactly EOF + 2 => (current_size, 1 + rng.rand_range(1024)), + // write past EOF with gap + 3 => { + let gap = 1 + rng.rand_range(2048); + let offset = current_size + gap; + let len = 1 + rng.rand_range(1024); + (offset, len) + } + // any random offset+len + _ => { + let offset = rng.rand_range(STRESS_MAX_SIZE.saturating_sub(1)); + let len = 1 + rng.rand_range(2048); + (offset, len) + } + }; + let len = len.min(STRESS_MAX_SIZE.saturating_sub(offset)); + if len == 0 || offset >= STRESS_MAX_SIZE { + StressOp::VerifyRead + } else { + StressOp::Write { offset, len } + } + } + // 15% setattr-grow + 50..=64 => StressOp::SetattrGrow { + delta: 1 + rng.rand_range(4096), + }, + // 15% setattr-shrink + 65..=79 if current_size > 1 => StressOp::SetattrShrink { + delta: 1 + rng.rand_range(current_size - 1), + }, + // 10% fsync (force flush + apply_commit_sparse cycle) + 80..=87 => StressOp::Fsync, + // 5% reopen + 88..=92 => StressOp::Reopen, + // 4% rogue remote rotation (only fires when the worker happens to be + // clean — see the impl); the new remote size is chosen here, but the + // actual rotation depends on the inode being clean at apply time. + 93..=96 => StressOp::RogueRotation { + new_size: 1 + rng.rand_range(STRESS_MAX_SIZE - 1), + }, + // remainder + fallback for invalid shrink: verification read + _ => StressOp::VerifyRead, + } +} + +async fn read_full(vfs: &std::sync::Arc, ino: u64, size: u64) -> Vec { + if size == 0 { + return Vec::new(); + } + // Read in chunks of 16 KB to mirror real FUSE behaviour. + let fh = vfs.open(ino, false, false, None).await.expect("open ro"); + let mut out = Vec::with_capacity(size as usize); + let chunk: u32 = 16 * 1024; + let mut off: u64 = 0; + while off < size { + let want = chunk.min((size - off) as u32); + let (bytes, _eof) = vfs.read(fh, off, want).await.expect("read"); + if bytes.is_empty() { + break; + } + out.extend_from_slice(&bytes); + off += bytes.len() as u64; + } + vfs.release(fh).await.expect("release ro"); + out +} + +async fn run_stress_worker( + vfs: std::sync::Arc, + xet: std::sync::Arc, + path: String, + seed: u64, + base: Vec, + n_ops: usize, +) -> Result<(), String> { + let mut shadow = base.clone(); + let mut rng = StressRng::new(seed); + + let ino = vfs + .lookup(ROOT_INODE, &path) + .await + .map_err(|e| format!("[{path}] initial lookup: {e}"))? + .ino; + + let mut fh = vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[{path}] initial open: {e}"))?; + + for op_idx in 0..n_ops { + let op = gen_stress_op(&mut rng, shadow.len()); + match op { + StressOp::Write { offset, len } => { + let buf: Vec = (0..len).map(|_| rng.rand_byte()).collect(); + let written = write_blocking(&vfs, ino, fh, offset as u64, &buf) + .await + .map_err(|e| format!("[{path}] op {op_idx} write({offset},{len}): {e}"))?; + // Mirror in shadow. The pwrite may have written fewer bytes + // than asked, but in practice on a regular file it writes + // all or errors — defend just in case. + let actual_len = written as usize; + if offset > shadow.len() { + shadow.resize(offset, 0); + } + let end = offset + actual_len; + if shadow.len() < end { + shadow.resize(end, 0); + } + shadow[offset..end].copy_from_slice(&buf[..actual_len]); + } + StressOp::SetattrGrow { delta } => { + let new_size = (shadow.len() + delta).min(STRESS_MAX_SIZE); + if new_size == shadow.len() { + continue; + } + vfs.setattr(ino, Some(new_size as u64), None, None, None, None, None) + .await + .map_err(|e| format!("[{path}] op {op_idx} setattr_grow({new_size}): {e}"))?; + shadow.resize(new_size, 0); + } + StressOp::SetattrShrink { delta } => { + if shadow.len() <= 1 { + continue; + } + let new_size = shadow.len().saturating_sub(delta).max(1); + vfs.setattr(ino, Some(new_size as u64), None, None, None, None, None) + .await + .map_err(|e| format!("[{path}] op {op_idx} setattr_shrink({new_size}): {e}"))?; + shadow.truncate(new_size); + } + StressOp::Fsync => { + vfs.fsync(ino, fh, None) + .await + .map_err(|e| format!("[{path}] op {op_idx} fsync: {e}"))?; + // Best-effort wait for the debounced commit so the next + // verify-read can hit a clean inode. wait_for_clean blocks up + // to 4s and panics on timeout — too aggressive for stress, so + // just sleep a bit. + tokio::time::sleep(Duration::from_millis(50)).await; + } + StressOp::Reopen => { + vfs.release(fh) + .await + .map_err(|e| format!("[{path}] op {op_idx} release-before-reopen: {e}"))?; + fh = vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[{path}] op {op_idx} reopen: {e}"))?; + } + StressOp::RogueRotation { new_size } => { + // Quiesce: fsync + wait clean, then drop the fd so the next + // reopen picks up the rotated state (update_remote_file + // refuses to rotate a dirty inode, so we MUST be clean here). + vfs.fsync(ino, fh, None) + .await + .map_err(|e| format!("[{path}] op {op_idx} rogue: fsync: {e}"))?; + vfs.release(fh) + .await + .map_err(|e| format!("[{path}] op {op_idx} rogue: pre-rotate release: {e}"))?; + wait_for_clean(&vfs, ino).await; + + // Build new remote content + register it as a fresh CAS hash. + let new_content: Vec = (0..new_size).map(|_| rng.rand_byte()).collect(); + let new_hash = format!("rogue_{path}_{op_idx}_{}", rng.next()); + xet.add_file(&new_hash, &new_content); + + // Apply the rotation. update_remote_file returns false if the + // inode is dirty — should never happen because we just + // quiesced, but handle defensively (skip the shadow update so + // we don't desync from the live state). + let rotated = { + let mut inodes = vfs.inode_table.write().expect("inodes poisoned"); + inodes.update_remote_file(ino, Some(new_hash), None, new_size as u64, std::time::SystemTime::now()) + }; + if rotated { + // Worker's view from the NEXT open onwards is the new + // remote content — shadow must match for the post-op + // verify reads and the final CAS-level check. + shadow = new_content; + } + + // Reopen to pick up the rotation. open_advanced_write detects + // the drift (sparse_write was keyed to the old hash) and + // rebuilds it against the new revision. + fh = vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[{path}] op {op_idx} rogue: reopen: {e}"))?; + } + StressOp::VerifyRead => { + // Read a random window and compare with shadow. Don't verify + // the full file mid-stream (too slow and the shadow is + // already mirroring every op) — just sanity-check a slice. + if shadow.is_empty() { + continue; + } + let len = 1 + rng.rand_range(shadow.len().min(2048)); + let offset = rng.rand_range(shadow.len() - 1); + let want = (len as u32).min((shadow.len() - offset) as u32); + let (bytes, _eof) = vfs + .read(fh, offset as u64, want) + .await + .map_err(|e| format!("[{path}] op {op_idx} mid-stream read: {e}"))?; + let expected = &shadow[offset..offset + bytes.len()]; + if bytes.as_ref() != expected { + return Err(format!( + "[{path}] op {op_idx} mid-stream read MISMATCH at offset {} (got {} bytes, expected {})\n first diff at {:?}", + offset, + bytes.len(), + expected.len(), + bytes.iter().zip(expected.iter()).position(|(a, b)| a != b), + )); + } + } + } + } + + // Quiesce: release, wait for clean, then full-file CAS-level verify. + vfs.release(fh) + .await + .map_err(|e| format!("[{path}] final release: {e}"))?; + wait_for_clean(&vfs, ino).await; + + let committed = read_full(&vfs, ino, shadow.len() as u64).await; + if committed != shadow { + return Err(format!( + "[{path}] FINAL CAS-LEVEL MISMATCH: committed_len={} shadow_len={} first_diff_at={:?}", + committed.len(), + shadow.len(), + committed.iter().zip(shadow.iter()).position(|(a, b)| a != b), + )); + } + Ok(()) +} + +/// Multi-worker stress test for the sparse-write feature. Each worker churns +/// random ops on its own file; the global FlushManager batches across files +/// so Pass A / Pass B / drift / lock-map paths all get exercised +/// concurrently. The shadow model catches any data divergence — a single +/// mismatch fails the test with the offending offset. +/// +/// Default: 12 files × 200 ops, ~2-4 s wall time. Override with +/// `STRESS_FILES` / `STRESS_OPS_PER_FILE` env vars for longer runs. +#[test] +fn sparse_writes_concurrent_stress() { + let n_files: usize = std::env::var("STRESS_FILES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(STRESS_FILES); + let ops_per_file: usize = std::env::var("STRESS_OPS_PER_FILE") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(STRESS_OPS_PER_FILE); + let seed: u64 = std::env::var("STRESS_SEED") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + }); + + let hub = MockHub::new(); + let xet = MockXet::new(); + let mut bases: Vec<(String, Vec)> = Vec::with_capacity(n_files); + for i in 0..n_files { + let path = format!("stress_{:03}.bin", i); + let base_size = STRESS_BASE_SIZE + (i * 32); + let mut base = vec![0u8; base_size]; + for (j, b) in base.iter_mut().enumerate() { + *b = ((i * 31 + j) % 251) as u8; + } + let hash = format!("stress_h_{:03}", i); + hub.add_file(&path, base_size as u64, Some(&hash), None); + xet.add_file(&hash, &base); + bases.push((path, base)); + } + let (rt, vfs) = vfs_sparse(&hub, &xet); + + eprintln!("sparse_writes_concurrent_stress: {n_files} files, seed={seed}"); + + rt.block_on(async { + let mut handles = Vec::with_capacity(n_files); + for (i, (path, base)) in bases.into_iter().enumerate() { + let vfs = vfs.clone(); + let xet = xet.clone(); + let worker_seed = seed.wrapping_add(i as u64).wrapping_mul(0x9E3779B97F4A7C15); + handles.push(tokio::spawn(async move { + run_stress_worker(vfs, xet, path, worker_seed, base, ops_per_file).await + })); + } + let mut errors = Vec::new(); + for h in handles { + match h.await.expect("worker panic") { + Ok(()) => {} + Err(e) => errors.push(e), + } + } + if !errors.is_empty() { + panic!( + "sparse-write stress FAILED ({} workers): seed={}\n{}", + errors.len(), + seed, + errors.join("\n ") + ); + } + }); + + vfs.shutdown(); +} + +// ── Same-inode concurrent stress ────────────────────────────────────── +// +// The cross-file stress above exercises the global FlushManager batching and +// inter-inode locking. This variant pins all workers to a SINGLE inode so the +// per-inode `io_lock`, sparse_write state machine, and dirty_generation guard +// take the brunt of the contention — the paths that are easiest to get wrong +// when multiple FUSE handles touch one file at the same time. +// +// Validation strategy is different from the cross-file stress: multiple +// workers writing to overlapping offsets makes the final byte content +// indeterminate (last-write-wins, but "last" depends on tokio scheduling). +// So this is a FUZZ-style test rather than an equivalence test — the +// pass condition is "no panic, no poison, no surprise EIO, and the inode's +// sparse state remains internally consistent". We verify: +// 1. No worker bubbles up an unexpected error (panics are fatal). +// 2. After quiesce, sparse_write (if Some) satisfies its invariants: +// coverage sorted/non-overlapping, dirty_ranges ⊆ coverage, every +// range within [0, entry.size). +// 3. A full-file read after quiesce returns exactly entry.size bytes +// without erroring. +const SAME_INODE_WORKERS: usize = 8; +const SAME_INODE_OPS_PER_WORKER: usize = 500; +const SAME_INODE_BASE_SIZE: usize = 8192; + +/// Lightweight per-handle op generator: drop RogueRotation (requires global +/// quiesce — incompatible with concurrent siblings on the same inode) and +/// add a heavy Read mix so the pread + sparse-coverage snapshot path gets +/// real contention. +#[derive(Debug, Clone, Copy)] +enum SameInodeOp { + Write { offset: usize, len: usize }, + Read { offset: usize, len: usize }, + SetattrGrow { delta: usize }, + SetattrShrink { delta: usize }, + Fsync, + Reopen, +} + +fn gen_same_inode_op(rng: &mut StressRng, observed_size: usize) -> SameInodeOp { + let pick = rng.next() % 100; + match pick { + // 40% writes, sampled across overwrite / past-EOF / random + 0..=39 => { + let kind = rng.next() % 4; + let (offset, len) = match kind { + 0 if observed_size >= 2 => { + let len = 1 + rng.rand_range(observed_size.min(1024)); + let offset = rng.rand_range(observed_size.saturating_sub(len).max(1)); + (offset, len) + } + 1 => (observed_size, 1 + rng.rand_range(1024)), + 2 => { + let gap = 1 + rng.rand_range(2048); + (observed_size + gap, 1 + rng.rand_range(1024)) + } + _ => { + let offset = rng.rand_range(STRESS_MAX_SIZE.saturating_sub(1)); + (offset, 1 + rng.rand_range(2048)) + } + }; + let len = len.min(STRESS_MAX_SIZE.saturating_sub(offset)); + if len == 0 || offset >= STRESS_MAX_SIZE { + SameInodeOp::Read { offset: 0, len: 1 } + } else { + SameInodeOp::Write { offset, len } + } + } + // 35% reads — many small ones to thrash the io_lock + sparse-fill + 40..=74 => { + if observed_size == 0 { + return SameInodeOp::Write { + offset: 0, + len: 1 + rng.rand_range(1024), + }; + } + let len = 1 + rng.rand_range(observed_size.min(2048)); + let offset = rng.rand_range(observed_size.saturating_sub(1).max(1)); + SameInodeOp::Read { offset, len } + } + 75..=84 => SameInodeOp::SetattrGrow { + delta: 1 + rng.rand_range(4096), + }, + 85..=92 if observed_size > 1 => SameInodeOp::SetattrShrink { + delta: 1 + rng.rand_range(observed_size - 1), + }, + 93..=96 => SameInodeOp::Fsync, + _ => SameInodeOp::Reopen, + } +} + +async fn run_same_inode_worker( + vfs: std::sync::Arc, + ino: u64, + worker_id: usize, + seed: u64, + n_ops: usize, +) -> Result<(), String> { + let mut rng = StressRng::new(seed); + + let mut fh = vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[w{worker_id}] initial open: {e}"))?; + + for op_idx in 0..n_ops { + // Re-read entry.size each iteration: another worker may have just + // grown/shrunk it and the op generator needs a recent view. + let observed_size = { + let inodes = vfs.inode_table.read().expect("inodes poisoned"); + inodes.get(ino).map(|e| e.size as usize).unwrap_or(0) + }; + let op = gen_same_inode_op(&mut rng, observed_size); + match op { + SameInodeOp::Write { offset, len } => { + let buf: Vec = (0..len).map(|_| rng.rand_byte()).collect(); + // ANY ok or controlled errno is accepted — the point is that + // the call doesn't panic / poison. Other workers may have + // raced our preconditions. + if let Err(e) = write_blocking(&vfs, ino, fh, offset as u64, &buf).await + && e != libc::EAGAIN + && e != libc::EBADF + { + return Err(format!( + "[w{worker_id}] op {op_idx} write({offset},{len}): unexpected errno {e}" + )); + } + } + SameInodeOp::Read { offset, len } => { + let want = len.min(STRESS_MAX_SIZE) as u32; + match vfs.read(fh, offset as u64, want).await { + Ok(_) => {} + Err(libc::EAGAIN) | Err(libc::EBADF) => {} + Err(e) => { + return Err(format!( + "[w{worker_id}] op {op_idx} read({offset},{len}): unexpected errno {e}" + )); + } + } + } + SameInodeOp::SetattrGrow { delta } => { + let new_size = (observed_size + delta).min(STRESS_MAX_SIZE); + if new_size == observed_size { + continue; + } + if let Err(e) = vfs + .setattr(ino, Some(new_size as u64), None, None, None, None, None) + .await + && e != libc::EAGAIN + { + return Err(format!( + "[w{worker_id}] op {op_idx} setattr_grow({new_size}): unexpected errno {e}" + )); + } + } + SameInodeOp::SetattrShrink { delta } => { + if observed_size <= 1 { + continue; + } + let new_size = observed_size.saturating_sub(delta).max(1); + if let Err(e) = vfs + .setattr(ino, Some(new_size as u64), None, None, None, None, None) + .await + && e != libc::EAGAIN + { + return Err(format!( + "[w{worker_id}] op {op_idx} setattr_shrink({new_size}): unexpected errno {e}" + )); + } + } + SameInodeOp::Fsync => { + if let Err(e) = vfs.fsync(ino, fh, None).await + && e != libc::EAGAIN + { + return Err(format!("[w{worker_id}] op {op_idx} fsync: unexpected errno {e}")); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + SameInodeOp::Reopen => { + if let Err(e) = vfs.release(fh).await { + return Err(format!("[w{worker_id}] op {op_idx} release: {e}")); + } + // EAGAIN is the drift sentinel; retry once internally so the + // worker doesn't bail on a benign race. + fh = match vfs.open(ino, true, false, None).await { + Ok(fh) => fh, + Err(libc::EAGAIN) => vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[w{worker_id}] op {op_idx} reopen retry: {e}"))?, + Err(e) => return Err(format!("[w{worker_id}] op {op_idx} reopen: {e}")), + }; + } + } + } + + vfs.release(fh) + .await + .map_err(|e| format!("[w{worker_id}] final release: {e}")) +} + +/// Multi-worker stress targeting ONE inode. Verifies that the per-inode +/// io_lock, sparse_write state, and dirty_generation guard survive +/// realistic concurrent FUSE handle traffic without panic, poison, or +/// invariant violation. +#[test] +fn sparse_writes_same_inode_concurrent_stress() { + let n_workers: usize = std::env::var("SAME_INODE_WORKERS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(SAME_INODE_WORKERS); + let ops_per_worker: usize = std::env::var("SAME_INODE_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(SAME_INODE_OPS_PER_WORKER); + let seed: u64 = std::env::var("SAME_INODE_SEED") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + }); + + let hub = MockHub::new(); + let xet = MockXet::new(); + let path = "shared.bin"; + let base: Vec = (0..SAME_INODE_BASE_SIZE).map(|j| (j % 251) as u8).collect(); + hub.add_file(path, base.len() as u64, Some("shared_h"), None); + xet.add_file("shared_h", &base); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + eprintln!("sparse_writes_same_inode_concurrent_stress: {n_workers} workers × {ops_per_worker} ops, seed={seed}"); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, path).await.unwrap().ino; + let mut handles = Vec::with_capacity(n_workers); + for w in 0..n_workers { + let vfs = vfs.clone(); + let worker_seed = seed.wrapping_add(w as u64).wrapping_mul(0x9E3779B97F4A7C15); + handles.push(tokio::spawn(async move { + run_same_inode_worker(vfs, ino, w, worker_seed, ops_per_worker).await + })); + } + let mut errors = Vec::new(); + for h in handles { + match h.await.expect("worker panic") { + Ok(()) => {} + Err(e) => errors.push(e), + } + } + if !errors.is_empty() { + panic!( + "same-inode stress FAILED ({} workers): seed={}\n{}", + errors.len(), + seed, + errors.join("\n ") + ); + } + + // Quiesce: wait for any in-flight flushes to land. + wait_for_clean(&vfs, ino).await; + + // Invariant checks on the final sparse_write state. + let (entry_size, sparse_snapshot) = { + let inodes = vfs.inode_table.read().expect("inodes poisoned"); + let entry = inodes.get(ino).expect("inode must exist"); + (entry.size, entry.sparse_write.clone()) + }; + if let Some(sw) = sparse_snapshot { + for window in sw.coverage.windows(2) { + let (a_s, a_e) = window[0]; + let (b_s, b_e) = window[1]; + assert!(a_s < a_e, "coverage entry not start d_e { + break; + } + } + assert!( + covered, + "dirty_range ({d_s},{d_e}) not covered by any coverage entry: {:?}", + sw.coverage + ); + } + for &(s, e) in &sw.coverage { + assert!( + e <= entry_size, + "coverage ({s},{e}) extends past entry.size {entry_size}" + ); + } + for &(s, e) in &sw.dirty_ranges { + assert!( + e <= entry_size, + "dirty_range ({s},{e}) extends past entry.size {entry_size}" + ); + } + } + + // Final read-back must not error and must return entry.size bytes. + let final_bytes = read_full(&vfs, ino, entry_size).await; + assert_eq!( + final_bytes.len() as u64, + entry_size, + "final read returned wrong length: got {} expected {}", + final_bytes.len(), + entry_size + ); + }); + + vfs.shutdown(); +} + +// ── Same-inode verified-integrity stress ───────────────────────────── +// +// Companion to `sparse_writes_same_inode_concurrent_stress`. The chaos +// variant maximises race coverage by letting workers write to overlapping +// offsets, at the cost of being unable to assert exact byte content (last- +// write-wins, "last" is scheduler-dependent). This variant trades some +// race surface for a STRONG oracle: +// +// - The file is split into N disjoint per-worker slots. +// - Each worker writes ONLY within its own slot, so its private shadow +// is the ground truth for that range. +// - Reads can target any slot — exercises concurrent pread vs. another +// worker's in-flight pwrite + sparse-coverage snapshot pairing on the +// io_lock — but their content isn't verified mid-stream (the target +// slot may be racing under another worker's writes). +// - After quiesce, the file's content MUST equal the concatenation of +// every worker's final shadow, byte-for-byte. A silent corruption +// (torn read, lost write, fill_sparse_holes clobbering user bytes, +// dirty_generation guard missing a write) fails this assertion. +// +// setattr is dropped: it would change the file size and complicate the +// per-slot ownership invariant. The chaos variant + the cross-file +// stress already exercise setattr extensively. +const VERIFIED_WORKERS: usize = 8; +const VERIFIED_OPS_PER_WORKER: usize = 1000; +const VERIFIED_SLOT_SIZE: usize = 4096; + +#[derive(Debug, Clone, Copy)] +enum VerifiedOp { + /// Write at random offset within own slot. + Write { + offset: usize, + len: usize, + }, + /// Read at random offset anywhere in the file (cross-slot reads exercise + /// reader/writer concurrency on a foreign slot's io_lock). + Read { + offset: usize, + len: usize, + }, + Fsync, + Reopen, +} + +fn gen_verified_op(rng: &mut StressRng, file_size: usize, slot_start: usize, slot_end: usize) -> VerifiedOp { + let pick = rng.next() % 100; + match pick { + // 55% writes within own slot + 0..=54 => { + let slot_len = slot_end - slot_start; + let len = 1 + rng.rand_range(slot_len.min(512)); + let offset = slot_start + rng.rand_range(slot_len - len + 1); + VerifiedOp::Write { offset, len } + } + // 35% reads (anywhere) + 55..=89 => { + let len = 1 + rng.rand_range(file_size.min(2048)); + let offset = rng.rand_range(file_size - len + 1); + VerifiedOp::Read { offset, len } + } + 90..=94 => VerifiedOp::Fsync, + _ => VerifiedOp::Reopen, + } +} + +/// Per-op-type counters, shared across all workers. Used to prove that ops +/// actually fire (and at what proportion) — without this it's easy for a +/// fast-running stress test to silently devolve into "the rng path never +/// picked write" or similar workload collapse. +#[derive(Default)] +struct VerifiedStats { + writes: std::sync::atomic::AtomicU64, + reads: std::sync::atomic::AtomicU64, + fsyncs: std::sync::atomic::AtomicU64, + reopens: std::sync::atomic::AtomicU64, +} + +/// All the inputs `run_verified_worker` needs in a single struct so the +/// clippy::too_many_arguments lint stays happy without adding allow attrs. +struct VerifiedWorker { + vfs: std::sync::Arc, + ino: u64, + worker_id: usize, + seed: u64, + n_ops: usize, + file_size: usize, + slot_start: usize, + slot_end: usize, + shadow: Vec, + stats: std::sync::Arc, +} + +/// Runs one worker; returns its final shadow for its slot. Errors propagate +/// up as Result::Err so the test fails with the offending worker's context. +async fn run_verified_worker(w: VerifiedWorker) -> Result, String> { + use std::sync::atomic::Ordering; + let VerifiedWorker { + vfs, + ino, + worker_id, + seed, + n_ops, + file_size, + slot_start, + slot_end, + mut shadow, + stats, + } = w; + let mut rng = StressRng::new(seed); + let mut fh = vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[w{worker_id}] initial open: {e}"))?; + + for op_idx in 0..n_ops { + let op = gen_verified_op(&mut rng, file_size, slot_start, slot_end); + match op { + VerifiedOp::Write { offset, len } => { + debug_assert!( + offset >= slot_start && offset + len <= slot_end, + "[w{worker_id}] write outside slot: [{offset},{}) not in [{slot_start},{slot_end})", + offset + len + ); + let buf: Vec = (0..len).map(|_| rng.rand_byte()).collect(); + let written = write_blocking(&vfs, ino, fh, offset as u64, &buf) + .await + .map_err(|e| format!("[w{worker_id}] op {op_idx} write({offset},{len}): {e}"))?; + let actual = written as usize; + let shadow_off = offset - slot_start; + shadow[shadow_off..shadow_off + actual].copy_from_slice(&buf[..actual]); + stats.writes.fetch_add(1, Ordering::Relaxed); + } + VerifiedOp::Read { offset, len } => { + let want = (len as u32).min((file_size - offset) as u32); + match vfs.read(fh, offset as u64, want).await { + Ok(_) => {} + Err(libc::EAGAIN) | Err(libc::EBADF) => {} + Err(e) => { + return Err(format!( + "[w{worker_id}] op {op_idx} read({offset},{len}): unexpected errno {e}" + )); + } + } + stats.reads.fetch_add(1, Ordering::Relaxed); + } + VerifiedOp::Fsync => { + if let Err(e) = vfs.fsync(ino, fh, None).await + && e != libc::EAGAIN + { + return Err(format!("[w{worker_id}] op {op_idx} fsync: unexpected errno {e}")); + } + tokio::time::sleep(Duration::from_millis(5)).await; + stats.fsyncs.fetch_add(1, Ordering::Relaxed); + } + VerifiedOp::Reopen => { + if let Err(e) = vfs.release(fh).await { + return Err(format!("[w{worker_id}] op {op_idx} release: {e}")); + } + fh = match vfs.open(ino, true, false, None).await { + Ok(fh) => fh, + Err(libc::EAGAIN) => vfs + .open(ino, true, false, None) + .await + .map_err(|e| format!("[w{worker_id}] op {op_idx} reopen retry: {e}"))?, + Err(e) => return Err(format!("[w{worker_id}] op {op_idx} reopen: {e}")), + }; + stats.reopens.fetch_add(1, Ordering::Relaxed); + } + } + } + + vfs.release(fh) + .await + .map_err(|e| format!("[w{worker_id}] final release: {e}"))?; + Ok(shadow) +} + +/// Strong-oracle multi-worker stress on a single inode. Disjoint per-worker +/// slots make the final byte content deterministic so we can assert exact +/// equality between the VFS-committed file and the concatenated per-worker +/// shadows — catching corruption that the chaos variant cannot. +#[test] +fn sparse_writes_same_inode_verified_integrity() { + let n_workers: usize = std::env::var("VERIFIED_WORKERS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(VERIFIED_WORKERS); + let ops_per_worker: usize = std::env::var("VERIFIED_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(VERIFIED_OPS_PER_WORKER); + let slot_size: usize = std::env::var("VERIFIED_SLOT_SIZE") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(VERIFIED_SLOT_SIZE); + let seed: u64 = std::env::var("VERIFIED_SEED") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + }); + + let file_size = n_workers * slot_size; + let hub = MockHub::new(); + let xet = MockXet::new(); + let path = "verified_shared.bin"; + let base: Vec = (0..file_size).map(|j| (j % 251) as u8).collect(); + hub.add_file(path, file_size as u64, Some("verified_h"), None); + xet.add_file("verified_h", &base); + let (rt, vfs) = vfs_sparse(&hub, &xet); + + eprintln!( + "sparse_writes_same_inode_verified_integrity: {n_workers} workers × {ops_per_worker} ops, slot={slot_size}B, file={file_size}B, seed={seed}" + ); + + let stats = std::sync::Arc::new(VerifiedStats::default()); + let wall_start = std::time::Instant::now(); + + rt.block_on(async { + let ino = vfs.lookup(ROOT_INODE, path).await.unwrap().ino; + let mut handles = Vec::with_capacity(n_workers); + for w in 0..n_workers { + let slot_start = w * slot_size; + let slot_end = slot_start + slot_size; + let shadow_slot = base[slot_start..slot_end].to_vec(); + let vfs = vfs.clone(); + let stats = stats.clone(); + let worker_seed = seed.wrapping_add(w as u64).wrapping_mul(0x9E3779B97F4A7C15); + handles.push(tokio::spawn(async move { + run_verified_worker(VerifiedWorker { + vfs, + ino, + worker_id: w, + seed: worker_seed, + n_ops: ops_per_worker, + file_size, + slot_start, + slot_end, + shadow: shadow_slot, + stats, + }) + .await + })); + } + + let mut errors = Vec::new(); + let mut expected = Vec::with_capacity(file_size); + for h in handles { + match h.await.expect("worker panic") { + Ok(slot_shadow) => expected.extend_from_slice(&slot_shadow), + Err(e) => errors.push(e), + } + } + if !errors.is_empty() { + panic!( + "verified-integrity stress FAILED ({} workers): seed={}\n{}", + errors.len(), + seed, + errors.join("\n ") + ); + } + assert_eq!(expected.len(), file_size, "shadow concatenation must equal file_size"); + + // Quiesce: wait for any in-flight flushes to land before reading. + wait_for_clean(&vfs, ino).await; + + // Workload-shape evidence: print per-op-type counts + MockXet + // activity. Proves the rng actually hit every op branch (otherwise + // a stress test can quietly degenerate into "only reads" or similar) + // and that the sparse code path was exercised (CAS streams > 0 + // proves fill_sparse_holes ran; range_upload-produced hashes prove + // sparse-aware commits ran). + use std::sync::atomic::Ordering; + let elapsed = wall_start.elapsed(); + let writes = stats.writes.load(Ordering::Relaxed); + let reads = stats.reads.load(Ordering::Relaxed); + let fsyncs = stats.fsyncs.load(Ordering::Relaxed); + let reopens = stats.reopens.load(Ordering::Relaxed); + let total_ops = writes + reads + fsyncs + reopens; + let cas_streams = xet.stream_calls.lock().unwrap().len(); + let cas_objects = xet.files.lock().unwrap().len(); + eprintln!( + " ops: writes={} reads={} fsyncs={} reopens={} (total={}, {:.0} ops/s)", + writes, + reads, + fsyncs, + reopens, + total_ops, + total_ops as f64 / elapsed.as_secs_f64(), + ); + eprintln!( + " MockXet: CAS streams={} files={} (1 seed + N commit hashes)", + cas_streams, cas_objects + ); + assert!(writes > 0, "rng never picked a Write op"); + assert!(reads > 0, "rng never picked a Read op"); + assert!(fsyncs > 0, "rng never picked an Fsync op"); + assert!(reopens > 0, "rng never picked a Reopen op"); + assert!(cas_streams > 0, "sparse CAS fill path never fired"); + + // Strong oracle: every byte of the committed file must equal the + // concatenation of each worker's final shadow. A divergence pinpoints + // the offset where corruption happened. + let actual = read_full(&vfs, ino, file_size as u64).await; + if actual != expected { + let first_diff = actual + .iter() + .zip(expected.iter()) + .position(|(a, e)| a != e) + .unwrap_or(actual.len().min(expected.len())); + let owner = first_diff / slot_size; + panic!( + "verified-integrity stress: BYTE-LEVEL MISMATCH at offset {} (owner=w{}), \ + actual=0x{:02x} expected=0x{:02x} (seed={})", + first_diff, + owner, + actual.get(first_diff).copied().unwrap_or(0), + expected.get(first_diff).copied().unwrap_or(0), + seed, + ); + } + }); + + vfs.shutdown(); +} diff --git a/src/xet.rs b/src/xet.rs index a3019f8f..594aba66 100644 --- a/src/xet.rs +++ b/src/xet.rs @@ -1,19 +1,42 @@ use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use bytes::Bytes; +use tokio::io::AsyncRead; +use tracing::info; use xet_client::cas_client::Client; use xet_client::cas_types::FileRange; use xet_client::chunk_cache::ChunkCache; use xet_core_structures::merklehash::MerkleHash; use xet_data::file_reconstruction::{DownloadStream, FileReconstructor}; use xet_data::processing::configurations::TranslatorConfig; -use xet_data::processing::{FileDownloadSession, FileUploadSession, Sha256Policy, SingleFileCleaner, XetFileInfo}; +use xet_data::processing::{ + DirtyInput, FileDownloadSession, FileUploadSession, Sha256Policy, SingleFileCleaner, XetFileInfo, +}; use xet_runtime::core::XetContext; use crate::error::{Error, Result}; +// ── RangeSnapshot ──────────────────────────────────────────────────── + +/// A self-contained snapshot of one dirty range, ready to feed `range_upload`. +/// +/// The caller (typically `flush_batch`) reads the bytes from the staging file +/// while holding the per-inode I/O lock, then releases the lock and hands the +/// snapshot to `range_upload`. `range_upload` itself never touches disk and +/// never holds the I/O lock, so concurrent writes to staging can proceed +/// during the upload without producing torn reads (codex review finding: the +/// previous per-chunk lock acquisition could let a concurrent pwrite interleave +/// between chunks, hashing chimeric content). +pub struct RangeSnapshot { + /// Offset in the new file where these bytes live. + pub offset: u64, + /// Pre-read bytes from staging at this offset. + pub data: Bytes, +} + // ── Traits ─────────────────────────────────────────────────────────── /// Trait abstracting CAS operations used by VirtualFs and FlushManager. @@ -31,6 +54,21 @@ pub trait XetOps: Send + Sync { /// Pre-warm the reconstruction cache for a file by fetching its full plan. /// Errors are silently ignored — this is best-effort. async fn warm_reconstruction_cache(&self, xet_hash: &str); + + /// Compose a new CAS file from an existing reconstruction (`original_hash`) + /// plus a set of pre-snapshotted dirty ranges. Used for sparse-write flushes + /// so unmodified bytes never round-trip the network. + /// + /// `new_file_size` is the size of the resulting file. When it is less than + /// `original_size`, a synthetic delete is appended past the last dirty range + /// so the original tail is dropped. + async fn range_upload( + &self, + original_hash: &str, + original_size: u64, + new_file_size: u64, + dirty_snapshots: Vec, + ) -> Result; } /// Append-only streaming writer trait (abstracts StreamingWriter for testing). @@ -155,6 +193,203 @@ impl XetOps for XetSessions { let _ = self.cas_client.get_reconstruction(&hash, None).await; } } + + async fn range_upload( + &self, + original_hash: &str, + original_size: u64, + new_file_size: u64, + dirty_snapshots: Vec, + ) -> Result { + let config = self + .upload_config + .as_ref() + .ok_or_else(|| Error::hub("no upload config (read-only mode)"))?; + + let original_merkle = + MerkleHash::from_hex(original_hash).map_err(|e| Error::Xet(format!("invalid original hash: {e}")))?; + + // No-op: no dirty bytes and file size unchanged means the new content + // is bit-for-bit the original. Skip the upload round-trip. + if dirty_snapshots.is_empty() && new_file_size == original_size { + return Ok(XetFileInfo::new(original_hash.to_string(), original_size)); + } + + // Plan the `original_range` each dirty input REPLACES (empty for inserts + // past EOF, partial for writes spanning EOF, full for overwrites within + // the original), plus any synthetic truncate-tail delete. Pure and + // unit-tested in `plan_original_ranges` so the composition mapping has + // coverage independent of a live CAS (the mock XetOps bypasses it). + let dirty_count = dirty_snapshots.len(); + let meta: Vec<(u64, u64)> = dirty_snapshots + .iter() + .map(|s| (s.offset, s.data.len() as u64)) + .collect(); + let plan = plan_original_ranges(&meta, original_size, new_file_size); + + // Pair plan entries with their readers by index: the first `meta.len()` + // entries are data-bearing (one per snapshot, same order); a trailing + // entry (if any) is the synthetic truncate tail with an empty reader. + let mut snaps = dirty_snapshots.into_iter(); + let mut dirty_inputs: Vec = Vec::with_capacity(plan.len()); + for (idx, (original_range, new_length)) in plan.into_iter().enumerate() { + let reader: Pin> = if idx < meta.len() { + let snap = snaps.next().expect("one snapshot per data plan entry"); + Box::pin(std::io::Cursor::new(snap.data)) + } else { + Box::pin(tokio::io::empty()) + }; + dirty_inputs.push(DirtyInput { + original_range, + reader, + new_length, + }); + } + + let result = xet_data::processing::upload_ranges( + config.clone(), + self.cas_client.clone(), + original_merkle, + original_size, + dirty_inputs, + ) + .await + .map_err(|e| Error::Xet(e.to_string()))?; + + info!( + "range_upload: hash={} size={:?} (original_size={}, {} dirty ranges)", + result.hash(), + result.file_size(), + original_size, + dirty_count + ); + + Ok(result) + } +} + +// ── range_upload composition planning ──────────────────────────────── + +/// For each dirty snapshot `(offset, new_length)`, compute the `original_range` +/// of the ORIGINAL file that the snapshot REPLACES, plus an optional synthetic +/// truncate-tail delete when the new file is shorter than the original. +/// +/// Returns one entry per snapshot (same order) followed by at most one extra +/// `(range, 0)` truncate entry, so callers can pair the first `snapshots.len()` +/// entries with their readers by index. Mapping rules per snapshot spanning +/// `[start, start+new_length)`: +/// - fully within the original (`end <= original_size`): replaces `start..end`; +/// - fully past EOF (`start >= original_size`): inserts (replaces an empty +/// `original_size..original_size`); +/// - spanning EOF: replaces `start..original_size` and extends. +/// +/// Pure function: the composition mapping is unit-tested here, independent of a +/// live CAS (the mock `XetOps::range_upload` bypasses this logic entirely). +fn plan_original_ranges( + snapshots: &[(u64, u64)], + original_size: u64, + new_file_size: u64, +) -> Vec<(std::ops::Range, u64)> { + let mut plan: Vec<(std::ops::Range, u64)> = Vec::with_capacity(snapshots.len() + 1); + for &(start, new_length) in snapshots { + let end = start + new_length; + let original_range = if end <= original_size { + start..end + } else if start >= original_size { + original_size..original_size + } else { + start..original_size + }; + plan.push((original_range, new_length)); + } + + // Truncate-past-end: if the new file is shorter than the original AND the + // truncation point isn't already covered by a dirty input, append a + // synthetic empty input over the cut tail so upload_ranges drops those + // original bytes. + if new_file_size < original_size { + let last_covered = plan.last().map(|(r, _)| r.end).unwrap_or(0); + let truncate_start = new_file_size.max(last_covered); + if truncate_start < original_size { + plan.push((truncate_start..original_size, 0)); + } + } + plan +} + +#[cfg(test)] +mod range_upload_tests { + use super::plan_original_ranges; + + #[test] + fn in_place_overwrite_maps_to_same_range() { + // Overwrite [10,20) of a 100-byte file: replaces exactly [10,20). + assert_eq!(plan_original_ranges(&[(10, 10)], 100, 100), vec![(10..20, 10)]); + } + + #[test] + fn write_past_eof_is_an_insert() { + // Write 5 bytes at offset 30 of a 20-byte file: inserts past EOF. + assert_eq!(plan_original_ranges(&[(30, 5)], 20, 35), vec![(20..20, 5)]); + } + + #[test] + fn write_spanning_eof_replaces_tail_and_extends() { + // Write [15,25) of a 20-byte file: replaces [15,20), extends to 25. + assert_eq!(plan_original_ranges(&[(15, 10)], 20, 25), vec![(15..20, 10)]); + } + + #[test] + fn pure_shrink_appends_truncate_tail() { + // No dirty bytes, shrink 50 -> 20: one synthetic delete of [20,50). + assert_eq!(plan_original_ranges(&[], 50, 20), vec![(20..50, 0)]); + } + + #[test] + fn shrink_with_dirty_prefix_truncates_after_last_dirty() { + // Overwrite [0,10) and shrink 50 -> 20: keep the overwrite, drop [20,50). + assert_eq!(plan_original_ranges(&[(0, 10)], 50, 20), vec![(0..10, 10), (20..50, 0)]); + } + + #[test] + fn shrink_below_a_dirty_range_does_not_double_count_tail() { + // Dirty [0,30) but file shrinks to 25: the dirty input already covers + // past new_file_size, so truncate_start = max(25, 30) = 30 == nothing + // extra to drop beyond the dirty range's original_range end. + assert_eq!(plan_original_ranges(&[(0, 30)], 50, 25), vec![(0..30, 30), (30..50, 0)]); + } + + #[test] + fn no_change_yields_empty_plan() { + assert!(plan_original_ranges(&[], 100, 100).is_empty()); + } + + #[test] + fn multiple_snapshots_preserve_order_one_entry_each() { + // Two disjoint in-place overwrites on a 50-byte file: one plan entry + // per snapshot, in order, no truncate tail. This pins the by-index + // reader-pairing range_upload relies on (data entries before the tail). + assert_eq!( + plan_original_ranges(&[(0, 10), (30, 5)], 50, 50), + vec![(0..10, 10), (30..35, 5)] + ); + } + + #[test] + fn multiple_snapshots_with_past_eof_then_shrink() { + // An in-file overwrite plus a past-EOF insert, on a file that also + // shrinks below the original tail: 2 data entries (order preserved) + + // a synthetic truncate tail after the last dirty original_range. + // original=40, write [0,10) in place, write [50,55) past EOF, new=55. + assert_eq!( + plan_original_ranges(&[(0, 10), (50, 5)], 40, 55), + vec![(0..10, 10), (40..40, 5)] + ); + // Same dirty inputs but the file shrinks to 35 (< original 40): the + // truncate tail drops [last_covered, original) where last_covered is + // the max original_range end across entries. + assert_eq!(plan_original_ranges(&[(0, 10)], 40, 35), vec![(0..10, 10), (35..40, 0)]); + } } // ── DownloadStreamWrapper ───────────────────────────────────────────── diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 89b96804..d21786f0 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -277,6 +277,22 @@ pub async fn upload_file(config: Arc, staged_path: &Path) -> X /// Spawn hf-mount-fuse as a child process, wait until the mountpoint is live. /// `extra_args` are appended to the command (e.g. `&["--read-only"]`). pub fn mount_bucket(bucket_id: &str, mount_point: &str, cache_dir: &str, extra_args: &[&str]) -> Child { + mount_bucket_with_env(bucket_id, mount_point, cache_dir, extra_args, &[]) +} + +/// Same as `mount_bucket` but also forwards `(key, value)` pairs as env vars +/// to the spawned child. Use this instead of `std::env::set_var` from a test: +/// `std::env::set_var` is unsafe and races with concurrent `cargo test` +/// threads that may be reading the env or spawning their own children at +/// the same time; passing through `Command::env` is racy-free because the +/// mutation lives on the child's `Command` only. +pub fn mount_bucket_with_env( + bucket_id: &str, + mount_point: &str, + cache_dir: &str, + extra_args: &[&str], + extra_env: &[(&str, &str)], +) -> Child { let token = std::env::var("HF_TOKEN").unwrap(); let binary = std::env::current_exe() @@ -293,11 +309,15 @@ pub fn mount_bucket(bucket_id: &str, mount_point: &str, cache_dir: &str, extra_a std::fs::create_dir_all(cache_dir).ok(); let ep = endpoint(); - let child = Command::new(binary) - .env( - "RUST_LOG", - std::env::var("RUST_LOG").unwrap_or_else(|_| "hf_mount=warn".to_string()), - ) + let mut cmd = Command::new(binary); + cmd.env( + "RUST_LOG", + std::env::var("RUST_LOG").unwrap_or_else(|_| "hf_mount=warn".to_string()), + ); + for (k, v) in extra_env { + cmd.env(k, v); + } + let child = cmd .args([ "--hf-token", &token, @@ -487,7 +507,7 @@ fn decode_proc_mount_path(path: &str) -> String { String::from_utf8_lossy(&decoded).into_owned() } -fn is_mounted(mount_point: &str) -> bool { +pub fn is_mounted(mount_point: &str) -> bool { #[cfg(target_os = "linux")] { let mount_point = std::fs::canonicalize(mount_point) diff --git a/tests/fsx_paranoid.rs b/tests/fsx_paranoid.rs new file mode 100644 index 00000000..2e1b6601 --- /dev/null +++ b/tests/fsx_paranoid.rs @@ -0,0 +1,242 @@ +//! Paranoid fsx variant: every mutation does a full CAS round-trip. +//! +//! After each write/truncate, the file is closed, we wait for the async flush to +//! commit to CAS, then re-open and read back. This catches composition bugs in +//! `range_upload` that the canonical fsx (in `fsx.rs`) misses since it reads from +//! the local staging file, not from CAS. +//! +//! Slow (~1.5s per op for flush debounce + CAS propagation). Use `FSX_PARANOID_OPS` +//! to control iteration count (default: 100). +//! +//! Requires HF_TOKEN. Run with: +//! cargo test --release --test fsx_paranoid -- --nocapture + +mod common; + +use std::io::{Seek, SeekFrom}; + +const MAX_SIZE: usize = 1 << 20; // 1 MB + +#[tokio::test] +async fn test_fsx_paranoid_cas_roundtrip() { + let guard = match common::setup_bucket("fsx-paranoid").await { + Some(g) => g, + None => return, + }; + let bucket_id = guard.bucket_id.clone(); + + let pid = std::process::id(); + let mount_point = format!("/tmp/hf-fsx-paranoid-{}", pid); + let cache_dir = format!("/tmp/hf-fsx-paranoid-cache-{}", pid); + + // Force sparse engagement for this 1 MiB workload. Without overriding the + // 256 MiB production default, the `--sparse-writes` CLI flag below would + // be a no-op: every file in this test stays well under the threshold and + // would silently fall back to the legacy download-then-upload path, + // defeating the test's stated purpose of catching range_upload + // composition regressions. + // + // The env var is passed through `Command::env` rather than + // `std::env::set_var` so the mutation lives on the child's command + // builder only — `set_var` is `unsafe` and races with sibling tests + // that read the parent's env or spawn their own children concurrently. + let child = common::mount_bucket_with_env( + &bucket_id, + &mount_point, + &cache_dir, + // --sparse-writes is required: the test's stated purpose is to catch + // range_upload composition bugs, and range_upload is only exercised + // on the sparse path. Without this flag the test runs the legacy + // download-then-upload path and never invokes range_upload. + // + // --direct-io is required for the round-trip to actually validate CAS: + // without it the kernel page cache (FOPEN_KEEP_CACHE) can serve the + // read-back from the just-written pages, so a range_upload that + // corrupted the committed CAS object would still match `reference`. + // direct-io forces every read through the FUSE handler, which on the + // now-clean inode reads from the committed CAS revision. + &[ + "--advanced-writes", + "--sparse-writes", + "--direct-io", + "--flush-debounce-ms", + "100", + ], + &[("HF_MOUNT_SPARSE_MIN_BYTES", "0")], + ); + + // mount_bucket_with_env only warns on a failed wait_for_mount and returns + // the Child anyway. Without this check, a mount failure (stale FUSE + // state, broken macFUSE, missing kernel module) lets the test pass + // trivially: every write hits the bare /tmp dir, every read returns the + // same local bytes, and no actual CAS round-trip happens despite the + // test's stated purpose. + if !common::is_mounted(&mount_point) { + common::unmount(&mount_point, child, 5); + panic!("mount not live at {mount_point} — see hf-mount-fuse output above"); + } + + let num_ops = std::env::var("FSX_PARANOID_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(100); + + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + | 1; // ensure non-zero for xorshift + + eprintln!("fsx-paranoid: {} ops, seed={}, mount={}", num_ops, seed, mount_point); + + let test_file = format!("{}/fsx_paranoid_{}", mount_point, pid); + // flush_debounce=100ms + upload + CAS propagation + let flush_wait = std::time::Duration::from_millis(1500); + + let mut reference = vec![0u8; MAX_SIZE]; + let mut file_size: usize = 0; + let mut rng_state = seed; + + let mut xorshift = || -> u64 { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 7; + rng_state ^= rng_state << 17; + rng_state + }; + + for op in 1..=num_ops { + match xorshift() % 3 { + 0 => { + // Write random bytes at random offset + let offset = (xorshift() as usize) % (MAX_SIZE / 4); + let mut len = 1 + (xorshift() as usize) % 4096; + if offset + len > MAX_SIZE { + len = MAX_SIZE - offset; + } + let mut wbuf = vec![0u8; len]; + for byte in &mut wbuf { + *byte = xorshift() as u8; + } + + { + use std::io::Write; + let mut f = if file_size == 0 { + std::fs::File::create(&test_file).expect("create") + } else { + std::fs::OpenOptions::new() + .write(true) + .open(&test_file) + .expect("open for write") + }; + f.seek(SeekFrom::Start(offset as u64)).expect("seek"); + f.write_all(&wbuf).expect("write"); + } + reference[offset..offset + len].copy_from_slice(&wbuf); + if offset + len > file_size { + file_size = offset + len; + } + eprintln!(" op {}/{}: write {} bytes at offset {}", op, num_ops, len, offset); + } + 1 => { + if file_size < 100 { + continue; + } + let new_size = (xorshift() as usize) % file_size; + { + let f = std::fs::OpenOptions::new() + .write(true) + .open(&test_file) + .expect("open for truncate"); + f.set_len(new_size as u64).expect("truncate"); + } + for byte in &mut reference[new_size..file_size] { + *byte = 0; + } + file_size = new_size; + eprintln!(" op {}/{}: truncate to {}", op, num_ops, new_size); + } + 2 => { + let new_size = file_size + 1 + (xorshift() as usize) % 2048; + let new_size = new_size.min(MAX_SIZE); + if new_size <= file_size { + continue; + } + { + let f = if file_size == 0 { + std::fs::File::create(&test_file).expect("create") + } else { + std::fs::OpenOptions::new() + .write(true) + .open(&test_file) + .expect("open for grow") + }; + f.set_len(new_size as u64).expect("grow"); + } + file_size = new_size; + eprintln!(" op {}/{}: grow to {}", op, num_ops, new_size); + } + _ => unreachable!(), + } + + if file_size == 0 { + continue; + } + + // Wait for async flush to commit to CAS, with retry. A flush can fail + // transiently if the previous mutation's upload is still in-flight when + // we modify the staging file (early EOF). The generation counter keeps + // the file dirty and the next flush retries. + let mut verified = false; + for attempt in 0..3 { + std::thread::sleep(flush_wait); + match std::fs::read(&test_file) { + Ok(content) if content.len() == file_size && content == reference[..file_size] => { + verified = true; + break; + } + Ok(content) if attempt < 2 => { + eprintln!( + " op {}/{}: verify attempt {} failed (size {}/{}), retrying...", + op, + num_ops, + attempt + 1, + content.len(), + file_size + ); + } + Ok(content) => { + if content.len() != file_size { + panic!( + "op {}: size mismatch after 3 CAS attempts: got {}, expected {}", + op, + content.len(), + file_size + ); + } + for i in 0..file_size { + if content[i] != reference[i] { + panic!( + "op {}: CAS MISMATCH at byte {}: got 0x{:02x} expected 0x{:02x} (file_size={})", + op, i, content[i], reference[i], file_size + ); + } + } + } + Err(e) if attempt < 2 => { + eprintln!(" op {}/{}: read failed ({}), retrying...", op, num_ops, e); + } + Err(e) => panic!("op {}: read failed after 3 attempts: {}", op, e), + } + } + assert!(verified, "op {}: CAS verify failed after 3 attempts", op); + eprintln!(" op {}/{}: CAS verify OK (size={})", op, num_ops, file_size); + } + + eprintln!("fsx-paranoid: PASSED {} ops (final size={})", num_ops, file_size); + std::fs::remove_file(&test_file).ok(); + + common::unmount(&mount_point, child, 10); + drop(guard); + std::fs::remove_dir_all(&mount_point).ok(); + std::fs::remove_dir_all(&cache_dir).ok(); +} diff --git a/tests/sparse_concurrent_real.rs b/tests/sparse_concurrent_real.rs new file mode 100644 index 00000000..5a344af0 --- /dev/null +++ b/tests/sparse_concurrent_real.rs @@ -0,0 +1,343 @@ +//! Multi-worker concurrent sparse-write stress against real CAS. +//! +//! Companion to `fsx_paranoid` (single-threaded random ops) and the mock-based +//! verified-integrity stress (no network). This one combines both axes: +//! +//! - N tokio workers writing concurrently on a SINGLE inode +//! - Real FUSE kernel path + real hf-mount-fuse async handler + real CAS +//! upload / `range_upload` composition / Hub commit +//! - Disjoint per-worker slots so the final byte content is deterministic +//! and we can byte-compare against the concatenation of per-worker shadows +//! +//! After workers finish: unmount and REMOUNT with a fresh cache so the +//! read-back forces a CAS-side load. Anything corrupted in the upload path +//! (lost write, torn range_upload composition, dirty_generation guard +//! missing a write) surfaces as a byte-level mismatch with offset + owner. +//! +//! Slow (real network): ~30 s default at 6 workers × 12 ops × 1 MiB. Bump +//! `CONCURRENT_REAL_WORKERS` / `CONCURRENT_REAL_OPS_PER_WORKER` for longer +//! runs. Requires `HF_TOKEN`. Run with: +//! cargo test --release --test sparse_concurrent_real -- --nocapture + +mod common; + +use std::os::unix::fs::FileExt; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +const DEFAULT_WORKERS: usize = 6; +const DEFAULT_OPS_PER_WORKER: usize = 12; +/// 1 MiB per slot default — large enough that the sparse path's per-window +/// upload (range_upload) has measurable wire savings vs. the full re-upload +/// baseline, while keeping the test under ~30 s. Override via +/// `CONCURRENT_REAL_SLOT_KB` to push toward GiB-scale files. +const DEFAULT_SLOT_KB: usize = 1024; +/// Default: no intermediate fsync (one flush at end of run). Set +/// `CONCURRENT_REAL_FSYNC_EVERY_N` to force a `sync_all` every N ops +/// per worker, simulating periodic checkpoint saves in a real training +/// workload and exercising multiple apply_commit_sparse cycles within a +/// single test run. +const DEFAULT_FSYNC_EVERY_N: usize = 0; + +struct Rng(u64); +impl Rng { + fn new(seed: u64) -> Self { + Self(seed | 1) + } + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn rand_byte(&mut self) -> u8 { + self.next() as u8 + } + fn rand_range(&mut self, hi: usize) -> usize { + if hi == 0 { 0 } else { (self.next() as usize) % hi } + } +} + +struct Worker { + path: String, + worker_id: usize, + seed: u64, + n_ops: usize, + slot_start: usize, + slot_end: usize, + shadow: Vec, + writes_counter: Arc, + fsyncs_counter: Arc, + fsync_every_n: usize, +} + +/// One worker: opens the shared file at `path` for write, does `n_ops` random +/// writes within its slot, and returns the final shadow for its slot. +/// +/// All file I/O is synchronous (std::fs + FileExt::write_at) — wrapped in +/// spawn_blocking from the caller so the tokio executor isn't starved. +fn run_worker(w: Worker) -> Result, String> { + let Worker { + path, + worker_id, + seed, + n_ops, + slot_start, + slot_end, + mut shadow, + writes_counter, + fsyncs_counter, + fsync_every_n, + } = w; + let mut rng = Rng::new(seed); + let slot_len = slot_end - slot_start; + + let file = std::fs::OpenOptions::new() + .write(true) + .open(&path) + .map_err(|e| format!("[w{worker_id}] open: {e}"))?; + + for op_idx in 0..n_ops { + let len = 1024 + rng.rand_range(64 * 1024 - 1024); + let len = len.min(slot_len); + let off_in_slot = rng.rand_range(slot_len - len + 1); + let file_offset = slot_start + off_in_slot; + + let buf: Vec = (0..len).map(|_| rng.rand_byte()).collect(); + file.write_at(&buf, file_offset as u64) + .map_err(|e| format!("[w{worker_id}] op {op_idx} write_at({file_offset},{len}): {e}"))?; + shadow[off_in_slot..off_in_slot + len].copy_from_slice(&buf); + writes_counter.fetch_add(1, Ordering::Relaxed); + + // Periodic fsync — when enabled, simulates the training-loop + // pattern where the app saves a checkpoint every N steps. Each + // sync_all triggers a flush + apply_commit_sparse cycle, so the + // test exercises the per-cycle sparse paths multiple times + // rather than just the final flush. Concurrent fsyncs from N + // workers are batched by the FlushManager's debounce window. + if fsync_every_n > 0 && (op_idx + 1) % fsync_every_n == 0 { + file.sync_all() + .map_err(|e| format!("[w{worker_id}] op {op_idx} mid-run sync_all: {e}"))?; + fsyncs_counter.fetch_add(1, Ordering::Relaxed); + } + } + + file.sync_all() + .map_err(|e| format!("[w{worker_id}] final sync_all: {e}"))?; + fsyncs_counter.fetch_add(1, Ordering::Relaxed); + Ok(shadow) +} + +fn read_full(path: &Path) -> std::io::Result> { + std::fs::read(path) +} + +#[tokio::test] +async fn test_sparse_concurrent_real_cas() { + let n_workers: usize = std::env::var("CONCURRENT_REAL_WORKERS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_WORKERS); + let ops_per_worker: usize = std::env::var("CONCURRENT_REAL_OPS_PER_WORKER") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_OPS_PER_WORKER); + let slot_kb: usize = std::env::var("CONCURRENT_REAL_SLOT_KB") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_SLOT_KB); + let slot_size: usize = slot_kb * 1024; + let fsync_every_n: usize = std::env::var("CONCURRENT_REAL_FSYNC_EVERY_N") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_FSYNC_EVERY_N); + + let guard = match common::setup_bucket("sparse-concurrent").await { + Some(g) => g, + None => { + eprintln!("skipping: HF_TOKEN not set"); + return; + } + }; + let bucket_id = guard.bucket_id.clone(); + + let pid = std::process::id(); + let mount_point = format!("/tmp/hf-sparse-concurrent-{}", pid); + let cache_dir_a = format!("/tmp/hf-sparse-concurrent-cache-a-{}", pid); + let cache_dir_b = format!("/tmp/hf-sparse-concurrent-cache-b-{}", pid); + + let file_size = n_workers * slot_size; + let test_file = format!("{}/shared.bin", mount_point); + + // Sparse engagement at any file size via HF_MOUNT_SPARSE_MIN_BYTES=0 on + // the child. direct-io forces every read through the FUSE handler so the + // read-back after remount can't be served by the kernel page cache. + let mount_args = &[ + "--advanced-writes", + "--sparse-writes", + "--direct-io", + "--flush-debounce-ms", + "100", + ]; + let env = &[("HF_MOUNT_SPARSE_MIN_BYTES", "0")]; + + // ── Mount #1: seed + concurrent workers + final flush ── + let child = common::mount_bucket_with_env(&bucket_id, &mount_point, &cache_dir_a, mount_args, env); + + // mount_bucket_with_env only warns on a failed wait_for_mount and returns + // the Child regardless — without this check, a mount failure (e.g. stale + // FUSE state on the host) would let the test pass trivially: std::fs::write + // would land on the bare /tmp dir, std::fs::read would read it back from + // the same /tmp dir, and the byte-compare would match because no remote + // upload ever happened. Fail loud instead. + if !common::is_mounted(&mount_point) { + common::unmount(&mount_point, child, 5); + panic!("mount #1 not live at {mount_point} — see hf-mount-fuse output above"); + } + + // Seed: write a known-content file. Each slot starts with a per-worker + // signature so a regression that swaps slot contents is immediately + // visible at byte-compare time. The shadow each worker carries into + // run_worker is initialised to this same signature so the byte-compare + // accounts for un-touched bytes inside the slot. + let mut seed_buf = vec![0u8; file_size]; + for w in 0..n_workers { + let start = w * slot_size; + let end = start + slot_size; + for (i, b) in seed_buf[start..end].iter_mut().enumerate() { + *b = ((w * 17 + i) % 251) as u8; + } + } + std::fs::write(&test_file, &seed_buf).expect("seed write"); + // Wait for the seed flush to settle so the workers start from a clean + // committed base. flush_debounce_ms=100 + upload + Hub commit ≈ 1.5 s. + std::thread::sleep(Duration::from_millis(2000)); + + eprintln!( + "sparse-concurrent-real: {n_workers} workers × {ops_per_worker} ops/worker, slot={} KiB, file={} MiB, fsync_every_n={}", + slot_size / 1024, + file_size / (1024 * 1024), + fsync_every_n, + ); + + let writes_counter = Arc::new(AtomicU64::new(0)); + let fsyncs_counter = Arc::new(AtomicU64::new(0)); + let wall_start = Instant::now(); + + let seed_base = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + | 1; + + let mut join_handles = Vec::with_capacity(n_workers); + for w in 0..n_workers { + let path = test_file.clone(); + let slot_start = w * slot_size; + let slot_end = slot_start + slot_size; + let shadow_slot = seed_buf[slot_start..slot_end].to_vec(); + let worker_seed = seed_base.wrapping_add(w as u64).wrapping_mul(0x9E3779B97F4A7C15); + let writes_counter = writes_counter.clone(); + let fsyncs_counter = fsyncs_counter.clone(); + join_handles.push(tokio::task::spawn_blocking(move || { + run_worker(Worker { + path, + worker_id: w, + seed: worker_seed, + n_ops: ops_per_worker, + slot_start, + slot_end, + shadow: shadow_slot, + writes_counter, + fsyncs_counter, + fsync_every_n, + }) + })); + } + + let mut errors = Vec::new(); + let mut expected = Vec::with_capacity(file_size); + for h in join_handles { + match h.await.expect("worker join panic") { + Ok(slot_shadow) => expected.extend_from_slice(&slot_shadow), + Err(e) => errors.push(e), + } + } + let workers_elapsed = wall_start.elapsed(); + if !errors.is_empty() { + common::unmount(&mount_point, child, 10); + panic!( + "concurrent-real stress FAILED ({} workers):\n {}", + errors.len(), + errors.join("\n ") + ); + } + assert_eq!(expected.len(), file_size, "shadow concatenation must equal file_size"); + + eprintln!( + " workers done: {} writes + {} fsyncs total in {:.1}s ({:.1} writes/s)", + writes_counter.load(Ordering::Relaxed), + fsyncs_counter.load(Ordering::Relaxed), + workers_elapsed.as_secs_f64(), + writes_counter.load(Ordering::Relaxed) as f64 / workers_elapsed.as_secs_f64(), + ); + + // Give the flush manager plenty of time to drain the post-worker dirty + // state to CAS + Hub. The bench's flush_debounce_ms=100 + per-upload + // network latency means a few seconds is a comfortable upper bound. + std::thread::sleep(Duration::from_millis(5000)); + common::unmount(&mount_point, child, 30); + + // ── Mount #2: fresh cache → read-back forces CAS load ── + let child = common::mount_bucket_with_env(&bucket_id, &mount_point, &cache_dir_b, mount_args, env); + if !common::is_mounted(&mount_point) { + common::unmount(&mount_point, child, 5); + panic!("mount #2 not live at {mount_point} — see hf-mount-fuse output above"); + } + + let test_file_b = format!("{}/shared.bin", mount_point); + let actual = match read_full(Path::new(&test_file_b)) { + Ok(v) => v, + Err(e) => { + common::unmount(&mount_point, child, 10); + panic!("read after remount failed: {e}"); + } + }; + + if actual != expected { + let first_diff = actual + .iter() + .zip(expected.iter()) + .position(|(a, e)| a != e) + .unwrap_or(actual.len().min(expected.len())); + let owner = first_diff / slot_size; + common::unmount(&mount_point, child, 10); + panic!( + "BYTE-LEVEL MISMATCH at offset {} (owner=w{}, slot offset={}), \ + actual=0x{:02x} expected=0x{:02x}, actual_len={}, expected_len={}", + first_diff, + owner, + first_diff % slot_size, + actual.get(first_diff).copied().unwrap_or(0), + expected.get(first_diff).copied().unwrap_or(0), + actual.len(), + expected.len(), + ); + } + + eprintln!( + "sparse-concurrent-real: PASSED {} ops total (file_size={} bytes, total_elapsed={:.1}s)", + n_workers * ops_per_worker, + file_size, + wall_start.elapsed().as_secs_f64(), + ); + + std::fs::remove_file(&test_file_b).ok(); + common::unmount(&mount_point, child, 10); + drop(guard); + std::fs::remove_dir_all(&mount_point).ok(); + std::fs::remove_dir_all(&cache_dir_a).ok(); + std::fs::remove_dir_all(&cache_dir_b).ok(); +} diff --git a/tests/sparse_delta_bench.rs b/tests/sparse_delta_bench.rs new file mode 100644 index 00000000..68d257d6 --- /dev/null +++ b/tests/sparse_delta_bench.rs @@ -0,0 +1,491 @@ +//! Benchmark: hf-mount sparse-writes vs full overwrite for the per-step +//! commit pattern from https://huggingface.co/blog/delta-weight-sync. +//! +//! The blog post showed Qwen 0.6B async-RL with ~1% bf16 element changes +//! per training step: 20-35 MB sparse delta vs 1.2 GB full checkpoint (~50× +//! reduction), per-step pause 1.1 s vs 9.4 s. They achieve this by +//! BATCHING the changed elements into a single sparse-safetensors blob +//! (one `(indices, values)` pair per tensor) and uploading that as a +//! separate `deltas/step_NNN.safetensors` file each step. +//! +//! This bench mirrors that batched pattern — the realistic application +//! workflow — instead of the pathological "issue 6M individual 2-byte +//! pwrites" worst case. Three variants: +//! +//! - `bench_sparse_delta_blob`: sparse-write mount, app writes a single +//! contiguous delta-sized blob in place each step. range_upload composes +//! exactly 1 dirty range against the old CAS base. This is what an app +//! following the blog's pattern would actually do if it kept the +//! checkpoint file canonical and patched it through hf-mount. +//! +//! - `bench_full_overwrite`: advanced-writes only (no sparse), app writes +//! the SAME blob each step but the legacy upload re-uploads the full +//! file. Baseline for the "ship the whole checkpoint every step" +//! approach the blog compares against. +//! +//! - `bench_sparse_delta_scattered`: sparse-write mount, app does N tiny +//! scattered pwrites (the pathological pattern). Kept as a stress test +//! for the upload_ranges compose path, NOT a realistic workflow — N tiny +//! inserts hit a quadratic cost in xet-core's window planner. Disabled +//! by default; enable with `BENCH_RUN_SCATTERED=1`. +//! +//! All variants run the same total-modified-bytes per step (`BENCH_DELTA_RATE` +//! × file size). Run with HF_TOKEN set: +//! +//! cargo test --release --test sparse_delta_bench -- --nocapture --test-threads=1 + +mod common; + +use std::io::{Read, Seek, SeekFrom, Write}; +use std::time::{Duration, Instant}; + +/// Size of the simulated checkpoint file. Default 64 MB keeps the bench under +/// a minute end-to-end while still being big enough that the "full upload" +/// baseline takes meaningful time. Bump via `BENCH_FILE_SIZE_MB` for a more +/// faithful Qwen-style run (1200 MB). +fn file_size_bytes() -> u64 { + let mb: u64 = std::env::var("BENCH_FILE_SIZE_MB") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(64); + mb * 1024 * 1024 +} + +/// How many delta-equivalent steps to run. +fn n_steps() -> usize { + std::env::var("BENCH_STEPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5) +} + +/// Fraction of the file's bytes that change per step. The blog says ~1% +/// of bf16 elements change at typical RL learning rates; we mirror that. +fn delta_rate() -> f64 { + std::env::var("BENCH_DELTA_RATE") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0.01) +} + +struct Rng(u64); +impl Rng { + fn new(seed: u64) -> Self { + Self(seed | 1) + } + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn rand_range(&mut self, hi: u64) -> u64 { + if hi == 0 { 0 } else { self.next() % hi } + } +} + +/// Generate a contiguous delta blob's offset for one step. Random offset so +/// successive steps don't dedupe trivially against each other. Returns +/// `(offset, len)`. +fn blob_for_step(rng: &mut Rng, file_size: u64, blob_len: u64) -> (u64, u64) { + let offset = rng.rand_range(file_size.saturating_sub(blob_len).max(1)); + (offset, blob_len) +} + +/// Generate N scattered tiny patches summing to `total_bytes`. Used only by +/// the stress variant. +fn scattered_patches(rng: &mut Rng, file_size: u64, total_bytes: u64) -> Vec<(u64, usize)> { + const AVG_RUN: usize = 8; + let mut patches: Vec<(u64, usize)> = Vec::new(); + let mut emitted: u64 = 0; + while emitted < total_bytes { + let len = 1 + (rng.next() as usize) % (3 * AVG_RUN); + let offset = rng.rand_range(file_size.saturating_sub(len as u64).max(1)); + patches.push((offset, len)); + emitted += len as u64; + } + patches.sort_by_key(|&(o, _)| o); + let mut deduped: Vec<(u64, usize)> = Vec::with_capacity(patches.len()); + for (o, l) in patches { + let end = o + l as u64; + if let Some((last_o, last_l)) = deduped.last_mut() { + let last_end = *last_o + *last_l as u64; + if o <= last_end { + *last_l = (end.max(last_end) - *last_o) as usize; + continue; + } + } + deduped.push((o, l)); + } + deduped +} + +#[derive(Default, Debug, Clone)] +struct StepStats { + bytes_modified: u64, + commit_secs: f64, +} + +/// Apply a single contiguous blob write at `offset` of length `len` and time +/// the round-trip to "verified durable on the mount" (close+reopen+read). +fn apply_blob_and_wait_durable(test_file: &str, offset: u64, len: u64, data_seed: u8) -> Duration { + let start = Instant::now(); + let buf: Vec = (0..len).map(|i| data_seed.wrapping_add(i as u8)).collect(); + { + let mut f = std::fs::OpenOptions::new() + .write(true) + .open(test_file) + .expect("open for blob"); + f.seek(SeekFrom::Start(offset)).expect("seek"); + f.write_all(&buf).expect("write blob"); + f.sync_all().expect("sync_all"); + } + // Verify the new bytes are visible via a fresh read handle. Brief sleep so + // the debounced background flush can apply_commit_sparse and clear dirty + // before we re-open. + std::thread::sleep(Duration::from_millis(200)); + { + let mut f = std::fs::File::open(test_file).expect("verify open"); + let mut head = [0u8; 16]; + let probe_len = head.len().min(len as usize); + f.seek(SeekFrom::Start(offset)).expect("verify seek"); + f.read_exact(&mut head[..probe_len]).expect("verify read"); + for (i, b) in head[..probe_len].iter().enumerate() { + assert_eq!(*b, data_seed.wrapping_add(i as u8), "verify byte mismatch at +{i}"); + } + } + start.elapsed() +} + +/// Apply N scattered tiny patches and time the same round-trip. +fn apply_scattered_and_wait_durable(test_file: &str, patches: &[(u64, usize)], data_seed: u8) -> Duration { + let start = Instant::now(); + { + let mut f = std::fs::OpenOptions::new() + .write(true) + .open(test_file) + .expect("open for scattered"); + for (i, &(offset, len)) in patches.iter().enumerate() { + let byte = data_seed.wrapping_add(i as u8); + let buf = vec![byte; len]; + f.seek(SeekFrom::Start(offset)).expect("seek"); + f.write_all(&buf).expect("write"); + } + f.sync_all().expect("sync_all"); + } + if let Some(&(offset, _)) = patches.last() { + std::thread::sleep(Duration::from_millis(200)); + let mut f = std::fs::File::open(test_file).expect("verify open"); + let mut byte = [0u8; 1]; + f.seek(SeekFrom::Start(offset)).expect("verify seek"); + f.read_exact(&mut byte).expect("verify read"); + let expected = data_seed.wrapping_add((patches.len() - 1) as u8); + assert_eq!(byte[0], expected, "verify byte at offset {offset}"); + } + start.elapsed() +} + +fn seed_checkpoint_file(test_file: &str, size: u64) { + eprintln!(" seeding {} MB checkpoint...", size / (1024 * 1024)); + let t0 = Instant::now(); + let mut f = std::fs::File::create(test_file).expect("create checkpoint"); + let chunk_size: usize = 1024 * 1024; + let mut buf = vec![0u8; chunk_size]; + let mut written: u64 = 0; + let mut block_idx: u32 = 0; + while written < size { + let want = ((size - written) as usize).min(chunk_size); + for (i, b) in buf[..want].iter_mut().enumerate() { + *b = ((block_idx as usize + i) as u8).wrapping_mul(31); + } + f.write_all(&buf[..want]).expect("seed write"); + written += want as u64; + block_idx = block_idx.wrapping_add(1); + } + f.sync_all().expect("seed sync_all"); + eprintln!(" seeded in {:.1}s", t0.elapsed().as_secs_f64()); +} + +#[derive(Copy, Clone)] +enum Workload { + Blob, + Scattered, +} + +async fn run_bench(label: &str, extra_mount_args: &[&str], workload: Workload) { + let guard = match common::setup_bucket(&format!("sparse-delta-{label}")).await { + Some(g) => g, + None => { + eprintln!("skipping {label}: HF_TOKEN not set"); + return; + } + }; + let bucket_id = guard.bucket_id.clone(); + let pid = std::process::id(); + let mount_point = format!("/tmp/hf-sparse-delta-{}-{}", label, pid); + let cache_dir = format!("/tmp/hf-sparse-delta-cache-{}-{}", label, pid); + + let mut mount_args: Vec<&str> = vec!["--advanced-writes", "--direct-io", "--flush-debounce-ms", "100"]; + mount_args.extend_from_slice(extra_mount_args); + // Force sparse engagement at any file size: the bench default (64 MiB) + // is below the production 256 MiB threshold, so without this override + // SPARSE_* labels would just be measuring the non-sparse fallback. + // Passed via `Command::env` (not `std::env::set_var`) because + // sparse_delta_bench has multiple `#[tokio::test]`s that cargo can run + // concurrently — process-wide env mutation is racy. No-op for runs + // that don't pass `--sparse-writes`. + // + // NOTE: with the fast-path commit fix (full-coverage sparse items + // route through upload_files), the warm SPARSE_BLOB variant ends up + // measuring upload_files, NOT range_upload — the seed leaves staging + // fully covered, so each subsequent delta open re-installs full + // coverage and flush picks the fast path. The cold variant + // (`run_cold_bench`) is the one that actually exercises range_upload + // because the post-remount staging is sparse (hole + dirty delta). + let child = common::mount_bucket_with_env( + &bucket_id, + &mount_point, + &cache_dir, + &mount_args, + &[("HF_MOUNT_SPARSE_MIN_BYTES", "0")], + ); + + let file_size = file_size_bytes(); + let n_steps = n_steps(); + let delta_rate = delta_rate(); + let delta_bytes = (file_size as f64 * delta_rate) as u64; + let test_file = format!("{}/ckpt.bin", mount_point); + + eprintln!( + "\n=== {} ===\n file={} MB, steps={}, delta_rate={:.2}% ({} bytes/step)", + label, + file_size / (1024 * 1024), + n_steps, + delta_rate * 100.0, + delta_bytes, + ); + + // Phase 1: seed the full checkpoint and wait for the initial upload to + // settle so per-step timings don't see seed-upload contention. + let t_seed = Instant::now(); + seed_checkpoint_file(&test_file, file_size); + std::thread::sleep(Duration::from_millis(1500)); + let seed_total = t_seed.elapsed(); + let post_seed_size = std::fs::metadata(&test_file).expect("stat after seed").len(); + eprintln!( + " initial commit total: {:.2}s; mount-side size after seed: {} bytes (expected {})", + seed_total.as_secs_f64(), + post_seed_size, + file_size, + ); + assert_eq!(post_seed_size, file_size, "post-seed size must match"); + + // Phase 2: N delta-style commits + let mut rng = Rng::new(0x00DE_ADBE_EFC0_FFEE); + let mut stats: Vec = Vec::with_capacity(n_steps); + for step in 0..n_steps { + let dur; + let bytes_modified; + match workload { + Workload::Blob => { + let (offset, len) = blob_for_step(&mut rng, file_size, delta_bytes); + dur = apply_blob_and_wait_durable(&test_file, offset, len, step as u8 + 1); + bytes_modified = len; + eprintln!( + " step {}: 1 blob, offset={} {:.2} MB, commit {:.2}s", + step + 1, + offset, + len as f64 / (1024.0 * 1024.0), + dur.as_secs_f64(), + ); + } + Workload::Scattered => { + let patches = scattered_patches(&mut rng, file_size, delta_bytes); + let apparent: u64 = patches.iter().map(|&(_, l)| l as u64).sum(); + dur = apply_scattered_and_wait_durable(&test_file, &patches, step as u8 + 1); + bytes_modified = apparent; + eprintln!( + " step {}: {} patches, {:.2} MB modified, commit {:.2}s", + step + 1, + patches.len(), + apparent as f64 / (1024.0 * 1024.0), + dur.as_secs_f64(), + ); + } + } + stats.push(StepStats { + bytes_modified, + commit_secs: dur.as_secs_f64(), + }); + } + + let mean = stats.iter().map(|s| s.commit_secs).sum::() / n_steps as f64; + let min_t = stats.iter().map(|s| s.commit_secs).fold(f64::INFINITY, f64::min); + let max_t = stats.iter().map(|s| s.commit_secs).fold(0.0, f64::max); + let total_modified: u64 = stats.iter().map(|s| s.bytes_modified).sum(); + + eprintln!( + "\n --- {} summary ---\n initial commit (full {} MB): {:.2}s\n per-step commit: mean {:.2}s, min {:.2}s, max {:.2}s\n bytes modified per step: {:.2} MB ({:.2}% of file)\n", + label, + file_size / (1024 * 1024), + seed_total.as_secs_f64(), + mean, + min_t, + max_t, + total_modified as f64 / (n_steps as f64 * 1024.0 * 1024.0), + (total_modified as f64 / n_steps as f64) / file_size as f64 * 100.0, + ); + + std::fs::remove_file(&test_file).ok(); + common::unmount(&mount_point, child, 10); + drop(guard); + std::fs::remove_dir_all(&mount_point).ok(); + std::fs::remove_dir_all(&cache_dir).ok(); +} + +/// Sparse-writes + one contiguous blob per step. Mirrors the realistic +/// delta-weight-sync workflow: the app encodes its delta into one blob and +/// writes it to a stable region of the checkpoint file. `range_upload` +/// composes 1 dirty range against the old CAS base — the path our sparse +/// design is optimized for. +#[tokio::test] +async fn bench_sparse_delta_blob() { + run_bench("SPARSE_BLOB", &["--sparse-writes"], Workload::Blob).await; +} + +/// Baseline: advanced-writes only, same blob workload. Every step +/// re-uploads the full file via `upload_files`. This is our equivalent of +/// shipping the whole checkpoint every step. +#[tokio::test] +async fn bench_full_overwrite() { + run_bench("FULL", &[], Workload::Blob).await; +} + +/// Stress variant: sparse-writes with N tiny scattered patches (the +/// pathological pattern where every changed bf16 element becomes its own +/// pwrite). Currently hits a quadratic cost in xet-core's `upload_ranges` +/// window planner — kept as a regression marker rather than a realistic +/// workflow. Off by default because a single step can take 10+ minutes. +#[tokio::test] +async fn bench_sparse_delta_scattered() { + if std::env::var("BENCH_RUN_SCATTERED").ok().as_deref() != Some("1") { + eprintln!("skipping bench_sparse_delta_scattered (set BENCH_RUN_SCATTERED=1 to enable)"); + return; + } + run_bench("SPARSE_SCATTERED", &["--sparse-writes"], Workload::Scattered).await; +} + +/// Cold-cache scenario: seed the file on one mount, unmount + wipe the +/// local cache, then remount fresh and time the first commit. This is +/// where sparse-write's wire-side win actually shows: the full-overwrite +/// path has to download the entire checkpoint into staging before it can +/// upload the modified version, while sparse just punches a hole and +/// composes against the existing CAS reconstruction. +/// +/// Real-world equivalent: an async-RL inference replica spinning up on a +/// fresh node, pulling a checkpoint that's already on the Hub. +async fn run_cold_bench(label: &str, extra_mount_args: &[&str]) { + let guard = match common::setup_bucket(&format!("sparse-cold-{label}")).await { + Some(g) => g, + None => { + eprintln!("skipping {label}: HF_TOKEN not set"); + return; + } + }; + let bucket_id = guard.bucket_id.clone(); + let pid = std::process::id(); + let mount_point = format!("/tmp/hf-sparse-cold-{}-{}", label, pid); + let cache_dir_seed = format!("/tmp/hf-sparse-cold-cache-seed-{}-{}", label, pid); + let cache_dir_bench = format!("/tmp/hf-sparse-cold-cache-bench-{}-{}", label, pid); + let file_size = file_size_bytes(); + let delta_rate = delta_rate(); + let delta_bytes = (file_size as f64 * delta_rate) as u64; + + eprintln!( + "\n=== {} (cold cache) ===\n file={} MB, delta_rate={:.2}% ({} bytes)", + label, + file_size / (1024 * 1024), + delta_rate * 100.0, + delta_bytes, + ); + + // Mount #1: seed the file, push it to CAS, unmount cleanly. + // HF_MOUNT_SPARSE_MIN_BYTES=0 passed through Command::env (not + // std::env::set_var) because cargo runs the cold/warm tests + // concurrently; see `run_bench` for the full rationale. + { + let mut mount_args: Vec<&str> = vec!["--advanced-writes", "--direct-io", "--flush-debounce-ms", "100"]; + mount_args.extend_from_slice(extra_mount_args); + let child = common::mount_bucket_with_env( + &bucket_id, + &mount_point, + &cache_dir_seed, + &mount_args, + &[("HF_MOUNT_SPARSE_MIN_BYTES", "0")], + ); + let test_file = format!("{}/ckpt.bin", mount_point); + let t_seed = Instant::now(); + seed_checkpoint_file(&test_file, file_size); + // Wait for the deferred flush to commit the seed to the Hub before + // unmounting (otherwise the bucket has no file for the remount to + // open against). + std::thread::sleep(Duration::from_millis(2500)); + eprintln!(" seed committed in {:.2}s", t_seed.elapsed().as_secs_f64()); + common::unmount(&mount_point, child, 10); + std::fs::remove_dir_all(&cache_dir_seed).ok(); + } + + // Mount #2: cold cache. The remote file exists on the Hub but nothing + // is cached locally — sparse path can punch a hole and compose; the + // non-sparse path must download the whole file first. + let mut mount_args: Vec<&str> = vec!["--advanced-writes", "--direct-io", "--flush-debounce-ms", "100"]; + mount_args.extend_from_slice(extra_mount_args); + let child = common::mount_bucket_with_env( + &bucket_id, + &mount_point, + &cache_dir_bench, + &mount_args, + &[("HF_MOUNT_SPARSE_MIN_BYTES", "0")], + ); + let test_file = format!("{}/ckpt.bin", mount_point); + + // Cold step: write a single delta-sized blob at a random offset and + // wait for the commit to settle. This is the headline number. + let mut rng = Rng::new(0x00DE_ADBE_EFC0_FFEE); + let (offset, len) = blob_for_step(&mut rng, file_size, delta_bytes); + let dur = apply_blob_and_wait_durable(&test_file, offset, len, 1); + eprintln!( + " cold step: 1 blob, offset={} {:.2} MB, commit {:.2}s", + offset, + len as f64 / (1024.0 * 1024.0), + dur.as_secs_f64(), + ); + eprintln!( + "\n --- {} cold summary ---\n cold-cache first commit: {:.2}s\n", + label, + dur.as_secs_f64(), + ); + + std::fs::remove_file(&test_file).ok(); + common::unmount(&mount_point, child, 10); + drop(guard); + std::fs::remove_dir_all(&mount_point).ok(); + std::fs::remove_dir_all(&cache_dir_bench).ok(); +} + +/// Cold-cache sparse-write commit. Expect: no full-file download — sparse +/// punches a hole locally and `range_upload` composes against CAS. +#[tokio::test] +async fn bench_sparse_delta_cold() { + run_cold_bench("SPARSE_COLD", &["--sparse-writes"]).await; +} + +/// Cold-cache full-overwrite commit. Expect: the full checkpoint has to +/// be downloaded into local staging before the bench can `open` it for +/// write, then re-uploaded via `upload_files` (CDC dedup wire-trims, but +/// the download itself is unavoidable). On a 1.2 GB checkpoint with a 100 +/// MB/s effective xet bandwidth this is ~12 s of pure download wait. +#[tokio::test] +async fn bench_full_cold() { + run_cold_bench("FULL_COLD", &[]).await; +}