Derive download buffer defaults from usable, cgroup-aware system memory

.github/workflows/bump-crates-version.yml

@@ -21,10 +21,10 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
           python-version: '3.10'

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
       - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
       - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - name: Install cargo-machete
         run: cargo install --locked cargo-machete@0.9.2
       - name: Run cargo-machete
@@ -48,7 +48,7 @@ jobs:
       - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
       - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: ./.github/actions/cache-rust-build
       - name: Compile benchmarks
         run: |
@@ -60,10 +60,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
           components: clippy
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         with:
@@ -117,10 +117,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: ./.github/actions/cache-rust-build
       - name: Build and Test
         run: |
@@ -150,10 +150,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - name: Set up Git LFS
         run: |
           brew install git-lfs
@@ -189,10 +189,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: ./.github/actions/build-wasm
       - name: Check hf_xet_thin_wasm Cargo.lock has no uncommitted changes
         working-directory: wasm/hf_xet_thin_wasm
@@ -228,7 +228,7 @@ jobs:
       - uses: huggingface/hf-workflows/.github/actions/setup-hf-registry-proxies@21845492ef47d4e46b07a3b2a6e14fd4748c39cc  # main
       - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - name: Install cargo-audit
         run: cargo install --locked cargo-audit
       - name: Run cargo audit

.github/workflows/crates-release.yml

@@ -26,10 +26,10 @@ jobs:
         with:
           ref: main
 
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
 
       - name: Authenticate to crates.io
         id: auth

.github/workflows/git-xet-release.yml

@@ -27,10 +27,10 @@ jobs:
             target: aarch64
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: ./.github/actions/cache-rust-build
       - name: Build
         run: |
@@ -52,10 +52,10 @@ jobs:
             target: aarch64
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: ./.github/actions/cache-rust-build
       - name: Build
         run: |
@@ -92,10 +92,10 @@ jobs:
             wix_arch: arm64
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - name: Install Rust 1.94
+      - name: Install Rust 1.95
         uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - uses: ./.github/actions/cache-rust-build
       - name: Install WiX
         run: |

.github/workflows/hf-xet-tests.yml

@@ -41,7 +41,7 @@ jobs:
           sudo apt-get install -y "libpython${py_ver}-dev"
       - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9  # master
         with:
-          toolchain: 1.94.1
+          toolchain: 1.95.0
       - name: Create venv
         run: python3 -m venv .venv
       - name: Build wheel

Cargo.toml

@@ -103,7 +103,7 @@ statrs = { version = "0.18", default-features = false }
 shellexpand = "3.1"
 smol = "2"
 static_assertions = "1.1"
-sysinfo = "0.38"
+sysinfo = "0.39"
 tempfile = "3.25"
 thiserror = "2.0"
 tokio = { version = "1.49" }

api_changes/update_260819_memory_derived_download_buffers.md

@@ -0,0 +1,77 @@
+# Memory-derived download buffer defaults
+
+Date: 2026-08-19
+Issue: #927 (related: huggingface_hub#3300)
+
+## Summary
+
+The three reconstruction download buffer defaults are no longer compile-time
+constants. They are derived at session startup from the memory usable by the
+process: `min(host physical RAM, effective cgroup memory limit)`.
+
+| knob | derivation | floor | ceiling | old constant |
+|---|---|---|---|---|
+| `reconstruction.download_buffer_size` | usable/16 | 32MB | 16GB | 2GB |
+| `reconstruction.download_buffer_perfile_size` | usable/64 | 8MB | 2GB | 512MB |
+| `reconstruction.download_buffer_limit` | usable/4 | 128MB | 64GB | 8GB |
+
+Values are rounded down to a multiple of 8MB (decimal units, matching
+`ByteSize`). 32GB of usable memory reproduces the old defaults (perfile lands
+at 496MB rather than 512MB); >= 256GB reproduces the old high-performance
+constants. The invariant `size + max_concurrent_file_downloads * perfile <=
+limit` holds across the whole range, so the three values describe one coherent
+allocation.
+
+High-performance mode (`HF_XET_HIGH_PERFORMANCE` / `HF_XET_HP`) uses fractions
+twice as aggressive (usable/8, usable/32, usable/2) with the same floors and
+ceilings — it no longer sets a 16GB base buffer on an 8GB machine.
+
+## Toolchain and dependency changes
+
+- **Rust toolchain: 1.94.1 -> 1.95.0** (all CI workflow pins). Required by
+  sysinfo 0.39.
+- **sysinfo: 0.38.4 -> 0.39.6** (workspace pin `0.39`; all four lockfiles).
+  0.39 adds `Process::cgroup_limits()`, which resolves the cgroup path from
+  `/proc/<pid>/cgroup` and takes the tightest memory limit across ancestor
+  cgroups (v1 and v2) — the container-aware probe this change needs, so no
+  hand-rolled `/proc` parsing is maintained in this repo.
+
+## New API
+
+- `xet_runtime::utils::system_memory` (new module): `usable_system_memory()`
+  (cached probe; host total via sysinfo, cgroup limit via sysinfo 0.39's
+  `Process::cgroup_limits` — cgroup v1 + v2, nested paths, namespaced roots),
+  `default_download_buffer_sizes()`, `high_performance_download_buffer_sizes()`,
+  `STATIC_DEFAULTS`, `STATIC_HP_DEFAULTS`.
+- `reconstruction::ConfigValueGroup::normalize()`: raises
+  `download_buffer_limit` to `download_buffer_size` if it is below it. Called
+  by `XetContext::new` before the config is frozen. Previously an env
+  configuration with `SIZE > LIMIT` panicked at context construction.
+
+## Behavior changes downstream consumers should know about
+
+1. **Defaults are machine-dependent.** Tests or tooling asserting the literal
+   values 2GB/512MB/8GB (or HP 16GB/2GB/64GB) will see derived values instead.
+   Set `HF_XET_MEMORY_DERIVED_DOWNLOAD_BUFFERS=0` to restore the static
+   defaults (kill switch; per-knob env vars also still override).
+2. **Env overrides now beat the high-performance preset.** `XetConfig::new()`
+   applies defaults -> HP preset -> env overrides. Previously HP silently
+   overwrote explicitly set `HF_XET_RECONSTRUCTION_*` values.
+3. **`apply_env_overrides` / `with_env_overrides` no longer reset fields.**
+   Fields without an env override keep their current value (e.g. from a
+   preset) instead of being reset to the macro default. Callers relying on the
+   old reset behavior must construct a fresh config instead.
+4. Wasm and any platform where memory cannot be determined keep the old static
+   defaults. Windows Job Object / Windows-container limits are not probed
+   (documented limitation; use the env vars). WSL2 and all Linux containers go
+   through the cgroup path.
+5. Known accounting slack (pre-existing, unchanged): up to one decompressed
+   xorb block (<= 64MiB) per active connection can be resident while only
+   partially covered by buffer permits. The floors leave room for this even in
+   very small containers.
+
+## Observability
+
+The first probe logs one `info!` line with host total, cgroup limit, usable
+memory, and the derived triple. Per-field `Config: <name> = <value> (default)`
+log lines and the Python `XetConfig` repr show the derived values.