From 9b655c7f17d39ffd18239d70e70827f09abd8efb Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Mon, 24 Aug 2026 11:42:32 +0200 Subject: [PATCH 01/21] add mutate_bucket_file() --- src/huggingface_hub/hf_api.py | 88 ++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 36f51072f8..3432292cb9 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -14701,7 +14701,7 @@ def get_bucket_file_metadata( if xet_file_data is None: raise ValueError(f"Could not parse xet file data for '{remote_path}' in bucket '{bucket_id}'.") - size = response.headers.get("Content-Length") + size = response.headers.get("X-Linked-Size") if size is None: raise ValueError(f"Could not get size for '{remote_path}' in bucket '{bucket_id}'.") @@ -14958,6 +14958,92 @@ def sync_bucket( token=token, ) + def mutate_bucket_file( + self, + *, + bucket_id: str, + remote_path: str, + edit: list[tuple[tuple[int, int], bytes]] | None = None, + insert: list[tuple[int, bytes]] | None = None, + delete: list[tuple[int, int]] | None = None, + token: bool | str | None = None, + ) -> None: + # TODO(QL): TRY IT OUT !!! (+ debug lol) + from .utils._xet import ( + XetTokenType, + abort_xet_session, + get_xet_session, + xet_connection_info_refresh_url, + xet_headers_without_auth, + ) + from .utils._xet_progress_reporting import XetUploadProgressReporter + + if edit: + edit = [((start, end), data) for (start, end), data in edit if start < end or (start == end and len(data) > 0)] + if insert: + insert = [(loc, data) for loc, data in insert if len(data) > 0] + if delete: + delete = [(loc, length) for loc, length in delete if length > 0] + if not (edit or insert or delete): + return + + headers = self._build_hf_headers(token=token) + + if not are_progress_bars_disabled(): + _progress = XetUploadProgressReporter(total_files=1) + else: + _progress = None + + refresh_url = xet_connection_info_refresh_url( + token_type=XetTokenType.WRITE, + repo_id=bucket_id, + repo_type="bucket", + endpoint=self.endpoint, + ) + xet_headers = xet_headers_without_auth(headers) + + owns_progress = _progress is None + if _progress is not None: + progress = _progress + progress.reset_for_next_commit() + progress_callback = progress.update_progress + elif not are_progress_bars_disabled(): + progress = XetUploadProgressReporter() + progress_callback = progress.update_progress + else: + progress, progress_callback = None, None + session = get_xet_session() + file_metadata = self.get_bucket_file_metadata( + bucket_id=bucket_id, + remote_path=remote_path, + token=token + ) + try: + with session.new_range_upload( + file_metadata.xet_file_data.file_hash, + file_metadata.size, + token_refresh_url=refresh_url, + token_refresh_headers=headers, + custom_headers=xet_headers, + progress_callback=progress_callback, + ) as commit: + if edit: + for (start, end), data in edit: + commit.insert(start, end).write(data) + if insert: + for loc, data in insert: + commit.insert(loc, len(data)).write(data) + if delete: + for loc, length in delete: + commit.delete(loc, loc + length) + except KeyboardInterrupt: + abort_xet_session() + raise + finally: + if owns_progress and progress is not None: + progress.close() + return + def _parse_revision_from_pr_url(pr_url: str) -> str: """Safely parse revision number from a PR url. From 53b3befb433a047474ca44f88f8d6edf737d5c88 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Mon, 24 Aug 2026 11:42:43 +0200 Subject: [PATCH 02/21] add HfFileSystemMutateFile() --- src/huggingface_hub/hf_file_system.py | 373 +++++++++++++++++++++++++- 1 file changed, 371 insertions(+), 2 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 89cccd62ee..14ee906601 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1,6 +1,7 @@ import os import tempfile import threading +import time from collections import deque from collections.abc import Iterable, Iterator from contextlib import ExitStack @@ -385,6 +386,56 @@ def invalidate_cache(self, path: str | None = None) -> None: else: self._bucket_exists_cache.pop(resolved_path.bucket_id, None) + def open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ): + """ + Return a file-like object from the filesystem + + The resultant instance must function correctly in a context ``with`` + block. + + Parameters + ---------- + path: str + Target file + mode: str like 'rb', 'w', 'a' + See builtin ``open()``. + There is an extra mode 'mb' (mutate bytes in-place) allows insert(), delete(), edit() and append(). + It is available thanks to Xet which stores files by chunks. + block_size: int + Some indication of buffering - this is a value in bytes + cache_options : dict, optional + Extra arguments to pass through to the cache. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding, errors, newline: passed on to TextIOWrapper for text mode + """ + mutate_mode = "m" if "m" in mode else ("a" if "a" in mode else None) + if mutate_mode: + if "b" not in mode: + raise NotImplementedError(f"Mode '{mutate_mode}' is not implemented, use 'mb' instead") + if compression is not None: + raise NotImplementedError(f"Mode '{mutate_mode}' with compression is not implemented") + if cache_options is not None or kwargs.get("cache_type") is not None: + raise NotImplementedError(f"Mode '{mutate_mode}' with cache is not implemented.") + return super().open( + path, + mode=mode, + block_size=block_size, + cache_options=cache_options, + compression=compression, + **kwargs, + ) + def _open( # type: ignore self, path: str, @@ -396,8 +447,8 @@ def _open( # type: ignore block_size = block_size if block_size is not None else self.block_size if block_size is not None: kwargs["block_size"] = block_size - if "a" in mode: - raise NotImplementedError("Appending to remote files is not yet supported.") + if "a" in mode or "m" in mode: + return HfFileSystemMutateFile(self, path, mode=mode, **kwargs) if block_size == 0: return HfFileSystemStreamFile(self, path, mode=mode, revision=revision, **kwargs) else: @@ -1391,6 +1442,324 @@ def _open_connection(self): self._stream_iterator = self.response.iter_bytes() +class HfFileSystemMutateFile(fsspec.spec.AbstractBufferedFile): + MIN_SECONDS_BETWEEN_UPDATES = 5. + + def __init__(self, + fs: HfFileSystem, + path: str, + mode="rb", + block_size="default", + autocommit=True, + size=None, + **kwargs, + ): + from fsspec.caching import BaseCache + + if mode not in {"mb", "ab"}: + raise NotImplementedError("File mode not supported") + try: + resolved_path = fs.resolve_path(path) + path = resolved_path.unresolve() + fs.info(path) + except FileNotFoundError as e: + raise FileNotFoundError( + f"{e}.\nMake sure the bucket and file exist before writing data." + ) from e + if not isinstance(resolved_path, HfFileSystemResolvedBucketPath): + raise ValueError(f"File mode '{mode}' is only available for Storage Buckets (hf://buckets/...)") + self.resolved_path = resolved_path + + # required by AbstractBufferedFile + self.path = path + self.fs = fs + self.mode = mode + self.blocksize = ( + self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size + ) + self.autocommit = autocommit + self.closed = False + self.forced = False + self.size = size if size is not None else self.details["size"] + self.original_size = self.size + self.cache = BaseCache(self.blocksize, self._fetch_range, self.size) + self.loc = self.size if "a" in mode else 0 + self.kwargs = kwargs + + # specific to HfFileSystemMutateFile + self.ranges: list[range | bytes] = [range(0, self.size)] + self.buffer_size = 0 + self.last_update_time: float | None = None + + def __del__(self): + if not hasattr(self, "resolved_path"): + # Means that the constructor failed. Nothing to do. + return + return super().__del__() + + def _fetch_range(self, start: int, end: int) -> bytes: + ranges_contents: list[bytes] = [] + offset = 0 + for range_ in self.ranges: + if isinstance(range_, range): + range_to_fetch = range(range_.start + max(0, start - offset), range_.stop + min(0, end - offset - len(range_))) + if range_to_fetch.start < range_to_fetch.stop: + headers = { + "range": f"bytes={range_to_fetch.start}-{range_to_fetch.stop - 1}", + **self.fs._api._build_hf_headers(), + } + url = self.url() + r = http_backoff("GET", url, headers=headers, timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT) + hf_raise_for_status(r) + ranges_contents.append(r.content) + else: + range_to_fetch = range(max(0, start - offset), len(range_) + min(0, end - offset - len(range_))) + if range_to_fetch.start < range_to_fetch.stop: + ranges_contents.append(range_[range_to_fetch.start:range_to_fetch.stop]) + offset += len(range_) + return b"".join(ranges_contents) + + def _upload_ranges(self) -> None: + original_offset = 0 + insert: list[tuple[int, bytes]] = [] + delete: list[tuple[int, int]] = [] + for range_ in self.ranges: + if isinstance(range_, range): + if range_.start > original_offset: + delete.append((original_offset, range_.start - original_offset)) + original_offset = range_.stop + else: + insert.append((original_offset, range_)) + if original_offset < self.original_size: + delete.append((original_offset, self.original_size)) + if insert or delete: + self.fs._api.mutate_bucket_file( + bucket_id=self.resolved_path.bucket_id, + remote_path=self.resolved_path.path, + insert=insert or None, + delete=delete or None, + ) + self.fs.invalidate_cache( + path=self.resolved_path.unresolve(), + ) + self.original_size = self.size + self.ranges = [range(0, self.size)] + self.buffer_size = 0 + self._details = None + + def url(self) -> str: + return self.fs.url(self.path) + + def readable(self): + """Whether opened for reading""" + return not self.closed + + def read(self, length=-1): + """Read remote file. + + If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is + loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. + """ + if self.mode == "rb" and (length is None or length == -1) and self.loc == 0 and self.ranges == [range(0, self.original_size)]: + with self.fs.open(self.path, "rb", block_size=0) as f: # block_size=0 enables fast streaming + out = f.read() + self.loc += len(out) + return out + + length = -1 if length is None else int(length) + if length < 0: + length = self.size - self.loc + if self.closed: + raise ValueError("I/O operation on closed file.") + if length == 0: + # don't even bother calling fetch + return b"" + out = self.cache._fetch(self.loc, self.loc + length) + self.loc += len(out) + return out + + def seek(self, loc, whence=0): + """Set current file location + + Parameters + ---------- + loc: int + byte location + whence: {0, 1, 2} + from start of file, current location or end of file, resp. + """ + loc = int(loc) + if whence == 0: + nloc = loc + elif whence == 1: + nloc = self.loc + loc + elif whence == 2: + nloc = self.size + loc + else: + raise ValueError(f"invalid whence ({whence}, should be 0, 1 or 2)") + if nloc < 0: + raise ValueError("Seek before start of file") + self.loc = nloc + return self.loc + + def writable(self): + """Whether opened for writing""" + return not self.closed + + def write(self, data: bytes | str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + return self.edit((self.loc, self.loc + len(data)), data) + + def edit(self, byte_range: tuple[int, int], data: bytes | str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + byte_range: tuple[int, int] + (start, end) where to edit the file. + data: bytes + Set of bytes to be placed in the specified range. + Size can be different than the range. + """ + if self.closed: + raise ValueError("I/O operation on closed file.") + if isinstance(data, str): + data = data.encode("utf-8") + + start, end = byte_range + if start == end and not data: + return + + new_ranges: list[range, bytes] = [] + offset = 0 + done = False + def add_range(range_: range | bytes): + if new_ranges and isinstance(new_ranges[-1], bytes) and isinstance(range_, bytes): + new_ranges[-1] += range_ + else: + new_ranges.append(range_) + + for range_ in self.ranges: + if offset <= start <= offset + len(range_) <= end: + if offset < start: + add_range(range_[:start - offset]) + if data and not done: + add_range(data) + done = True + elif start <= offset <= offset + len(range_) <= end: + pass + elif start <= offset <= end <= offset + len(range_): + if data and not done: + add_range(data) + done = True + if end < offset + len(range_): + add_range(range_[end - offset - len(range_):]) + elif offset <= start <= end <= offset + len(range_): + if offset < start: + add_range(range_[:start - offset]) + if data and not done: + add_range(data) + done = True + if end < offset + len(range_): + add_range(range_[end - offset - len(range_):]) + else: + add_range(range_) + offset += len(range_) + self.ranges = new_ranges + + self.loc = byte_range[0] + len(data) + self.buffer_size += len(data) + self.size += end - start + len(data) + if self.last_update_time is None: + self.last_update_time = time.time() + if self.flush(): + self.last_update_time = time.time() + return len(data) + + def insert(self, loc: int, data: bytes | str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + loc: int + Where to insert the data. + data: bytes + Set of bytes to be inserted. + """ + return self.edit((loc, loc), data) + + def append(self, data: bytes | str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be appended at the end of the file. + """ + return self.edit((self.size, self.size + len(data)), data) + + def delete(self, loc: int, length: int): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + loc: int + Where to insert the data. + length: int + Number of bytes to delete. + """ + return self.edit((loc, loc + length), b"") + + def flush(self, force=False): + """ + Write buffered data to backend store. + + Writes the current buffer, if it is larger than the block-size, or if + the file is being closed. + + Parameters + ---------- + force: bool + When closing, write the last block even if it is smaller than + blocks are allowed to be. Disallows further writing to this file. + """ + + if self.closed: + raise ValueError("Flush on closed file") + if force or (self.buffer_size >= self.blocksize and self.last_update_time and (time.time() - self.last_update_time) > self.MIN_SECONDS_BETWEEN_UPDATES): + self._upload_ranges() + return True + else: + # Defer write on small block or quick update + return False + + def safe_revision(revision: str) -> str: return revision if SPECIAL_REFS_REVISION_REGEX.match(revision) else safe_quote(revision) From ceac90221b4252fe2de97e0de65442c1c1cb540d Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Mon, 24 Aug 2026 17:36:54 +0200 Subject: [PATCH 03/21] fix edit() + update file reference --- src/huggingface_hub/hf_api.py | 36 ++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 3432292cb9..37ed04a085 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -15019,29 +15019,59 @@ def mutate_bucket_file( token=token ) try: - with session.new_range_upload( + commit = session.new_range_upload( file_metadata.xet_file_data.file_hash, file_metadata.size, token_refresh_url=refresh_url, token_refresh_headers=headers, custom_headers=xet_headers, progress_callback=progress_callback, - ) as commit: + ) + try: if edit: for (start, end), data in edit: - commit.insert(start, end).write(data) + commit.edit(start, end).write(data) if insert: for loc, data in insert: commit.insert(loc, len(data)).write(data) if delete: for loc, length in delete: commit.delete(loc, loc + length) + report = commit.commit() + except KeyboardInterrupt: + commit.abort() + raise + finally: + # Clean up the commit resources + try: + commit.close() + except Exception: + pass except KeyboardInterrupt: abort_xet_session() raise finally: if owns_progress and progress is not None: progress.close() + + # Update the Hub file reference with the new hash and size + if report is not None and report.file_info is not None: + from huggingface_hub._buckets import _BucketCopyFile + + # Use _BucketCopyFile to update the file reference. + # We copy from the same bucket (self-referential) to update the file hash. + self._batch_bucket_files( + bucket_id=bucket_id, + copy=[ + _BucketCopyFile( + destination=remote_path, + xet_hash=report.file_info.hash, + source_repo_type="bucket", + source_repo_id=bucket_id, + ) + ], + token=token, + ) return From e1e224364eff7a6c547e755a002302ea2b67f34f Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Mon, 24 Aug 2026 17:41:47 +0200 Subject: [PATCH 04/21] typing --- src/huggingface_hub/hf_file_system.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 14ee906601..07ea27a4bd 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -10,7 +10,7 @@ from datetime import datetime from itertools import chain from pathlib import Path, PurePosixPath -from typing import Any, NoReturn, Union +from typing import Any, Literal, NoReturn, Union, overload from urllib.parse import quote, unquote import fsspec @@ -386,6 +386,18 @@ def invalidate_cache(self, path: str | None = None) -> None: else: self._bucket_exists_cache.pop(resolved_path.bucket_id, None) + @overload + def open( + self, + path, + mode: Literal["ab", "mb"], + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ) -> "HfFileSystemMutateFile": + ... + def open( self, path, @@ -394,7 +406,7 @@ def open( cache_options=None, compression=None, **kwargs, - ): + ) -> fsspec.spec.AbstractBufferedFile: """ Return a file-like object from the filesystem From c4e46688075f0a10da9799b3d2b56e894cef2f8f Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Mon, 24 Aug 2026 17:43:01 +0200 Subject: [PATCH 05/21] more typing --- src/huggingface_hub/hf_file_system.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 07ea27a4bd..6cfebef916 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -398,6 +398,18 @@ def open( ) -> "HfFileSystemMutateFile": ... + @overload + def open( + self, + path, + mode: str, + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ) -> fsspec.spec.AbstractBufferedFile: + ... + def open( self, path, From 8366066d7a81898d75ce7b0c5efda86925bff5a7 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 26 Aug 2026 17:43:54 +0200 Subject: [PATCH 06/21] defer send and support text wrapper --- src/huggingface_hub/hf_api.py | 117 +++++++-- src/huggingface_hub/hf_file_system.py | 365 ++++++++++++++++++++++---- 2 files changed, 413 insertions(+), 69 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 37ed04a085..94a52c4f5e 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -14968,7 +14968,69 @@ def mutate_bucket_file( delete: list[tuple[int, int]] | None = None, token: bool | str | None = None, ) -> None: - # TODO(QL): TRY IT OUT !!! (+ debug lol) + """Mutate an existing file in a bucket in-place, only re-uploading the parts the caller actually rewrites. + + See [`HfFileSystemMutateFile`] to use this feature with a file-like API. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + remote_path (`str`): + the file to mutate. + edit (`list[tuple[tuple[int, int], bytes]]`, *optional*): + List edits to apply, in the form `((start, end), data)`. + Ranges [`start`, `end`) are replaced with `data`, which can + be of any size (not necessarily the size of the replaced range). + insert (`list[tuple[int, bytes]]`, *optional*): + List of inserts to apply, in the form `(loc, data)`. + The content of `data` is inserted at location `loc`, and shifts + the rest of the file. + delete (`list[tuple[int, bytes]]`, *optional*): + List of deletes to apply, in the form `(loc, length)`. + The range [`loc`, `loc + length`) is deleted. + """ + from .utils._xet_progress_reporting import XetUploadProgressReporter + + + if not are_progress_bars_disabled(): + _progress = XetUploadProgressReporter(total_files=1) + else: + _progress = None + + file_metadata = self.get_bucket_file_metadata( + bucket_id=bucket_id, + remote_path=remote_path, + token=token + ) + self._mutate_bucket_file( + bucket_id=bucket_id, + remote_path=remote_path, + edit=edit, + insert=insert, + delete=delete, + _progress=_progress, + _file_hash=file_metadata.xet_file_data.file_hash, + _file_size=file_metadata.size + ) + + + def _mutate_bucket_file( + self, + *, + bucket_id: str, + remote_path: str, + edit: list[tuple[tuple[int, int], bytes]] | None = None, + insert: list[tuple[int, bytes]] | None = None, + delete: list[tuple[int, int]] | None = None, + token: bool | str | None = None, + _progress: XetUploadProgressReporter | None = None, + _file_hash: str | None = None, + _file_size: int | None = None, + ) -> str: + """ + Internal method: process a single batch of bucket file mutate operations (upload to XET + call /batch). + Returns the new file's xet hash. + """ from .utils._xet import ( XetTokenType, abort_xet_session, @@ -14987,13 +15049,30 @@ def mutate_bucket_file( if not (edit or insert or delete): return - headers = self._build_hf_headers(token=token) + owns_progress = _progress is None + if _progress is not None: + progress = _progress + progress.reset_for_next_commit() + progress_callback = progress.update_progress + elif not are_progress_bars_disabled(): + progress = XetUploadProgressReporter(total_files=1) + progress_callback = progress.update_progress + else: + progress, progress_callback = None, None - if not are_progress_bars_disabled(): - _progress = XetUploadProgressReporter(total_files=1) + if _file_hash is None or _file_size is None: + file_metadata = self.get_bucket_file_metadata( + bucket_id=bucket_id, + remote_path=remote_path, + token=token + ) + file_hash = file_metadata.xet_file_data.file_hash + file_size = file_metadata.size else: - _progress = None + file_hash = _file_hash + file_size = _file_size + headers = self._build_hf_headers(token=token) refresh_url = xet_connection_info_refresh_url( token_type=XetTokenType.WRITE, repo_id=bucket_id, @@ -15002,31 +15081,19 @@ def mutate_bucket_file( ) xet_headers = xet_headers_without_auth(headers) - owns_progress = _progress is None - if _progress is not None: - progress = _progress - progress.reset_for_next_commit() - progress_callback = progress.update_progress - elif not are_progress_bars_disabled(): - progress = XetUploadProgressReporter() - progress_callback = progress.update_progress - else: - progress, progress_callback = None, None - session = get_xet_session() - file_metadata = self.get_bucket_file_metadata( - bucket_id=bucket_id, - remote_path=remote_path, - token=token - ) try: - commit = session.new_range_upload( - file_metadata.xet_file_data.file_hash, - file_metadata.size, + commit = get_xet_session().new_range_upload( + file_hash, + file_size, token_refresh_url=refresh_url, token_refresh_headers=headers, custom_headers=xet_headers, progress_callback=progress_callback, ) + logger.debug( + f"About to commit to a file on the hub: {len(edit or [])} edit(s), {len(insert or [])} insert(s) and" + f" {len(delete or [])} deletion(s)." + ) try: if edit: for (start, end), data in edit: @@ -15072,7 +15139,7 @@ def mutate_bucket_file( ], token=token, ) - return + return report.file_info.hash def _parse_revision_from_pr_url(pr_url: str) -> str: diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 6cfebef916..269f239606 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1,13 +1,16 @@ +import io import os import tempfile import threading import time from collections import deque from collections.abc import Iterable, Iterator +from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack from copy import deepcopy from dataclasses import dataclass, field from datetime import datetime +from functools import partial from itertools import chain from pathlib import Path, PurePosixPath from typing import Any, Literal, NoReturn, Union, overload @@ -398,6 +401,30 @@ def open( ) -> "HfFileSystemMutateFile": ... + @overload + def open( + self, + path, + mode: Literal["a", "m", "at", "mt"], + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ) -> "MutableTextIOWrapper": + ... + + @overload + def open( + self, + path, + mode: Literal["r", "rt", "w", "wb"], + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ) -> io.TextIOWrapper: + ... + @overload def open( self, @@ -445,12 +472,24 @@ def open( """ mutate_mode = "m" if "m" in mode else ("a" if "a" in mode else None) if mutate_mode: - if "b" not in mode: - raise NotImplementedError(f"Mode '{mutate_mode}' is not implemented, use 'mb' instead") if compression is not None: raise NotImplementedError(f"Mode '{mutate_mode}' with compression is not implemented") - if cache_options is not None or kwargs.get("cache_type") is not None: - raise NotImplementedError(f"Mode '{mutate_mode}' with cache is not implemented.") + if "b" not in mode: + mode = mode.replace("t", "") + "b" + text_kwargs = { + k: kwargs.pop(k) + for k in ["encoding", "errors", "newline"] + if k in kwargs + } + buffer = super().open( + path, + mode=mode, + block_size=block_size, + cache_options=cache_options, + compression=compression, + **kwargs, + ) + return MutableTextIOWrapper(buffer, **text_kwargs, write_through=True) return super().open( path, mode=mode, @@ -1467,31 +1506,81 @@ def _open_connection(self): class HfFileSystemMutateFile(fsspec.spec.AbstractBufferedFile): - MIN_SECONDS_BETWEEN_UPDATES = 5. + """Mutate a file in a bucket in-place, only re-uploading the parts + the caller actually rewrites. + + Supported mutate operations: append, edit, insert, delete, truncate. + + It uses a buffer that is only sent on flush(force=True) or if 10 + seconds passed since last send and buffer is greater than or equal + to block_size. + + Examples: + + - Write logs progressively using the append mode "a": + + ```py + from huggingface_hub import hffs + + with hffs.open("buckets/username/my-bucket/logs.txt", "a") as f: + for log in logs: + f.write(log) + ``` + + - Edit a file header using the mutate mode "m": + + ```py + from huggingface_hub import hffs + + header_length = 16 + new_header = b"MY_NEW_HEADER_00" + with hffs.open("buckets/username/my-bucket/data.bin", "mb") as f: + f.edit((0, header_length), new_header) + ``` + + - Remove a certain line using the mutate mode "m": + + ```py + from huggingface_hub import hffs + + line_idx_to_remove = 42 + with hffs.open("buckets/username/my-bucket/doc.txt", "m") as f: + for i, line in enumerate(f): + if i == line_idx_to_remove: + line_loc = f.loc - len(line) + line_length = len(line) + f.delete(line_loc, line_length) + break + ``` + """ + + DEFAULT_SEND_INTERVAL = 10. def __init__(self, fs: HfFileSystem, path: str, - mode="rb", - block_size="default", - autocommit=True, - size=None, + mode: str = "rb", + block_size: str | int | None = "default", + send_interval: str | int | None = "default", + autocommit: bool=True, + cache_type: str | None ="readahead", + cache_options: dict | None =None, + size: int | None = None, + file_hash: str | None = None, **kwargs, ): - from fsspec.caching import BaseCache + from fsspec.core import caches if mode not in {"mb", "ab"}: raise NotImplementedError("File mode not supported") - try: - resolved_path = fs.resolve_path(path) - path = resolved_path.unresolve() - fs.info(path) - except FileNotFoundError as e: - raise FileNotFoundError( - f"{e}.\nMake sure the bucket and file exist before writing data." - ) from e + resolved_path = fs.resolve_path(path) if not isinstance(resolved_path, HfFileSystemResolvedBucketPath): raise ValueError(f"File mode '{mode}' is only available for Storage Buckets (hf://buckets/...)") + path = resolved_path.unresolve() + try: + fs.info(path) + except FileNotFoundError: + fs.touch(path) self.resolved_path = resolved_path # required by AbstractBufferedFile @@ -1506,14 +1595,23 @@ def __init__(self, self.forced = False self.size = size if size is not None else self.details["size"] self.original_size = self.size - self.cache = BaseCache(self.blocksize, self._fetch_range, self.size) + self.cache_type = cache_type + self.cache_options = cache_options + self.cache = caches[cache_type]( + self.blocksize, self._fetch_range, self.size, **(cache_options or {}) + ) self.loc = self.size if "a" in mode else 0 self.kwargs = kwargs # specific to HfFileSystemMutateFile + self.file_hash = file_hash if file_hash is not None else self.details["xet_hash"] self.ranges: list[range | bytes] = [range(0, self.size)] self.buffer_size = 0 self.last_update_time: float | None = None + self.send_interval = ( + self.DEFAULT_SEND_INTERVAL if send_interval in ["default", None] else send_interval + ) + self.task = None def __del__(self): if not hasattr(self, "resolved_path"): @@ -1522,6 +1620,8 @@ def __del__(self): return super().__del__() def _fetch_range(self, start: int, end: int) -> bytes: + if self.task is not None and not self.task.done(): + raise NotImplementedError("Attempted to read a file while blocks are being sent but this is not implemented. Use f.flush(force=True) to send blocks first.") ranges_contents: list[bytes] = [] offset = 0 for range_ in self.ranges: @@ -1543,32 +1643,44 @@ def _fetch_range(self, start: int, end: int) -> bytes: offset += len(range_) return b"".join(ranges_contents) - def _upload_ranges(self) -> None: + def _upload_ranges(self, defer: bool) -> None: + if self.task is not None: + while not self.task.done(): + time.sleep(0.1) + if defer: + self.task = _get_deferred_executor().submit(partial(self._upload_ranges_inner, self.ranges, self.original_size)) + else: + self._upload_ranges_inner(self.ranges, self.original_size) + self.task = None + self.ranges = [range(0, self.size)] + self.original_size = self.size + self.buffer_size = 0 + + def _upload_ranges_inner(self, ranges: list[range, bytes], original_size: int) -> None: original_offset = 0 insert: list[tuple[int, bytes]] = [] delete: list[tuple[int, int]] = [] - for range_ in self.ranges: + for range_ in ranges: if isinstance(range_, range): if range_.start > original_offset: delete.append((original_offset, range_.start - original_offset)) original_offset = range_.stop else: insert.append((original_offset, range_)) - if original_offset < self.original_size: - delete.append((original_offset, self.original_size)) + if original_offset < original_size: + delete.append((original_offset, original_size - original_offset)) if insert or delete: - self.fs._api.mutate_bucket_file( + self.file_hash = self.fs._api._mutate_bucket_file( bucket_id=self.resolved_path.bucket_id, remote_path=self.resolved_path.path, insert=insert or None, delete=delete or None, + _file_hash=self.file_hash, + _file_size=self.original_size, ) self.fs.invalidate_cache( path=self.resolved_path.unresolve(), ) - self.original_size = self.size - self.ranges = [range(0, self.size)] - self.buffer_size = 0 self._details = None def url(self) -> str: @@ -1630,7 +1742,7 @@ def writable(self): """Whether opened for writing""" return not self.closed - def write(self, data: bytes | str): + def write(self, data: bytes): """ Write data to buffer. @@ -1644,7 +1756,7 @@ def write(self, data: bytes | str): """ return self.edit((self.loc, self.loc + len(data)), data) - def edit(self, byte_range: tuple[int, int], data: bytes | str): + def edit(self, byte_range: tuple[int, int], data: bytes): """ Write data to buffer. @@ -1661,10 +1773,10 @@ def edit(self, byte_range: tuple[int, int], data: bytes | str): """ if self.closed: raise ValueError("I/O operation on closed file.") - if isinstance(data, str): - data = data.encode("utf-8") + from fsspec.core import caches start, end = byte_range + end = min(self.size, end) if start == end and not data: return @@ -1705,16 +1817,19 @@ def add_range(range_: range | bytes): offset += len(range_) self.ranges = new_ranges - self.loc = byte_range[0] + len(data) + self.loc = start + len(data) self.buffer_size += len(data) - self.size += end - start + len(data) + self.size += len(data) - (end - start) + self.cache = caches[self.cache_type]( + self.blocksize, self._fetch_range, self.size, **(self.cache_options or {}) + ) if self.last_update_time is None: self.last_update_time = time.time() - if self.flush(): + if self.flush(defer=True): self.last_update_time = time.time() return len(data) - def insert(self, loc: int, data: bytes | str): + def insert(self, loc: int, data: bytes): """ Write data to buffer. @@ -1730,7 +1845,7 @@ def insert(self, loc: int, data: bytes | str): """ return self.edit((loc, loc), data) - def append(self, data: bytes | str): + def append(self, data: bytes): """ Write data to buffer. @@ -1742,7 +1857,7 @@ def append(self, data: bytes | str): data: bytes Set of bytes to be appended at the end of the file. """ - return self.edit((self.size, self.size + len(data)), data) + return self.insert(self.size, data) def delete(self, loc: int, length: int): """ @@ -1760,30 +1875,177 @@ def delete(self, loc: int, length: int): """ return self.edit((loc, loc + length), b"") - def flush(self, force=False): + def truncate(self, size: int | None = None): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + size: int | None + Resize the file to the given size in bytes. + """ + if size is None: + size = self.loc + if size < self.size: + self.delete(size, self.size - size) + elif self.size < size: + self.append(b"\0" * (size - self.size)) + return self.size + + def flush(self, force=False, defer=False): """ Write buffered data to backend store. - Writes the current buffer, if it is larger than the block-size, or if - the file is being closed. + Writes the current buffer if it being closed, or if: + - the buffer size is larger than blocksize + - AND the last update was more than 10 seconds ago + - AND the last update has finished Parameters ---------- force: bool - When closing, write the last block even if it is smaller than - blocks are allowed to be. Disallows further writing to this file. + Send the buffer even if it is smaller than + blocks are allowed to be. + defer: bool + Send the buffer in the background, non-blocking. """ if self.closed: raise ValueError("Flush on closed file") - if force or (self.buffer_size >= self.blocksize and self.last_update_time and (time.time() - self.last_update_time) > self.MIN_SECONDS_BETWEEN_UPDATES): - self._upload_ranges() + if force or ( + self.buffer_size >= self.blocksize + and self.last_update_time + and (time.time() - self.last_update_time) > self.send_interval + and (self.task is None or self.task.done()) + ): + self._upload_ranges(defer=defer) return True else: # Defer write on small block or quick update return False +class MutableTextIOWrapper(io.TextIOWrapper): + buffer: HfFileSystemMutateFile + + @property + def loc(self) -> int: + return self.buffer.loc + + def write(self, data: str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + return self.buffer.write(data.encode(self.encoding, errors=self.errors)) + + def edit(self, byte_range: tuple[int, int], data: str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + byte_range: tuple[int, int] + (start, end) where to edit the file. + data: bytes + Set of bytes to be placed in the specified range. + Size can be different than the range. + """ + return self.buffer.edit(byte_range, data.encode(self.encoding, errors=self.errors)) + + def insert(self, loc: int, data: str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + loc: int + Where to insert the data. + data: bytes + Set of bytes to be inserted. + """ + return self.buffer.insert(loc, data.encode(self.encoding, errors=self.errors)) + + def append(self, data: str): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be appended at the end of the file. + """ + return self.buffer.append(data.encode(self.encoding, errors=self.errors)) + + def delete(self, loc: int, length: int): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + loc: int + Where to insert the data. + length: int + Number of bytes to delete. + """ + return self.buffer.delete(loc, length) + + def truncate(self, size: int | None = None): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + size: int | None + Resize the file to the given size in bytes. + """ + return self.buffer.truncate(size) + + def flush(self, force=False, defer=False): + """ + Write buffered data to backend store. + + Writes the current buffer if it being closed, or if: + - the buffer size is larger than blocksize + - AND the last update was more than 10 seconds ago + - AND the last update has finished + + Parameters + ---------- + force: bool + Send the buffer even if it is smaller than + blocks are allowed to be. + defer: bool + Send the buffer in the background, non-blocking. + """ + return self.buffer.flush(force=force, defer=defer) + + def safe_revision(revision: str) -> str: return revision if SPECIAL_REFS_REVISION_REGEX.match(revision) else safe_quote(revision) @@ -1814,4 +2076,19 @@ def make_instance(cls, args, kwargs, instance_state): return fs +_DEFERRED_CLOSE_THREAD_NAME = "hffs-deferred" +_deferred_executor = None +_deferred_executor_lock = threading.Lock() + + +def _get_deferred_executor(): + global _deferred_executor + with _deferred_executor_lock: + if _deferred_executor is None: + _deferred_executor = ThreadPoolExecutor( + thread_name_prefix=_DEFERRED_CLOSE_THREAD_NAME + ) + return _deferred_executor + + hffs = HfFileSystem() From cea8d06d4e5bf69bce7a334736ca9701c69c5e61 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 26 Aug 2026 18:36:30 +0200 Subject: [PATCH 07/21] use bytearray to avoid copies --- src/huggingface_hub/hf_file_system.py | 38 ++++++++++++++++++--------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 269f239606..c446e4283f 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1605,7 +1605,7 @@ def __init__(self, # specific to HfFileSystemMutateFile self.file_hash = file_hash if file_hash is not None else self.details["xet_hash"] - self.ranges: list[range | bytes] = [range(0, self.size)] + self.ranges: list[range | bytearray] = [range(0, self.size)] self.buffer_size = 0 self.last_update_time: float | None = None self.send_interval = ( @@ -1639,7 +1639,7 @@ def _fetch_range(self, start: int, end: int) -> bytes: else: range_to_fetch = range(max(0, start - offset), len(range_) + min(0, end - offset - len(range_))) if range_to_fetch.start < range_to_fetch.stop: - ranges_contents.append(range_[range_to_fetch.start:range_to_fetch.stop]) + ranges_contents.append(bytes(range_[range_to_fetch.start:range_to_fetch.stop])) offset += len(range_) return b"".join(ranges_contents) @@ -1656,7 +1656,7 @@ def _upload_ranges(self, defer: bool) -> None: self.original_size = self.size self.buffer_size = 0 - def _upload_ranges_inner(self, ranges: list[range, bytes], original_size: int) -> None: + def _upload_ranges_inner(self, ranges: list[range | bytearray], original_size: int) -> None: original_offset = 0 insert: list[tuple[int, bytes]] = [] delete: list[tuple[int, int]] = [] @@ -1666,7 +1666,7 @@ def _upload_ranges_inner(self, ranges: list[range, bytes], original_size: int) - delete.append((original_offset, range_.start - original_offset)) original_offset = range_.stop else: - insert.append((original_offset, range_)) + insert.append((original_offset, bytes(range_))) if original_offset < original_size: delete.append((original_offset, original_size - original_offset)) if insert or delete: @@ -1780,19 +1780,31 @@ def edit(self, byte_range: tuple[int, int], data: bytes): if start == end and not data: return - new_ranges: list[range, bytes] = [] + new_ranges: list[range, bytearray] = [] offset = 0 done = False - def add_range(range_: range | bytes): - if new_ranges and isinstance(new_ranges[-1], bytes) and isinstance(range_, bytes): - new_ranges[-1] += range_ + + def add_range(range_or_bytes: range | bytearray | bytes): + """Add a new range and use byterarray appends when possible (this would make the list of ranges very long)""" + if new_ranges and isinstance(new_ranges[-1], bytearray) and isinstance(range_or_bytes, bytes): + new_ranges[-1] += range_or_bytes + else: + new_ranges.append(range_or_bytes if isinstance(range_or_bytes, (range, bytearray)) else bytearray(range_or_bytes)) + + def fast_slice(range_: range | bytearray, start=None, end=None): + """slice a range and avoid slicing a bytearray when possible (this would cause a copy)""" + if start is not None and start > 0: + return range_[start:] + elif end is not None and end < len(range_): + return range_[:end] else: - new_ranges.append(range_) + return range_ + # note: this could be optimized with an offset index for range_ in self.ranges: if offset <= start <= offset + len(range_) <= end: if offset < start: - add_range(range_[:start - offset]) + add_range(fast_slice(range_, end=start - offset)) if data and not done: add_range(data) done = True @@ -1803,15 +1815,15 @@ def add_range(range_: range | bytes): add_range(data) done = True if end < offset + len(range_): - add_range(range_[end - offset - len(range_):]) + add_range(fast_slice(range_, start=end - offset - len(range_))) elif offset <= start <= end <= offset + len(range_): if offset < start: - add_range(range_[:start - offset]) + add_range(fast_slice(range_, end=start - offset)) if data and not done: add_range(data) done = True if end < offset + len(range_): - add_range(range_[end - offset - len(range_):]) + add_range(fast_slice(range_, start=end - offset - len(range_))) else: add_range(range_) offset += len(range_) From 081d84fae035fedf95e2d42e570c259ddcf52cab Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 26 Aug 2026 18:51:02 +0200 Subject: [PATCH 08/21] docs --- src/huggingface_hub/hf_file_system.py | 45 +++++++++++++++++---------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index c446e4283f..c5c8ad9cf0 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1511,9 +1511,9 @@ class HfFileSystemMutateFile(fsspec.spec.AbstractBufferedFile): Supported mutate operations: append, edit, insert, delete, truncate. - It uses a buffer that is only sent on flush(force=True) or if 10 - seconds passed since last send and buffer is greater than or equal - to block_size. + It uses a buffer that is only sent on flush(force=True) or if buffer + is greater than or equal to block_size (and there is also a minimum + 10 second interval between sends to avoid doing too many requests). Examples: @@ -1746,8 +1746,9 @@ def write(self, data: bytes): """ Write data to buffer. - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. + Buffer only sent on flush() or if buffer is greater than or + equal to blocksize (and there is also a minimum 10 second + interval between sends to avoid doing too many requests). Parameters ---------- @@ -1760,8 +1761,9 @@ def edit(self, byte_range: tuple[int, int], data: bytes): """ Write data to buffer. - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. + Buffer only sent on flush() or if buffer is greater than or + equal to blocksize (and there is also a minimum 10 second + interval between sends to avoid doing too many requests). Parameters ---------- @@ -1845,8 +1847,9 @@ def insert(self, loc: int, data: bytes): """ Write data to buffer. - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. + Buffer only sent on flush() or if buffer is greater than or + equal to blocksize (and there is also a minimum 10 second + interval between sends to avoid doing too many requests). Parameters ---------- @@ -1861,8 +1864,9 @@ def append(self, data: bytes): """ Write data to buffer. - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. + Buffer only sent on flush() or if buffer is greater than or + equal to blocksize (and there is also a minimum 10 second + interval between sends to avoid doing too many requests). Parameters ---------- @@ -1875,8 +1879,9 @@ def delete(self, loc: int, length: int): """ Write data to buffer. - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. + Buffer only sent on flush() or if buffer is greater than or + equal to blocksize (and there is also a minimum 10 second + interval between sends to avoid doing too many requests). Parameters ---------- @@ -1891,8 +1896,9 @@ def truncate(self, size: int | None = None): """ Write data to buffer. - Buffer only sent on flush() or if buffer is greater than - or equal to blocksize. + Buffer only sent on flush() or if buffer is greater than or + equal to blocksize (and there is also a minimum 10 second + interval between sends to avoid doing too many requests). Parameters ---------- @@ -1911,7 +1917,7 @@ def flush(self, force=False, defer=False): """ Write buffered data to backend store. - Writes the current buffer if it being closed, or if: + Writes the current buffer if force=True, or if: - the buffer size is larger than blocksize - AND the last update was more than 10 seconds ago - AND the last update has finished @@ -1920,7 +1926,12 @@ def flush(self, force=False, defer=False): ---------- force: bool Send the buffer even if it is smaller than - blocks are allowed to be. + blocks are allowed to be and even if last block + was sent less than 10 seconds ago. + + If the last update wasn't finished, it waits for + it to finish before flushing. + defer: bool Send the buffer in the background, non-blocking. """ From 36b54b8cdbecd61ce75182583fe46b087a785243 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 26 Aug 2026 19:09:27 +0200 Subject: [PATCH 09/21] docs --- docs/source/en/guides/hf_file_system.md | 12 +++++++++++- docs/source/en/package_reference/hf_file_system.md | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/source/en/guides/hf_file_system.md b/docs/source/en/guides/hf_file_system.md index 0edd7cf5f4..e3156e0f3a 100644 --- a/docs/source/en/guides/hf_file_system.md +++ b/docs/source/en/guides/hf_file_system.md @@ -39,11 +39,21 @@ In addition to the [`HfApi`], the `huggingface_hub` library provides [`HfFileSys >>> with hffs.open("datasets/my-username/my-dataset-repo/data/validation.csv", "w") as f: ... f.write("text,label") ... f.write("Fantastic movie!,good") + +>>> # Append to a remote file (uses Xet to only send the new data without downloading the file - only for buckets) +>>> with hffs.open("buckets/my-username/my-bucket/validation.csv", "a") as f: +... f.write("Another fantastic review,good") + +>>> # Edit a remote file (uses Xet to only send the new data without downloading the file - only for buckets) +>>> with hffs.open("buckets/my-username/my-bucket/validation.csv", "m") as f: +... f.edit((f.size - 4, f.size), "bad") ``` The optional `revision` argument can be passed to run an operation from a specific commit such as a branch, tag name, or a commit hash. Note that `revision` is not compatible with Buckets. -Unlike Python's built-in `open`, `fsspec`'s `open` defaults to binary mode, `"rb"`. This means you must explicitly set mode as `"r"` for reading and `"w"` for writing in text mode. Appending to a file (modes `"a"` and `"ab"`) is not supported yet. +Unlike Python's built-in `open`, `fsspec`'s `open` defaults to binary mode, `"rb"`. This means you must explicitly set mode as `"r"` for reading and `"w"` for writing in text mode. + +Appending to a file (modes `"a"` and `"ab"`) is supported and very efficient thanks to Xet. Similarly, editing a file in place (mutate modes `"m"` and `"mb"` - not available in Python built-in `open`) is supported and allows efficient operations `edit`, `append`, `insert`, `delete` and `truncate` via [`HfFileSystemMutateFile`]. ## Integrations diff --git a/docs/source/en/package_reference/hf_file_system.md b/docs/source/en/package_reference/hf_file_system.md index eefc2c2234..8305345b29 100644 --- a/docs/source/en/package_reference/hf_file_system.md +++ b/docs/source/en/package_reference/hf_file_system.md @@ -11,3 +11,9 @@ The `HfFileSystem` class provides a pythonic file interface to the Hugging Face `HfFileSystem` is based on [fsspec](https://filesystem-spec.readthedocs.io/en/latest/), so it is compatible with most of the APIs that it offers. For more details, check out [our guide](../guides/hf_file_system) and fsspec's [API Reference](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem). [[autodoc]] HfFileSystem + +## HfFileSystemMutateFile + +In addition to regular file-like objects obtained using open modes "w", "wb", "r" or "rb" to read and overwrite files, `HfFileSystem` also offers open modes "a" and "ab" to append to an existing file and "m" and "mb" to edit an existing file in-place. + +[[autodoc]] HfFileSystemMutateFile From 230a974cc62c5d2d6bc9fc961ada3694249dced3 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 26 Aug 2026 19:20:59 +0200 Subject: [PATCH 10/21] style and quality --- src/huggingface_hub/hf_api.py | 35 +++------ src/huggingface_hub/hf_file_system.py | 109 +++++++++++++------------- 2 files changed, 67 insertions(+), 77 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 94a52c4f5e..9d278761af 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -14989,19 +14989,24 @@ def mutate_bucket_file( List of deletes to apply, in the form `(loc, length)`. The range [`loc`, `loc + length`) is deleted. """ + if edit: + edit = [ + ((start, end), data) for (start, end), data in edit if start < end or (start == end and len(data) > 0) + ] + if insert: + insert = [(loc, data) for loc, data in insert if len(data) > 0] + if delete: + delete = [(loc, length) for loc, length in delete if length > 0] + if not (edit or insert or delete): + return from .utils._xet_progress_reporting import XetUploadProgressReporter - if not are_progress_bars_disabled(): _progress = XetUploadProgressReporter(total_files=1) else: _progress = None - file_metadata = self.get_bucket_file_metadata( - bucket_id=bucket_id, - remote_path=remote_path, - token=token - ) + file_metadata = self.get_bucket_file_metadata(bucket_id=bucket_id, remote_path=remote_path, token=token) self._mutate_bucket_file( bucket_id=bucket_id, remote_path=remote_path, @@ -15010,10 +15015,9 @@ def mutate_bucket_file( delete=delete, _progress=_progress, _file_hash=file_metadata.xet_file_data.file_hash, - _file_size=file_metadata.size + _file_size=file_metadata.size, ) - def _mutate_bucket_file( self, *, @@ -15040,15 +15044,6 @@ def _mutate_bucket_file( ) from .utils._xet_progress_reporting import XetUploadProgressReporter - if edit: - edit = [((start, end), data) for (start, end), data in edit if start < end or (start == end and len(data) > 0)] - if insert: - insert = [(loc, data) for loc, data in insert if len(data) > 0] - if delete: - delete = [(loc, length) for loc, length in delete if length > 0] - if not (edit or insert or delete): - return - owns_progress = _progress is None if _progress is not None: progress = _progress @@ -15061,11 +15056,7 @@ def _mutate_bucket_file( progress, progress_callback = None, None if _file_hash is None or _file_size is None: - file_metadata = self.get_bucket_file_metadata( - bucket_id=bucket_id, - remote_path=remote_path, - token=token - ) + file_metadata = self.get_bucket_file_metadata(bucket_id=bucket_id, remote_path=remote_path, token=token) file_hash = file_metadata.xet_file_data.file_hash file_size = file_metadata.size else: diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index c5c8ad9cf0..58e611042e 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -13,7 +13,7 @@ from functools import partial from itertools import chain from pathlib import Path, PurePosixPath -from typing import Any, Literal, NoReturn, Union, overload +from typing import Any, Literal, NoReturn, Union, overload, override from urllib.parse import quote, unquote import fsspec @@ -398,8 +398,7 @@ def open( cache_options=None, compression=None, **kwargs, - ) -> "HfFileSystemMutateFile": - ... + ) -> "HfFileSystemMutateFile": ... @overload def open( @@ -410,8 +409,7 @@ def open( cache_options=None, compression=None, **kwargs, - ) -> "MutableTextIOWrapper": - ... + ) -> "MutableTextIOWrapper": ... @overload def open( @@ -422,8 +420,7 @@ def open( cache_options=None, compression=None, **kwargs, - ) -> io.TextIOWrapper: - ... + ) -> io.TextIOWrapper: ... @overload def open( @@ -434,10 +431,10 @@ def open( cache_options=None, compression=None, **kwargs, - ) -> fsspec.spec.AbstractBufferedFile: - ... + ) -> fsspec.spec.AbstractBufferedFile: ... - def open( + @override + def open( # ty: ignore[invalid-method-override] self, path, mode="rb", @@ -445,7 +442,7 @@ def open( cache_options=None, compression=None, **kwargs, - ) -> fsspec.spec.AbstractBufferedFile: + ) -> fsspec.spec.AbstractBufferedFile | "HfFileSystemMutateFile" | io.TextIOWrapper | "MutableTextIOWrapper": """ Return a file-like object from the filesystem @@ -476,11 +473,7 @@ def open( raise NotImplementedError(f"Mode '{mutate_mode}' with compression is not implemented") if "b" not in mode: mode = mode.replace("t", "") + "b" - text_kwargs = { - k: kwargs.pop(k) - for k in ["encoding", "errors", "newline"] - if k in kwargs - } + text_kwargs = {k: kwargs.pop(k) for k in ["encoding", "errors", "newline"] if k in kwargs} buffer = super().open( path, mode=mode, @@ -506,7 +499,7 @@ def _open( # type: ignore block_size: int | None = None, revision: str | None = None, **kwargs, - ) -> Union["HfFileSystemFile", "HfFileSystemStreamFile"]: + ) -> Union["HfFileSystemFile", "HfFileSystemStreamFile", "HfFileSystemMutateFile"]: block_size = block_size if block_size is not None else self.block_size if block_size is not None: kwargs["block_size"] = block_size @@ -1554,21 +1547,22 @@ class HfFileSystemMutateFile(fsspec.spec.AbstractBufferedFile): ``` """ - DEFAULT_SEND_INTERVAL = 10. - - def __init__(self, - fs: HfFileSystem, - path: str, - mode: str = "rb", - block_size: str | int | None = "default", - send_interval: str | int | None = "default", - autocommit: bool=True, - cache_type: str | None ="readahead", - cache_options: dict | None =None, - size: int | None = None, - file_hash: str | None = None, - **kwargs, - ): + DEFAULT_SEND_INTERVAL = 10.0 + + def __init__( + self, + fs: HfFileSystem, + path: str, + mode: str = "rb", + block_size: Literal["default"] | int | None = "default", + send_interval: Literal["default"] | int | None = "default", + autocommit: bool = True, + cache_type: str | None = "readahead", + cache_options: dict | None = None, + size: int | None = None, + file_hash: str | None = None, + **kwargs, + ): from fsspec.core import caches if mode not in {"mb", "ab"}: @@ -1587,9 +1581,7 @@ def __init__(self, self.path = path self.fs = fs self.mode = mode - self.blocksize = ( - self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size - ) + self.blocksize = self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size self.autocommit = autocommit self.closed = False self.forced = False @@ -1597,9 +1589,7 @@ def __init__(self, self.original_size = self.size self.cache_type = cache_type self.cache_options = cache_options - self.cache = caches[cache_type]( - self.blocksize, self._fetch_range, self.size, **(cache_options or {}) - ) + self.cache = caches[cache_type](self.blocksize, self._fetch_range, self.size, **(cache_options or {})) self.loc = self.size if "a" in mode else 0 self.kwargs = kwargs @@ -1608,9 +1598,7 @@ def __init__(self, self.ranges: list[range | bytearray] = [range(0, self.size)] self.buffer_size = 0 self.last_update_time: float | None = None - self.send_interval = ( - self.DEFAULT_SEND_INTERVAL if send_interval in ["default", None] else send_interval - ) + self.send_interval = self.DEFAULT_SEND_INTERVAL if send_interval in ["default", None] else send_interval self.task = None def __del__(self): @@ -1621,12 +1609,16 @@ def __del__(self): def _fetch_range(self, start: int, end: int) -> bytes: if self.task is not None and not self.task.done(): - raise NotImplementedError("Attempted to read a file while blocks are being sent but this is not implemented. Use f.flush(force=True) to send blocks first.") + raise NotImplementedError( + "Attempted to read a file while blocks are being sent but this is not implemented. Use f.flush(force=True) to send blocks first." + ) ranges_contents: list[bytes] = [] offset = 0 for range_ in self.ranges: if isinstance(range_, range): - range_to_fetch = range(range_.start + max(0, start - offset), range_.stop + min(0, end - offset - len(range_))) + range_to_fetch = range( + range_.start + max(0, start - offset), range_.stop + min(0, end - offset - len(range_)) + ) if range_to_fetch.start < range_to_fetch.stop: headers = { "range": f"bytes={range_to_fetch.start}-{range_to_fetch.stop - 1}", @@ -1639,7 +1631,7 @@ def _fetch_range(self, start: int, end: int) -> bytes: else: range_to_fetch = range(max(0, start - offset), len(range_) + min(0, end - offset - len(range_))) if range_to_fetch.start < range_to_fetch.stop: - ranges_contents.append(bytes(range_[range_to_fetch.start:range_to_fetch.stop])) + ranges_contents.append(bytes(range_[range_to_fetch.start : range_to_fetch.stop])) offset += len(range_) return b"".join(ranges_contents) @@ -1648,7 +1640,9 @@ def _upload_ranges(self, defer: bool) -> None: while not self.task.done(): time.sleep(0.1) if defer: - self.task = _get_deferred_executor().submit(partial(self._upload_ranges_inner, self.ranges, self.original_size)) + self.task = _get_deferred_executor().submit( + partial(self._upload_ranges_inner, self.ranges, self.original_size) + ) else: self._upload_ranges_inner(self.ranges, self.original_size) self.task = None @@ -1696,7 +1690,12 @@ def read(self, length=-1): If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. """ - if self.mode == "rb" and (length is None or length == -1) and self.loc == 0 and self.ranges == [range(0, self.original_size)]: + if ( + self.mode == "rb" + and (length is None or length == -1) + and self.loc == 0 + and self.ranges == [range(0, self.original_size)] + ): with self.fs.open(self.path, "rb", block_size=0) as f: # block_size=0 enables fast streaming out = f.read() self.loc += len(out) @@ -1782,7 +1781,7 @@ def edit(self, byte_range: tuple[int, int], data: bytes): if start == end and not data: return - new_ranges: list[range, bytearray] = [] + new_ranges: list[range | bytearray] = [] offset = 0 done = False @@ -1791,7 +1790,9 @@ def add_range(range_or_bytes: range | bytearray | bytes): if new_ranges and isinstance(new_ranges[-1], bytearray) and isinstance(range_or_bytes, bytes): new_ranges[-1] += range_or_bytes else: - new_ranges.append(range_or_bytes if isinstance(range_or_bytes, (range, bytearray)) else bytearray(range_or_bytes)) + new_ranges.append( + range_or_bytes if isinstance(range_or_bytes, (range, bytearray)) else bytearray(range_or_bytes) + ) def fast_slice(range_: range | bytearray, start=None, end=None): """slice a range and avoid slicing a bytearray when possible (this would cause a copy)""" @@ -1970,7 +1971,7 @@ def write(self, data: str): data: bytes Set of bytes to be written. """ - return self.buffer.write(data.encode(self.encoding, errors=self.errors)) + return self.buffer.write(data.encode(self.encoding, errors=self.errors or "strict")) def edit(self, byte_range: tuple[int, int], data: str): """ @@ -1987,7 +1988,7 @@ def edit(self, byte_range: tuple[int, int], data: str): Set of bytes to be placed in the specified range. Size can be different than the range. """ - return self.buffer.edit(byte_range, data.encode(self.encoding, errors=self.errors)) + return self.buffer.edit(byte_range, data.encode(self.encoding, errors=self.errors or "strict")) def insert(self, loc: int, data: str): """ @@ -2003,7 +2004,7 @@ def insert(self, loc: int, data: str): data: bytes Set of bytes to be inserted. """ - return self.buffer.insert(loc, data.encode(self.encoding, errors=self.errors)) + return self.buffer.insert(loc, data.encode(self.encoding, errors=self.errors or "strict")) def append(self, data: str): """ @@ -2017,7 +2018,7 @@ def append(self, data: str): data: bytes Set of bytes to be appended at the end of the file. """ - return self.buffer.append(data.encode(self.encoding, errors=self.errors)) + return self.buffer.append(data.encode(self.encoding, errors=self.errors or "strict")) def delete(self, loc: int, length: int): """ @@ -2108,9 +2109,7 @@ def _get_deferred_executor(): global _deferred_executor with _deferred_executor_lock: if _deferred_executor is None: - _deferred_executor = ThreadPoolExecutor( - thread_name_prefix=_DEFERRED_CLOSE_THREAD_NAME - ) + _deferred_executor = ThreadPoolExecutor(thread_name_prefix=_DEFERRED_CLOSE_THREAD_NAME) return _deferred_executor From ab8007a1281324b6e58b38e08cb437b6151c484a Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 2 Sep 2026 14:31:07 +0200 Subject: [PATCH 11/21] docstrings --- src/huggingface_hub/hf_file_system.py | 202 ++++++++++++-------------- 1 file changed, 93 insertions(+), 109 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 58e611042e..cfb51a4cbf 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -41,8 +41,8 @@ class HfFileSystemResolvedPath: """Top level Data structure containing information about a resolved Hugging Face file system path.""" - root: str - path: str + root (`str`): + path (`str`): def unresolve(self) -> str: return f"{self.root}/{self.path}".rstrip("/") @@ -52,10 +52,10 @@ def unresolve(self) -> str: class HfFileSystemResolvedRepositoryPath(HfFileSystemResolvedPath): """Data structure containing information about a resolved path in a repository.""" - repo_type: str - repo_id: str - revision: str - path_in_repo: str + repo_type (`str`): + repo_id (`str`): + revision (`str`): + path_in_repo (`str`): root: str = field(init=False) path: str = field(init=False) # The part placed after '@' in the initial path. It can be a quoted or unquoted refs revision. @@ -77,7 +77,7 @@ def __post_init__(self): class HfFileSystemResolvedBucketPath(HfFileSystemResolvedPath): """Data structure containing information about a resolved path in a bucket.""" - bucket_id: str + bucket_id (`str`): root: str = field(init=False) def __post_init__(self): @@ -449,23 +449,22 @@ def open( # ty: ignore[invalid-method-override] The resultant instance must function correctly in a context ``with`` block. - Parameters - ---------- - path: str - Target file - mode: str like 'rb', 'w', 'a' - See builtin ``open()``. - There is an extra mode 'mb' (mutate bytes in-place) allows insert(), delete(), edit() and append(). - It is available thanks to Xet which stores files by chunks. - block_size: int - Some indication of buffering - this is a value in bytes - cache_options : dict, optional - Extra arguments to pass through to the cache. - compression: string or None - If given, open file using compression codec. Can either be a compression - name (a key in ``fsspec.compression.compr``) or "infer" to guess the - compression from the filename suffix. - encoding, errors, newline: passed on to TextIOWrapper for text mode + Args: + path (`str`): + Target file + mode: str like 'rb', 'w', 'a' + See builtin ``open()``. + There is an extra mode 'mb' (mutate bytes in-place) allows insert(), delete(), edit() and append(). + It is available thanks to Xet which stores files by chunks. + block_size (`int`): + Some indication of buffering - this is a value in bytes + cache_options : dict, optional + Extra arguments to pass through to the cache. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding, errors, newline: passed on to TextIOWrapper for text mode """ mutate_mode = "m" if "m" in mode else ("a" if "a" in mode else None) if mutate_mode: @@ -1716,12 +1715,11 @@ def read(self, length=-1): def seek(self, loc, whence=0): """Set current file location - Parameters - ---------- - loc: int - byte location - whence: {0, 1, 2} - from start of file, current location or end of file, resp. + Args: + loc (`int`): + byte location + whence (`int`, one of 0, 1 or 2): + from start of file, current location or end of file, resp. """ loc = int(loc) if whence == 0: @@ -1749,10 +1747,9 @@ def write(self, data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). - Parameters - ---------- - data: bytes - Set of bytes to be written. + Args: + data (`bytes`): + Set of bytes to be written. """ return self.edit((self.loc, self.loc + len(data)), data) @@ -1764,13 +1761,12 @@ def edit(self, byte_range: tuple[int, int], data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). - Parameters - ---------- - byte_range: tuple[int, int] - (start, end) where to edit the file. - data: bytes - Set of bytes to be placed in the specified range. - Size can be different than the range. + Args: + byte_range (`tuple[int, int]`): + (start, end) where to edit the file. + data (`bytes`): + Set of bytes to be placed in the specified range. + Size can be different than the range. """ if self.closed: raise ValueError("I/O operation on closed file.") @@ -1852,12 +1848,11 @@ def insert(self, loc: int, data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). - Parameters - ---------- - loc: int - Where to insert the data. - data: bytes - Set of bytes to be inserted. + Args: + loc (`int`): + Where to insert the data. + data (`bytes`): + Set of bytes to be inserted. """ return self.edit((loc, loc), data) @@ -1869,10 +1864,9 @@ def append(self, data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). - Parameters - ---------- - data: bytes - Set of bytes to be appended at the end of the file. + Args: + data (`bytes`): + Set of bytes to be appended at the end of the file. """ return self.insert(self.size, data) @@ -1884,12 +1878,11 @@ def delete(self, loc: int, length: int): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). - Parameters - ---------- - loc: int - Where to insert the data. - length: int - Number of bytes to delete. + Args: + loc (`int`): + Where to insert the data. + length (`int`): + Number of bytes to delete. """ return self.edit((loc, loc + length), b"") @@ -1901,10 +1894,9 @@ def truncate(self, size: int | None = None): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). - Parameters - ---------- - size: int | None - Resize the file to the given size in bytes. + Args: + size (`int`, *optional*): + Resize the file to the given size in bytes. """ if size is None: size = self.loc @@ -1923,18 +1915,17 @@ def flush(self, force=False, defer=False): - AND the last update was more than 10 seconds ago - AND the last update has finished - Parameters - ---------- - force: bool - Send the buffer even if it is smaller than - blocks are allowed to be and even if last block - was sent less than 10 seconds ago. + Args: + force (`bool`): + Send the buffer even if it is smaller than + blocks are allowed to be and even if last block + was sent less than 10 seconds ago. - If the last update wasn't finished, it waits for - it to finish before flushing. + If the last update wasn't finished, it waits for + it to finish before flushing. - defer: bool - Send the buffer in the background, non-blocking. + defer (`bool`): + Send the buffer in the background, non-blocking. """ if self.closed: @@ -1966,10 +1957,9 @@ def write(self, data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. - Parameters - ---------- - data: bytes - Set of bytes to be written. + Args: + data (`str`): + String to be written. """ return self.buffer.write(data.encode(self.encoding, errors=self.errors or "strict")) @@ -1980,13 +1970,12 @@ def edit(self, byte_range: tuple[int, int], data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. - Parameters - ---------- - byte_range: tuple[int, int] - (start, end) where to edit the file. - data: bytes - Set of bytes to be placed in the specified range. - Size can be different than the range. + Args: + byte_range (`tuple[int, int]`): + (start, end) where to edit the file. + data (`str`): + String to be placed in the specified range. + Size can be different than the range. """ return self.buffer.edit(byte_range, data.encode(self.encoding, errors=self.errors or "strict")) @@ -1997,12 +1986,11 @@ def insert(self, loc: int, data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. - Parameters - ---------- - loc: int - Where to insert the data. - data: bytes - Set of bytes to be inserted. + Args: + loc (`int`): + Where to insert the data. + data (`str`): + String to be inserted. """ return self.buffer.insert(loc, data.encode(self.encoding, errors=self.errors or "strict")) @@ -2013,10 +2001,9 @@ def append(self, data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. - Parameters - ---------- - data: bytes - Set of bytes to be appended at the end of the file. + Args: + data (`str`): + String to be appended at the end of the file. """ return self.buffer.append(data.encode(self.encoding, errors=self.errors or "strict")) @@ -2027,12 +2014,11 @@ def delete(self, loc: int, length: int): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. - Parameters - ---------- - loc: int - Where to insert the data. - length: int - Number of bytes to delete. + Args: + loc (`int`): + Where to insert the data. + length (`int`): + Number of bytes to delete. """ return self.buffer.delete(loc, length) @@ -2043,10 +2029,9 @@ def truncate(self, size: int | None = None): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. - Parameters - ---------- - size: int | None - Resize the file to the given size in bytes. + Args: + size (`int`, *optional*): + Resize the file to the given size in bytes. """ return self.buffer.truncate(size) @@ -2059,13 +2044,12 @@ def flush(self, force=False, defer=False): - AND the last update was more than 10 seconds ago - AND the last update has finished - Parameters - ---------- - force: bool - Send the buffer even if it is smaller than - blocks are allowed to be. - defer: bool - Send the buffer in the background, non-blocking. + Args: + force (`bool`): + Send the buffer even if it is smaller than + blocks are allowed to be. + defer (`bool`): + Send the buffer in the background, non-blocking. """ return self.buffer.flush(force=force, defer=defer) From c419b263253d817ce84ce75975c4e1e6f656f193 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 2 Sep 2026 14:42:48 +0200 Subject: [PATCH 12/21] mutate "m" -> edit "e" --- docs/source/en/guides/hf_file_system.md | 4 +- .../en/package_reference/hf_file_system.md | 6 +-- src/huggingface_hub/hf_api.py | 12 +++--- src/huggingface_hub/hf_file_system.py | 42 +++++++++---------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/docs/source/en/guides/hf_file_system.md b/docs/source/en/guides/hf_file_system.md index e3156e0f3a..1ab9fd3e6f 100644 --- a/docs/source/en/guides/hf_file_system.md +++ b/docs/source/en/guides/hf_file_system.md @@ -45,7 +45,7 @@ In addition to the [`HfApi`], the `huggingface_hub` library provides [`HfFileSys ... f.write("Another fantastic review,good") >>> # Edit a remote file (uses Xet to only send the new data without downloading the file - only for buckets) ->>> with hffs.open("buckets/my-username/my-bucket/validation.csv", "m") as f: +>>> with hffs.open("buckets/my-username/my-bucket/validation.csv", "e") as f: ... f.edit((f.size - 4, f.size), "bad") ``` @@ -53,7 +53,7 @@ The optional `revision` argument can be passed to run an operation from a specif Unlike Python's built-in `open`, `fsspec`'s `open` defaults to binary mode, `"rb"`. This means you must explicitly set mode as `"r"` for reading and `"w"` for writing in text mode. -Appending to a file (modes `"a"` and `"ab"`) is supported and very efficient thanks to Xet. Similarly, editing a file in place (mutate modes `"m"` and `"mb"` - not available in Python built-in `open`) is supported and allows efficient operations `edit`, `append`, `insert`, `delete` and `truncate` via [`HfFileSystemMutateFile`]. +Appending to a file (modes `"a"` and `"ab"`) is supported and very efficient thanks to Xet. Similarly, editing a file in place (edit modes `"e"` and `"eb"` - not available in Python built-in `open`) is supported and allows efficient operations `edit`, `append`, `insert`, `delete` and `truncate` via [`HfFileSystemEditFile`]. ## Integrations diff --git a/docs/source/en/package_reference/hf_file_system.md b/docs/source/en/package_reference/hf_file_system.md index 8305345b29..c6d2a94b78 100644 --- a/docs/source/en/package_reference/hf_file_system.md +++ b/docs/source/en/package_reference/hf_file_system.md @@ -12,8 +12,8 @@ The `HfFileSystem` class provides a pythonic file interface to the Hugging Face [[autodoc]] HfFileSystem -## HfFileSystemMutateFile +## HfFileSystemEditFile -In addition to regular file-like objects obtained using open modes "w", "wb", "r" or "rb" to read and overwrite files, `HfFileSystem` also offers open modes "a" and "ab" to append to an existing file and "m" and "mb" to edit an existing file in-place. +In addition to regular file-like objects obtained using open modes "w", "wb", "r" or "rb" to read and overwrite files, `HfFileSystem` also offers open modes "a" and "ab" to append to an existing file and "e" and "eb" to edit an existing file in-place. -[[autodoc]] HfFileSystemMutateFile +[[autodoc]] HfFileSystemEditFile diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 9d278761af..6fc339d491 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -14958,7 +14958,7 @@ def sync_bucket( token=token, ) - def mutate_bucket_file( + def edit_bucket_file( self, *, bucket_id: str, @@ -14968,15 +14968,15 @@ def mutate_bucket_file( delete: list[tuple[int, int]] | None = None, token: bool | str | None = None, ) -> None: - """Mutate an existing file in a bucket in-place, only re-uploading the parts the caller actually rewrites. + """Edit an existing file in a bucket in-place, only re-uploading the parts the caller actually rewrites. - See [`HfFileSystemMutateFile`] to use this feature with a file-like API. + See [`HfFileSystemEditFile`] to use this feature with a file-like API. Args: bucket_id (`str`): The ID of the bucket (e.g. `"username/my-bucket"`). remote_path (`str`): - the file to mutate. + The file to edit. edit (`list[tuple[tuple[int, int], bytes]]`, *optional*): List edits to apply, in the form `((start, end), data)`. Ranges [`start`, `end`) are replaced with `data`, which can @@ -15007,7 +15007,7 @@ def mutate_bucket_file( _progress = None file_metadata = self.get_bucket_file_metadata(bucket_id=bucket_id, remote_path=remote_path, token=token) - self._mutate_bucket_file( + self._edit_bucket_file( bucket_id=bucket_id, remote_path=remote_path, edit=edit, @@ -15018,7 +15018,7 @@ def mutate_bucket_file( _file_size=file_metadata.size, ) - def _mutate_bucket_file( + def _edit_bucket_file( self, *, bucket_id: str, diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index cfb51a4cbf..5741717db9 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -393,18 +393,18 @@ def invalidate_cache(self, path: str | None = None) -> None: def open( self, path, - mode: Literal["ab", "mb"], + mode: Literal["ab", "eb"], block_size=None, cache_options=None, compression=None, **kwargs, - ) -> "HfFileSystemMutateFile": ... + ) -> "HfFileSystemEditFile": ... @overload def open( self, path, - mode: Literal["a", "m", "at", "mt"], + mode: Literal["a", "e", "at", "mt"], block_size=None, cache_options=None, compression=None, @@ -442,7 +442,7 @@ def open( # ty: ignore[invalid-method-override] cache_options=None, compression=None, **kwargs, - ) -> fsspec.spec.AbstractBufferedFile | "HfFileSystemMutateFile" | io.TextIOWrapper | "MutableTextIOWrapper": + ) -> fsspec.spec.AbstractBufferedFile | "HfFileSystemEditFile" | io.TextIOWrapper | "MutableTextIOWrapper": """ Return a file-like object from the filesystem @@ -454,7 +454,7 @@ def open( # ty: ignore[invalid-method-override] Target file mode: str like 'rb', 'w', 'a' See builtin ``open()``. - There is an extra mode 'mb' (mutate bytes in-place) allows insert(), delete(), edit() and append(). + There is an extra mode 'mb' (edit bytes in-place) allows insert(), delete(), edit() and append(). It is available thanks to Xet which stores files by chunks. block_size (`int`): Some indication of buffering - this is a value in bytes @@ -466,10 +466,10 @@ def open( # ty: ignore[invalid-method-override] compression from the filename suffix. encoding, errors, newline: passed on to TextIOWrapper for text mode """ - mutate_mode = "m" if "m" in mode else ("a" if "a" in mode else None) - if mutate_mode: + edit_mode = "e" if "e" in mode else ("a" if "a" in mode else None) + if edit_mode: if compression is not None: - raise NotImplementedError(f"Mode '{mutate_mode}' with compression is not implemented") + raise NotImplementedError(f"Mode '{edit_mode}' with compression is not implemented") if "b" not in mode: mode = mode.replace("t", "") + "b" text_kwargs = {k: kwargs.pop(k) for k in ["encoding", "errors", "newline"] if k in kwargs} @@ -498,12 +498,12 @@ def _open( # type: ignore block_size: int | None = None, revision: str | None = None, **kwargs, - ) -> Union["HfFileSystemFile", "HfFileSystemStreamFile", "HfFileSystemMutateFile"]: + ) -> Union["HfFileSystemFile", "HfFileSystemStreamFile", "HfFileSystemEditFile"]: block_size = block_size if block_size is not None else self.block_size if block_size is not None: kwargs["block_size"] = block_size - if "a" in mode or "m" in mode: - return HfFileSystemMutateFile(self, path, mode=mode, **kwargs) + if "a" in mode or "e" in mode: + return HfFileSystemEditFile(self, path, mode=mode, **kwargs) if block_size == 0: return HfFileSystemStreamFile(self, path, mode=mode, revision=revision, **kwargs) else: @@ -1497,11 +1497,11 @@ def _open_connection(self): self._stream_iterator = self.response.iter_bytes() -class HfFileSystemMutateFile(fsspec.spec.AbstractBufferedFile): +class HfFileSystemEditFile(fsspec.spec.AbstractBufferedFile): """Mutate a file in a bucket in-place, only re-uploading the parts the caller actually rewrites. - Supported mutate operations: append, edit, insert, delete, truncate. + Supported edit operations: append(), edit(), insert(), delete(), truncate(). It uses a buffer that is only sent on flush(force=True) or if buffer is greater than or equal to block_size (and there is also a minimum @@ -1519,24 +1519,24 @@ class HfFileSystemMutateFile(fsspec.spec.AbstractBufferedFile): f.write(log) ``` - - Edit a file header using the mutate mode "m": + - Edit a file header using the edit mode "e": ```py from huggingface_hub import hffs header_length = 16 new_header = b"MY_NEW_HEADER_00" - with hffs.open("buckets/username/my-bucket/data.bin", "mb") as f: + with hffs.open("buckets/username/my-bucket/data.bin", "eb") as f: f.edit((0, header_length), new_header) ``` - - Remove a certain line using the mutate mode "m": + - Remove a certain line using the edit mode "e": ```py from huggingface_hub import hffs line_idx_to_remove = 42 - with hffs.open("buckets/username/my-bucket/doc.txt", "m") as f: + with hffs.open("buckets/username/my-bucket/doc.txt", "e") as f: for i, line in enumerate(f): if i == line_idx_to_remove: line_loc = f.loc - len(line) @@ -1564,7 +1564,7 @@ def __init__( ): from fsspec.core import caches - if mode not in {"mb", "ab"}: + if mode not in {"eb", "ab"}: raise NotImplementedError("File mode not supported") resolved_path = fs.resolve_path(path) if not isinstance(resolved_path, HfFileSystemResolvedBucketPath): @@ -1592,7 +1592,7 @@ def __init__( self.loc = self.size if "a" in mode else 0 self.kwargs = kwargs - # specific to HfFileSystemMutateFile + # specific to HfFileSystemEditFile self.file_hash = file_hash if file_hash is not None else self.details["xet_hash"] self.ranges: list[range | bytearray] = [range(0, self.size)] self.buffer_size = 0 @@ -1663,7 +1663,7 @@ def _upload_ranges_inner(self, ranges: list[range | bytearray], original_size: i if original_offset < original_size: delete.append((original_offset, original_size - original_offset)) if insert or delete: - self.file_hash = self.fs._api._mutate_bucket_file( + self.file_hash = self.fs._api._edit_bucket_file( bucket_id=self.resolved_path.bucket_id, remote_path=self.resolved_path.path, insert=insert or None, @@ -1944,7 +1944,7 @@ def flush(self, force=False, defer=False): class MutableTextIOWrapper(io.TextIOWrapper): - buffer: HfFileSystemMutateFile + buffer: HfFileSystemEditFile @property def loc(self) -> int: From fc8ae36db93e3f1b3816783909db678efcbb9a10 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 2 Sep 2026 15:42:52 +0200 Subject: [PATCH 13/21] minor --- src/huggingface_hub/hf_file_system.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 5741717db9..616da8a797 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -409,7 +409,7 @@ def open( cache_options=None, compression=None, **kwargs, - ) -> "MutableTextIOWrapper": ... + ) -> "EditTextIOWrapper": ... @overload def open( @@ -442,7 +442,7 @@ def open( # ty: ignore[invalid-method-override] cache_options=None, compression=None, **kwargs, - ) -> fsspec.spec.AbstractBufferedFile | "HfFileSystemEditFile" | io.TextIOWrapper | "MutableTextIOWrapper": + ) -> fsspec.spec.AbstractBufferedFile | "HfFileSystemEditFile" | io.TextIOWrapper | "EditTextIOWrapper": """ Return a file-like object from the filesystem @@ -481,7 +481,7 @@ def open( # ty: ignore[invalid-method-override] compression=compression, **kwargs, ) - return MutableTextIOWrapper(buffer, **text_kwargs, write_through=True) + return EditTextIOWrapper(buffer, **text_kwargs, write_through=True) return super().open( path, mode=mode, @@ -1530,19 +1530,16 @@ class HfFileSystemEditFile(fsspec.spec.AbstractBufferedFile): f.edit((0, header_length), new_header) ``` - - Remove a certain line using the edit mode "e": + - Remove a line using the edit mode "e": ```py from huggingface_hub import hffs - line_idx_to_remove = 42 with hffs.open("buckets/username/my-bucket/doc.txt", "e") as f: - for i, line in enumerate(f): - if i == line_idx_to_remove: - line_loc = f.loc - len(line) - line_length = len(line) - f.delete(line_loc, line_length) + for line in f: + if line == "this is a bad line\n": break + f.delete(loc=f.loc - len(line), length=len(line)) ``` """ @@ -1943,7 +1940,7 @@ def flush(self, force=False, defer=False): return False -class MutableTextIOWrapper(io.TextIOWrapper): +class EditTextIOWrapper(io.TextIOWrapper): buffer: HfFileSystemEditFile @property From 5395515d5a250e34b2a9c9ac3b94237c70653b70 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 2 Sep 2026 15:58:20 +0200 Subject: [PATCH 14/21] fix bad search and replace --- src/huggingface_hub/hf_file_system.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 616da8a797..2b53ed9cb2 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -41,8 +41,8 @@ class HfFileSystemResolvedPath: """Top level Data structure containing information about a resolved Hugging Face file system path.""" - root (`str`): - path (`str`): + root: str + path: str def unresolve(self) -> str: return f"{self.root}/{self.path}".rstrip("/") @@ -52,10 +52,10 @@ def unresolve(self) -> str: class HfFileSystemResolvedRepositoryPath(HfFileSystemResolvedPath): """Data structure containing information about a resolved path in a repository.""" - repo_type (`str`): - repo_id (`str`): - revision (`str`): - path_in_repo (`str`): + repo_type: str + repo_id: str + revision: str + path_in_repo: str root: str = field(init=False) path: str = field(init=False) # The part placed after '@' in the initial path. It can be a quoted or unquoted refs revision. @@ -77,7 +77,7 @@ def __post_init__(self): class HfFileSystemResolvedBucketPath(HfFileSystemResolvedPath): """Data structure containing information about a resolved path in a bucket.""" - bucket_id (`str`): + bucket_id: str root: str = field(init=False) def __post_init__(self): From d5a4fd5aaabee84a122e393357f6b2233c2ebd2b Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 2 Sep 2026 15:59:11 +0200 Subject: [PATCH 15/21] fix typing --- src/huggingface_hub/hf_file_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 2b53ed9cb2..3a29b78f6f 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -442,7 +442,7 @@ def open( # ty: ignore[invalid-method-override] cache_options=None, compression=None, **kwargs, - ) -> fsspec.spec.AbstractBufferedFile | "HfFileSystemEditFile" | io.TextIOWrapper | "EditTextIOWrapper": + ) -> Union[fsspec.spec.AbstractBufferedFile, "HfFileSystemEditFile", io.TextIOWrapper, "EditTextIOWrapper"]: """ Return a file-like object from the filesystem From 9e215227b468a19f7c54e14a1b71393aaf177168 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 2 Sep 2026 17:55:03 +0200 Subject: [PATCH 16/21] implement cp_file() --- src/huggingface_hub/hf_api.py | 1 + src/huggingface_hub/hf_file_system.py | 29 ++++++++++++++++++++++++++- tests/test_buckets_hf_file_system.py | 4 ---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 6fc339d491..04a576ce5b 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -14635,6 +14635,7 @@ def _payload_as_ndjson() -> Iterable[bytes]: "xetHash": op.xet_hash, "sourceRepoType": op.source_repo_type, "sourceRepoId": op.source_repo_id, + "mtime": op.mtime, } else: payload = { diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 3a29b78f6f..506ea1e596 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -920,7 +920,34 @@ def cp_file(self, path1: str, path2: str, revision: str | None = None, **kwargs) if isinstance(resolved_path1, HfFileSystemResolvedBucketPath) or isinstance( resolved_path2, HfFileSystemResolvedBucketPath ): - raise NotImplementedError("Copy from/to buckets is not available yet") + if not isinstance(resolved_path1, HfFileSystemResolvedBucketPath) or not isinstance( + resolved_path2, HfFileSystemResolvedBucketPath + ): + raise NotImplementedError("Copy between repos and buckets is not available yet") + if resolved_path1.bucket_id != resolved_path2.bucket_id: + raise NotImplementedError("Copy between different buckets is not available yet") + from huggingface_hub._buckets import _BucketCopyFile + + # Use _BucketCopyFile to update the file reference. + # We copy from the same bucket (self-referential) to update the file hash. + file_metadata = self._api.get_bucket_file_metadata( + bucket_id=resolved_path1.bucket_id, remote_path=resolved_path1.path, token=self.token + ) + self._api._batch_bucket_files( + bucket_id=resolved_path2.bucket_id, + copy=[ + _BucketCopyFile( + destination=resolved_path2.path, + xet_hash=file_metadata.xet_file_data.file_hash, + source_repo_type="bucket", + source_repo_id=resolved_path2.bucket_id, + ) + ], + token=self.token, + ) + self.invalidate_cache(path=resolved_path1.unresolve()) + self.invalidate_cache(path=resolved_path2.unresolve()) + return same_repo = ( resolved_path1.repo_type == resolved_path2.repo_type and resolved_path1.repo_id == resolved_path2.repo_id diff --git a/tests/test_buckets_hf_file_system.py b/tests/test_buckets_hf_file_system.py index 9b6ea19389..9edfcad7ef 100644 --- a/tests/test_buckets_hf_file_system.py +++ b/tests/test_buckets_hf_file_system.py @@ -82,7 +82,3 @@ def _bucket(self): self.text_file = self.hf_path + "/" + self.text_file_path yield self.api.delete_bucket(self.bucket_id) - - @pytest.mark.skip("Not implemented yet") - def test_copy_file(self): - pass From 0b465c894ac41b6f1f12be1ad3f1b343b4231f51 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 4 Sep 2026 17:05:05 +0200 Subject: [PATCH 17/21] add tests --- src/huggingface_hub/hf_api.py | 2 +- src/huggingface_hub/hf_file_system.py | 53 +++++++++++++++++++++------ tests/test_buckets_hf_file_system.py | 20 ++++++++++ 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index 04a576ce5b..a56bc09d93 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -14980,7 +14980,7 @@ def edit_bucket_file( The file to edit. edit (`list[tuple[tuple[int, int], bytes]]`, *optional*): List edits to apply, in the form `((start, end), data)`. - Ranges [`start`, `end`) are replaced with `data`, which can + Ranges [`start`, `end`] are replaced with `data`, which can be of any size (not necessarily the size of the replaced range). insert (`list[tuple[int, bytes]]`, *optional*): List of inserts to apply, in the form `(loc, data)`. diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 506ea1e596..d3c95e7cb3 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1664,7 +1664,7 @@ def _upload_ranges(self, defer: bool) -> None: time.sleep(0.1) if defer: self.task = _get_deferred_executor().submit( - partial(self._upload_ranges_inner, self.ranges, self.original_size) + partial(self._upload_ranges_inner, list(self.ranges), self.original_size) ) else: self._upload_ranges_inner(self.ranges, self.original_size) @@ -1771,6 +1771,8 @@ def write(self, data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). + Write `data` at the current location. + Args: data (`bytes`): Set of bytes to be written. @@ -1785,6 +1787,9 @@ def edit(self, byte_range: tuple[int, int], data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). + Replace the range [`start`, `end`] with `data`, which can + be of any size (not necessarily the size of the replaced range). + Args: byte_range (`tuple[int, int]`): (start, end) where to edit the file. @@ -1816,7 +1821,13 @@ def add_range(range_or_bytes: range | bytearray | bytes): def fast_slice(range_: range | bytearray, start=None, end=None): """slice a range and avoid slicing a bytearray when possible (this would cause a copy)""" - if start is not None and start > 0: + if start is not None and start < 0: + start += len(range_) + if end is not None and end < 0: + end += len(range_) + if start is not None and start > 0 and end is not None and end < len(range_): + return range_[start:end] + elif start is not None and start > 0: return range_[start:] elif end is not None and end < len(range_): return range_[:end] @@ -1825,31 +1836,32 @@ def fast_slice(range_: range | bytearray, start=None, end=None): # note: this could be optimized with an offset index for range_ in self.ranges: - if offset <= start <= offset + len(range_) <= end: + length = len(range_) + if offset <= start <= offset + length <= end: if offset < start: add_range(fast_slice(range_, end=start - offset)) if data and not done: add_range(data) done = True - elif start <= offset <= offset + len(range_) <= end: + elif start <= offset <= offset + length <= end: pass - elif start <= offset <= end <= offset + len(range_): + elif start <= offset <= end <= offset + length: if data and not done: add_range(data) done = True - if end < offset + len(range_): - add_range(fast_slice(range_, start=end - offset - len(range_))) - elif offset <= start <= end <= offset + len(range_): + if end < offset + length: + add_range(fast_slice(range_, start=end - offset - length)) + elif offset <= start <= end <= offset + length: if offset < start: add_range(fast_slice(range_, end=start - offset)) if data and not done: add_range(data) done = True - if end < offset + len(range_): - add_range(fast_slice(range_, start=end - offset - len(range_))) + if end < offset + length: + add_range(fast_slice(range_, start=end - offset - length)) else: add_range(range_) - offset += len(range_) + offset += length self.ranges = new_ranges self.loc = start + len(data) @@ -1872,6 +1884,9 @@ def insert(self, loc: int, data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). + Insert the content of `data` at location `loc`, and shift + the rest of the file. + Args: loc (`int`): Where to insert the data. @@ -1888,6 +1903,8 @@ def append(self, data: bytes): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). + Append `data` at the end of the file. + Args: data (`bytes`): Set of bytes to be appended at the end of the file. @@ -1902,6 +1919,8 @@ def delete(self, loc: int, length: int): equal to blocksize (and there is also a minimum 10 second interval between sends to avoid doing too many requests). + Delete the range [`loc`, `loc + length`). + Args: loc (`int`): Where to insert the data. @@ -1981,6 +2000,8 @@ def write(self, data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. + Write `data` at the current location. + Args: data (`str`): String to be written. @@ -1994,6 +2015,9 @@ def edit(self, byte_range: tuple[int, int], data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. + Replace the range [`start`, `end`] with `data`, which can + be of any size (not necessarily the size of the replaced range). + Args: byte_range (`tuple[int, int]`): (start, end) where to edit the file. @@ -2010,6 +2034,9 @@ def insert(self, loc: int, data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. + Insert the content of `data` at location `loc`, and shift + the rest of the file. + Args: loc (`int`): Where to insert the data. @@ -2025,6 +2052,8 @@ def append(self, data: str): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. + Append `data` at the end of the file. + Args: data (`str`): String to be appended at the end of the file. @@ -2038,6 +2067,8 @@ def delete(self, loc: int, length: int): Buffer only sent on flush() or if buffer is greater than or equal to blocksize. + Delete the range [`loc`, `loc + length`). + Args: loc (`int`): Where to insert the data. diff --git a/tests/test_buckets_hf_file_system.py b/tests/test_buckets_hf_file_system.py index 9edfcad7ef..cab08b7495 100644 --- a/tests/test_buckets_hf_file_system.py +++ b/tests/test_buckets_hf_file_system.py @@ -82,3 +82,23 @@ def _bucket(self): self.text_file = self.hf_path + "/" + self.text_file_path yield self.api.delete_bucket(self.bucket_id) + + def test_append_file(self): + with self.hffs.open(self.text_file, "a") as f: + f.write(" appended text") + + with self.hffs.open(self.text_file, "r") as f: + assert f.read() == "dummy text data appended text" + + def test_edit_file(self): + with self.hffs.open(self.text_file, "e") as f: + f.insert(0, "this is ") + f.edit((8, 13), "a fantastic") + f.delete(24, 5) + f.append("!") + import warnings + + warnings.warn(str(f.buffer.ranges)) + + with self.hffs.open(self.text_file, "r") as f: + assert f.read() == "this is a fantastic text!" From f580a36185650d4803f6d827293e7add105235f1 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 4 Sep 2026 17:23:49 +0200 Subject: [PATCH 18/21] remove unnecessary override --- src/huggingface_hub/hf_file_system.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index d3c95e7cb3..c5b377fa83 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -13,7 +13,7 @@ from functools import partial from itertools import chain from pathlib import Path, PurePosixPath -from typing import Any, Literal, NoReturn, Union, overload, override +from typing import Any, Literal, NoReturn, Union, overload from urllib.parse import quote, unquote import fsspec @@ -433,7 +433,6 @@ def open( **kwargs, ) -> fsspec.spec.AbstractBufferedFile: ... - @override def open( # ty: ignore[invalid-method-override] self, path, From ab04da7432989245ff664f412fcd1671de4ec2a5 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 4 Sep 2026 17:33:23 +0200 Subject: [PATCH 19/21] fix docs reference --- docs/source/en/guides/hf_file_system.md | 2 +- docs/source/en/package_reference/hf_file_system.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/guides/hf_file_system.md b/docs/source/en/guides/hf_file_system.md index 1ab9fd3e6f..098c928f16 100644 --- a/docs/source/en/guides/hf_file_system.md +++ b/docs/source/en/guides/hf_file_system.md @@ -53,7 +53,7 @@ The optional `revision` argument can be passed to run an operation from a specif Unlike Python's built-in `open`, `fsspec`'s `open` defaults to binary mode, `"rb"`. This means you must explicitly set mode as `"r"` for reading and `"w"` for writing in text mode. -Appending to a file (modes `"a"` and `"ab"`) is supported and very efficient thanks to Xet. Similarly, editing a file in place (edit modes `"e"` and `"eb"` - not available in Python built-in `open`) is supported and allows efficient operations `edit`, `append`, `insert`, `delete` and `truncate` via [`HfFileSystemEditFile`]. +Appending to a file (modes `"a"` and `"ab"`) is supported and very efficient thanks to Xet. Similarly, editing a file in place (edit modes `"e"` and `"eb"` - not available in Python built-in `open`) is supported and allows efficient operations `edit`, `append`, `insert`, `delete` and `truncate` via [`huggingface_hub.hf_file_system.HfFileSystemEditFile`]. ## Integrations diff --git a/docs/source/en/package_reference/hf_file_system.md b/docs/source/en/package_reference/hf_file_system.md index c6d2a94b78..4091fddbdb 100644 --- a/docs/source/en/package_reference/hf_file_system.md +++ b/docs/source/en/package_reference/hf_file_system.md @@ -16,4 +16,4 @@ The `HfFileSystem` class provides a pythonic file interface to the Hugging Face In addition to regular file-like objects obtained using open modes "w", "wb", "r" or "rb" to read and overwrite files, `HfFileSystem` also offers open modes "a" and "ab" to append to an existing file and "e" and "eb" to edit an existing file in-place. -[[autodoc]] HfFileSystemEditFile +[[autodoc]] huggingface_hub.hf_file_system.HfFileSystemEditFile From 54783337efe2e3090258159e1fdd8b34aa2ee349 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Mon, 7 Sep 2026 15:05:40 +0200 Subject: [PATCH 20/21] fix ranges --- src/huggingface_hub/hf_file_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index c5b377fa83..5ada6a5c74 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1849,7 +1849,7 @@ def fast_slice(range_: range | bytearray, start=None, end=None): add_range(data) done = True if end < offset + length: - add_range(fast_slice(range_, start=end - offset - length)) + add_range(fast_slice(range_, start=end - offset)) elif offset <= start <= end <= offset + length: if offset < start: add_range(fast_slice(range_, end=start - offset)) @@ -1857,7 +1857,7 @@ def fast_slice(range_: range | bytearray, start=None, end=None): add_range(data) done = True if end < offset + length: - add_range(fast_slice(range_, start=end - offset - length)) + add_range(fast_slice(range_, start=end - offset)) else: add_range(range_) offset += length From 07c7b46558351ec52381aaee062566d465374b4e Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Tue, 8 Sep 2026 15:08:45 +0200 Subject: [PATCH 21/21] fix edit --- src/huggingface_hub/hf_api.py | 2 +- src/huggingface_hub/hf_file_system.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py index a56bc09d93..185956aa98 100644 --- a/src/huggingface_hub/hf_api.py +++ b/src/huggingface_hub/hf_api.py @@ -15089,7 +15089,7 @@ def _edit_bucket_file( try: if edit: for (start, end), data in edit: - commit.edit(start, end).write(data) + commit.edit((start, end), len(data)).write(data) if insert: for loc, data in insert: commit.insert(loc, len(data)).write(data) diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py index 5ada6a5c74..e692b9c9c9 100644 --- a/src/huggingface_hub/hf_file_system.py +++ b/src/huggingface_hub/hf_file_system.py @@ -1674,22 +1674,24 @@ def _upload_ranges(self, defer: bool) -> None: def _upload_ranges_inner(self, ranges: list[range | bytearray], original_size: int) -> None: original_offset = 0 - insert: list[tuple[int, bytes]] = [] delete: list[tuple[int, int]] = [] + edit: list[tuple[tuple[int, int], bytes]] = [] # ((start, end), replacement_data) for range_ in ranges: if isinstance(range_, range): if range_.start > original_offset: delete.append((original_offset, range_.start - original_offset)) original_offset = range_.stop else: - insert.append((original_offset, bytes(range_))) + data = bytes(range_) + edit.append(((original_offset, original_offset + len(data)), data)) + original_offset += len(data) if original_offset < original_size: delete.append((original_offset, original_size - original_offset)) - if insert or delete: + if edit or delete: self.file_hash = self.fs._api._edit_bucket_file( bucket_id=self.resolved_path.bucket_id, remote_path=self.resolved_path.path, - insert=insert or None, + edit=edit or None, delete=delete or None, _file_hash=self.file_hash, _file_size=self.original_size,