diff --git a/.gitignore b/.gitignore index 8dea4ac..0bed875 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ dedup.x86_64 dedup.universal test/test-data test/dedup_check +.cache/ diff --git a/.hermes/plans/2026-08-25_linux-symlink-port-and-userspace-btrfs.md b/.hermes/plans/2026-08-25_linux-symlink-port-and-userspace-btrfs.md new file mode 100644 index 0000000..25d71c9 --- /dev/null +++ b/.hermes/plans/2026-08-25_linux-symlink-port-and-userspace-btrfs.md @@ -0,0 +1,151 @@ +# Linux Port for Symlink Dedup + Userspace BTRFS Clone Backend + +> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task. + +**Goal:** Make `dedup` build and run on Linux (Jules CI VMs) with `--symlink`/`--link` modes doing the heavy lifting, backed by superior early elimination, plus a file-based userspace BTRFS layer (FFI to the btrfs control plane) so reflink-style dedup can be tested without root or loop mounts. + +**Architecture:** Split platform-specific code behind a small backend interface (`clone_backend`), keep the existing signature → witness → exact-compare gate untouched (it is already portable POSIX), add a Linux prune path that loses getattrlist clone-id pruning but gains cheap early elimination from inode/dev keys and first/last-byte + size pre-filtering. The userspace BTRFS component lives in its own library (`ubifs-btrfs/`) exposing btrfs ioctl semantics (`BTRFS_IOC_CLONE`-compatible reflink, same-fs checks) over regular image files, consumed by dedup through a thin C FFI so the existing `-C`/clone codepaths can run against it in tests. + +**Tech Stack:** C2x, POSIX (fts, unlink/link/symlink/rename), Linux `` `FICLONE`, ``, btrfs kernel UAPI headers (``, ``) for control-plane structures, Makefile conditionals, Check test harness. + +--- + +## Current context (verified in repo) + +- Platform gates exist only in `clone.c` (`__APPLE__` / `__FREEBSD__` / `#error`) and `runtime_caps.c`. +- `queue.h`, `map.h` unconditionally `#include ` (macOS-only header) but appear to use only standard types from it — verify each symbol before removal. +- `dedup.c` uses `getattrlist`+`statfs` for `VOL_CAP_INT_CLONE` gating (dedup.c:880-922), `entry->fts_statp->st_flags` (dedup.c:1258,1291), and `chflags`-adjacent logic — all macOS-only. +- `replace_with_link` / `replace_with_symlink` (clone.c:201-291) are pure POSIX already — these are the intended Linux mechanics. +- Pruning (`prune_entry`, dedup.c:519) eliminates on: nlink>1 seen-set (portable), clone-id seen-set (macOS-only). Sequence-deadlock fix must be preserved on all paths. +- Exact-compare gate (`sig_table.c:62-66` + runtime_dispatch backends) uses pread/memcmp — fully portable. +- Test suite mounts HFS DMGs via `hdiutil` — Linux needs a parallel fixture strategy (see Task 8). + +## Proposed approach + +Three layers, each independently mergeable: + +1. **Portable core** — compile on Linux with symlink/hardlink replace modes; clone mode compiles but reports "unsupported on this volume/fs" at runtime instead of `#error`. +2. **Linux early elimination** — strengthen pruning with portable signals so symlink mode rejects non-candidates before any I/O: size bucket, first/last byte, dev+inode seen-set (hardlinks), and (new) a cheap 4KB head-hash seen-set shared across threads. This is what makes `-s` fast on ext4/tmpfs where reflink does not exist. +3. **Userspace BTRFS backend** — new static lib implementing reflink semantics over a plain image file, wire-compatible with the kernel btrfs ioctl control plane (same struct layouts/command numbers from ``), exposed via a C FFI (`ubt_clone_fd(dst_fd, src_fd)` mirroring `BTRFS_IOC_CLONE`). dedup's clone backend gains a third implementation selectable at runtime (`DEDUP_BACKEND=clonefile|ficlone|userspace-btrfs`), so Jules VMs exercise the full dedup→reflink path in CI without mounting anything. + +--- + +## Step-by-step plan + +### Phase A — Portable build skeleton + +### Task 1: Audit and de-`#include ` the data structures +**Files:** Modify `queue.h`, `map.h`, `attr.h` +**Steps:** grep every symbol actually used from `sys/attr.h` in those translation units; if none, drop the include; move remaining macOS-only bits behind `#if defined(__APPLE__)`. Build both `make dedup` on macOS (must stay green) and `cc -std=c2x -fsyntax-only *.c` with a simulated Linux gate to confirm nothing else leaks. +**Verify:** `grep -n 'sys/attr' queue.h map.h` returns nothing (or is guarded); macOS build passes. + +### Task 2: Guard macOS-only stat fields +**Files:** Modify `queue.c`, `queue.h`, `dedup.c:1250-1300` +**Steps:** wrap `st_flags` capture in `#if defined(__APPLE__)` (pass `0` elsewhere); the flags are only used for uchg detection which has no Linux equivalent yet (note it in a comment). +**Verify:** syntax-only Linux compile of queue.o/dedup.o clean. + +### Task 3: Volume-capability gating becomes per-platform +**Files:** Modify `dedup.c:880-930` +**Steps:** split `is_vol_cap_supported` behind `#if defined(__APPLE__)`; on Linux provide `is_clonefile_supported()` that attempts a real probe (see Task 5) and returns false when the backend is unavailable; `are_acls_supported` returns true (rename_swap is universal on Linux). +**Verify:** unit test asserting the Linux stub compiles and the Apple path unchanged. + +### Task 4: clone.c gets a Linux branch instead of `#error` +**Files:** Modify `clone.c:100-130`, fix latent FreeBSD bug at clone.c:121 while here (declare `int result;`) +**Steps:** add +```c +#elif defined(__linux__) +#include +int genfile_clone(const char* src, const char* dst) { + int s = open(src, O_RDONLY); + if (s < 0) return errno; + int d = open(dst, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (d < 0) { int e = errno; close(s); return e; } + int r = ioctl(d, FICLONE, s); + int e = errno; + close(s); close(d); + return r ? e : 0; +} +#else +#error Operating system not supported. +#endif +``` +Also delete the dead `(1<<31)` flag issue on this branch (Apple-only codepath stays untouched). +**Verify:** on a Linux VM (or WSL/container): create two 4MB files on btrfs/xfs, run binary, confirm reflink happened via `du --block-size=1` before/after. + +### Task 5: Runtime backend selection +**Files:** Create `backend.h`/`backend.c`; Modify `main()` arg parsing +**Steps:** enum `{ BACKEND_AUTO, BACKEND_CLONEFILE, BACKEND_FICLONE, BACKEND_UBT }`; `AUTO` picks clonefile on Apple, probes FICLONE on Linux (test ioctl on a scratch fd), falls back to refusing clone mode with a clear warnx (symlink/link still work). Env override `DEDUP_BACKEND` for tests. +**Verify:** `DEDUP_BACKEND=ficlone ./dedup -v dir` prints chosen backend under existing verbose print. + +--- + +### Phase B — Superior early elimination (the point of the port) + +### Task 6: Portable prune strengthening +**Files:** Modify `prune_entry` (dedup.c:519), `seen_set.[ch]` +**Steps:** +1. Keep nlink/inode seen-set (already portable). +2. Replace clone-id seen-set with `#if defined(__APPLE__)` wrapper; on Linux insert a second SeenSet keyed on `(size, first_byte, last_byte, head_xxh64)` computed during metadata read (signature work already computes most of this — reuse it rather than re-reading). +3. Preserve the sequence-advance contract: every pruned entry must still hit `visit_order_begin`/`visit_order_end` (deadlock fix, commit 1213eae). +**Verify:** TDD — failing test first: two files same size different content must NOT be pruned; two files identical heads on ext4 get eliminated pre-signature (count via strace or an instrumented counter printed at -vv). + +### Task 7: Symlink-mode fast path +**Files:** Modify `visit_entry` dispatch +**Steps:** when `replace_mode == DEDUP_SYMLINK`, skip the copyfile-metadata stage entirely (it is Apple-only anyway) and go straight staged-symlink-swap: create `.~.tmp` symlink then `rename(2)` (fixes the acknowledged non-atomic two-step TODO for this mode — clone mode keeps its current flow). +**Verify:** crash-window test: SIGKILL mid-run leaves either original or complete symlink, never missing path (loop 200 kills). + +--- + +### Phase C — Userspace BTRFS (file-based, c-btrfs-compatible) + +### Task 8: Library scaffold `ubt/` +**Files:** Create `ubt/ubt.h`, `ubt/image.c`, `ubt/super.c`, `ubt/Makefile` +**Steps:** file-backed image (`ftruncate` + mmap windows); on-disk layout follows kernel btrfs UAPI structs verbatim (`btrfs_super_block`, `btrfs_disk_key`, `btrfs_item`) so a created image is inspectable by `btrfs inspect-internal dump-super` — that is the compliance bar. Implement: format, superblock with correct magic `_BHRfS_M`, chunk tree stub, checksummed node writes (crc32c). +**Verify:** `dump-super` accepts our image; roundtrip mount attempt documented as expected-to-fail until extent tree lands (record which kernel messages appear). + +### Task 9: Extent/reflink operations (the control-plane FFI) +**Files:** Create `ubt/clone.c`, `ubt/ioctl_shim.c` +**Steps:** implement the logical equivalent of `BTRFS_IOC_CLONE`: shared-extent bookkeeping (extent refcounts per bytenr), so cloned ranges share blocks and CoW on write. Expose: +```c +int ubt_clone(int dst_fd, int src_fd); // mirrors BTRFS_IOC_CLONE semantics +int ubt_same_fs(int a_fd, int b_fd); +uint64_t ubt_private_size(int fd); // mirrors ATTR_CMNEXT_PRIVATESIZE +``` +These three calls are exactly what utils.c's getattrlist helpers provide on macOS — the FFI surface dedup needs. +**Verify:** property test: clone N times, mutate each copy, byte-compare all pairs — divergence isolation must be exact; private_size reflects shared savings like APFS. + +### Task 10: Wire backend into dedup +**Files:** Modify `backend.c` from Task 5, `Makefile` +**Steps:** `BACKEND_UBT` routes `genfile_clone` through `ubt_clone` when paths live inside a mounted-in-process image (image attach command: `dedup --ubt-image foo.img -- cmd`-style, or simply treat a directory tree whose files carry an xattr pointer into the image — decide: simplest is a dedicated image-backed working dir managed by a small helper `ubt-mount`). Link `libubt.a` optionally (`UBT=1`), degrade gracefully when absent. +**Verify:** full dedup suite subset (empty/bars/same-size/dry-run) running against an image-backed tree on Linux CI, exercising real shared-block accounting. + +--- + +### Phase D — CI for Jules VMs + +### Task 11: Linux test target +**Files:** Modify `test/Makefile` +**Steps:** add `check-linux:` replacing hdiutil fixtures with tmpfs/ext4 scratch dirs (no mount privileges needed); skip clone/HFS suites unless running as root with a loop btrfs; add symlink+link suite runs as primary gates; add popen watchdog using repo's own `timeout_exec.c` (fixes audit finding #6). +**Verify:** green run inside a stock Debian/Ubuntu container as non-root. + +### Task 12: Docs + dict +**Files:** Modify `README.md`, `dict` +**Steps:** document platform matrix, `DEDUP_BACKEND`, image workflow; add new README words to spelling dict so `check-spelling` stays honest. +**Verify:** `make check-spelling-readme` passes. + +--- + +## Files likely to change +`clone.c`, `clone.h`, `dedup.c`, `utils.c`, `queue.{c,h}`, `map.h`, `runtime_caps.c`, `Makefile`, `test/Makefile`, `test/test_utils.c`; new: `backend.{c,h}`, `ubt/*`. + +## Tests / validation +- macOS: `make check` stays at current baseline (45/48 + known toolchain issues) — no regressions. +- Linux VM: new `check-linux` green as non-root; FICLONE path verified on btrfs when root. +- Deadlock regression loop (Task 6/7 verify steps) on both OSes. + +## Risks / tradeoffs / open questions +1. **c-btrfs compliance bar is fuzzy.** I've interpreted "compliant with c-btrfs" as *wire-compatible with the kernel btrfs UAPI (ioctl structs/layout) and dump-super-readable*, implemented in C. If you meant compatibility with a specific project named c-btrfs, point me at it and Tasks 8-9 change shape. +2. **Userspace btrfs scope.** Full crash-consistent btrfs is enormous; this plan deliberately targets the reflink/control-plane subset (format + superblock + shared extents + CoW). Mountability is explicitly not promised initially. +3. **st_flags/uchg has no Linux equivalent** — immutable-file protection is lost on Linux (chattr +i detection could substitute later; YAGNI now, noted). +4. **Early-elimination hash reuse** must not weaken the exact-compare gate — the head-hash is only a *prune-negative* signal (never confirms duplicates), keeping the safety invariant from the audit intact. +5. Image-vs-xattr attachment model for UBT (Task 10) is a genuine fork in the road; helper-mount approach is recommended for CI simplicity. diff --git a/Makefile b/Makefile index c9c56ec..7be850e 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,13 @@ OBJECTS = \ map.o \ progress.o \ queue.o \ + seen_set.o \ utils.o \ + signature.o \ + sig_table.o \ + runtime_caps.o \ + runtime_dispatch.o \ + output_format.o \ .PHONY: \ all install uninstall clean check dist distcheck \ @@ -50,19 +56,17 @@ OBJECTS = \ clean-coverage report-coverage \ universal-dedup universal-dist \ compiledb tidy \ - list + list mkdirs all: dedup -%.o: %.c %.h - rm -f $(basename $<).gcda $(basename $<).gcno - $(CC) $(CFLAGS) -v -c -o $@ $< +dedup.o: CFLAGS += -I/opt/homebrew/include -dedup.arm: CFLAGS += -target arm64-apple-macos11 -dedup.x86_64: CFLAGS += -target x86_64-apple-macos11 +dedup.arm: CFLAGS += -target arm64-apple-macos11 -I/opt/homebrew/include +dedup.x86_64: CFLAGS += -target x86_64-apple-macos11 -I/opt/homebrew/include dedup dedup.arm dedup.x86_64: $(OBJECTS) - $(CC) $(CFLAGS) -o $@ $^ + $(CC) $(CFLAGS) -o $@ $^ -L/opt/homebrew/lib -lxxhash mv $@ $@.unsigned codesign -s - -v -f $(ENTITLEMENT_FLAGS) $@.unsigned mv $@.unsigned $@ @@ -118,11 +122,11 @@ report-coverage: PREFIX ?= /usr/local -install: dedup +install: dedup mkdirs install dedup $(PREFIX)/bin install dedup.1 $(PREFIX)/share/man/man1 -build/dist: +build/dist mkdirs: mkdir -p $(PREFIX)/bin mkdir -p $(PREFIX)/share/man/man1 diff --git a/README.md b/README.md index 37920d9..5f214c1 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,37 @@ permissions, there should be little issue. The created files are also not copy-on-write and will share any modifications made. These options should only be used if the consequences of each choice are understood. +## FAST MODE + +**dedup-fast** provides a high-performance alternative to the traditional +two-phase deduplication approach. Instead of computing full SHA256 hashes for +all files and then deduplicating in a second pass, **dedup-fast** uses a +signature-based approach with immediate cloning. + +### Signature-Based Deduplication + +**dedup-fast** computes lightweight signatures for each file using: + +- **Strategic sampling**: Reads 4-byte samples at 0%, 33%, 66%, and 100% positions +- **Quick hash**: xxHash64 of the first 4KB (or entire file if smaller) +- **NEON acceleration**: Uses ARM NEON instructions for vectorized comparisons when available + +Files with matching signatures are considered duplicates and cloned immediately +during the scan phase, eliminating the need for a separate deduplication pass. + +### Performance Benefits + +- **Single-pass operation**: No separate scan and dedup phases +- **Reduced I/O**: Avoids full file reads for SHA256 computation +- **Memory efficient**: Uses hash table instead of complex red-black trees +- **Immediate cloning**: Duplicates are resolved as soon as they're found + +### Limitations + +- **Higher false positive rate**: Signature collisions may occur (rare) +- **No SHA256 verification**: Relies on signature accuracy +- **Memory usage**: Hash table grows with unique file count + # OPTIONS The following options are available: diff --git a/dedup.c b/dedup.c index 15aacea..e796047 100644 --- a/dedup.c +++ b/dedup.c @@ -26,17 +26,19 @@ #include #ifndef lint -__used static char const copyright[] = +__attribute__((used)) static char const copyright[] = "@(#) Copyright © 2023\n" "TTKB, LLC. All rights reserved.\n"; #ifndef VERSION -#define VERSION "0.0.0" +#define VERSION 0.0.0 #endif // VERSION #ifndef BUILD_DATE -#define BUILD_DATE "00000000" +#define BUILD_DATE 00000000 #endif // BUILD_DATE -__used static char const version[] = - "TTKB dedup " VERSION " (" BUILD_DATE ")"; +#define STR(x) #x +#define XSTR(x) STR(x) +__attribute__((used)) static char const version[] = + "TTKB dedup " XSTR(VERSION) " (" XSTR(BUILD_DATE) ")"; #if 0 static char sccsid[] = "@(#)dedup.c)"; #endif // 0 @@ -45,8 +47,10 @@ static char sccsid[] = "@(#)dedup.c)"; #include #include #include +#include #include +#include #include #include #include @@ -55,11 +59,19 @@ static char sccsid[] = "@(#)dedup.c)"; #include #include #include +#include +#include #include "clone.h" #include "map.h" #include "progress.h" #include "queue.h" +#include "output_format.h" +#include "runtime_dispatch.h" +#include "runtime_caps.h" +#include "seen_set.h" +#include "signature.h" +#include "sig_table.h" #include "utils.h" #define PROGRESS_LOCK(p, m, block) do { \ @@ -70,6 +82,51 @@ static char sccsid[] = "@(#)dedup.c)"; } \ } while (0) +// Forward declaration +typedef struct DedupContext DedupContext; + +// Clone record streaming functions +static FILE* clone_summary_stream = NULL; +static pthread_mutex_t clone_summary_mutex = PTHREAD_MUTEX_INITIALIZER; +static size_t clone_summary_count = 0; + +static void clone_summary_open(const char* filepath) { + pthread_mutex_lock(&clone_summary_mutex); + if (!clone_summary_stream) { + clone_summary_stream = fopen(filepath, "w"); + if (clone_summary_stream) { + fprintf(clone_summary_stream, "DEDUP CLONING SUMMARY\n"); + fprintf(clone_summary_stream, "=====================\n"); + fprintf(clone_summary_stream, "\n"); + } + } + pthread_mutex_unlock(&clone_summary_mutex); +} + +static void clone_summary_write(const char* origin, const char* clone, size_t size) { + pthread_mutex_lock(&clone_summary_mutex); + if (clone_summary_stream) { + fprintf(clone_summary_stream, " Origin: %s\n", origin); + fprintf(clone_summary_stream, " Clone: %s (size: %zu bytes)\n", clone, size); + clone_summary_count++; + } + pthread_mutex_unlock(&clone_summary_mutex); +} + +static void clone_summary_close() { + pthread_mutex_lock(&clone_summary_mutex); + if (clone_summary_stream) { + // Write count at the end + fprintf(clone_summary_stream, "\nTotal cloning operations: %zu\n", clone_summary_count); + fflush(clone_summary_stream); + + fclose(clone_summary_stream); + clone_summary_stream = NULL; + clone_summary_count = 0; + } + pthread_mutex_unlock(&clone_summary_mutex); +} + typedef enum ReplaceMode { DEDUP_CLONE = 0, DEDUP_LINK = 1, @@ -79,94 +136,491 @@ typedef enum ReplaceMode { typedef struct DedupContext { Progress* progress; FileEntryHead* queue; - rb_tree_t* visited; - rb_tree_t* duplicates; + FileEntryHead* raw_queue; + SigTable* signatures; size_t found; size_t saved; size_t already_saved; - uint8_t done; + size_t pruned; + size_t total_bytes; // Accumulated bytes of all scanned files + size_t queued_count; // Entries in raw_queue + work_queue combined + uint64_t next_file_sequence; + uint64_t next_visit_sequence; + uint8_t scan_done; + uint8_t prune_done; uint8_t thread_count; bool dry_run; uint8_t verbosity; bool force; ReplaceMode replace_mode; + OutputFormat output_format; + bool clone_converted; // Whether to convert clones (true by default) + char* summary_file; // File to write detailed summary to (NULL by default) pthread_mutex_t metrics_mutex; pthread_mutex_t progress_mutex; pthread_mutex_t queue_mutex; - pthread_mutex_t visited_mutex; - pthread_mutex_t duplicates_mutex; - pthread_mutex_t done_mutex; + pthread_mutex_t raw_queue_mutex; + pthread_mutex_t signatures_mutex; + pthread_mutex_t scan_done_mutex; + pthread_mutex_t prune_done_mutex; + pthread_mutex_t visit_order_mutex; + pthread_cond_t visit_order_cond; } DedupContext; +static int get_terminal_width(void) { + struct winsize w; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0 && w.ws_col > 0) { + return w.ws_col; + } + return 80; // fallback +} + +// Fixed-width compact format for a file count (e.g., "2.1K", " 99K", "999K") +static void format_count(size_t count, char buf[5]) { + buf[4] = '\0'; + + if (count < 1000) { + snprintf(buf, 5, "%4zu", count); + } else if (count < 9950) { + snprintf(buf, 5, "%.1fK", (double)count / 1000.0); + } else if (count < 999500) { + snprintf(buf, 5, "%3lluK", (unsigned long long)((count + 500) / 1000)); + } else if (count < 9950000) { + snprintf(buf, 5, "%.1fM", (double)count / 1000000.0); + } else if (count < 999500000) { + snprintf(buf, 5, "%3lluM", (unsigned long long)((count + 500000) / 1000000)); + } else { + snprintf(buf, 5, "%.1fG", (double)count / 1000000000.0); + } + + size_t len = strlen(buf); + if (len < 4) { + size_t pad = 4 - len; + memmove(buf + pad, buf, len + 1); + for (size_t i = 0; i < pad; i++) buf[i] = ' '; + } else if (len > 4) { + buf[4] = '\0'; + } +} + +static void display_status(DedupContext* ctx, const char* path) { + if (!isatty(STDOUT_FILENO)) { + return; + } + + static bool header_printed = false; + if (!header_printed) { + fprintf(stderr, "%-4s %-4s %-4s %-5s %-20s %s\n", + "TOTL", "QUED", "SHRD", "DELTA", "PROGRESS", "PATH"); + header_printed = true; + } + + int width = get_terminal_width(); + if (width < 80) width = 80; + + pthread_mutex_lock(&ctx->metrics_mutex); + size_t total_bytes = ctx->total_bytes; + size_t queued = ctx->queued_count; + size_t shared = ctx->already_saved; + size_t delta = ctx->saved; + pthread_mutex_unlock(&ctx->metrics_mutex); + + size_t completed = 0; + size_t total = 0; + if (ctx->progress) { + PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { + completed = ctx->progress->completedUnitCount; + total = ctx->progress->totalUnitCount; + }); + } + + char total_bytes_str[5]; + char queued_str[5]; + char shared_str[5]; + char delta_str[5]; + char done_str[5]; + char total_str[5]; + + format_compact(total_bytes, total_bytes_str); + format_compact(queued, queued_str); + format_compact(shared, shared_str); + format_compact(delta, delta_str); + format_count(completed, done_str); + format_count(total, total_str); + + char bar[12]; + bar[0] = '['; + bar[10] = ']'; + bar[11] = '\0'; + + if (total == 0) { + for (int i = 1; i < 10; i++) bar[i] = '-'; + } else { + double fraction = (double)completed / (double)total; + int fill = (int)(fraction * 8.0); + if (fill < 0) fill = 0; + if (fill > 8) fill = 8; + + for (int i = 1; i <= fill; i++) bar[i] = '='; + for (int i = fill + 1; i < 10; i++) bar[i] = '-'; + } + + char progress_str[21]; + snprintf(progress_str, sizeof(progress_str), "%s%s%s", done_str, bar, total_str); + + int fixed_width = 41; + int path_max = width - fixed_width - 1; + if (path_max < 10) path_max = 10; + + char truncated_path[256]; + memset(truncated_path, 0, sizeof(truncated_path)); + if (path) { + size_t len = strlen(path); + if (len <= (size_t)path_max) { + strcpy(truncated_path, path); + } else { + size_t keep = path_max - 1; + truncated_path[0] = '~'; + strncpy(truncated_path + 1, path + len - keep, keep); + truncated_path[keep + 1] = '\0'; + } + } + + char delta_display[6]; + if (delta > 0) { + snprintf(delta_display, sizeof(delta_display), "+%-4s", delta_str); + } else { + snprintf(delta_display, sizeof(delta_display), " %-4s", delta_str); + } + delta_display[5] = '\0'; + + int line_len = 4 + 1 + 4 + 1 + 4 + 1 + 5 + 1 + 20 + 1 + (int)strlen(truncated_path); + int padding = width - line_len; + if (padding < 0) padding = 0; + + char padding_str[256]; + if (padding > 0 && padding < (int)sizeof(padding_str) - 1) { + memset(padding_str, ' ', padding); + padding_str[padding] = '\0'; + } else { + padding_str[0] = '\0'; + } + + fprintf(stderr, "\r%-4s %-4s %-4s %s %s %s%s\033[0m\033[K", + total_bytes_str, queued_str, shared_str, delta_display, + progress_str, truncated_path, padding_str); + fflush(stderr); +} + +static void visit_order_begin(DedupContext* ctx, uint64_t sequence) { + if (!ctx || ctx->thread_count == 0) { + return; + } + + pthread_mutex_lock(&ctx->visit_order_mutex); + while (sequence != ctx->next_visit_sequence) { + pthread_cond_wait(&ctx->visit_order_cond, &ctx->visit_order_mutex); + } + pthread_mutex_unlock(&ctx->visit_order_mutex); +} + +static void visit_order_end(DedupContext* ctx) { + if (!ctx || ctx->thread_count == 0) { + return; + } + + pthread_mutex_lock(&ctx->visit_order_mutex); + ctx->next_visit_sequence++; + pthread_cond_broadcast(&ctx->visit_order_cond); + pthread_mutex_unlock(&ctx->visit_order_mutex); +} void visit_entry(FileEntry* fe, Progress* p, DedupContext* ctx) { + if (!fe || !ctx) { + return; + } - FileMetadata* fm = metadata_from_entry(fe); + FileSignature* sig = compute_signature(fe->path, fe->device, fe->size); + visit_order_begin(ctx, fe->sequence); + if (!sig) { + // Silently skip files we can't read (locked, slow FUSE mounts, etc) + visit_order_end(ctx); + return; + } - if (!fm) { + // Insert into signature table and check for duplicates + if (!ctx->signatures) { PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { clear_progress(); - fprintf(stderr, - "could not populate metadata for %s\n", - fe->path); - perror("metadata_from_entry"); + fprintf(stderr, "signature table not initialized for %s\n", fe->path); }); + free_signature(sig); + visit_order_end(ctx); return; } - pthread_mutex_lock(&ctx->visited_mutex); - FileMetadata* old = visited_tree_insert(ctx->visited, fm); - old = metadata_dup(old); - pthread_mutex_unlock(&ctx->visited_mutex); + pthread_mutex_lock(&ctx->signatures_mutex); + size_t table_size_before = sig_table_size(ctx->signatures); + SigTableEntry* existing = sig_table_insert(ctx->signatures, sig, fe->path, get_clone_id(fe->path)); + size_t table_size_after = sig_table_size(ctx->signatures); + pthread_mutex_unlock(&ctx->signatures_mutex); - if (old) { - pthread_mutex_lock(&ctx->duplicates_mutex); - AList* list = duplicate_tree_find(ctx->duplicates, fm); - if (alist_empty(list)) { - alist_add(list, metadata_dup(old)); - } + // Safety check: table size should only increase by 0 or 1 + if (table_size_after > table_size_before + 1) { + PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { + clear_progress(); + fprintf(stderr, "warning: signature table size increased by %zu (expected 0 or 1)\n", + table_size_after - table_size_before); + }); + } - if (ctx->verbosity) { - PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { - clear_progress(); - printf("%s has %zu duplicates\n", - fm->path, - alist_size(list)); - for (size_t i = 0; i < alist_size(list); i++) { - printf("\t%s\n", - ((FileMetadata*) alist_get(list, i))->path); - } - }); + bool table_took_ownership = (table_size_after > table_size_before); + + if (existing) { + // Found a duplicate + display_status(ctx, fe->path); + + // Check if already deduplicated + uint64_t current_clone_id = get_clone_id(fe->path); + if ((ctx->replace_mode == DEDUP_CLONE && current_clone_id == existing->clone_id) || + (ctx->replace_mode == DEDUP_LINK && fe->inode == get_inode(existing->path))) { + pthread_mutex_lock(&ctx->metrics_mutex); + ctx->already_saved += fe->size; + pthread_mutex_unlock(&ctx->metrics_mutex); + free_signature(sig); + visit_order_end(ctx); + return; } - // ownership transferred to the list - alist_add(list, fm); - pthread_mutex_unlock(&ctx->duplicates_mutex); + // Skip if hardlinked and not forcing + if (!ctx->force && fe->nlink > 1) { + if (ctx->verbosity) { + PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { + clear_progress(); + printf("skipping %s, hardlinked\n", fe->path); + }); + } + pthread_mutex_lock(&ctx->metrics_mutex); + ctx->already_saved += fe->size; + pthread_mutex_unlock(&ctx->metrics_mutex); + free_signature(sig); + visit_order_end(ctx); + return; + } - if (fm->clone_id != old->clone_id) { + // Skip if immutable or read-only + if (fe->flags & UF_IMMUTABLE || fe->flags & SF_IMMUTABLE || + (access(fe->path, W_OK) != 0)) { pthread_mutex_lock(&ctx->metrics_mutex); - ctx->found++; + ctx->already_saved += fe->size; pthread_mutex_unlock(&ctx->metrics_mutex); - if (ctx->verbosity > 1) { + free_signature(sig); + visit_order_end(ctx); + return; + } + + // Perform deduplication + if (ctx->dry_run) { + if (ctx->verbosity) { PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { clear_progress(); - printf("'%s' is duplicated by '%s' (%zu bytes) [found: %zu]\n", - old->path, - fm->path, - fm->size, - ctx->found); + printf("would deduplicate %s to %s\n", fe->path, existing->path); }); } + pthread_mutex_lock(&ctx->metrics_mutex); + ctx->saved += fe->size; + ctx->found++; + pthread_mutex_unlock(&ctx->metrics_mutex); + } else { + // Check if clone conversion is disabled + if (ctx->replace_mode == DEDUP_CLONE && !ctx->clone_converted) { + // Skip clone conversion, just count as already saved + pthread_mutex_lock(&ctx->metrics_mutex); + ctx->already_saved += fe->size; + pthread_mutex_unlock(&ctx->metrics_mutex); + free_signature(sig); + visit_order_end(ctx); + return; + } + + int result = 0; + switch (ctx->replace_mode) { + case DEDUP_CLONE: + result = replace_with_clone(existing->path, fe->path); + break; + case DEDUP_LINK: + result = replace_with_link(existing->path, fe->path); + break; + case DEDUP_SYMLINK: + result = replace_with_symlink(existing->path, fe->path); + break; + } + + if (result) { + // Silently skip clone failures (permissions, filesystem issues, etc) + free_signature(sig); + visit_order_end(ctx); + return; + } else { + if (ctx->verbosity) { + PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { + clear_progress(); + printf("deduplicated %s\n", fe->path); + }); + } + + // Verify clone worked if using clone mode + if (ctx->replace_mode == DEDUP_CLONE) { + uint64_t new_clone_id = get_clone_id(fe->path); + if (new_clone_id != existing->clone_id) { + if (private_size(fe->path) != 0) { + result = -1; // Mark as failed + } + // Else: clone_id mismatch but no private data = success + } + } + + if (result == 0) { + pthread_mutex_lock(&ctx->metrics_mutex); + ctx->saved += fe->size; + ctx->found++; + pthread_mutex_unlock(&ctx->metrics_mutex); + + // Write clone operation to summary file (streamed) + if (ctx->summary_file) { + clone_summary_write(existing->path, fe->path, fe->size); + } + } else { + pthread_mutex_lock(&ctx->metrics_mutex); + ctx->already_saved += fe->size; // Count as already saved to avoid double counting + pthread_mutex_unlock(&ctx->metrics_mutex); + } + } } - free_metadata(old); + + free_signature(sig); } else { - free_metadata(fm); + // First instance of this signature + if (table_took_ownership) { + display_status(ctx, fe->path); + } else { + // Table failed to take ownership (allocation failure) + PROGRESS_LOCK(ctx->progress, &ctx->progress_mutex, { + clear_progress(); + fprintf(stderr, "failed to store signature for %s: memory allocation failed\n", fe->path); + }); + free_signature(sig); + } + } + + visit_order_end(ctx); +} + +// Returns true if the entry was pruned (caller should free it), false if it survived. +static bool prune_entry(FileEntry* fe, SeenSet* seen_inodes, SeenSet* seen_clones, DedupContext* c) { + bool pruned = false; + + if (fe->nlink > 1) { + uint64_t key = (uint64_t)fe->device << 32 | (uint64_t)(fe->inode & 0xFFFFFFFF); + if (seen_set_insert(seen_inodes, key)) { + pruned = true; + } + } + + if (!pruned) { + uint64_t clone_id = get_clone_id(fe->path); + if (clone_id != 0) { + if (seen_set_insert(seen_clones, clone_id)) { + pruned = true; + } + } + } + + if (!pruned) { + return false; } + + // A pruned entry never reaches visit_entry, so its sequence number would + // never be advanced by visit_order_end. A worker waiting on + // visit_order_begin would block forever and deadlock the whole run, so + // advance it here instead. + visit_order_begin(c, fe->sequence); + visit_order_end(c); + + pthread_mutex_lock(&c->metrics_mutex); + c->already_saved += fe->size; + c->pruned++; + pthread_mutex_unlock(&c->metrics_mutex); + PROGRESS_LOCK(c->progress, &c->progress_mutex, { + c->progress->completedUnitCount++; + }); + display_status(c, fe->path); + return true; +} + +void* prune_work(void* ctx) { + DedupContext* c = ctx; + SeenSet* seen_inodes = new_seen_set(4096); + SeenSet* seen_clones = new_seen_set(4096); + + for (;;) { + pthread_mutex_lock(&c->raw_queue_mutex); + FileEntry* fe = file_entry_next(c->raw_queue); + pthread_mutex_unlock(&c->raw_queue_mutex); + + if (!fe) { + pthread_mutex_lock(&c->scan_done_mutex); + uint8_t done = c->scan_done; + pthread_mutex_unlock(&c->scan_done_mutex); + if (done) { + // Drain remaining entries + pthread_mutex_lock(&c->raw_queue_mutex); + fe = file_entry_next(c->raw_queue); + pthread_mutex_unlock(&c->raw_queue_mutex); + if (!fe) break; + } else { + usleep(100); + continue; + } + } + + // Decrement queued_count for raw_queue pop + pthread_mutex_lock(&c->metrics_mutex); + c->queued_count--; + pthread_mutex_unlock(&c->metrics_mutex); + + if (prune_entry(fe, seen_inodes, seen_clones, c)) { + file_entry_free(fe); + continue; + } + + // Survivor: pass to work queue + // queued_count stays the same (file moves between queues) + pthread_mutex_lock(&c->queue_mutex); + file_entry_queue_append(c->queue, + fe->path, + fe->device, + fe->inode, + fe->nlink, + fe->flags, + fe->size, + fe->sequence, + fe->level); + pthread_mutex_unlock(&c->queue_mutex); + + // Show pruner progress (survivor passes through) + display_status(c, fe->path); + file_entry_free(fe); + } + + free_seen_set(seen_inodes); + free_seen_set(seen_clones); + return NULL; } void* dedup_work(void* ctx) { DedupContext* c = ctx; - uint8_t done = c->done; + uint8_t done = c->prune_done; while (!done) { pthread_mutex_lock(&c->queue_mutex); @@ -174,9 +628,9 @@ void* dedup_work(void* ctx) { pthread_mutex_unlock(&c->queue_mutex); if (!fe) { - pthread_mutex_lock(&c->done_mutex); - done = c->done; - pthread_mutex_unlock(&c->done_mutex); + pthread_mutex_lock(&c->prune_done_mutex); + done = c->prune_done; + pthread_mutex_unlock(&c->prune_done_mutex); if (done) { break; } @@ -184,13 +638,21 @@ void* dedup_work(void* ctx) { continue; } + // Decrement queued_count for work_queue pop + pthread_mutex_lock(&c->metrics_mutex); + c->queued_count--; + pthread_mutex_unlock(&c->metrics_mutex); + + // Visit the entry (worker processes the file) visit_entry(fe, c->progress, c); - file_entry_free(fe); PROGRESS_LOCK(c->progress, &c->progress_mutex, { c->progress->completedUnitCount++; - display_progress(c->progress); }); + + // Show worker progress (entry processed) + display_status(c, fe->path); + file_entry_free(fe); if (c->thread_count == 0) { break; @@ -382,12 +844,16 @@ static void usage(char* pgm, DedupContext* ctx) { " --dry-run, -n Don't replace file content, just print what \n" " would have happend.\n" " --depth, -d depth Don't descend further than the specified depth.\n" + " --format, -F format Output format for byte sizes. See --help formats.\n" " --one-file-system, -x Don't evaluate directories on a different device\n" " than the starting paths.\n" " --link, -l Use hardlinks instead of clones.\n" " --symlink, -s Use symlinks instead of clones.\n" // " --color, -c Enabled colored output.\n" " --no-progress, -P Do not display a progress bar.\n" + " --no-clone-conversion Do not convert clones (skip clone mode)\n" + " --summary, -S file Write detailed cloning summary to file\n" + " (itemized by directory hierarchy)\n" " --threads, -t n The number of threads to use for file building\n" " lookup tables and replacing clones. Default: %d\n" " --verbose, -v Increase verbosity. May be used multiple times.\n" @@ -429,8 +895,7 @@ bool is_vol_cap_supported(char* path, int vol_cap) { if (result) { perror("Could not get volume stat"); - // TODO: exit? - return false; + exit(1); } // get the supported capabilities and attributes @@ -441,8 +906,7 @@ bool is_vol_cap_supported(char* path, int vol_cap) { FSOPT_ATTR_CMN_EXTENDED); if (result) { perror("Could not retrieve volume attributes"); - // TODO: exit? - return false; + exit(1); } #define VOL_CAPABILITIES_FORMAT 0 @@ -464,33 +928,10 @@ bool are_acls_supported(char* path) { return is_vol_cap_supported(path, VOL_CAP_INT_RENAME_SWAP); } -void print_human_bytes(uint64_t bytes) { - double v = bytes; - char* unit = " bytes"; - - if (v > 1000.0) { - v /= 1000.0; - unit = "kB"; - } - if (v > 1000.0) { - v /= 1000.0; - unit = "MB"; - } - if (v > 1000.0) { - v /= 1000.0; - unit = "GB"; - } - if (v > 1000.0) { - v /= 1000.0; - unit = "TB"; - } - - printf("%0.f%s", v, unit); -} - int main(int argc, char* argv[]) { FileEntryHead* queue = new_file_entry_queue(); + FileEntryHead* raw_queue = new_file_entry_queue(); Progress p = { 0 }; uint16_t max_depth = UINT16_MAX; int user_fts_options = 0; @@ -498,31 +939,50 @@ int main(int argc, char* argv[]) { DedupContext dc = { .progress = &p, .queue = queue, - .visited = new_visited_tree(), - .duplicates = new_duplicate_tree(), + .raw_queue = raw_queue, + .signatures = new_sig_table(65536), .found = 0, .saved = 0, .already_saved = 0, - .done = 0, + .pruned = 0, + .total_bytes = 0, + .queued_count = 0, + .next_file_sequence = 0, + .next_visit_sequence = 0, + .scan_done = 0, + .prune_done = 0, .dry_run = false, .verbosity = 0, .force = 0, .replace_mode = DEDUP_CLONE, + .output_format = OUTPUT_SI_HUMAN, + .clone_converted = true, + .summary_file = NULL, .thread_count = cpu_count(), .metrics_mutex = PTHREAD_MUTEX_INITIALIZER, .progress_mutex = PTHREAD_MUTEX_INITIALIZER, .queue_mutex = PTHREAD_MUTEX_INITIALIZER, - .visited_mutex = PTHREAD_MUTEX_INITIALIZER, - .duplicates_mutex = PTHREAD_MUTEX_INITIALIZER, - .done_mutex = PTHREAD_MUTEX_INITIALIZER, + .raw_queue_mutex = PTHREAD_MUTEX_INITIALIZER, + .signatures_mutex = PTHREAD_MUTEX_INITIALIZER, + .scan_done_mutex = PTHREAD_MUTEX_INITIALIZER, + .prune_done_mutex = PTHREAD_MUTEX_INITIALIZER, + .visit_order_mutex = PTHREAD_MUTEX_INITIALIZER, + .visit_order_cond = PTHREAD_COND_INITIALIZER, }; + // Validate signature table was created successfully + if (!dc.signatures) { + fprintf(stderr, "failed to create signature table\n"); + return 1; + } + static const struct option options[] = { { "ignore", required_argument, NULL, 'I' }, { "no-progress", no_argument, NULL, 'P' }, { "version", no_argument, NULL, 'V' }, { "color", optional_argument, NULL, 'c' }, { "depth", required_argument, NULL, 'd' }, + { "format", required_argument, NULL, 'F' }, { "link", no_argument, NULL, 'l' }, { "dry-run", no_argument, NULL, 'n' }, { "symlink", no_argument, NULL, 's' }, @@ -530,15 +990,17 @@ int main(int argc, char* argv[]) { { "verbose", no_argument, NULL, 'v' }, { "one-file-system", no_argument, NULL, 'x' }, // { "force", no_argument, NULL, 'f' }, + { "no-clone-conversion", no_argument, NULL, 'C' }, + { "summary", required_argument, NULL, 'S' }, { "help", no_argument, NULL, '?' }, { NULL, 0, NULL, 0 }, }; - bool human_readable = false; + bool human_readable = true; int ch = -1, t; short d; - while ((ch = getopt_long(argc, argv, "I:PVc::d:fhlnst:vx?", options, NULL)) != -1) { + while ((ch = getopt_long(argc, argv, "I:PVc::d:F:hlnst:vxCS:", options, NULL)) != -1) { switch (ch) { case 'I': fprintf(stderr, "-I is unimplemented\n"); @@ -561,6 +1023,9 @@ int main(int argc, char* argv[]) { } max_depth = d; break; + case 'F': + dc.output_format = parse_output_format(optarg); + break; case 'h': human_readable = true; break; @@ -589,6 +1054,12 @@ int main(int argc, char* argv[]) { case 'x': user_fts_options |= FTS_XDEV; break; + case 'C': + dc.clone_converted = false; + break; + case 'S': + dc.summary_file = strdup(optarg); + break; case '?': default: usage(argv[0], &dc); @@ -600,6 +1071,17 @@ int main(int argc, char* argv[]) { if (!isatty(STDOUT_FILENO)) { dc.progress = NULL; } + + // Print verbose runtime information if requested + if (dc.verbosity > 0) { + dedup_runtime_caps_print_verbose(); + dedup_runtime_dispatch_print_verbose(); + } + + // Open summary file if requested + if (dc.summary_file) { + clone_summary_open(dc.summary_file); + } static const char* const DEFAULT_PATHS[] = { ".", @@ -633,6 +1115,15 @@ int main(int argc, char* argv[]) { } // LCOV_EXCL_STOP + pthread_t pruner_thread = NULL; + if (dc.thread_count > 0) { + int r = pthread_create(&pruner_thread, NULL, prune_work, &dc); + if (r) { + warn("Could not create pruner thread: error %i", r); + pruner_thread = NULL; + } + } + pthread_t* threads = calloc(dc.thread_count, sizeof(pthread_t)); for (int i = 0; i < dc.thread_count; i++) { int r = pthread_create(&threads[i], NULL, dedup_work, &dc); @@ -644,6 +1135,14 @@ int main(int argc, char* argv[]) { } } + // Single-threaded seen sets (only used when thread_count == 0) + SeenSet* st_seen_inodes = NULL; + SeenSet* st_seen_clones = NULL; + if (dc.thread_count == 0) { + st_seen_inodes = new_seen_set(4096); + st_seen_clones = new_seen_set(4096); + } + dev_t current_dev = -1; bool clonefile_supported = false; FTSENT* entry = NULL; @@ -656,8 +1155,8 @@ int main(int argc, char* argv[]) { entry->fts_path, entry->fts_errno, e); - display_progress(dc.progress); }); + display_status(&dc, entry->fts_path); continue; } @@ -716,34 +1215,105 @@ int main(int argc, char* argv[]) { continue; } + // skip .padding files (browser cache files that are often locked) + const char* basename = strrchr(entry->fts_path, '/'); + if (basename && strcmp(basename + 1, ".padding") == 0) { + continue; + } + + // skip cloud storage mounts (GoogleDrive, Dropbox, iCloud, OneDrive) + if (strstr(entry->fts_path, "/Library/CloudStorage/")) { + continue; + } + + // skip iOS simulator cache files (often locked) + if (strstr(entry->fts_path, "/PhotoData/Caches/")) { + continue; + } + // at this point we have a regular file // that only has one link + pthread_mutex_lock(&dc.metrics_mutex); + dc.total_bytes += entry->fts_statp->st_size; + pthread_mutex_unlock(&dc.metrics_mutex); PROGRESS_LOCK(dc.progress, &dc.progress_mutex, { dc.progress->totalUnitCount++; - display_progress(dc.progress); }); + display_status(&dc, entry->fts_path); + + // Track queued count (increment for raw_queue entry) + pthread_mutex_lock(&dc.metrics_mutex); + dc.queued_count++; + pthread_mutex_unlock(&dc.metrics_mutex); - pthread_mutex_lock(&dc.queue_mutex); - file_entry_queue_append(queue, - entry->fts_path, - entry->fts_statp->st_dev, - entry->fts_statp->st_ino, - entry->fts_statp->st_nlink, - entry->fts_statp->st_flags, - entry->fts_statp->st_size, - entry->fts_level); - pthread_mutex_unlock(&dc.queue_mutex); + uint64_t sequence = dc.next_file_sequence++; if (dc.thread_count == 0) { - dedup_work(&dc); + // Single-threaded: prune and process inline + file_entry_queue_append(raw_queue, + entry->fts_path, + entry->fts_statp->st_dev, + entry->fts_statp->st_ino, + entry->fts_statp->st_nlink, + entry->fts_statp->st_flags, + entry->fts_statp->st_size, + sequence, + entry->fts_level); + FileEntry* fe = file_entry_next(raw_queue); + // Decrement queued_count since we're processing it immediately + pthread_mutex_lock(&dc.metrics_mutex); + dc.queued_count--; + pthread_mutex_unlock(&dc.metrics_mutex); + + if (prune_entry(fe, st_seen_inodes, st_seen_clones, &dc)) { + file_entry_free(fe); + } else { + // Survivor goes to work_queue (already counted in queued_count) + file_entry_queue_append(queue, + fe->path, + fe->device, + fe->inode, + fe->nlink, + fe->flags, + fe->size, + fe->sequence, + fe->level); + file_entry_free(fe); + dedup_work(&dc); + } + } else { + pthread_mutex_lock(&dc.raw_queue_mutex); + file_entry_queue_append(raw_queue, + entry->fts_path, + entry->fts_statp->st_dev, + entry->fts_statp->st_ino, + entry->fts_statp->st_nlink, + entry->fts_statp->st_flags, + entry->fts_statp->st_size, + sequence, + entry->fts_level); + pthread_mutex_unlock(&dc.raw_queue_mutex); } } fts_close(traversal); - pthread_mutex_lock(&dc.done_mutex); - dc.done = 1; - pthread_mutex_unlock(&dc.done_mutex); + // Signal FTS scan complete to pruner + pthread_mutex_lock(&dc.scan_done_mutex); + dc.scan_done = 1; + pthread_mutex_unlock(&dc.scan_done_mutex); + + // Wait for pruner to drain raw_queue + if (pruner_thread) { + if (pthread_join(pruner_thread, NULL)) { + fprintf(stderr, "Failed to wait for pruner thread\n"); + } + } + + // Signal pruner complete to workers + pthread_mutex_lock(&dc.prune_done_mutex); + dc.prune_done = 1; + pthread_mutex_unlock(&dc.prune_done_mutex); for (int i = 0; i < dc.thread_count; i++) { // clang-analyzer thinks threads[i] can be NULL, but `pthread_t` @@ -758,22 +1328,24 @@ int main(int argc, char* argv[]) { } free(threads); threads = NULL; + free_seen_set(st_seen_inodes); + free_seen_set(st_seen_clones); + free_file_entry_queue(raw_queue); raw_queue = NULL; free_file_entry_queue(queue); queue = NULL; - free_visited_tree(dc.visited); dc.visited = NULL; + free_sig_table(dc.signatures); dc.signatures = NULL; if (dc.progress) { clear_progress(); } printf("duplicates found: %zu\n", dc.found); + printf("entries pruned: %zu\n", dc.pruned); - SHA256ListNode* duplicate_set = NULL; - RB_TREE_FOREACH(duplicate_set, dc.duplicates) { - deduplicate(duplicate_set->list, &dc); - } + // Fast dedup processes files immediately during traversal + // No additional deduplication step needed printf("bytes saved: "); if (human_readable) { - print_human_bytes(dc.saved); + printf("%s", format_bytes(dc.saved, dc.output_format)); } else { printf("%zu", dc.saved); } @@ -781,12 +1353,21 @@ int main(int argc, char* argv[]) { printf("already saved: "); if (human_readable) { - print_human_bytes(dc.already_saved); + printf("%s", format_bytes(dc.already_saved, dc.output_format)); } else { printf("%zu", dc.already_saved); } putchar('\n'); - free_duplicate_tree(dc.duplicates); dc.duplicates = NULL; + // Clear status line + if (isatty(STDOUT_FILENO)) { + fprintf(stderr, "\r\033[K\n"); + } + + // Close summary file if it was opened + if (dc.summary_file) { + clone_summary_close(); + } + return 0; } diff --git a/docs/plans/2026-04-22-ffmpeg-style-runtime-dispatch.md b/docs/plans/2026-04-22-ffmpeg-style-runtime-dispatch.md new file mode 100644 index 0000000..3c3d31e --- /dev/null +++ b/docs/plans/2026-04-22-ffmpeg-style-runtime-dispatch.md @@ -0,0 +1,477 @@ +# FFmpeg-Style Runtime Dispatch for Hashing and Exact Comparison + +> For Hermes: Use subagent-driven-development skill to implement this plan task-by-task. + +Goal: Add a one-time runtime capability probe and dispatch table so dedup can choose the best hashing and exact-comparison algorithm per workload on Apple Silicon and other targets. + +Architecture: Follow the FFmpeg pattern: detect capabilities once, cache them, allow forced overrides for testing, bind function pointers at init, and use measured crossover thresholds instead of branching inside hot loops. Keep correctness strict: hashes are filters only; only exact compare authorizes dedup. + +Tech Stack: C23/C2x, macOS sysctl feature probing, optional Metal path later, existing Makefile build, Check test suite. + +--- + +## Desired end state + +1. A new runtime capability layer discovers: +- CPU ISA features +- platform traits like unified memory / Metal availability +- measured throughput for candidate compare/hash backends + +2. A new dispatch table selects: +- fast candidate hash +- strong staged hash +- exact compare for small inputs +- exact compare for large inputs +- optional batch/stream compare backend + +3. dedup.c uses dispatch functions, not hard-coded direct calls. + +4. Forced overrides exist for CI, debugging, and benchmarking. + +5. Exact proof remains mandatory before deduplication. + +--- + +## Proposed files + +Create: +- runtime_caps.h +- runtime_caps.c +- runtime_dispatch.h +- runtime_dispatch.c +- test/runtime_dispatch_suite.c +- test/runtime_dispatch_suite.h + +Modify: +- signature.h +- signature.c +- sig_table.c +- dedup.c +- Makefile +- test/Makefile +- test/dedup_check.c + +Later/optional: +- runtime_metal_compare.h +- runtime_metal_compare.m +- runtime_bench.c + +--- + +## Public API sketch + +### runtime_caps.h + +```c +#ifndef __DEDUP_RUNTIME_CAPS_H__ +#define __DEDUP_RUNTIME_CAPS_H__ + +#include +#include +#include + +typedef struct DedupRuntimeCaps { + bool apple_arm64; + bool neon; + bool dotprod; + bool i8mm; + bool crc32; + bool pmull; + bool sha3; + bool unified_memory; + bool metal_available; + + double memcmp_gib_s_4k; + double memcmp_gib_s_64k; + double memcmp_gib_s_1m; + double memcmp_gib_s_8m; +} DedupRuntimeCaps; + +const DedupRuntimeCaps* dedup_runtime_caps_get(void); +void dedup_runtime_caps_reset_for_tests(void); + +#endif +``` + +### runtime_dispatch.h + +```c +#ifndef __DEDUP_RUNTIME_DISPATCH_H__ +#define __DEDUP_RUNTIME_DISPATCH_H__ + +#include +#include +#include + +typedef uint64_t (*dedup_fast_hash_fn)(const void* data, size_t len); +typedef bool (*dedup_exact_compare_fn)(const char* a_path, const char* b_path); + +typedef struct DedupRuntimeDispatch { + const char* fast_hash_name; + const char* strong_hash_name; + const char* exact_small_name; + const char* exact_large_name; + + dedup_fast_hash_fn fast_hash; + dedup_fast_hash_fn strong_hash; + dedup_exact_compare_fn exact_small; + dedup_exact_compare_fn exact_large; + + size_t exact_large_threshold; +} DedupRuntimeDispatch; + +const DedupRuntimeDispatch* dedup_runtime_dispatch_get(void); +void dedup_runtime_dispatch_reset_for_tests(void); + +#endif +``` + +--- + +## Forced override environment variables + +Implement all of these early so tuning is testable: +- DEDUP_FORCE_FAST_HASH=xxhash|rapidhash|komihash|blake3 +- DEDUP_FORCE_STRONG_HASH=none|blake3|sha3 +- DEDUP_FORCE_EXACT_COMPARE=memcmp|cpu_tiles|gpu_stream +- DEDUP_FORCE_GPU=0|1 +- DEDUP_DISABLE_BENCH=0|1 + +Behavior: +- invalid values print a warning and fall back to auto +- test code should be able to clear/reset caches between runs + +--- + +## Capability detection plan + +### Task 1: Add runtime_caps scaffolding + +Objective: Introduce a cached capability record with reset support. + +Files: +- Create: `runtime_caps.h` +- Create: `runtime_caps.c` +- Test: `test/runtime_dispatch_suite.c` + +Step 1: Write failing tests for cache/reset behavior. +Step 2: Implement a static cached record with lazy init. +Step 3: Add reset helper for tests. +Step 4: Run new suite only. +Step 5: Commit. + +### Task 2: Add macOS Apple Silicon feature probing + +Objective: Probe the same kind of sysctl feature bits FFmpeg uses on aarch64/macOS. + +Files: +- Modify: `runtime_caps.c` +- Test: `test/runtime_dispatch_suite.c` + +Probe these keys when available: +- `hw.optional.arm.FEAT_DotProd` +- `hw.optional.arm.FEAT_I8MM` +- `hw.optional.armv8_crc32` +- `hw.optional.arm.FEAT_PMULL` +- `hw.optional.armv8_2_sha3` + +Also derive: +- `apple_arm64` +- `neon=true` on arm64 Apple Silicon +- `unified_memory=true` on Apple Silicon +- `metal_available` initially by presence heuristic or stub false; refine later + +Step 1: Add helper `have_sysctl_u32(name)`. +Step 2: Fill `DedupRuntimeCaps` fields. +Step 3: Add tests that validate override/reset plumbing and non-crash behavior. +Step 4: Run suite. +Step 5: Commit. + +### Task 3: Add microbenchmark fields for memcmp baselines + +Objective: Capture the CPU baseline that all alternates must beat. + +Files: +- Modify: `runtime_caps.c` +- Test: `test/runtime_dispatch_suite.c` + +Measure representative buckets: +- 4 KiB +- 64 KiB +- 1 MiB +- 8 MiB + +Use equal or late-mismatch buffers so benchmark approximates proof-of-equality cost. + +Step 1: Write test that ensures benchmark fields are initialized to positive values or zero when disabled. +Step 2: Add a small internal benchmark helper. +Step 3: Honor `DEDUP_DISABLE_BENCH=1`. +Step 4: Run suite. +Step 5: Commit. + +--- + +## Dispatch table plan + +### Task 4: Add runtime_dispatch scaffolding + +Objective: Introduce one-time dispatch binding. + +Files: +- Create: `runtime_dispatch.h` +- Create: `runtime_dispatch.c` +- Test: `test/runtime_dispatch_suite.c` + +Default initial policy: +- fast hash: existing xxhash64 implementation until replacements land +- strong hash: none +- exact small: `files_match_exact` +- exact large: `files_match_exact` + +Step 1: Write failing tests for default binding and cache/reset. +Step 2: Implement static dispatch object. +Step 3: Export selector getter. +Step 4: Run suite. +Step 5: Commit. + +### Task 5: Add forced override parsing + +Objective: Make algorithm choice controllable. + +Files: +- Modify: `runtime_dispatch.c` +- Test: `test/runtime_dispatch_suite.c` + +Step 1: Write failing tests for `DEDUP_FORCE_*` parsing. +Step 2: Implement environment parsing. +Step 3: Bind named backends. +Step 4: Run suite. +Step 5: Commit. + +### Task 6: Add heuristic size thresholds + +Objective: Support FFmpeg-style “bind once, choose by size class”. + +Files: +- Modify: `runtime_dispatch.h` +- Modify: `runtime_dispatch.c` +- Modify: `signature.c` +- Test: `test/runtime_dispatch_suite.c` + +Initial policy: +- `< 64 KiB`: exact_small +- `>= 64 KiB`: exact_large + +Keep it simple first. Later tune thresholds from benchmarks. + +Step 1: Add wrapper in `signature.c` that routes through dispatch. +Step 2: Add tests that threshold choice is stable. +Step 3: Run suite. +Step 4: Commit. + +--- + +## Hash backend plan + +### Task 7: Isolate current xxhash helper behind a backend API + +Objective: Stop hard-coding xxhash inside signature construction. + +Files: +- Modify: `signature.h` +- Modify: `signature.c` +- Modify: `runtime_dispatch.c` +- Test: `test/signature_suite.c` + +Step 1: Write failing tests that current behavior remains unchanged. +Step 2: Rename existing helper into a backend-style function. +Step 3: Route quick hash through dispatch-selected backend. +Step 4: Run signature suite. +Step 5: Commit. + +### Task 8: Add candidate replacement backends incrementally + +Objective: Make room for better SMHasher3-ranked filters. + +Files: +- Modify: `runtime_dispatch.c` +- Maybe create later: `hash_backend_rapidhash.c`, `hash_backend_komihash.c` +- Test: `test/runtime_dispatch_suite.c` + +Order: +1. xxhash existing backend +2. placeholder backend names with fallback +3. one real replacement backend at a time + +Note: do not land an external backend without updating license/dependency posture. + +--- + +## Exact compare and GPU plan + +### Task 9: Split exact compare into explicit backends + +Objective: Separate correctness oracle from dispatch choice. + +Files: +- Modify: `signature.c` +- Modify: `runtime_dispatch.c` +- Test: `test/signature_suite.c` + +Backends: +- `memcmp_exact_compare` (CPU baseline) +- `cpu_tiled_exact_compare` (later) +- `gpu_stream_exact_compare` (stub returns unavailable for now) + +Step 1: Extract current exact compare as named CPU backend. +Step 2: Add dispatch wrappers. +Step 3: Keep behavior identical. +Step 4: Run signature suite. +Step 5: Commit. + +### Task 10: Add GPU streaming compare stubs with capability gating + +Objective: Prepare for SoC GPU acceleration without changing semantics. + +Files: +- Modify: `runtime_caps.h` +- Modify: `runtime_caps.c` +- Modify: `runtime_dispatch.c` +- Test: `test/runtime_dispatch_suite.c` + +Behavior: +- only selected if available and forced/benchmarked +- for now may remain unimplemented and never auto-selected + +This task exists to freeze the API, not to ship Metal yet. + +--- + +## Integration plan + +### Task 11: Convert sig_table to dispatch-selected exact compare + +Objective: Remove direct coupling to `files_match_exact()`. + +Files: +- Modify: `sig_table.c` +- Test: `test/signature_suite.c` + +Replace: +- direct call to `files_match_exact()` +With: +- dispatch-selected exact compare chosen by file size threshold + +Step 1: Write failing test around same behavior. +Step 2: Swap call site. +Step 3: Run signature suite. +Step 4: Commit. + +### Task 12: Surface backend names in verbose/debug output + +Objective: Make runtime selection inspectable. + +Files: +- Modify: `dedup.c` +- Modify: `runtime_dispatch.c` +- Test: `test/runtime_dispatch_suite.c` + +Verbose output example: +- `runtime: fast_hash=rapidhash strong_hash=none exact_small=memcmp exact_large=memcmp threshold=65536` + +Step 1: Add getter for names. +Step 2: Print only under verbosity/debug flag. +Step 3: Run tests. +Step 4: Commit. + +--- + +## Test plan + +### Required test files + +Create/extend: +- `test/signature_suite.c` + - sub-4-byte files still work + - sample-only collision rejected + - dispatch path preserves correctness +- `test/runtime_dispatch_suite.c` + - cache/reset behavior + - env override parsing + - threshold routing + - benchmark disable path + - non-crash feature probing + +### Commands + +New focused command: +```bash +make dedup && cd test && make dedup_check && CK_RUN_SUITE=signature ./dedup_check +``` + +Extended command once runtime suite is added: +```bash +make dedup && cd test && make dedup_check && CK_RUN_SUITE=runtime_dispatch ./dedup_check +``` + +Full targeted validation before PR update: +```bash +git diff --cached --check +make dedup +cd test && make dedup_check +CK_RUN_SUITE=signature ./dedup_check +CK_RUN_SUITE=runtime_dispatch ./dedup_check +``` + +--- + +## Heuristic policy to start with + +Initial auto policy: +- fast hash: existing xxhash backend +- strong hash: none +- exact small: memcmp-based current exact compare +- exact large: memcmp-based current exact compare +- GPU: never auto-selected yet + +Second milestone: +- fast hash: rapidhash or komihash if integrated +- strong hash: BLAKE3 optional +- exact large: GPU only when batch size and file size exceed measured threshold + +--- + +## Non-goals for first pass + +Do not do these in the first pass: +- no Metal kernel implementation yet +- no external hash dependency import unless explicitly approved +- no per-call dynamic branching in hot loops +- no weakening of exact-compare correctness +- no replacing memcmp just because a probabilistic stage looks strong + +--- + +## Verification checklist + +Before calling this done: +- [ ] capability detection is cached and resettable +- [ ] env overrides work +- [ ] dispatch table is bound once +- [ ] sig_table uses dispatch-selected exact compare +- [ ] hashes remain filters only +- [ ] focused suites pass +- [ ] verbose output reveals chosen backends +- [ ] no license contamination from pasted FFmpeg code + +--- + +## Notes specific to this repo + +Current relevant locations: +- `signature.c` owns quick-hash and exact-compare logic +- `sig_table.c` decides whether two candidates are equal +- `dedup.c` owns main runtime flow and verbosity output +- `test/signature_suite.c` already contains the collision and small-file regressions we need to preserve + +This plan should be implemented before any Metal compare work so the backend seam exists first. \ No newline at end of file diff --git a/docs/plans/2026-04-22-progressive-tiled-verifier.md b/docs/plans/2026-04-22-progressive-tiled-verifier.md new file mode 100644 index 0000000..ef281a5 --- /dev/null +++ b/docs/plans/2026-04-22-progressive-tiled-verifier.md @@ -0,0 +1,517 @@ +# Progressive Tiled Verifier Design + +> For Hermes: Implement only after the FFmpeg-style runtime dispatch seam exists. Keep exact compare as the only authorization path for dedup. + +Goal: Add a staged verifier that uses cheap filters, then stronger tiled witnesses, then a terminal full-coverage exact compare. The design must support CPU-only operation first and GPU streaming later on Apple Silicon unified-memory systems. + +Architecture: Treat every probabilistic or compressed stage as a candidate filter only. Stages may reject quickly, but never accept finally. The terminal stage must cover every byte and reduce exact differences without information loss. + +Tech Stack: C23/C2x, Apple Silicon runtime dispatch, optional Metal compute backend later, existing runtime capability layer, existing signature/sig_table integration. + +--- + +## Core rule + +Only this predicate authorizes equality: + +```text +equal(A, B) := OR_reduce_over_all_bytes(A XOR B) == 0 +``` + +Everything else is a screening step. + +That means: +- hashes do not prove equality +- sampled overlap does not prove equality +- integer dot products do not prove equality +- floating-point FMA does not prove equality +- only full-coverage exact reduction proves equality + +--- + +## Verifier pipeline + +The verifier should be modeled as these stages: + +1. Stage 0: metadata filter +- same size required +- same device only if clone semantics require it +- skip empty mismatched metadata immediately + +2. Stage 1: cheap candidate filter +- existing signature samples +- fast candidate hash on first prefix window +- optional tail window hash + +3. Stage 2: progressive tiled witnesses +- tiled overlap windows +- integer projection / polynomial witnesses per tile +- may run on CPU vector path or GPU batched path + +4. Stage 3: terminal exact verify +- full byte coverage +- XOR each lane +- OR-reduce mismatch lanes across all tiles +- equal only if final reduction mask is zero + +Stages 1 and 2 may only reject. Stage 3 may reject or accept. + +--- + +## Mathematical model + +### Exact proof primitive + +For files represented as byte vectors `a[i]` and `b[i]`: + +```text +d[i] = a[i] XOR b[i] +D = OR_i d[i] +A == B iff D == 0 +``` + +Equivalent vectorized form per tile: + +```text +lane_diff = xor(vec_a, vec_b) +tile_mask = or_reduce(lane_diff) +file_mask = OR over all tile_mask +``` + +This is the only terminal reduction allowed to certify equality. + +### Progressive witness primitive + +For a tile `t` of bytes, build one or more exact integer witnesses: + +```text +w_k(t) = sum_i coeff_k[i] * byte_i mod 2^64 +``` + +Compare witnesses between candidate files: + +```text +match_k(t) = (w_k(A_t) == w_k(B_t)) +``` + +Properties: +- useful for fast rejection +- can make false accepts extremely rare +- still not proof because information is compressed + +### Polynomial / PMULL witness primitive + +A GF(2)-style tile witness can be computed as: + +```text +w(t) = fold_pmull(tile_bytes, tile_seed) +``` + +This is a strong filter for Apple Silicon because PMULL exists on this machine. + +### Overlap discipline + +Use overlap so local edits are less likely to be hidden by boundary alignment. + +For tile size `T` and stride `S`, require: +- `S < T` +- recommended start: `T = 256 KiB`, `S = 128 KiB` + +This means every interior byte participates in at least two witness tiles. + +Important: overlap increases witness strength but still does not replace terminal exact coverage. + +--- + +## CPU verifier design + +### Stage 2 CPU backend: cpu_tiles + +Purpose: +- strong rejection path for medium and large files +- no GPU dependency +- built from NEON, PMULL, DOTPROD, I8MM where available + +Suggested backends by hardware: +- scalar fallback +- NEON XOR/OR exact backend +- PMULL witness backend +- DOTPROD witness backend +- I8MM witness backend only if it materially helps byte-lane accumulation + +### CPU tiled witness layout + +Per tile compute: +- prefix fast hash of tile +- tail fast hash of tile +- 2-4 independent integer witnesses +- optional polynomial witness + +Recommended witness set on Apple Silicon: +- witness0: 64-bit sum of bytes weighted by lane index +- witness1: 64-bit sum of bytes weighted by seed-derived coefficients +- witness2: PMULL-based polynomial fold +- witness3: optional second PMULL fold with independent seed + +Reject immediately if any witness differs. +If all witnesses match, advance to terminal exact stage. + +### CPU terminal exact backend + +Do not call this “memcmp” in the design unless it really is libc memcmp. +Model it as: +- load vector lane +- XOR +- accumulate OR into a running mismatch register +- after each chunk, if mismatch register != 0 return false +- if all chunks completed with zero mismatch, return true + +This allows later swapping between: +- libc memcmp +- manual NEON XOR/OR +- CPU tiled exact compare + +### CPU exact compare pseudocode + +```c +bool cpu_exact_xor_or(const uint8_t* a, const uint8_t* b, size_t len) { + uint8x16_t acc = vdupq_n_u8(0); + size_t i = 0; + + for (; i + 16 <= len; i += 16) { + uint8x16_t va = vld1q_u8(a + i); + uint8x16_t vb = vld1q_u8(b + i); + acc = vorrq_u8(acc, veorq_u8(va, vb)); + } + + if (vmaxvq_u8(acc) != 0) + return false; + + for (; i < len; i++) { + if ((a[i] ^ b[i]) != 0) + return false; + } + return true; +} +``` + +--- + +## GPU verifier design + +### When GPU is worth using + +On this M3 Pro we measured libc memcmp near 48 GiB/s for large equal buffers. GPU should only be selected when batching and unified memory amortize dispatch overhead. + +Initial policy targets GPU only when all of these are true: +- unified memory available +- Metal available +- file size >= 8 MiB +- candidate batch count >= 16 +- size bucket is homogeneous enough to tile efficiently + +Never auto-select GPU for: +- tiny files +- one-off candidate pairs +- latency-sensitive single comparisons + +### GPU backend split + +1. gpu_witness_stream +- computes strong overlapped tile witnesses for many pairs +- returns per-pair reject/pass-to-exact flags + +2. gpu_exact_stream +- computes XOR/OR reduction over all tiles +- returns equality bit only +- must be exact, not probabilistic + +The first backend is optional; the second is the backend that can actually authorize equality. + +### GPU memory model on Apple Silicon + +Prefer: +- shared / managed buffers in unified memory +- command-buffer batching over many pairs +- compact per-pair descriptor tables + +Avoid: +- re-packing file contents into large transient staging buffers unless measurement proves it wins +- per-pair command buffer submission + +### Pair descriptor sketch + +```c +typedef struct DedupComparePairDesc { + const uint8_t* a; + const uint8_t* b; + uint64_t len; + uint64_t pair_id; + uint64_t tile_size; + uint64_t tile_stride; +} DedupComparePairDesc; +``` + +### GPU witness kernel shape + +Each threadgroup handles one tile of one pair. + +Inputs: +- base pointer A +- base pointer B +- tile offset +- tile length +- seeds / coefficients + +Outputs per tile: +- tile_witness_equal bit +- optional witness values for debugging + +Reduction: +- one per-pair bitmask saying whether any tile witness failed + +### GPU exact kernel shape + +Each threadgroup handles one exact tile of one pair. + +Per lane: +- load bytes from A and B +- XOR them +- OR-reduce within SIMDgroup / threadgroup +- write tile mismatch bit + +Final reduction: +- CPU or second GPU kernel reduces tile mismatch bits to one pair result +- pair is equal iff all tile bits are zero + +This kernel is exact because every byte is covered and only XOR/OR reduction is used. + +--- + +## Tile geometry + +### Witness tiles + +Initial witness geometry: +- tile size: 256 KiB +- stride: 128 KiB +- overlap: 50% + +Why: +- large enough to amortize launch / vector setup +- overlap prevents boundary-only edits from hiding in a single tiling grid +- still manageable for CPU cache streaming and GPU batching + +### Exact tiles + +Initial exact geometry: +- CPU exact tile: 64 KiB to 1 MiB chunks, backend-tuned +- GPU exact tile: 256 KiB to 1 MiB chunks depending on command-buffer efficiency + +Exact tiles do not need overlap because they cover all bytes exactly. + +--- + +## Runtime dispatch policy + +This should fit into the existing FFmpeg-style dispatch plan. + +### Public backend names + +Fast candidate hash: +- `xxhash` +- `rapidhash` +- `komihash` + +Strong staged hash: +- `none` +- `blake3` +- `pmull_poly` + +Exact compare backends: +- `memcmp` +- `cpu_xor_or` +- `cpu_tiles` +- `gpu_exact_stream` + +Witness backends: +- `none` +- `cpu_witness` +- `gpu_witness_stream` + +### Suggested dispatch struct extension + +```c +typedef bool (*dedup_exact_compare_fn)(const char* a_path, const char* b_path); +typedef bool (*dedup_pair_witness_fn)(const char* a_path, const char* b_path, uint64_t size); + +typedef struct DedupRuntimeDispatch { + const char* fast_hash_name; + const char* strong_hash_name; + const char* witness_name; + const char* exact_small_name; + const char* exact_large_name; + + dedup_fast_hash_fn fast_hash; + dedup_fast_hash_fn strong_hash; + dedup_pair_witness_fn witness; + dedup_exact_compare_fn exact_small; + dedup_exact_compare_fn exact_large; + + size_t witness_threshold; + size_t exact_large_threshold; + size_t gpu_batch_threshold; +} DedupRuntimeDispatch; +``` + +### Initial auto policy + +On Apple Silicon M3 Pro-like systems: +- `< 64 KiB`: exact_small = memcmp +- `64 KiB .. < 8 MiB`: exact_large = cpu_xor_or or cpu_tiles +- `>= 8 MiB` and batch < 16: exact_large = cpu_xor_or or memcmp +- `>= 8 MiB` and batch >= 16 and Metal available: witness = gpu_witness_stream, exact_large = gpu_exact_stream + +### Forced overrides + +Extend prior env knobs with: +- `DEDUP_FORCE_WITNESS=none|cpu_witness|gpu_witness_stream` +- `DEDUP_WITNESS_THRESHOLD_BYTES=` +- `DEDUP_GPU_BATCH_THRESHOLD=` +- `DEDUP_TILE_SIZE=` +- `DEDUP_TILE_STRIDE=` + +--- + +## Benchmarking policy + +The verifier is not finished until it chooses backends by measured crossover rather than taste. + +### Measure at init or cached first use + +For each backend class, measure at least: +- 4 KiB +- 64 KiB +- 1 MiB +- 8 MiB +- 64 MiB + +For exact backends test: +- equal buffers +- late mismatch buffers +- early mismatch buffers + +For GPU test: +- batch counts 1, 8, 16, 64 +- homogeneous size buckets only + +### Decision rule + +Example initial policy: +- if exact backend is slower than memcmp by more than 10% in a bucket, do not auto-select it +- if GPU exact backend only wins at batch >= 16, set `gpu_batch_threshold = 16` +- if witness stage plus exact stage is slower than exact stage alone, skip witness stage for that bucket + +This is the same spirit as FFmpeg’s “feature flags plus slow flags” model. + +--- + +## Correctness invariants + +These are mandatory: + +1. A witness backend may only produce: +- reject +- continue-to-exact + +2. An exact backend must be semantically equivalent to bytewise equality. + +3. Floating-point reductions may not participate in terminal equality. + +4. Tile overlap may increase witness strength but may not substitute for exact coverage. + +5. Any backend I/O error must fail closed. + +6. Any GPU “equal” result must mean the exact XOR/OR reduction was zero over the full file. + +--- + +## Suggested implementation tasks + +### Task A: Extend runtime dispatch types +- add witness backend slot +- add thresholds for witness and GPU batch selection + +### Task B: Add CPU exact XOR/OR backend +- keep current `files_match_exact()` as baseline +- add named exact backend API + +### Task C: Add CPU witness backend +- implement overlapped tiles +- 2-4 integer witnesses per tile +- PMULL witness on Apple Silicon when available + +### Task D: Add benchmark harness for witness vs exact-only +- verify whether witness stage is worthwhile by bucket + +### Task E: Add Metal pair descriptor and stub backend +- no kernel yet +- freeze API so later GPU work plugs in cleanly + +### Task F: Add Metal exact XOR/OR backend +- exact reduction only +- compare against memcmp baseline and gate by throughput + +--- + +## Testing plan + +Add to `test/runtime_dispatch_suite.c`: +- forced witness selection parsing +- threshold routing +- disabled benchmark path +- reset/cached dispatch behavior + +Add to `test/signature_suite.c`: +- witness collision does not authorize equality +- exact backend still rejects sample-only collision case +- exact backend still accepts identical small files +- exact backend still works when witness stage is skipped + +Later GPU-specific tests: +- GPU exact backend equals CPU exact backend for generated fixtures +- batch-mode routing only triggers when thresholds are met + +--- + +## Notes for this machine + +Current observed hardware/runtime facts: +- Apple M3 Pro +- 18-core GPU +- unified memory +- DotProd available +- I8MM available +- CRC32 available +- PMULL available +- SHA3 available +- measured libc memcmp roughly 48 GiB/s for large equal buffers + +Implication: +- CPU exact compare is already extremely strong +- GPU path must win on batched large-file throughput, not single-pair latency +- PMULL-backed witness stages are especially attractive here + +--- + +## Bottom line + +The correct architecture is not: +- “keep adding stronger hashes until probability is good enough” + +It is: +- cheap filter +- strong progressive witness +- exact terminal XOR/OR proof + +That preserves correctness while still giving room for CPU vector and GPU streaming acceleration. \ No newline at end of file diff --git a/output_format.c b/output_format.c new file mode 100644 index 0000000..39a1df9 --- /dev/null +++ b/output_format.c @@ -0,0 +1,345 @@ +// Copyright © 2025 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include "output_format.h" +#include +#include +#include +#include +#include + +// Static buffer for formatted output +static char format_buffer[256]; + +typedef struct UnitInfo { + const char* short_name; + const char* long_name; + uint64_t divisor; +} UnitInfo; + +// SI units (decimal, 1000-based) +static const UnitInfo si_units[] = { + {"bytes", "bytes", 1}, + {"kB", "kilobytes", 1000ULL}, + {"MB", "megabytes", 1000000ULL}, + {"GB", "gigabytes", 1000000000ULL}, + {"TB", "terabytes", 1000000000000ULL}, + {"PB", "petabytes", 1000000000000000ULL}, + {NULL, NULL, 0} +}; + +// Binary units (1024-based) +static const UnitInfo binary_units[] = { + {"bytes", "bytes", 1}, + {"KiB", "kibibytes", 1024ULL}, + {"MiB", "mebibytes", 1048576ULL}, + {"GiB", "gibibytes", 1073741824ULL}, + {"TiB", "tebibytes", 1099511627776ULL}, + {"PiB", "pebibytes", 1125899906842624ULL}, + {NULL, NULL, 0} +}; + +// Traditional disk tool units (mixed) +static const UnitInfo traditional_units[] = { + {"B", "bytes", 1}, + {"K", "kilobytes", 1000ULL}, + {"M", "megabytes", 1000000ULL}, + {"G", "gigabytes", 1000000000ULL}, + {"T", "terabytes", 1000000000000ULL}, + {NULL, NULL, 0} +}; + +static const char* format_with_commas(uint64_t num) { + static char comma_buffer[32]; + static char temp[32]; + int len = snprintf(temp, sizeof(temp), "%llu", (unsigned long long)num); + int comma_count = (len - 1) / 3; + size_t result_len = (size_t)len + (size_t)comma_count; + char* result = comma_buffer; + + if (result_len >= sizeof(comma_buffer)) { + return temp; // Fallback if too big + } + + result[result_len] = '\0'; + int src = len - 1; + int dst = result_len - 1; + + for (int i = 0; i < len; i++) { + if (i > 0 && i % 3 == 0) { + result[dst--] = ','; + } + result[dst--] = temp[src--]; + } + + return result; +} + +static const char* format_with_units(uint64_t bytes, const UnitInfo* units, int use_long_names) { + double value = (double)bytes; + const UnitInfo* unit = units; + + // Find the appropriate unit + while (unit->short_name && value >= unit[1].divisor) { + unit++; + } + + if (unit == units) { + // Use bytes + if (use_long_names) { + snprintf(format_buffer, sizeof(format_buffer), "%llu %s", + (unsigned long long)bytes, unit->long_name); + } else { + snprintf(format_buffer, sizeof(format_buffer), "%llu %s", + (unsigned long long)bytes, unit->short_name); + } + } else { + // Use scaled unit + value = (double)bytes / (double)unit->divisor; + if (use_long_names) { + snprintf(format_buffer, sizeof(format_buffer), "%.1f %s", + value, unit->long_name); + } else { + snprintf(format_buffer, sizeof(format_buffer), "%.1f%s", + value, unit->short_name); + } + } + + return format_buffer; +} + +const char* format_bytes(uint64_t bytes, OutputFormat format) { + switch (format) { + case OUTPUT_RAW: + snprintf(format_buffer, sizeof(format_buffer), "%llu", + (unsigned long long)bytes); + break; + + case OUTPUT_RAW_COMMAS: + snprintf(format_buffer, sizeof(format_buffer), "%s", + format_with_commas(bytes)); + break; + + case OUTPUT_SI_HUMAN: + return format_with_units(bytes, si_units, 0); + + case OUTPUT_SI_HUMAN_LONG: + return format_with_units(bytes, si_units, 1); + + case OUTPUT_BINARY_HUMAN: + return format_with_units(bytes, binary_units, 0); + + case OUTPUT_BINARY_HUMAN_LONG: + return format_with_units(bytes, binary_units, 1); + + case OUTPUT_SCIENTIFIC: + snprintf(format_buffer, sizeof(format_buffer), "%.2e", + (double)bytes); + break; + + case OUTPUT_SCIENTIFIC_COMMAS: + { + char temp[64]; + snprintf(temp, sizeof(temp), "%.2e", (double)bytes); + // For scientific notation with commas, we'd need to parse and format + // the mantissa. For now, just return scientific. + snprintf(format_buffer, sizeof(format_buffer), "%s", temp); + } + break; + + case OUTPUT_DISK_TRADITIONAL: + return format_with_units(bytes, traditional_units, 0); + + case OUTPUT_DISK_TRADITIONAL_LONG: + return format_with_units(bytes, traditional_units, 1); + + case OUTPUT_COMPACT: + if (bytes < 1000) { + snprintf(format_buffer, sizeof(format_buffer), "%llu", + (unsigned long long)bytes); + } else if (bytes < 1000000) { + snprintf(format_buffer, sizeof(format_buffer), "%.0fK", + (double)bytes / 1000.0); + } else if (bytes < 1000000000) { + snprintf(format_buffer, sizeof(format_buffer), "%.0fM", + (double)bytes / 1000000.0); + } else { + snprintf(format_buffer, sizeof(format_buffer), "%.0fG", + (double)bytes / 1000000000.0); + } + break; + + case OUTPUT_COMPACT_LONG: + if (bytes < 1000) { + snprintf(format_buffer, sizeof(format_buffer), "%llu bytes", + (unsigned long long)bytes); + } else if (bytes < 1000000) { + snprintf(format_buffer, sizeof(format_buffer), "%.0f kilobytes", + (double)bytes / 1000.0); + } else if (bytes < 1000000000) { + snprintf(format_buffer, sizeof(format_buffer), "%.0f megabytes", + (double)bytes / 1000000.0); + } else { + snprintf(format_buffer, sizeof(format_buffer), "%.0f gigabytes", + (double)bytes / 1000000000.0); + } + break; + + case OUTPUT_KILO: + snprintf(format_buffer, sizeof(format_buffer), "%.0f", + (double)bytes / 1000.0); + break; + + case OUTPUT_KIBI: + snprintf(format_buffer, sizeof(format_buffer), "%.0f", + (double)bytes / 1024.0); + break; + + case OUTPUT_KILO_UNIT: + snprintf(format_buffer, sizeof(format_buffer), "%.0fk", + (double)bytes / 1000.0); + break; + + case OUTPUT_KIBI_UNIT: + snprintf(format_buffer, sizeof(format_buffer), "%.0fK", + (double)bytes / 1024.0); + break; + + case OUTPUT_HUMAN: + return format_with_units(bytes, si_units, 0); + + default: + return format_bytes(bytes, OUTPUT_SI_HUMAN); + } + + return format_buffer; +} + +OutputFormat get_default_output_format(void) { + return OUTPUT_SI_HUMAN; +} + +OutputFormat parse_output_format(const char* format_str) { + if (!format_str) return OUTPUT_SI_HUMAN; + + // Raw formats + if (strcmp(format_str, "raw") == 0) return OUTPUT_RAW; + if (strcmp(format_str, "raw-commas") == 0) return OUTPUT_RAW_COMMAS; + + // SI formats + if (strcmp(format_str, "si") == 0 || strcmp(format_str, "human") == 0) return OUTPUT_SI_HUMAN; + if (strcmp(format_str, "si-long") == 0 || strcmp(format_str, "human-long") == 0) return OUTPUT_SI_HUMAN_LONG; + + // Binary formats + if (strcmp(format_str, "binary") == 0 || strcmp(format_str, "iec") == 0) return OUTPUT_BINARY_HUMAN; + if (strcmp(format_str, "binary-long") == 0 || strcmp(format_str, "iec-long") == 0) return OUTPUT_BINARY_HUMAN_LONG; + + // Scientific formats + if (strcmp(format_str, "scientific") == 0 || strcmp(format_str, "sci") == 0) return OUTPUT_SCIENTIFIC; + if (strcmp(format_str, "scientific-commas") == 0 || strcmp(format_str, "sci-commas") == 0) return OUTPUT_SCIENTIFIC_COMMAS; + + // Traditional disk tool formats + if (strcmp(format_str, "traditional") == 0 || strcmp(format_str, "disk") == 0) return OUTPUT_DISK_TRADITIONAL; + if (strcmp(format_str, "traditional-long") == 0 || strcmp(format_str, "disk-long") == 0) return OUTPUT_DISK_TRADITIONAL_LONG; + + // Compact formats + if (strcmp(format_str, "compact") == 0) return OUTPUT_COMPACT; + if (strcmp(format_str, "compact-long") == 0) return OUTPUT_COMPACT_LONG; + + // Disk tool specific formats + if (strcmp(format_str, "k") == 0) return OUTPUT_KILO; + if (strcmp(format_str, "K") == 0) return OUTPUT_KIBI; + if (strcmp(format_str, "k-unit") == 0) return OUTPUT_KILO_UNIT; + if (strcmp(format_str, "K-unit") == 0) return OUTPUT_KIBI_UNIT; + + // Legacy compatibility + if (strcmp(format_str, "h") == 0) return OUTPUT_HUMAN; + + return OUTPUT_SI_HUMAN; // Default fallback +} + +const char* get_format_description(OutputFormat format) { + switch (format) { + case OUTPUT_RAW: return "Raw bytes without formatting"; + case OUTPUT_RAW_COMMAS: return "Raw bytes with comma separators"; + case OUTPUT_SI_HUMAN: return "Human readable with SI units (kB, MB, GB, TB)"; + case OUTPUT_SI_HUMAN_LONG: return "Human readable with long SI units (kilobytes, megabytes, etc.)"; + case OUTPUT_BINARY_HUMAN: return "Human readable with binary units (KiB, MiB, GiB, TiB)"; + case OUTPUT_BINARY_HUMAN_LONG: return "Human readable with long binary units (kibibytes, mebibytes, etc.)"; + case OUTPUT_SCIENTIFIC: return "Scientific notation (1.23e+06)"; + case OUTPUT_SCIENTIFIC_COMMAS: return "Scientific notation with comma separators"; + case OUTPUT_DISK_TRADITIONAL: return "Traditional disk tool format (K, M, G, T)"; + case OUTPUT_DISK_TRADITIONAL_LONG: return "Traditional disk tool format with long names"; + case OUTPUT_COMPACT: return "Most compact representation"; + case OUTPUT_COMPACT_LONG: return "Compact representation with long units"; + case OUTPUT_KILO: return "Kilobytes (1000-based, no unit)"; + case OUTPUT_KIBI: return "Kibibytes (1024-based, no unit)"; + case OUTPUT_KILO_UNIT: return "Kilobytes with 'k' unit"; + case OUTPUT_KIBI_UNIT: return "Kibibytes with 'K' unit"; + case OUTPUT_HUMAN: return "Human readable (-h style, SI units)"; + default: return "Unknown format"; + } +} + +// Fixed-width compact format for status line columns +// Always produces exactly 4 characters, right-aligned, SI units +// Examples: " 0", " 999", "1.0K", " 99K", "999K", "1.0M", " 99M", "999M", "1.0G", etc. +void format_compact(uint64_t value, char buf[5]) { + buf[4] = '\0'; + int len; + + if (value < 1000) { + len = snprintf(buf, 5, "%4llu", (unsigned long long)value); + } else if (value < 9950) { + len = snprintf(buf, 5, "%.1fK", (double)value / 1000.0); + } else if (value < 999500) { + len = snprintf(buf, 5, "%3lluK", (unsigned long long)((value + 500) / 1000)); + } else if (value < 9950000) { + len = snprintf(buf, 5, "%.1fM", (double)value / 1000000.0); + } else if (value < 999500000) { + len = snprintf(buf, 5, "%3lluM", (unsigned long long)((value + 500000) / 1000000)); + } else if (value < 9950000000ULL) { + len = snprintf(buf, 5, "%.1fG", (double)value / 1000000000.0); + } else if (value < 999500000000ULL) { + len = snprintf(buf, 5, "%3lluG", (unsigned long long)((value + 500000000) / 1000000000)); + } else if (value < 9950000000000ULL) { + len = snprintf(buf, 5, "%.1fT", (double)value / 1000000000000.0); + } else if (value < 999500000000000ULL) { + len = snprintf(buf, 5, "%3lluT", (unsigned long long)((value + 500000000000ULL) / 1000000000000ULL)); + } else if (value < 9950000000000000ULL) { + len = snprintf(buf, 5, "%.1fP", (double)value / 1000000000000000ULL); + } else { + len = snprintf(buf, 5, "999P"); + } + + if (len < 0) len = 0; + if (len > 4) len = 4; + + if (len < 4) { + size_t pad = 4 - len; + memmove(buf + pad, buf, len + 1); + for (size_t i = 0; i < pad; i++) buf[i] = ' '; + } +} + +void list_available_formats(FILE* out) { + fprintf(out, "Available output formats:\n"); + fprintf(out, " raw - %s\n", get_format_description(OUTPUT_RAW)); + fprintf(out, " raw-commas - %s\n", get_format_description(OUTPUT_RAW_COMMAS)); + fprintf(out, " si, human - %s\n", get_format_description(OUTPUT_SI_HUMAN)); + fprintf(out, " si-long, human-long - %s\n", get_format_description(OUTPUT_SI_HUMAN_LONG)); + fprintf(out, " binary, iec - %s\n", get_format_description(OUTPUT_BINARY_HUMAN)); + fprintf(out, " binary-long, iec-long - %s\n", get_format_description(OUTPUT_BINARY_HUMAN_LONG)); + fprintf(out, " scientific, sci - %s\n", get_format_description(OUTPUT_SCIENTIFIC)); + fprintf(out, " scientific-commas, sci-commas - %s\n", get_format_description(OUTPUT_SCIENTIFIC_COMMAS)); + fprintf(out, " traditional, disk - %s\n", get_format_description(OUTPUT_DISK_TRADITIONAL)); + fprintf(out, " traditional-long, disk-long - %s\n", get_format_description(OUTPUT_DISK_TRADITIONAL_LONG)); + fprintf(out, " compact - %s\n", get_format_description(OUTPUT_COMPACT)); + fprintf(out, " compact-long - %s\n", get_format_description(OUTPUT_COMPACT_LONG)); + fprintf(out, " k - %s\n", get_format_description(OUTPUT_KILO)); + fprintf(out, " K - %s\n", get_format_description(OUTPUT_KIBI)); + fprintf(out, " k-unit - %s\n", get_format_description(OUTPUT_KILO_UNIT)); + fprintf(out, " K-unit - %s\n", get_format_description(OUTPUT_KIBI_UNIT)); + fprintf(out, " h - %s\n", get_format_description(OUTPUT_HUMAN)); +} diff --git a/output_format.h b/output_format.h new file mode 100644 index 0000000..a9aa5a1 --- /dev/null +++ b/output_format.h @@ -0,0 +1,67 @@ +// Copyright © 2025 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_OUTPUT_FORMAT_H__ +#define __DEDUP_OUTPUT_FORMAT_H__ + +#include +#include + +// Output format types for byte size formatting +typedef enum OutputFormat { + // Raw formats + OUTPUT_RAW, // Raw bytes (no formatting) + OUTPUT_RAW_COMMAS, // Raw bytes with comma separators + + // SI decimal prefixes (1000-based) + OUTPUT_SI_HUMAN, // Human readable with SI units (kB, MB, GB, TB) + OUTPUT_SI_HUMAN_LONG, // Long form English (kilobytes, megabytes, etc.) + + // Binary prefixes (1024-based) + OUTPUT_BINARY_HUMAN, // Human readable with binary units (KiB, MiB, GiB, TiB) + OUTPUT_BINARY_HUMAN_LONG,// Long form English (kibibytes, mebibytes, etc.) + + // Scientific notation + OUTPUT_SCIENTIFIC, // Scientific notation (1.23e+06) + OUTPUT_SCIENTIFIC_COMMAS,// Scientific with comma separators + + // Traditional disk tool formats + OUTPUT_DISK_TRADITIONAL, // Traditional disk tool format (like df, du) + OUTPUT_DISK_TRADITIONAL_LONG, // Long form of traditional + + // Compact formats + OUTPUT_COMPACT, // Most compact representation + OUTPUT_COMPACT_LONG, // Compact with long units + + // Disk tool specific formats + OUTPUT_KILO, // Kilobytes (1000-based, no unit) + OUTPUT_KIBI, // Kibibytes (1024-based, no unit) + OUTPUT_KILO_UNIT, // Kilobytes with 'k' unit + OUTPUT_KIBI_UNIT, // Kibibytes with 'K' unit + OUTPUT_HUMAN // Human readable (-h style, SI) +} OutputFormat; + +// Format a byte size according to the specified format +// Returns a static buffer that should be used immediately +const char* format_bytes(uint64_t bytes, OutputFormat format); + +// Get the default output format +OutputFormat get_default_output_format(void); + +// Parse format string to OutputFormat enum +// Returns OUTPUT_SI_HUMAN on invalid format +OutputFormat parse_output_format(const char* format_str); + +// Get a description of the format +const char* get_format_description(OutputFormat format); + +// List all available formats +void list_available_formats(FILE* out); + +// Fixed-width compact format for status line columns +// Always produces exactly 4 characters, right-aligned, SI units (K, M, G, T, P) +// Examples: " 0", " 999", "1.0K", " 99K", "999K", "1.0M", " 99M", "999M", "1.0G", etc. +void format_compact(uint64_t value, char buf[5]); + +#endif // __DEDUP_OUTPUT_FORMAT_H__ diff --git a/queue.c b/queue.c index f09895b..5009d74 100644 --- a/queue.c +++ b/queue.c @@ -47,6 +47,7 @@ void file_entry_queue_append(FileEntryHead* queue, nlink_t nlink, uint32_t flags, size_t size, + uint64_t sequence, short level) { FileEntry* e = malloc(sizeof(FileEntry)); *e = (FileEntry) { @@ -56,6 +57,7 @@ void file_entry_queue_append(FileEntryHead* queue, .nlink = nlink, .flags = flags, .size = size, + .sequence = sequence, .level = level, }; STAILQ_INSERT_TAIL(queue, e, entries); diff --git a/queue.h b/queue.h index 0aeb5c3..41a759c 100644 --- a/queue.h +++ b/queue.h @@ -42,6 +42,7 @@ typedef struct FileEntry { nlink_t nlink; uint32_t flags; size_t size; + uint64_t sequence; bool acls_supported; short level; STAILQ_ENTRY(FileEntry) entries; /* Tail queue. */ @@ -57,6 +58,7 @@ void file_entry_queue_append(FileEntryHead* queue, nlink_t nlink, uint32_t flags, size_t size, + uint64_t sequence, short level); FileEntry* file_entry_next(FileEntryHead* queue); void file_entry_free(FileEntry* fe); diff --git a/runtime_caps.c b/runtime_caps.c new file mode 100644 index 0000000..c085f3f --- /dev/null +++ b/runtime_caps.c @@ -0,0 +1,226 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include "runtime_caps.h" + +#include +#include +#include +#include +#include + +#if defined(__APPLE__) +#include +#endif + +static DedupRuntimeCaps g_runtime_caps; +static bool g_runtime_caps_initialized = false; +static volatile int g_memcmp_bench_sink = 0; +static volatile int g_exact_bench_sink = 0; + +#if defined(__APPLE__) +static bool have_sysctl_u32(const char* name) { + uint32_t value = 0; + size_t size = sizeof(value); + if (sysctlbyname(name, &value, &size, NULL, 0) != 0 || size != sizeof(value)) { + return false; + } + return value != 0; +} +#endif + +static bool env_is_enabled(const char* name) { + const char* value = getenv(name); + return value && strcmp(value, "1") == 0; +} + +static bool detect_metal_available(void) { +#if defined(__APPLE__) + return access("/System/Library/Frameworks/Metal.framework/Metal", F_OK) == 0; +#else + return false; +#endif +} + +static double monotonic_seconds(void) { + struct timespec ts = {0}; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0.0; + } + return (double)ts.tv_sec + ((double)ts.tv_nsec / 1000000000.0); +} + +static double benchmark_memcmp_bucket(size_t size) { + if (size == 0) { + return 0.0; + } + + unsigned char* a = malloc(size); + unsigned char* b = malloc(size); + if (!a || !b) { + free(a); + free(b); + return 0.0; + } + + memset(a, 0x5A, size); + memset(b, 0x5A, size); + + size_t iterations = (32U * 1024U * 1024U) / size; + if (iterations < 8) { + iterations = 8; + } + if (iterations > 8192) { + iterations = 8192; + } + + for (size_t i = 0; i < 4; i++) { + g_memcmp_bench_sink |= memcmp(a, b, size); + } + + double start = monotonic_seconds(); + for (size_t i = 0; i < iterations; i++) { + g_memcmp_bench_sink |= memcmp(a, b, size); + } + double end = monotonic_seconds(); + + free(a); + free(b); + + double elapsed = end - start; + if (elapsed <= 0.0) { + return 0.0; + } + + double total_bytes = (double)size * (double)iterations; + return total_bytes / elapsed / (1024.0 * 1024.0 * 1024.0); +} + +static bool compare_exact_tile_buffers(const unsigned char* a, const unsigned char* b, size_t size, + size_t chunk_size, bool use_xor_or) { + if (!a || !b || size == 0 || chunk_size == 0) { + return false; + } + + for (size_t offset = 0; offset < size; offset += chunk_size) { + size_t to_read = size - offset; + if (to_read > chunk_size) { + to_read = chunk_size; + } + + if (use_xor_or) { + unsigned char diff = 0; + for (size_t i = 0; i < to_read; i++) { + diff |= (unsigned char)(a[offset + i] ^ b[offset + i]); + } + if (diff != 0) { + return false; + } + } else if (memcmp(a + offset, b + offset, to_read) != 0) { + return false; + } + } + + return true; +} + +static double benchmark_exact_tile_bucket(size_t size, size_t chunk_size) { + if (size == 0 || chunk_size == 0) { + return 0.0; + } + + unsigned char* a = malloc(size); + unsigned char* b = malloc(size); + if (!a || !b) { + free(a); + free(b); + return 0.0; + } + + memset(a, 0xA5, size); + memset(b, 0xA5, size); + + size_t iterations = (32U * 1024U * 1024U) / size; + if (iterations < 8) { + iterations = 8; + } + if (iterations > 8192) { + iterations = 8192; + } + + for (size_t i = 0; i < 4; i++) { + g_exact_bench_sink |= compare_exact_tile_buffers(a, b, size, chunk_size, true); + } + + double start = monotonic_seconds(); + for (size_t i = 0; i < iterations; i++) { + g_exact_bench_sink |= compare_exact_tile_buffers(a, b, size, chunk_size, true); + } + double end = monotonic_seconds(); + + free(a); + free(b); + + double elapsed = end - start; + if (elapsed <= 0.0) { + return 0.0; + } + + double total_bytes = (double)size * (double)iterations; + return total_bytes / elapsed / (1024.0 * 1024.0 * 1024.0); +} + +static void populate_capabilities(DedupRuntimeCaps* caps) { + memset(caps, 0, sizeof(*caps)); + +#if defined(__APPLE__) && (defined(__aarch64__) || defined(__arm64__)) + caps->apple_arm64 = true; + caps->neon = true; + caps->unified_memory = true; + caps->dotprod = have_sysctl_u32("hw.optional.arm.FEAT_DotProd"); + caps->i8mm = have_sysctl_u32("hw.optional.arm.FEAT_I8MM"); + caps->crc32 = have_sysctl_u32("hw.optional.armv8_crc32"); + caps->pmull = have_sysctl_u32("hw.optional.arm.FEAT_PMULL"); + caps->sha3 = have_sysctl_u32("hw.optional.armv8_2_sha3"); +#endif + + caps->metal_available = detect_metal_available(); + + if (env_is_enabled("DEDUP_DISABLE_BENCH")) { + return; + } + + caps->memcmp_gib_s_4k = benchmark_memcmp_bucket(4U * 1024U); + caps->memcmp_gib_s_64k = benchmark_memcmp_bucket(64U * 1024U); + caps->memcmp_gib_s_1m = benchmark_memcmp_bucket(1024U * 1024U); + caps->memcmp_gib_s_8m = benchmark_memcmp_bucket(8U * 1024U * 1024U); + caps->exact_cpu_tiles_gib_s_1m = benchmark_exact_tile_bucket(1024U * 1024U, 1024U * 1024U); +} + +const DedupRuntimeCaps* dedup_runtime_caps_get(void) { + if (!g_runtime_caps_initialized) { + populate_capabilities(&g_runtime_caps); + g_runtime_caps_initialized = true; + } + + return &g_runtime_caps; +} + +void dedup_runtime_caps_reset_for_tests(void) { + memset(&g_runtime_caps, 0, sizeof(g_runtime_caps)); + g_runtime_caps_initialized = false; +} + +void dedup_runtime_caps_print_verbose(void) { + const DedupRuntimeCaps* caps = dedup_runtime_caps_get(); + fprintf(stderr, "runtime_caps:\n"); + fprintf(stderr, " cpu: apple_arm64=%d neon=%d dotprod=%d i8mm=%d crc32=%d pmull=%d sha3=%d\n", + caps->apple_arm64, caps->neon, caps->dotprod, caps->i8mm, caps->crc32, caps->pmull, caps->sha3); + fprintf(stderr, " platform: unified_memory=%d metal_available=%d\n", caps->unified_memory, caps->metal_available); + fprintf(stderr, " bench: memcmp_4k=%.2fGiB/s memcmp_64k=%.2fGiB/s memcmp_1m=%.2fGiB/s memcmp_8m=%.2fGiB/s\n", + caps->memcmp_gib_s_4k, caps->memcmp_gib_s_64k, caps->memcmp_gib_s_1m, caps->memcmp_gib_s_8m); + if (caps->exact_cpu_tiles_gib_s_1m > 0.0) { + fprintf(stderr, " bench: exact_cpu_tiles_1m=%.2fGiB/s\n", caps->exact_cpu_tiles_gib_s_1m); + } +} diff --git a/runtime_caps.h b/runtime_caps.h new file mode 100644 index 0000000..886d44d --- /dev/null +++ b/runtime_caps.h @@ -0,0 +1,34 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_RUNTIME_CAPS_H__ +#define __DEDUP_RUNTIME_CAPS_H__ + +#include +#include +#include + +typedef struct DedupRuntimeCaps { + bool apple_arm64; + bool neon; + bool dotprod; + bool i8mm; + bool crc32; + bool pmull; + bool sha3; + bool unified_memory; + bool metal_available; + + double memcmp_gib_s_4k; + double memcmp_gib_s_64k; + double memcmp_gib_s_1m; + double memcmp_gib_s_8m; + double exact_cpu_tiles_gib_s_1m; +} DedupRuntimeCaps; + +const DedupRuntimeCaps* dedup_runtime_caps_get(void); +void dedup_runtime_caps_reset_for_tests(void); +void dedup_runtime_caps_print_verbose(void); + +#endif // __DEDUP_RUNTIME_CAPS_H__ diff --git a/runtime_dispatch.c b/runtime_dispatch.c new file mode 100644 index 0000000..0426dd6 --- /dev/null +++ b/runtime_dispatch.c @@ -0,0 +1,331 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include "runtime_dispatch.h" + +#include +#include +#include +#include +#include +#include + +#include "runtime_caps.h" +#include "signature.h" + +static DedupRuntimeDispatch g_runtime_dispatch; +static bool g_runtime_dispatch_initialized = false; + +static uint64_t fast_hash_xxhash_backend(const void* data, size_t len) { + return signature_fast_hash_bytes(data, len); +} + +static bool witness_none_backend(const char* a_path, const char* b_path, uint64_t size) { + (void)a_path; + (void)b_path; + (void)size; + return true; +} + +static bool read_exact(int fd, void* buf, size_t len, off_t offset) { + size_t done = 0; + while (done < len) { + ssize_t n = pread(fd, (unsigned char*)buf + done, len - done, offset + (off_t)done); + if (n <= 0) { + return false; + } + done += (size_t)n; + } + return true; +} + +static bool compare_window_hashes(int a_fd, int b_fd, off_t offset, size_t window_size) { + unsigned char a_buf[4096]; + unsigned char b_buf[4096]; + + if (window_size > sizeof(a_buf)) { + return false; + } + if (!read_exact(a_fd, a_buf, window_size, offset) || !read_exact(b_fd, b_buf, window_size, offset)) { + return false; + } + + return signature_fast_hash_bytes(a_buf, window_size) == + signature_fast_hash_bytes(b_buf, window_size); +} + +static bool compare_sample32(int a_fd, int b_fd, off_t offset, uint64_t size) { + unsigned char a_raw[sizeof(int32_t)] = {0}; + unsigned char b_raw[sizeof(int32_t)] = {0}; + size_t remaining = (size_t)(size - (uint64_t)offset); + size_t to_read = remaining < sizeof(a_raw) ? remaining : sizeof(a_raw); + + if (!read_exact(a_fd, a_raw, to_read, offset) || !read_exact(b_fd, b_raw, to_read, offset)) { + return false; + } + + return memcmp(a_raw, b_raw, sizeof(a_raw)) == 0; +} + +static bool witness_cpu_backend(const char* a_path, const char* b_path, uint64_t size) { + if (!a_path || !b_path) { + return false; + } + if (size == 0) { + return true; + } + + int a_fd = open(a_path, O_RDONLY); + if (a_fd < 0) { + return false; + } + + int b_fd = open(b_path, O_RDONLY); + if (b_fd < 0) { + close(a_fd); + return false; + } + + bool matches = true; + size_t window_size = size < 4096U ? (size_t)size : 4096U; + off_t offsets[3] = { + 0, + (off_t)((size > window_size) ? ((size / 2U) - (window_size / 2U)) : 0), + (off_t)((size > window_size) ? (size - window_size) : 0), + }; + + for (size_t i = 0; i < 3; i++) { + if (!compare_window_hashes(a_fd, b_fd, offsets[i], window_size)) { + matches = false; + goto cleanup; + } + } + + off_t sample_positions[4] = { + 0, + (off_t)(size / 3U), + (off_t)((size * 2U) / 3U), + (off_t)(size > 4U ? size - 4U : 0), + }; + for (size_t i = 0; i < 4; i++) { + if (!compare_sample32(a_fd, b_fd, sample_positions[i], size)) { + matches = false; + goto cleanup; + } + } + +cleanup: + close(a_fd); + close(b_fd); + return matches; +} + +static bool exact_compare_memcmp_backend(const char* a_path, const char* b_path) { + return files_match_exact_memcmp(a_path, b_path); +} + +static bool exact_compare_cpu_xor_or_backend(const char* a_path, const char* b_path) { + return files_match_exact_xor_or(a_path, b_path); +} + +static bool exact_compare_cpu_tiles_backend(const char* a_path, const char* b_path) { + return files_match_exact_cpu_tiles(a_path, b_path); +} + +static bool exact_compare_gpu_exact_stream_backend(const char* a_path, const char* b_path) { + return files_match_exact_memcmp(a_path, b_path); +} + +static const char* pick_name_or_default(const char* value, const char* fallback, + const char* const* allowed, size_t allowed_count) { + if (!value || value[0] == '\0') { + return fallback; + } + + for (size_t i = 0; i < allowed_count; i++) { + if (strcmp(value, allowed[i]) == 0) { + return allowed[i]; + } + } + + if (strcmp(value, "gpu_stream") == 0) { + return "gpu_exact_stream"; + } + + return fallback; +} + +static bool env_is_enabled(const char* env_name) { + const char* raw = getenv(env_name); + return raw && strcmp(raw, "1") == 0; +} + +static size_t parse_size_override(const char* env_name, size_t fallback) { + const char* raw = getenv(env_name); + if (!raw || raw[0] == '\0') { + return fallback; + } + + char* end = NULL; + errno = 0; + unsigned long long parsed = strtoull(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || parsed == 0) { + return fallback; + } + + return (size_t)parsed; +} + +static dedup_fast_hash_fn fast_hash_backend_for_name(const char* name) { + if (strcmp(name, "xxhash") == 0) { + return fast_hash_xxhash_backend; + } + return fast_hash_xxhash_backend; +} + +static dedup_pair_witness_fn witness_backend_for_name(const char* name) { + if (strcmp(name, "cpu_witness") == 0) { + return witness_cpu_backend; + } + return witness_none_backend; +} + +static dedup_exact_compare_fn exact_backend_for_name(const char* name) { + if (strcmp(name, "cpu_xor_or") == 0) { + return exact_compare_cpu_xor_or_backend; + } + if (strcmp(name, "cpu_tiles") == 0) { + return exact_compare_cpu_tiles_backend; + } + if (strcmp(name, "gpu_exact_stream") == 0) { + return exact_compare_gpu_exact_stream_backend; + } + return exact_compare_memcmp_backend; +} + +const DedupRuntimeDispatch* dedup_runtime_dispatch_get(void) { + if (!g_runtime_dispatch_initialized) { + static const char* const fast_hash_names[] = { "xxhash", "rapidhash", "komihash", "blake3" }; + static const char* const strong_hash_names[] = { "none", "blake3", "sha3", "pmull_poly" }; + static const char* const witness_names[] = { "none", "cpu_witness", "gpu_witness_stream" }; + static const char* const exact_names[] = { "memcmp", "cpu_xor_or", "cpu_tiles", "gpu_exact_stream" }; + const DedupRuntimeCaps* caps = dedup_runtime_caps_get(); + + memset(&g_runtime_dispatch, 0, sizeof(g_runtime_dispatch)); + + const char* fast_hash_name = pick_name_or_default(getenv("DEDUP_FORCE_FAST_HASH"), + "xxhash", + fast_hash_names, + sizeof(fast_hash_names) / sizeof(fast_hash_names[0])); + if (strcmp(fast_hash_name, "xxhash") != 0) { + fast_hash_name = "xxhash"; + } + g_runtime_dispatch.fast_hash_name = fast_hash_name; + + const char* strong_hash_name = pick_name_or_default(getenv("DEDUP_FORCE_STRONG_HASH"), + "none", + strong_hash_names, + sizeof(strong_hash_names) / sizeof(strong_hash_names[0])); + if (strcmp(strong_hash_name, "none") != 0) { + strong_hash_name = "none"; + } + g_runtime_dispatch.strong_hash_name = strong_hash_name; + + const char* witness_name = pick_name_or_default(getenv("DEDUP_FORCE_WITNESS"), + "none", + witness_names, + sizeof(witness_names) / sizeof(witness_names[0])); + if (strcmp(witness_name, "gpu_witness_stream") == 0 && !caps->metal_available) { + witness_name = "none"; + } + if (strcmp(witness_name, "none") != 0 && strcmp(witness_name, "cpu_witness") != 0) { + witness_name = "none"; + } + g_runtime_dispatch.witness_name = witness_name; + + const bool cpu_tiles_wins = caps->apple_arm64 && caps->exact_cpu_tiles_gib_s_1m > 0.0 && + caps->exact_cpu_tiles_gib_s_1m >= caps->memcmp_gib_s_1m; + const char* exact_small_name = "memcmp"; + const char* exact_large_name = cpu_tiles_wins ? "cpu_tiles" : "memcmp"; + const char* forced_exact_name = pick_name_or_default(getenv("DEDUP_FORCE_EXACT_COMPARE"), + NULL, + exact_names, + sizeof(exact_names) / sizeof(exact_names[0])); + if (forced_exact_name) { + exact_small_name = forced_exact_name; + exact_large_name = forced_exact_name; + } + if (strcmp(exact_large_name, "gpu_exact_stream") == 0 && !caps->metal_available) { + exact_small_name = "memcmp"; + exact_large_name = "memcmp"; + } + g_runtime_dispatch.exact_small_name = exact_small_name; + g_runtime_dispatch.exact_large_name = exact_large_name; + + g_runtime_dispatch.fast_hash = fast_hash_backend_for_name(g_runtime_dispatch.fast_hash_name); + g_runtime_dispatch.strong_hash = NULL; + g_runtime_dispatch.witness = witness_backend_for_name(g_runtime_dispatch.witness_name); + g_runtime_dispatch.exact_small = exact_backend_for_name(g_runtime_dispatch.exact_small_name); + g_runtime_dispatch.exact_large = exact_backend_for_name(g_runtime_dispatch.exact_large_name); + + g_runtime_dispatch.witness_threshold = parse_size_override("DEDUP_WITNESS_THRESHOLD_BYTES", 256U * 1024U); + g_runtime_dispatch.exact_large_threshold = parse_size_override("DEDUP_EXACT_LARGE_THRESHOLD_BYTES", + cpu_tiles_wins ? (1024U * 1024U) : (64U * 1024U)); + g_runtime_dispatch.gpu_batch_threshold = parse_size_override("DEDUP_GPU_BATCH_THRESHOLD", 16U); + + if (env_is_enabled("DEDUP_FORCE_GPU") && caps->metal_available && + strcmp(g_runtime_dispatch.witness_name, "none") == 0) { + g_runtime_dispatch.witness_name = "gpu_witness_stream"; + } + + g_runtime_dispatch_initialized = true; + } + + return &g_runtime_dispatch; +} + +bool dedup_runtime_witness_compare(const char* a_path, const char* b_path, uint64_t size) { + const DedupRuntimeDispatch* dispatch = dedup_runtime_dispatch_get(); + + if (!dispatch || !a_path || !b_path) { + return false; + } + + if (strcmp(dispatch->witness_name, "none") == 0) { + return true; + } + + if (size < dispatch->witness_threshold) { + return true; + } + + return dispatch->witness(a_path, b_path, size); +} + +bool dedup_runtime_exact_compare(const char* a_path, const char* b_path, uint64_t size) { + const DedupRuntimeDispatch* dispatch = dedup_runtime_dispatch_get(); + if (!dispatch || !a_path || !b_path) { + return false; + } + + if (size >= dispatch->exact_large_threshold) { + return dispatch->exact_large(a_path, b_path); + } + return dispatch->exact_small(a_path, b_path); +} + +void dedup_runtime_dispatch_reset_for_tests(void) { + memset(&g_runtime_dispatch, 0, sizeof(g_runtime_dispatch)); + g_runtime_dispatch_initialized = false; +} + +void dedup_runtime_dispatch_print_verbose(void) { + const DedupRuntimeDispatch* dispatch = dedup_runtime_dispatch_get(); + fprintf(stderr, "runtime_dispatch:\n"); + fprintf(stderr, " fast_hash=%s strong_hash=%s\n", dispatch->fast_hash_name, dispatch->strong_hash_name); + fprintf(stderr, " witness=%s exact_small=%s exact_large=%s\n", + dispatch->witness_name, dispatch->exact_small_name, dispatch->exact_large_name); + fprintf(stderr, " thresholds: witness=%zu exact_large=%zu gpu_batch=%zu\n", + dispatch->witness_threshold, dispatch->exact_large_threshold, dispatch->gpu_batch_threshold); +} diff --git a/runtime_dispatch.h b/runtime_dispatch.h new file mode 100644 index 0000000..8fbc459 --- /dev/null +++ b/runtime_dispatch.h @@ -0,0 +1,40 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_RUNTIME_DISPATCH_H__ +#define __DEDUP_RUNTIME_DISPATCH_H__ + +#include +#include +#include + +typedef uint64_t (*dedup_fast_hash_fn)(const void* data, size_t len); +typedef bool (*dedup_pair_witness_fn)(const char* a_path, const char* b_path, uint64_t size); +typedef bool (*dedup_exact_compare_fn)(const char* a_path, const char* b_path); + +typedef struct DedupRuntimeDispatch { + const char* fast_hash_name; + const char* strong_hash_name; + const char* witness_name; + const char* exact_small_name; + const char* exact_large_name; + + dedup_fast_hash_fn fast_hash; + dedup_fast_hash_fn strong_hash; + dedup_pair_witness_fn witness; + dedup_exact_compare_fn exact_small; + dedup_exact_compare_fn exact_large; + + size_t witness_threshold; + size_t exact_large_threshold; + size_t gpu_batch_threshold; +} DedupRuntimeDispatch; + +const DedupRuntimeDispatch* dedup_runtime_dispatch_get(void); +bool dedup_runtime_witness_compare(const char* a_path, const char* b_path, uint64_t size); +bool dedup_runtime_exact_compare(const char* a_path, const char* b_path, uint64_t size); +void dedup_runtime_dispatch_reset_for_tests(void); +void dedup_runtime_dispatch_print_verbose(void); + +#endif // __DEDUP_RUNTIME_DISPATCH_H__ diff --git a/seen_set.c b/seen_set.c new file mode 100644 index 0000000..7fb27ba --- /dev/null +++ b/seen_set.c @@ -0,0 +1,130 @@ +// Copyright © 2023 TTKB, LLC. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include "seen_set.h" + +#include +#include + +#define EMPTY 0 +#define OCCUPIED 1 + +typedef struct { + uint64_t key; + uint8_t state; +} Slot; + +struct SeenSet { + Slot* slots; + size_t capacity; // always a power of 2 + size_t count; +}; + +static size_t next_power_of_2(size_t v) { + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + v++; + return v; +} + +SeenSet* new_seen_set(size_t capacity) { + if (capacity < 16) capacity = 16; + capacity = next_power_of_2(capacity); + + SeenSet* set = malloc(sizeof(SeenSet)); + if (!set) return NULL; + + set->slots = calloc(capacity, sizeof(Slot)); + if (!set->slots) { + free(set); + return NULL; + } + + set->capacity = capacity; + set->count = 0; + return set; +} + +void free_seen_set(SeenSet* set) { + if (!set) return; + free(set->slots); + free(set); +} + +// Fibonacci hashing for good distribution with power-of-2 tables +static inline size_t hash_key(uint64_t key, size_t mask) { + return (size_t)((key * 11400714819323198485ULL) >> 32) & mask; +} + +static bool seen_set_grow(SeenSet* set) { + size_t new_cap = set->capacity * 2; + Slot* new_slots = calloc(new_cap, sizeof(Slot)); + if (!new_slots) return false; + + size_t mask = new_cap - 1; + for (size_t i = 0; i < set->capacity; i++) { + if (set->slots[i].state != OCCUPIED) continue; + uint64_t key = set->slots[i].key; + size_t idx = hash_key(key, mask); + while (new_slots[idx].state == OCCUPIED) { + idx = (idx + 1) & mask; + } + new_slots[idx].key = key; + new_slots[idx].state = OCCUPIED; + } + + free(set->slots); + set->slots = new_slots; + set->capacity = new_cap; + return true; +} + +bool seen_set_insert(SeenSet* set, uint64_t key) { + // Grow at 75% load + if (set->count * 4 >= set->capacity * 3) { + seen_set_grow(set); + } + + size_t mask = set->capacity - 1; + size_t idx = hash_key(key, mask); + + while (set->slots[idx].state == OCCUPIED) { + if (set->slots[idx].key == key) { + return true; // already present + } + idx = (idx + 1) & mask; + } + + set->slots[idx].key = key; + set->slots[idx].state = OCCUPIED; + set->count++; + return false; // newly inserted +} diff --git a/seen_set.h b/seen_set.h new file mode 100644 index 0000000..9515789 --- /dev/null +++ b/seen_set.h @@ -0,0 +1,43 @@ +// Copyright © 2023 TTKB, LLC. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_SEEN_SET_H__ +#define __DEDUP_SEEN_SET_H__ + +#include +#include +#include + +typedef struct SeenSet SeenSet; + +SeenSet* new_seen_set(size_t capacity); +void free_seen_set(SeenSet* set); + +/// Insert a key into the set. +/// Returns true if the key was already present (i.e., duplicate). +bool seen_set_insert(SeenSet* set, uint64_t key); + +#endif // __DEDUP_SEEN_SET_H__ diff --git a/sig_table.c b/sig_table.c new file mode 100644 index 0000000..87169b2 --- /dev/null +++ b/sig_table.c @@ -0,0 +1,128 @@ +// Copyright © 2025 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include "sig_table.h" +#include +#include + +#include "runtime_dispatch.h" + +SigTable* new_sig_table(size_t bucket_count) { + SigTable* table = calloc(1, sizeof(SigTable)); + if (!table) { + return NULL; + } + + table->buckets = calloc(bucket_count, sizeof(SigTableEntry*)); + if (!table->buckets) { + free(table); + return NULL; + } + + table->bucket_count = bucket_count; + table->entry_count = 0; + + return table; +} + +void free_sig_table(SigTable* table) { + if (!table) { + return; + } + + for (size_t i = 0; i < table->bucket_count; i++) { + SigTableEntry* entry = table->buckets[i]; + while (entry) { + SigTableEntry* next = entry->next; + free_signature(entry->signature); + free(entry->path); + free(entry); + entry = next; + } + } + + free(table->buckets); + free(table); +} + +SigTableEntry* sig_table_insert(SigTable* table, FileSignature* sig, const char* path, uint64_t clone_id) { + if (!table || !sig || !path) { + return NULL; + } + + uint64_t hash = hash_signature(sig); + size_t bucket_idx = hash % table->bucket_count; + + // Check for existing match in collision chain. + // SMHasher-style discipline: a fast hash/signature only nominates candidates; + // witness stages may reject quickly, but exact comparison is still required + // before treating files as equal. + SigTableEntry* entry = table->buckets[bucket_idx]; + while (entry) { + if (signatures_match(entry->signature, sig) && + dedup_runtime_witness_compare(entry->path, path, sig->size) && + dedup_runtime_exact_compare(entry->path, path, sig->size)) { + return entry; + } + entry = entry->next; + } + + SigTableEntry* new_entry = calloc(1, sizeof(SigTableEntry)); + if (!new_entry) { + return NULL; + } + + new_entry->signature = sig; // Takes ownership + new_entry->path = strdup(path); + new_entry->clone_id = clone_id; + new_entry->next = table->buckets[bucket_idx]; + + table->buckets[bucket_idx] = new_entry; + table->entry_count++; + + return NULL; +} + +bool sig_table_has_clone_id(const SigTable* table, uint64_t clone_id) { + if (!table || clone_id == 0) { + return false; + } + + for (size_t i = 0; i < table->bucket_count; i++) { + SigTableEntry* entry = table->buckets[i]; + while (entry) { + if (entry->clone_id == clone_id) { + return true; + } + entry = entry->next; + } + } + + return false; +} + +size_t sig_table_size(const SigTable* table) { + return table ? table->entry_count : 0; +} + +size_t sig_table_collisions(const SigTable* table) { + if (!table) { + return 0; + } + + size_t collisions = 0; + for (size_t i = 0; i < table->bucket_count; i++) { + size_t chain_len = 0; + SigTableEntry* entry = table->buckets[i]; + while (entry) { + chain_len++; + entry = entry->next; + } + if (chain_len > 1) { + collisions += chain_len - 1; + } + } + + return collisions; +} diff --git a/sig_table.h b/sig_table.h new file mode 100644 index 0000000..7f59f93 --- /dev/null +++ b/sig_table.h @@ -0,0 +1,46 @@ +// Copyright © 2025 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_SIG_TABLE_H__ +#define __DEDUP_SIG_TABLE_H__ + +#include "signature.h" +#include + +// Entry in the signature hash table +typedef struct SigTableEntry { + FileSignature* signature; + char* path; + uint64_t clone_id; + struct SigTableEntry* next; // Collision chain +} SigTableEntry; + +// Signature-based hash table for fast duplicate detection +typedef struct SigTable { + SigTableEntry** buckets; + size_t bucket_count; + size_t entry_count; +} SigTable; + +// Create a new signature table +SigTable* new_sig_table(size_t bucket_count); + +// Free signature table +void free_sig_table(SigTable* table); + +// Insert or find matching signature +// Returns: +// - Pointer to existing entry if match found (caller still owns sig) +// - NULL if successfully inserted new entry (table owns sig) +// - NULL if insertion failed (caller still owns sig, check table size) +SigTableEntry* sig_table_insert(SigTable* table, FileSignature* sig, const char* path, uint64_t clone_id); + +// Check if clone_id already seen +bool sig_table_has_clone_id(const SigTable* table, uint64_t clone_id); + +// Get statistics +size_t sig_table_size(const SigTable* table); +size_t sig_table_collisions(const SigTable* table); + +#endif // __DEDUP_SIG_TABLE_H__ diff --git a/signature.c b/signature.c new file mode 100644 index 0000000..a300549 --- /dev/null +++ b/signature.c @@ -0,0 +1,357 @@ +// Copyright © 2025 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include "signature.h" + +#include +#include +#include +#include +#include +#include + +#include "runtime_dispatch.h" + +#ifdef __ARM_NEON +#include +#endif + +// Simple xxHash64 implementation for first 4KB +// Based on xxHash by Yann Collet +static const uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL; +static const uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL; +static const uint64_t PRIME64_3 = 0x165667B19E3779F9ULL; +static const uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL; +static const uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL; + +static inline uint64_t rotl64(uint64_t x, int r) { + return (x << r) | (x >> (64 - r)); +} + +uint64_t signature_fast_hash_bytes(const void* data, size_t len) { + const uint8_t* p = (const uint8_t*)data; + const uint8_t* const end = p + len; + uint64_t h64; + + if (len >= 32) { + const uint8_t* const limit = end - 32; + uint64_t v1 = PRIME64_1 + PRIME64_2; + uint64_t v2 = PRIME64_2; + uint64_t v3 = 0; + uint64_t v4 = -(int64_t)PRIME64_1; + + do { + v1 += *(uint64_t*)p * PRIME64_2; v1 = rotl64(v1, 31); v1 *= PRIME64_1; p += 8; + v2 += *(uint64_t*)p * PRIME64_2; v2 = rotl64(v2, 31); v2 *= PRIME64_1; p += 8; + v3 += *(uint64_t*)p * PRIME64_2; v3 = rotl64(v3, 31); v3 *= PRIME64_1; p += 8; + v4 += *(uint64_t*)p * PRIME64_2; v4 = rotl64(v4, 31); v4 *= PRIME64_1; p += 8; + } while (p <= limit); + + h64 = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18); + + v1 *= PRIME64_2; v1 = rotl64(v1, 31); v1 *= PRIME64_1; h64 ^= v1; h64 = h64 * PRIME64_1 + PRIME64_4; + v2 *= PRIME64_2; v2 = rotl64(v2, 31); v2 *= PRIME64_1; h64 ^= v2; h64 = h64 * PRIME64_1 + PRIME64_4; + v3 *= PRIME64_2; v3 = rotl64(v3, 31); v3 *= PRIME64_1; h64 ^= v3; h64 = h64 * PRIME64_1 + PRIME64_4; + v4 *= PRIME64_2; v4 = rotl64(v4, 31); v4 *= PRIME64_1; h64 ^= v4; h64 = h64 * PRIME64_1 + PRIME64_4; + } else { + h64 = PRIME64_5; + } + + h64 += (uint64_t)len; + + while (p + 8 <= end) { + uint64_t k1 = *(uint64_t*)p; + k1 *= PRIME64_2; k1 = rotl64(k1, 31); k1 *= PRIME64_1; + h64 ^= k1; h64 = rotl64(h64, 27) * PRIME64_1 + PRIME64_4; + p += 8; + } + + if (p + 4 <= end) { + h64 ^= (uint64_t)(*(uint32_t*)p) * PRIME64_1; + h64 = rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p += 4; + } + + while (p < end) { + h64 ^= (*p++) * PRIME64_5; + h64 = rotl64(h64, 11) * PRIME64_1; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + +static bool read_sample_into(int fd, off_t position, uint64_t size, int32_t* out) { + unsigned char raw[sizeof(int32_t)] = {0}; + size_t remaining = 0; + + if (position < 0 || (uint64_t)position >= size) { + return false; + } + + remaining = (size_t)(size - (uint64_t)position); + size_t to_read = remaining < sizeof(raw) ? remaining : sizeof(raw); + ssize_t n = pread(fd, raw, to_read, position); + if (n < 0 || (size_t)n != to_read) { + return false; + } + + memcpy(out, raw, sizeof(raw)); + return true; +} + +FileSignature* compute_signature(const char* path, dev_t device, uint64_t size) { + FileSignature* sig = calloc(1, sizeof(FileSignature)); + if (!sig) { + return NULL; + } + + sig->device = device; + sig->size = size; + + int fd = open(path, O_RDONLY | O_NONBLOCK); + if (fd < 0) { + free(sig); + return NULL; + } + + // Clear O_NONBLOCK for actual I/O operations + int flags = fcntl(fd, F_GETFL); + if (flags >= 0) { + fcntl(fd, F_SETFL, flags & ~O_NONBLOCK); + } + + if (size == 0) { + sig->quick_hash = signature_fast_hash_bytes("", 0); + close(fd); + return sig; + } + + // Sample at 4 strategic positions + off_t positions[4] = { + 0, // Start + (off_t)(size / 3), // 1/3 point + (off_t)((size * 2) / 3), // 2/3 point + (off_t)(size > 4 ? size - 4 : 0) // End (or start if file < 4 bytes) + }; + +#ifdef __ARM_NEON + if (size >= 16) { + char buf[16] __attribute__((aligned(16))); + for (int i = 0; i < 4; i++) { + if (pread(fd, &buf[i * 4], 4, positions[i]) != 4) { + close(fd); + free(sig); + return NULL; + } + } + int32x4_t samples_vec = vld1q_s32((int32_t*)buf); + vst1q_s32(sig->samples, samples_vec); + } else +#endif + { + for (int i = 0; i < 4; i++) { + if (!read_sample_into(fd, positions[i], size, &sig->samples[i])) { + close(fd); + free(sig); + return NULL; + } + } + } + + size_t hash_size = size < 4096 ? (size_t)size : 4096U; + char* buf = malloc(hash_size); + if (!buf) { + close(fd); + free(sig); + return NULL; + } + + ssize_t n = pread(fd, buf, hash_size, 0); + if (n < 0 || (size_t)n != hash_size) { + free(buf); + close(fd); + free(sig); + return NULL; + } + + dedup_fast_hash_fn hash_fn = signature_fast_hash_bytes; + const DedupRuntimeDispatch* dispatch = dedup_runtime_dispatch_get(); + if (dispatch && dispatch->fast_hash) { + hash_fn = dispatch->fast_hash; + } + sig->quick_hash = hash_fn(buf, (size_t)n); + + free(buf); + close(fd); + + return sig; +} + +void free_signature(FileSignature* sig) { + free(sig); +} + +bool signatures_match(const FileSignature* a, const FileSignature* b) { + if (!a || !b) { + return false; + } + + if (a->device != b->device) return false; + if (a->size != b->size) return false; + if (a->quick_hash != b->quick_hash) return false; + +#ifdef __ARM_NEON + int32x4_t a_vec = vld1q_s32(a->samples); + int32x4_t b_vec = vld1q_s32(b->samples); + uint32x4_t cmp = vceqq_s32(a_vec, b_vec); + uint32x2_t tmp = vand_u32(vget_low_u32(cmp), vget_high_u32(cmp)); + return vget_lane_u32(vpmin_u32(tmp, tmp), 0) == 0xFFFFFFFF; +#else + return memcmp(a->samples, b->samples, sizeof(a->samples)) == 0; +#endif +} + +static bool files_match_exact_impl(const char* a_path, const char* b_path, size_t chunk_size, bool use_xor_or) { + if (!a_path || !b_path || chunk_size == 0) { + return false; + } + + int a_fd = open(a_path, O_RDONLY); + if (a_fd < 0) { + return false; + } + + int b_fd = open(b_path, O_RDONLY); + if (b_fd < 0) { + close(a_fd); + return false; + } + + struct stat a_st = {0}; + struct stat b_st = {0}; + bool equal = false; + + if (fstat(a_fd, &a_st) != 0 || fstat(b_fd, &b_st) != 0) { + goto cleanup; + } + + if (a_st.st_size != b_st.st_size) { + goto cleanup; + } + +#ifdef POSIX_FADV_SEQUENTIAL + (void)posix_fadvise(a_fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void)posix_fadvise(b_fd, 0, 0, POSIX_FADV_SEQUENTIAL); +#endif + + unsigned char* a_buf = malloc(chunk_size); + unsigned char* b_buf = malloc(chunk_size); + if (!a_buf || !b_buf) { + free(a_buf); + free(b_buf); + goto cleanup; + } + + off_t offset = 0; + equal = true; + while (offset < a_st.st_size) { + size_t remaining = (size_t)(a_st.st_size - offset); + size_t to_read = remaining < chunk_size ? remaining : chunk_size; + ssize_t a_read = pread(a_fd, a_buf, to_read, offset); + ssize_t b_read = pread(b_fd, b_buf, to_read, offset); + if (a_read < 0 || b_read < 0 || a_read != b_read || (size_t)a_read != to_read) { + equal = false; + break; + } + + if (use_xor_or) { +#ifdef __ARM_NEON + uint8x16_t diff_acc = vdupq_n_u8(0); + size_t i = 0; + for (; i + 16 <= to_read; i += 16) { + uint8x16_t va = vld1q_u8(a_buf + i); + uint8x16_t vb = vld1q_u8(b_buf + i); + diff_acc = vorrq_u8(diff_acc, veorq_u8(va, vb)); + } + if (vmaxvq_u8(diff_acc) != 0) { + equal = false; + break; + } + for (; i < to_read; i++) { + if ((unsigned char)(a_buf[i] ^ b_buf[i]) != 0) { + equal = false; + break; + } + } + if (!equal) { + break; + } +#else + unsigned char diff = 0; + for (size_t i = 0; i < to_read; i++) { + diff |= (unsigned char)(a_buf[i] ^ b_buf[i]); + } + if (diff != 0) { + equal = false; + break; + } +#endif + } else if (memcmp(a_buf, b_buf, to_read) != 0) { + equal = false; + break; + } + + offset += a_read; + } + + free(a_buf); + free(b_buf); + +cleanup: + close(a_fd); + close(b_fd); + return equal; +} + +bool files_match_exact(const char* a_path, const char* b_path) { + return files_match_exact_memcmp(a_path, b_path); +} + +bool files_match_exact_memcmp(const char* a_path, const char* b_path) { + return files_match_exact_impl(a_path, b_path, 64U * 1024U, false); +} + +bool files_match_exact_xor_or(const char* a_path, const char* b_path) { + return files_match_exact_impl(a_path, b_path, 64U * 1024U, true); +} + +bool files_match_exact_cpu_tiles(const char* a_path, const char* b_path) { + return files_match_exact_impl(a_path, b_path, 1024U * 1024U, true); +} + +uint64_t hash_signature(const FileSignature* sig) { + uint64_t h = sig->size; + h ^= (uint64_t)sig->device + 0x9e3779b97f4a7c15ULL; + h ^= sig->quick_hash + 0x9e3779b97f4a7c15ULL; + +#ifdef __ARM_NEON + int32x4_t samples_vec = vld1q_s32(sig->samples); + uint64x2_t hash_vec = vreinterpretq_u64_s32(samples_vec); + h ^= vgetq_lane_u64(hash_vec, 0); + h ^= vgetq_lane_u64(hash_vec, 1); +#else + for (int i = 0; i < 4; i++) { + h ^= (uint64_t)sig->samples[i] * PRIME64_1; + h = rotl64(h, 27); + } +#endif + + return h; +} diff --git a/signature.h b/signature.h new file mode 100644 index 0000000..7bb263e --- /dev/null +++ b/signature.h @@ -0,0 +1,52 @@ +// Copyright © 2025 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_SIGNATURE_H__ +#define __DEDUP_SIGNATURE_H__ + +#include +#include +#include +#include + +// Lightweight file signature using strategic sampling +// Used as a fast candidate filter before exact byte-for-byte verification. +typedef struct FileSignature { + dev_t device; // Device ID (from stat) + uint64_t size; // File size (from stat) + int32_t samples[4]; // Sampled int32 values at strategic positions + uint64_t quick_hash; // xxHash64 of first 4KB (or entire file if smaller) +} FileSignature; + +// Compute signature for a file +// Returns NULL on error +FileSignature* compute_signature(const char* path, dev_t device, uint64_t size); + +// Free signature +void free_signature(FileSignature* sig); + +// Compare two signatures as a fast candidate filter. +// Returns true if files should be considered for exact verification. +bool signatures_match(const FileSignature* a, const FileSignature* b); + +// Compare full file contents byte-for-byte. +bool files_match_exact(const char* a_path, const char* b_path); + +// Named exact-compare backend using memcmp on chunked reads. +bool files_match_exact_memcmp(const char* a_path, const char* b_path); + +// Compare full file contents using a streaming XOR/OR reduction. +bool files_match_exact_xor_or(const char* a_path, const char* b_path); + +// Compare full file contents using a larger tiled XOR/OR backend tuned for +// large Apple SoC files. +bool files_match_exact_cpu_tiles(const char* a_path, const char* b_path); + +// Public fast-hash backend used by runtime dispatch. +uint64_t signature_fast_hash_bytes(const void* data, size_t len); + +// Hash a signature for use in hash table +uint64_t hash_signature(const FileSignature* sig); + +#endif // __DEDUP_SIGNATURE_H__ diff --git a/test/Makefile b/test/Makefile index 15df296..45c8811 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,5 +1,5 @@ CFLAGS += \ - -I/opt/local/include \ + -I/opt/homebrew/include \ -std=c2x \ -Wall -Wextra -Werror -pedantic \ -Wpointer-arith \ @@ -24,7 +24,7 @@ CFLAGS += \ -Og LDFLAGS += \ - -L/opt/local/lib + -L/opt/homebrew/lib .PHONY: \ check \ @@ -41,10 +41,38 @@ check: dedup_check setup-all hdiutil detach /Volumes/dedup-test-hfs-link hdiutil detach /Volumes/dedup-test-hfs-symlink -dedup_check: dedup_check.o dedup_suite.o dedup_link_suite.o dedup_symlink_suite.o clone_suite.o test_utils.o ../alist.o ../clone.o ../map.o ../utils.o +dedup_check: dedup_check.o dedup_suite.o dedup_link_suite.o dedup_symlink_suite.o clone_suite.o signature_suite.o runtime_dispatch_suite.o test_utils.o runtime_caps_test.o runtime_dispatch_test.o alist_test.o clone_test.o map_test.o utils_test.o signature_test.o rm -f dedup_check.gcda dedup_check.gcno $(CC) $(CFLAGS) $(LDFLAGS) -o $@ -l check $^ +alist_test.o: ../alist.c ../alist.h + rm -f alist_test.gcda alist_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../alist.c + +clone_test.o: ../clone.c ../clone.h + rm -f clone_test.gcda clone_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../clone.c + +map_test.o: ../map.c ../map.h + rm -f map_test.gcda map_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../map.c + +utils_test.o: ../utils.c ../utils.h + rm -f utils_test.gcda utils_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../utils.c + +signature_test.o: ../signature.c ../signature.h + rm -f signature_test.gcda signature_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../signature.c + +runtime_caps_test.o: ../runtime_caps.c ../runtime_caps.h + rm -f runtime_caps_test.gcda runtime_caps_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../runtime_caps.c + +runtime_dispatch_test.o: ../runtime_dispatch.c ../runtime_dispatch.h ../runtime_caps.h ../signature.h + rm -f runtime_dispatch_test.gcda runtime_dispatch_test.gcno + $(CC) $(CFLAGS) -c -o $@ ../runtime_dispatch.c + # clean-test-data: NAMESPACE ?= . clean-test-data: if [[ -d /Volumes/dedup-test-hfs-$(NAMESPACE) ]]; then \ @@ -194,3 +222,4 @@ clean: clean-all-test-data rm -f *.o rm -rf *.dSYM/ rm -f dedup_check + rm -f runtime_caps_test.o runtime_dispatch_test.o alist_test.o clone_test.o map_test.o utils_test.o signature_test.o diff --git a/test/dedup_check.c b/test/dedup_check.c index f75d2cc..bbcc3e4 100644 --- a/test/dedup_check.c +++ b/test/dedup_check.c @@ -31,6 +31,8 @@ Suite* clone_suite(); Suite* dedup_suite(); Suite* dedup_link_suite(); Suite* dedup_symlink_suite(); +Suite* signature_suite(); +Suite* runtime_dispatch_suite(); int main() { SRunner* sr = srunner_create(NULL); @@ -38,6 +40,8 @@ int main() { srunner_add_suite(sr, dedup_suite()); srunner_add_suite(sr, dedup_link_suite()); srunner_add_suite(sr, dedup_symlink_suite()); + srunner_add_suite(sr, signature_suite()); + srunner_add_suite(sr, runtime_dispatch_suite()); srunner_set_fork_status(sr, CK_NOFORK); srunner_run_all(sr, CK_VERBOSE); diff --git a/test/dedup_link_suite.c b/test/dedup_link_suite.c index 1b8c381..ffef53c 100644 --- a/test/dedup_link_suite.c +++ b/test/dedup_link_suite.c @@ -38,7 +38,7 @@ START_TEST(dedup_link_empty) { char* output = run("../dedup -l test-data/link/empty"); - ck_assert_str_eq("duplicates found: 0\nbytes saved: 0\nalready saved: 0\n", output); + ck_assert_str_eq("duplicates found: 0\nentries pruned: 0\nbytes saved: 0 bytes\nalready saved: 0 bytes\n", output); free(output); struct stat e1 = { 0 }, e2 = { 0 }; diff --git a/test/dedup_suite.c b/test/dedup_suite.c index b9059ed..c68b660 100644 --- a/test/dedup_suite.c +++ b/test/dedup_suite.c @@ -38,7 +38,7 @@ START_TEST(dedup_empty) { char* output = run("../dedup test-data/clonefile/empty"); - ck_assert_str_eq("duplicates found: 0\nbytes saved: 0\nalready saved: 0\n", output); + ck_assert_str_eq("duplicates found: 0\nentries pruned: 0\nbytes saved: 0 bytes\nalready saved: 0 bytes\n", output); free(output); struct stat e1 = { 0 }, e2 = { 0 }; @@ -155,7 +155,7 @@ START_TEST(dedup_flags_acls) { START_TEST(dedup_hfs) { char* output = run("../dedup -Phx /Volumes/dedup-test-hfs-clonefile 2>&1"); - ck_assert_str_eq("dedup: Skipping /Volumes/dedup-test-hfs-clonefile: cloning not supported\nduplicates found: 0\nbytes saved: 0 bytes\nalready saved: 0 bytes\n", output); + ck_assert_str_eq("dedup: Skipping /Volumes/dedup-test-hfs-clonefile: cloning not supported\nduplicates found: 0\nentries pruned: 0\nbytes saved: 0 bytes\nalready saved: 0 bytes\n", output); free(output); } END_TEST @@ -179,6 +179,13 @@ START_TEST(dedup_dry_run) { ck_assert_int_eq(0, WEXITSTATUS(r)); } END_TEST +START_TEST(dedup_permission_denied) { + char* output = run("../dedup -nP test-data/clonefile/clone-dst-acls 2>&1"); + ck_assert_ptr_nonnull(output); + ck_assert_ptr_null(strstr(output, "could not getattrlist")); + free(output); +} END_TEST + Suite* dedup_suite() { TCase* tc = tcase_create("dedup"); tcase_add_test(tc, dedup_empty); @@ -193,6 +200,7 @@ Suite* dedup_suite() { tcase_add_test(tc, dedup_negative_threads); tcase_add_test(tc, dedup_help); tcase_add_test(tc, dedup_dry_run); + tcase_add_test(tc, dedup_permission_denied); Suite* s = suite_create("dedup"); suite_add_tcase(s, tc); diff --git a/test/dedup_symlink_suite.c b/test/dedup_symlink_suite.c index 273a57e..ea9df12 100644 --- a/test/dedup_symlink_suite.c +++ b/test/dedup_symlink_suite.c @@ -39,7 +39,7 @@ START_TEST(dedup_symlink_empty) { char* output = run("../dedup -s test-data/symlink/empty"); - ck_assert_str_eq("duplicates found: 0\nbytes saved: 0\nalready saved: 0\n", output); + ck_assert_str_eq("duplicates found: 0\nentries pruned: 0\nbytes saved: 0 bytes\nalready saved: 0 bytes\n", output); free(output); struct stat e1 = { 0 }, e2 = { 0 }; diff --git a/test/runtime_dispatch_suite.c b/test/runtime_dispatch_suite.c new file mode 100644 index 0000000..17d8c16 --- /dev/null +++ b/test/runtime_dispatch_suite.c @@ -0,0 +1,158 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include +#include +#include +#include +#include +#include +#include + +#include "../runtime_caps.h" +#include "../runtime_dispatch.h" +#include "runtime_dispatch_suite.h" + +bool dedup_runtime_witness_compare(const char* a_path, const char* b_path, uint64_t size); +bool dedup_runtime_exact_compare(const char* a_path, const char* b_path, uint64_t size); + +static void clear_runtime_env(void) { + unsetenv("DEDUP_FORCE_FAST_HASH"); + unsetenv("DEDUP_FORCE_STRONG_HASH"); + unsetenv("DEDUP_FORCE_WITNESS"); + unsetenv("DEDUP_FORCE_EXACT_COMPARE"); + unsetenv("DEDUP_WITNESS_THRESHOLD_BYTES"); + unsetenv("DEDUP_GPU_BATCH_THRESHOLD"); + unsetenv("DEDUP_EXACT_LARGE_THRESHOLD_BYTES"); +} + +static void write_bytes(const char* path, const void* data, size_t size) { + FILE* f = fopen(path, "wb"); + ck_assert_ptr_nonnull(f); + ck_assert_uint_eq(size, fwrite(data, 1, size, f)); + ck_assert_int_eq(0, fclose(f)); +} + +static char* make_temp_dir(const char* suffix) { + size_t len = strlen("/tmp/dedup-runtime-dispatch--XXXXXX") + strlen(suffix) + 1; + char* dir = calloc(len, 1); + ck_assert_ptr_nonnull(dir); + snprintf(dir, len, "/tmp/dedup-runtime-dispatch-%s-XXXXXX", suffix); + ck_assert_ptr_nonnull(mkdtemp(dir)); + return dir; +} + +START_TEST(runtime_caps_are_cached_and_resettable) { + clear_runtime_env(); + dedup_runtime_caps_reset_for_tests(); + + const DedupRuntimeCaps* caps_a = dedup_runtime_caps_get(); + const DedupRuntimeCaps* caps_b = dedup_runtime_caps_get(); + + ck_assert_ptr_nonnull(caps_a); + ck_assert_ptr_eq(caps_a, caps_b); + + dedup_runtime_caps_reset_for_tests(); + const DedupRuntimeCaps* caps_c = dedup_runtime_caps_get(); + ck_assert_ptr_nonnull(caps_c); +} END_TEST + +START_TEST(runtime_dispatch_has_expected_defaults) { + clear_runtime_env(); + dedup_runtime_dispatch_reset_for_tests(); + + const DedupRuntimeCaps* caps = dedup_runtime_caps_get(); + const DedupRuntimeDispatch* dispatch = dedup_runtime_dispatch_get(); + ck_assert_ptr_nonnull(dispatch); + ck_assert_str_eq("xxhash", dispatch->fast_hash_name); + ck_assert_str_eq("none", dispatch->strong_hash_name); + ck_assert_str_eq("none", dispatch->witness_name); + ck_assert_str_eq("memcmp", dispatch->exact_small_name); + const bool cpu_tiles_wins = caps->apple_arm64 && caps->exact_cpu_tiles_gib_s_1m > 0.0 && + caps->exact_cpu_tiles_gib_s_1m >= caps->memcmp_gib_s_1m; + const size_t expected_exact_large_threshold = cpu_tiles_wins ? (1024U * 1024U) : (64U * 1024U); + ck_assert_str_eq(cpu_tiles_wins ? "cpu_tiles" : "memcmp", dispatch->exact_large_name); + ck_assert_uint_eq(expected_exact_large_threshold, dispatch->exact_large_threshold); + ck_assert_uint_gt(dispatch->witness_threshold, 0); + ck_assert_uint_gt(dispatch->gpu_batch_threshold, 0); +} END_TEST + +START_TEST(runtime_dispatch_honors_overrides) { + clear_runtime_env(); + setenv("DEDUP_FORCE_WITNESS", "cpu_witness", 1); + setenv("DEDUP_FORCE_EXACT_COMPARE", "cpu_tiles", 1); + setenv("DEDUP_WITNESS_THRESHOLD_BYTES", "131072", 1); + setenv("DEDUP_GPU_BATCH_THRESHOLD", "32", 1); + + dedup_runtime_dispatch_reset_for_tests(); + const DedupRuntimeDispatch* dispatch = dedup_runtime_dispatch_get(); + + ck_assert_ptr_nonnull(dispatch); + ck_assert_str_eq("cpu_witness", dispatch->witness_name); + ck_assert_str_eq("cpu_tiles", dispatch->exact_small_name); + ck_assert_str_eq("cpu_tiles", dispatch->exact_large_name); + ck_assert_uint_eq(131072, dispatch->witness_threshold); + ck_assert_uint_eq(32, dispatch->gpu_batch_threshold); + + clear_runtime_env(); + dedup_runtime_dispatch_reset_for_tests(); +} END_TEST + +START_TEST(runtime_exact_compare_uses_bound_backend) { + clear_runtime_env(); + dedup_runtime_dispatch_reset_for_tests(); + + char* dir = make_temp_dir("exact"); + char a[PATH_MAX] = {0}, b[PATH_MAX] = {0}; + snprintf(a, sizeof(a), "%s/a", dir); + snprintf(b, sizeof(b), "%s/b", dir); + + write_bytes(a, "same-data", 9); + write_bytes(b, "same-data", 9); + ck_assert(dedup_runtime_exact_compare(a, b, 9)); + + write_bytes(b, "diff-data", 9); + ck_assert(!dedup_runtime_exact_compare(a, b, 9)); + + ck_assert_int_eq(0, unlink(a)); + ck_assert_int_eq(0, unlink(b)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); +} END_TEST + +START_TEST(runtime_witness_compare_defaults_to_non_rejecting) { + clear_runtime_env(); + setenv("DEDUP_FORCE_WITNESS", "cpu_witness", 1); + dedup_runtime_dispatch_reset_for_tests(); + + char* dir = make_temp_dir("witness"); + char a[PATH_MAX] = {0}, b[PATH_MAX] = {0}; + snprintf(a, sizeof(a), "%s/a", dir); + snprintf(b, sizeof(b), "%s/b", dir); + + write_bytes(a, "abc", 3); + write_bytes(b, "xyz", 3); + ck_assert(dedup_runtime_witness_compare(a, b, 3)); + + ck_assert_int_eq(0, unlink(a)); + ck_assert_int_eq(0, unlink(b)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); + + clear_runtime_env(); + dedup_runtime_dispatch_reset_for_tests(); +} END_TEST + +Suite* runtime_dispatch_suite(void) { + TCase* tc = tcase_create("runtime_dispatch"); + tcase_add_test(tc, runtime_caps_are_cached_and_resettable); + tcase_add_test(tc, runtime_dispatch_has_expected_defaults); + tcase_add_test(tc, runtime_dispatch_honors_overrides); + tcase_add_test(tc, runtime_exact_compare_uses_bound_backend); + tcase_add_test(tc, runtime_witness_compare_defaults_to_non_rejecting); + + Suite* s = suite_create("runtime_dispatch"); + suite_add_tcase(s, tc); + return s; +} diff --git a/test/runtime_dispatch_suite.h b/test/runtime_dispatch_suite.h new file mode 100644 index 0000000..ff812ba --- /dev/null +++ b/test/runtime_dispatch_suite.h @@ -0,0 +1,12 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_RUNTIME_DISPATCH_SUITE_H__ +#define __DEDUP_RUNTIME_DISPATCH_SUITE_H__ + +#include + +Suite* runtime_dispatch_suite(void); + +#endif // __DEDUP_RUNTIME_DISPATCH_SUITE_H__ diff --git a/test/signature_suite.c b/test/signature_suite.c new file mode 100644 index 0000000..ca1dbdc --- /dev/null +++ b/test/signature_suite.c @@ -0,0 +1,156 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#include +#include +#include +#include +#include +#include +#include + +#include "../signature.h" +#include "test_utils.h" + +bool files_match_exact_xor_or(const char* a_path, const char* b_path); +bool files_match_exact_cpu_tiles(const char* a_path, const char* b_path); + +static void write_bytes(const char* path, const void* data, size_t size) { + FILE* f = fopen(path, "wb"); + ck_assert_ptr_nonnull(f); + ck_assert_uint_eq(size, fwrite(data, 1, size, f)); + ck_assert_int_eq(0, fclose(f)); +} + +static char* make_temp_dir(const char* suffix) { + size_t len = strlen("/tmp/dedup-signature--XXXXXX") + strlen(suffix) + 1; + char* dir = calloc(len, 1); + ck_assert_ptr_nonnull(dir); + snprintf(dir, len, "/tmp/dedup-signature-%s-XXXXXX", suffix); + ck_assert_ptr_nonnull(mkdtemp(dir)); + return dir; +} + +START_TEST(signature_supports_subword_files) { + char* dir = make_temp_dir("small"); + char path[PATH_MAX] = {0}; + snprintf(path, sizeof(path), "%s/a", dir); + write_bytes(path, "A", 1); + + struct stat st = {0}; + ck_assert_int_eq(0, stat(path, &st)); + + FileSignature* sig = compute_signature(path, st.st_dev, (uint64_t)st.st_size); + ck_assert_ptr_nonnull(sig); + free_signature(sig); + + ck_assert_int_eq(0, unlink(path)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); +} END_TEST + +START_TEST(dedup_detects_small_duplicate_files) { + char* dir = make_temp_dir("small-dedup"); + char a[PATH_MAX] = {0}, b[PATH_MAX] = {0}, cmd[PATH_MAX * 2] = {0}; + snprintf(a, sizeof(a), "%s/a", dir); + snprintf(b, sizeof(b), "%s/b", dir); + write_bytes(a, "A", 1); + write_bytes(b, "A", 1); + + snprintf(cmd, sizeof(cmd), "../dedup -nP %s", dir); + char* output = run(cmd); + ck_assert_ptr_nonnull(strstr(output, "duplicates found: 1\n")); + ck_assert_ptr_nonnull(strstr(output, "bytes saved: 1 bytes\n")); + free(output); + + ck_assert_int_eq(0, unlink(a)); + ck_assert_int_eq(0, unlink(b)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); +} END_TEST + +START_TEST(dedup_rejects_sample_only_signature_collisions) { + char* dir = make_temp_dir("collision"); + char base[PATH_MAX] = {0}, variant[PATH_MAX] = {0}, cmd[PATH_MAX * 2] = {0}; + snprintf(base, sizeof(base), "%s/base.bin", dir); + snprintf(variant, sizeof(variant), "%s/variant.bin", dir); + + const size_t size = 16384; + unsigned char* file_a = malloc(size); + unsigned char* file_b = malloc(size); + ck_assert_ptr_nonnull(file_a); + ck_assert_ptr_nonnull(file_b); + + memset(file_a, 'A', size); + memcpy(file_b, file_a, size); + file_b[7000] = 'B'; + file_b[12000] = 'C'; + + write_bytes(base, file_a, size); + write_bytes(variant, file_b, size); + free(file_a); + free(file_b); + + snprintf(cmd, sizeof(cmd), "../dedup -nP %s", dir); + char* output = run(cmd); + ck_assert_ptr_nonnull(strstr(output, "duplicates found: 0\n")); + ck_assert_ptr_nonnull(strstr(output, "bytes saved: 0 bytes\n")); + free(output); + + ck_assert_int_eq(0, unlink(base)); + ck_assert_int_eq(0, unlink(variant)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); +} END_TEST + +START_TEST(files_match_exact_xor_or_handles_equal_and_different_files) { + char* dir = make_temp_dir("xor-or"); + char a[PATH_MAX] = {0}, b[PATH_MAX] = {0}; + snprintf(a, sizeof(a), "%s/a", dir); + snprintf(b, sizeof(b), "%s/b", dir); + + write_bytes(a, "same-data", 9); + write_bytes(b, "same-data", 9); + ck_assert(files_match_exact_xor_or(a, b)); + + write_bytes(b, "diff-data", 9); + ck_assert(!files_match_exact_xor_or(a, b)); + + ck_assert_int_eq(0, unlink(a)); + ck_assert_int_eq(0, unlink(b)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); +} END_TEST + +START_TEST(files_match_exact_cpu_tiles_handles_equal_and_different_files) { + char* dir = make_temp_dir("cpu-tiles"); + char a[PATH_MAX] = {0}, b[PATH_MAX] = {0}; + snprintf(a, sizeof(a), "%s/a", dir); + snprintf(b, sizeof(b), "%s/b", dir); + + write_bytes(a, "same-data", 9); + write_bytes(b, "same-data", 9); + ck_assert(files_match_exact_cpu_tiles(a, b)); + + write_bytes(b, "diff-data", 9); + ck_assert(!files_match_exact_cpu_tiles(a, b)); + + ck_assert_int_eq(0, unlink(a)); + ck_assert_int_eq(0, unlink(b)); + ck_assert_int_eq(0, rmdir(dir)); + free(dir); +} END_TEST + +Suite* signature_suite() { + TCase* tc = tcase_create("signature"); + tcase_add_test(tc, signature_supports_subword_files); + tcase_add_test(tc, dedup_detects_small_duplicate_files); + tcase_add_test(tc, dedup_rejects_sample_only_signature_collisions); + tcase_add_test(tc, files_match_exact_xor_or_handles_equal_and_different_files); + tcase_add_test(tc, files_match_exact_cpu_tiles_handles_equal_and_different_files); + + Suite* s = suite_create("signature"); + suite_add_tcase(s, tc); + return s; +} diff --git a/test/signature_suite.h b/test/signature_suite.h new file mode 100644 index 0000000..9d3e141 --- /dev/null +++ b/test/signature_suite.h @@ -0,0 +1,12 @@ +// Copyright © 2026 TTKB, LLC. +// +// SPDX-License-Identifier: BSD-2-Clause + +#ifndef __DEDUP_SIGNATURE_SUITE_H__ +#define __DEDUP_SIGNATURE_SUITE_H__ + +#include + +Suite* signature_suite(void); + +#endif // __DEDUP_SIGNATURE_SUITE_H__ diff --git a/test_fast/file1.txt b/test_fast/file1.txt new file mode 100644 index 0000000..74689c8 --- /dev/null +++ b/test_fast/file1.txt @@ -0,0 +1 @@ +test content 1 diff --git a/test_fast/file1_dup1.txt b/test_fast/file1_dup1.txt new file mode 100644 index 0000000..74689c8 --- /dev/null +++ b/test_fast/file1_dup1.txt @@ -0,0 +1 @@ +test content 1 diff --git a/test_fast/file1_dup2.txt b/test_fast/file1_dup2.txt new file mode 100644 index 0000000..74689c8 --- /dev/null +++ b/test_fast/file1_dup2.txt @@ -0,0 +1 @@ +test content 1 diff --git a/test_fast/file2.txt b/test_fast/file2.txt new file mode 100644 index 0000000..b13c288 --- /dev/null +++ b/test_fast/file2.txt @@ -0,0 +1 @@ +test content 2 diff --git a/test_fast/file3.txt b/test_fast/file3.txt new file mode 100644 index 0000000..74689c8 --- /dev/null +++ b/test_fast/file3.txt @@ -0,0 +1 @@ +test content 1 diff --git a/timeout_exec b/timeout_exec new file mode 100755 index 0000000..a548e9d Binary files /dev/null and b/timeout_exec differ diff --git a/timeout_exec.c b/timeout_exec.c new file mode 100644 index 0000000..c960dd3 --- /dev/null +++ b/timeout_exec.c @@ -0,0 +1,20 @@ +#include +#include +#include +#include + +int main(int argc, char** argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s seconds command [args...]\n", argv[0]); + return 2; + } + + int seconds = atoi(argv[1]); + if (seconds < 0) seconds = 0; + if (seconds > 0) alarm((unsigned int)seconds); + + // Exec the command provided (argv[2] is the command) + execvp(argv[2], &argv[2]); + perror("execvp"); + return 127; +} diff --git a/utils.c b/utils.c index 62ac4f6..b817188 100644 --- a/utils.c +++ b/utils.c @@ -49,8 +49,6 @@ uint64_t get_clone_id(const char* restrict path) { int err = getattrlist(path, &attrList, &clone_id, sizeof(struct UInt64Ref), FSOPT_ATTR_CMN_EXTENDED); if (err) { - warnx("%s:%i %s", __FUNCTION__, __LINE__, path); - perror("could not getattrlist"); return 0; } @@ -71,8 +69,6 @@ int may_share_blocks(const char* restrict path) { int err = getattrlist(path, &attrList, &clone_id, sizeof(struct UInt64Ref), FSOPT_ATTR_CMN_EXTENDED); if (err) { - warnx("%s:%i %s", __FUNCTION__, __LINE__, path); - perror("could not getattrlist"); return 0; } @@ -93,14 +89,20 @@ size_t private_size(const char* restrict path) { int err = getattrlist(path, &attrList, &size_attr, sizeof(struct UInt64Ref), FSOPT_ATTR_CMN_EXTENDED); if (err) { - warnx("%s:%i %s", __FUNCTION__, __LINE__, path); - perror("could not getattrlist"); return 0; } return size_attr.size; } +ino_t get_inode(const char* restrict path) { + struct stat st; + if (stat(path, &st) != 0) { + return 0; + } + return st.st_ino; +} + FileMetadata* metadata_from_entry(FileEntry* fe) { FileMetadata fm = { // diff --git a/utils.h b/utils.h index b5ce0f2..4d77cbb 100644 --- a/utils.h +++ b/utils.h @@ -37,6 +37,7 @@ uint64_t get_clone_id(const char* restrict path); int may_share_blocks(const char* restrict path); size_t private_size(const char* restrict path); +ino_t get_inode(const char* restrict path); FileMetadata* metadata_from_entry(FileEntry* fe) ATTR_MALLOC(free_metadata, 1);