diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6c009f22..6ff958ad 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -178,6 +178,9 @@ jobs: - name: Run C tests (embedded find_best_match) run: make c-test-embedded + - name: Run C tests (prefilter find_best_match) + run: make c-test-prefilter + - name: Verify CLI compression/decompression integrity run: | source .venv/bin/activate diff --git a/CLAUDE.md b/CLAUDE.md index da8059d3..d848cfed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,9 +31,13 @@ different platforms: - `compressor.h/c` - Compression implementation (sink/poll low-level API and higher-level compress/flush API) - `decompressor.h/c` - Decompression implementation -- `compressor_find_match_desktop.c` - Desktop-optimized match finding (included - by `compressor.c` on 64-bit targets: x86_64, aarch64, unless - `TAMP_USE_EMBEDDED_MATCH=1`) +- `common.c`/`compressor.c`/`decompressor.c` must compile standalone with only + the headers (users vendor these three files), so every implementation + reachable on embedded targets is defined inline. Only variants unreachable + there may live in `#include`'d files (`compressor_find_match_desktop.c` on + `x86_64`/`aarch64`, `compressor_find_match_swar32.c` opt-in) or come from a + platform component ESP32-style (extern `find_best_match`, + `private/tamp_copy.h`). ## Development Commands @@ -234,8 +238,34 @@ make website-clean # Clean website build artifacts (default: 32 bytes, 256+ recommended for performance) - `TAMP_STREAM_MEMORY` / `TAMP_STREAM_STDIO` / `TAMP_STREAM_LITTLEFS` / `TAMP_STREAM_FATFS` - Enable built-in I/O handlers for specific backends -- `TAMP_USE_EMBEDDED_MATCH=1` - Force embedded `find_best_match` implementation - on desktop (for testing) +- Platform tuning flags (see `common.h`'s "Platform performance tuning" + section): the core sources never select architecture-specific code on their + own - every flag defaults to the portable implementation, and each build + system opts into its platform's measured configuration (`setup.py` sets + `TAMP_USE_DESKTOP_MATCH=1` on 64-bit hosts, espidf Kconfig defaults + `TAMP_ESP32=y`, the STM32H7B0 harness sets `TAMP_ARMV7EM=1`): + - `TAMP_ARMV7EM=1` - profile for Cortex-M4/M7: enables the prefilter match + finder plus all six decompressor fast-path flags below (measured on + STM32H7B0/M7 vs the portable build: 1.31x compression, 1.92x decompression, + ~5.2 KB additional flash; M4 unmeasured on hardware) + - `TAMP_USE_EMBEDDED_MATCH=1` - the portable `find_best_match` (selections are + mutually exclusive, including `TAMP_ESP32`; conflicts are a compile error) + - `TAMP_USE_PREFILTER_MATCH` - first-byte prefilter (slower on 64-bit hosts) + - `TAMP_USE_DESKTOP_MATCH` - 64-bit SWAR for 64-bit hosts + - `TAMP_USE_SWAR32_MATCH` - experimental 32-bit SWAR (candidate for + single-issue cores like Cortex-M33) + - `TAMP_FAST_WINDOW_COPY` - no-wrap fast path in `tamp_window_copy` (+14% + decompression on M7; -3% Xtensa LX7, +160B Cortex-M0+) + - `TAMP_FAST_BIT_REFILL` - locals-based `refill_bit_buffer` (+5% decompression + on M7; -3% Xtensa LX7, +324B Cortex-M0+) + - `TAMP_FAST_OUTPUT_COPY` - word-at-a-time copy to the output buffer + - `TAMP_WINDOW_FROM_OUTPUT` - window update sourced from the just-written + output snapshot instead of `tamp_window_copy` + - `TAMP_FAST_DECODE_LOOP` - checked-once fast inner decode loop over a 64-bit + bit reservoir (the largest single decompression win) + - `TAMP_COMPACT_CAREFUL_BODY` - compile the non-fast-loop careful body -Os + (GCC-only; only sensible with `TAMP_FAST_DECODE_LOOP`) See `common.h` for + each flag's measured numbers. - `TAMP_USE_MEMSET=1` - Use libc `memset` (default: 1). Set to `0` for environments without libc (e.g. MicroPython native modules). diff --git a/Makefile b/Makefile index a8c5ea9d..8572e451 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ help-main: @echo " make test Run Python and MicroPython tests" @echo " make c-test Run C unit tests" @echo " make c-test-embedded Run C unit tests with embedded find_best_match" + @echo " make c-test-prefilter Run C unit tests with prefilter find_best_match" @echo " make clean Clean all build artifacts" @echo "" @echo "MicroPython native module:" @@ -29,10 +30,12 @@ help-extra: @echo "Fuzzing (requires LLVM clang; on macOS: brew install llvm):" @echo " make fuzz-decompressor Fuzz decompressor with random input" @echo " make fuzz-round-trip Fuzz compress->decompress round-trip" + @echo " make fuzz-matrix Short decompressor fuzz of every flag configuration" @echo " make fuzz-clean Clean fuzz artifacts and corpora" @echo "" @echo "Other targets:" @echo " make binary-size Show binary sizes for README table" + @echo " make benchmark-code-sizes Reproduce the BENCHMARKS.md 'Code size (B)' column" @echo " make v1-compressed-datasets Regenerate ground-truth v1 (--no-extended) .tamp binaries" @echo " make extended-compressed-datasets Regenerate ground-truth extended .tamp binaries" @echo " make c-benchmark-stream Benchmark stream API with various temporary working buffer sizes" @@ -408,6 +411,11 @@ CTEST_DEFINES = -DTAMP_STREAM_STDIO=1 -DTAMP_STREAM_MEMORY=1 \ -DTAMP_STREAM_FATFS=1 -DTEST_FATFS=1 \ -DTAMP_LAZY_MATCHING=1 \ -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR +# c-test covers the same match finder the pip/Cython build opts into on +# 64-bit hosts (the core defaults to the portable one); c-test-embedded +# covers the portable path. Selections are mutually exclusive (compile +# error), so this is applied only to the non-embedded tamp objects. +CTEST_MATCH_DEFINE = -DTAMP_USE_DESKTOP_MATCH=1 CTEST_CFLAGS = $(CTEST_INCLUDES) $(CTEST_SANITIZER_FLAGS) $(CTEST_DEFINES) # Strict warnings applied only to first-party tamp sources, not third-party (Unity/LittleFS/FatFs) CTEST_WARN_FLAGS = -Wall -Wextra -Wtype-limits -Werror @@ -439,7 +447,7 @@ CTEST_TEST_OBJS = \ # Build tamp source files for testing build/ctests/%.o: tamp/_c_src/tamp/%.c @mkdir -p build/ctests - $(CTEST_CC) $(CTEST_CFLAGS) $(CTEST_WARN_FLAGS) -c $< -o $@ + $(CTEST_CC) $(CTEST_CFLAGS) $(CTEST_MATCH_DEFINE) $(CTEST_WARN_FLAGS) -c $< -o $@ # Build Unity framework build/unity/unity.o: ctests/Unity/src/unity.c ctests/Unity/src/unity.h @@ -475,20 +483,62 @@ build/ctests/fatfs_ramdisk.o: ctests/fatfs_ramdisk.c # Build test runner (includes test files via #include) build/ctests/test_runner.o: ctests/test_runner.c ctests/test_compressor.c ctests/test_decompressor.c ctests/test_stream.c ctests/test_stream_filesystems.c @mkdir -p build/ctests - $(CTEST_CC) $(CTEST_CFLAGS) -c $< -o $@ + $(CTEST_CC) $(CTEST_CFLAGS) $(CTEST_MATCH_DEFINE) -c $< -o $@ # Link test executable build/test_runner: $(CTEST_TAMP_OBJS) $(CTEST_LFS_OBJS) $(CTEST_FATFS_OBJS) $(CTEST_TEST_OBJS) $(CTEST_CC) $(CTEST_LDFLAGS) -o $@ $^ -c-test: build/test_runner +c-test: build/test_runner c-compile-matrix ./build/test_runner +# Compile the vendored trio under every documented decompressor flag +# combination. Regression guard: flag-gated code paths must always compile +# (e.g. TAMP_FAST_DECODE_LOOP=1 with TAMP_EXTENDED=0 once referenced the +# extended-only token_state field and only broke in that combination). +C_COMPILE_MATRIX_CONFIGS = \ + "" \ + "-DTAMP_ARMV7EM=1" \ + "-DTAMP_FAST_DECODE_LOOP=1" \ + "-DTAMP_WINDOW_FROM_OUTPUT=1" \ + "-DTAMP_EXTENDED=0" \ + "-DTAMP_EXTENDED=0 -DTAMP_FAST_DECODE_LOOP=1" \ + "-DTAMP_EXTENDED=0 -DTAMP_ARMV7EM=1" \ + "-DTAMP_ARMV7EM=1 -DTAMP_COMPACT_CAREFUL_BODY=0" \ + "-DTAMP_FAST_DECODE_LOOP=1 -DTAMP_COMPACT_CAREFUL_BODY=1" \ + "-DTAMP_USE_MEMSET=0" \ + "-DTAMP_STREAM=0" \ + "-DTAMP_FIXED_WINDOW_BITS=10 -DTAMP_FIXED_LITERAL_BITS=8" \ + "-DTAMP_FIXED_WINDOW_BITS=10 -DTAMP_FIXED_LITERAL_BITS=8 -DTAMP_ARMV7EM=1" + +.PHONY: c-compile-matrix +c-compile-matrix: + @mkdir -p build + @set -e; for cfg in $(C_COMPILE_MATRIX_CONFIGS); do \ + echo "c-compile-matrix: $$cfg"; \ + for src in tamp/_c_src/tamp/common.c tamp/_c_src/tamp/compressor.c tamp/_c_src/tamp/decompressor.c; do \ + $(CC) -O2 -Wall -Itamp/_c_src $$cfg -c $$src -o build/c_compile_matrix.o.tmp; \ + done; \ + done; rm -f build/c_compile_matrix.o.tmp + @set -e; for cfg in \ + "-DTAMP_ESP32=1 -DTAMP_USE_DESKTOP_MATCH=1" \ + "-DTAMP_USE_EMBEDDED_MATCH=1 -DTAMP_USE_DESKTOP_MATCH=1"; do \ + echo "c-compile-matrix (must NOT compile): $$cfg"; \ + if $(CC) -O2 -Wall -Itamp/_c_src $$cfg -c tamp/_c_src/tamp/common.c \ + -o build/c_compile_matrix.o.tmp 2>/dev/null; then \ + echo "c-compile-matrix: ERROR: conflicting match selection compiled: $$cfg"; \ + rm -f build/c_compile_matrix.o.tmp; exit 1; \ + fi; \ + done; rm -f build/c_compile_matrix.o.tmp + @echo "c-compile-matrix: all configurations compile" + clean-c-test: @rm -f build/test_runner @rm -f build/test_runner_embedded + @rm -f build/test_runner_prefilter @rm -f build/ctests/*.o @rm -f build/ctests-embedded/*.o + @rm -f build/ctests-prefilter/*.o @rm -f build/unity/*.o # Embedded implementation tests (forces embedded find_best_match on desktop) @@ -517,6 +567,34 @@ build/test_runner_embedded: $(CTEST_EMBEDDED_TAMP_OBJS) $(CTEST_EMBEDDED_TEST_OB c-test-embedded: build/test_runner_embedded ./build/test_runner_embedded +# Prefilter implementation tests (forces the ARMV7EM profile's find_best_match +# on desktop; without this leg the shipping M4/M7 matcher is compile-checked +# but never behaviorally run on host). +.PHONY: c-test-prefilter + +CTEST_PREFILTER_TAMP_OBJS = \ + build/ctests-prefilter/common.o \ + build/ctests-prefilter/compressor.o \ + build/ctests-prefilter/decompressor.o + +CTEST_PREFILTER_TEST_OBJS = \ + build/unity/unity.o \ + build/ctests-prefilter/test_runner.o + +build/ctests-prefilter/%.o: tamp/_c_src/tamp/%.c + @mkdir -p build/ctests-prefilter + $(CTEST_CC) $(CTEST_CFLAGS) $(CTEST_WARN_FLAGS) -DTAMP_USE_PREFILTER_MATCH=1 -c $< -o $@ + +build/ctests-prefilter/test_runner.o: ctests/test_runner.c ctests/test_compressor.c ctests/test_decompressor.c + @mkdir -p build/ctests-prefilter + $(CTEST_CC) $(CTEST_CFLAGS) -DTAMP_USE_PREFILTER_MATCH=1 -c $< -o $@ + +build/test_runner_prefilter: $(CTEST_PREFILTER_TAMP_OBJS) $(CTEST_PREFILTER_TEST_OBJS) $(CTEST_LFS_OBJS) $(CTEST_FATFS_OBJS) + $(CTEST_CC) $(CTEST_LDFLAGS) -o $@ $^ + +c-test-prefilter: build/test_runner_prefilter + ./build/test_runner_prefilter + ############ # Fuzzing @@ -558,9 +636,18 @@ fuzz-round-trip: build/fuzz_round_trip @mkdir -p fuzz/corpus_round_trip ./build/fuzz_round_trip fuzz/corpus_round_trip +# Builds the malicious-input decompressor fuzzer in every flag-gated code-path +# configuration (portable, ARMV7EM profile, fast-loop-only, ESP32, extended +# off, memset off, ...), replays the shared corpus, and fuzzes each briefly. +# FUZZ_MATRIX_SECONDS overrides the per-config fuzz duration (default 90). +.PHONY: fuzz-matrix +fuzz-matrix: + @mkdir -p fuzz/corpus_decompressor + FUZZ_CC=$(FUZZ_CC) fuzz/fuzz-matrix.sh $(or $(FUZZ_MATRIX_SECONDS),90) + fuzz-clean: @rm -f build/fuzz_decompressor build/fuzz_round_trip - @rm -rf fuzz/corpus_decompressor fuzz/corpus_round_trip + @rm -rf fuzz/corpus_decompressor fuzz/corpus_round_trip build/fuzz_matrix @rm -rf build/esp32_host build/fuzz_round_trip_esp32 build/esp32_host_differential fuzz/corpus_round_trip_esp32 ################################## @@ -763,6 +850,10 @@ binary-size: @output=$$($(MAKE) -s mpy-native-size 2>&1) && echo "$$output" || echo "Tamp (MicroPython Native) (requires MPY_DIR)" @output=$$($(MAKE) -s c-size 2>&1) && echo "$$output" || echo "Tamp (C) (requires arm-none-eabi-gcc)" +.PHONY: benchmark-code-sizes +benchmark-code-sizes: + @tools/benchmark-code-size.sh + ########## # Website diff --git a/devices/BENCHMARKS.md b/devices/BENCHMARKS.md index a81b5f37..3d9e3c8f 100644 --- a/devices/BENCHMARKS.md +++ b/devices/BENCHMARKS.md @@ -2,21 +2,33 @@ All rows measure the same workload: compressing and decompressing the **first 100 KB of [enwik8](https://mattmahoney.net/dc/textdata.html)** with a 1 KB -window (`window=10`, `literal=8`). Throughput is input bytes per second for -compression and output bytes per second for decompression. Reproduce with the -`*-device-benchmark` Makefile targets (see the per-device directories here). +window (`window=10`, `literal=8`), in the classic (v1, non-extended) stream +format. Throughput is input bytes per second for compression and output bytes +per second for decompression. Reproduce with the `*-device-benchmark` Makefile +targets (see the per-device directories here). -| Device | Core | Clock | Runtime / build | Tamp options | Compression (s) | Compression (bytes/s) | Decompression (s) | Decompression (bytes/s) | -| ------------------------------------------------------------------------------------- | -------------- | ------- | --------------------------------- | ----------------------- | --------------- | --------------------- | ----------------- | ----------------------- | -| [Raspberry Pi Pico](https://www.raspberrypi.com/products/raspberry-pi-pico/) (RP2040) | Cortex-M0+ | 125 MHz | C, `-O3` | — | 2.77 | 36,127 | 0.071 | 1,400,600 | -| [Raspberry Pi Pico](https://www.raspberrypi.com/products/raspberry-pi-pico/) (RP2040) | Cortex-M0+ | 125 MHz | MicroPython v1.26.1 native module | — | 2.90 | 34,510 | 0.102 | 980,392 | -| ESP32 | Xtensa LX6 | 160 MHz | ESP-IDF v6.0.2, `-O2` | — | 1.756 | 56,900 | 0.068 | 1,470,000 | -| ESP32 | Xtensa LX6 | 160 MHz | ESP-IDF v6.0.2, `-O2` | `TAMP_ESP32` | 1.708 | 58,500 | 0.067 | 1,490,000 | -| ESP32-S3 | Xtensa LX7 | 160 MHz | ESP-IDF v6.0.2, `-O2` | — | 1.487 | 67,200 | 0.053 | 1,882,000 | -| ESP32-S3 | Xtensa LX7 | 160 MHz | ESP-IDF v6.0.2, `-O2` | `TAMP_ESP32` (PIE SIMD) | 0.255 | 392,600 | 0.050 | 1,987,000 | -| ESP32-C3 | RISC-V RV32IMC | 160 MHz | ESP-IDF v6.0.2, `-O2` | — | 1.413 | 70,800 | 0.040 | 2,500,000 | -| ESP32-C3 | RISC-V RV32IMC | 160 MHz | ESP-IDF v6.0.2, `-O2` | `TAMP_ESP32` | 0.961 | 104,100 | 0.036 | 2,777,800 | -| STM32H7B0 [^sram] | Cortex-M7 | 280 MHz | C, `-O3`, I+D cache | — | 0.518 | 192,900 | 0.017 | 5,907,000 | +The "Tamp options" column lists the compile-time flags that reproduce that build +when using Tamp as a library (`—` = none: the portable defaults). The available +flags are documented in the platform tuning section of +`tamp/_c_src/tamp/common.h`; `TAMP_ESP32=1` additionally requires the espidf +component's platform sources. + +"Code size (B)" [^codesize] is the sum of `text` (includes `.rodata`) across the +vendored objects, built with that row's exact flags. + +| Device | Core | Clock | Runtime / build | Tamp options | Code size (B) | Compression (s) | Compression (bytes/s) | Decompression (s) | Decompression (bytes/s) | +| ------------------------------------------------------------------------------------- | -------------- | ------- | --------------------------------- | ------------------------- | ------------- | --------------- | --------------------- | ----------------- | ----------------------- | +| [Raspberry Pi Pico](https://www.raspberrypi.com/products/raspberry-pi-pico/) (RP2040) | Cortex-M0+ | 125 MHz | C, `-O3` | — | 6,655 | 2.25 | 44,523 | 0.081 | 1,234,004 | +| [Raspberry Pi Pico](https://www.raspberrypi.com/products/raspberry-pi-pico/) (RP2040) | Cortex-M0+ | 125 MHz | C, `-O3` | `TAMP_FAST_DECODE_LOOP=1` | 7,387 | 2.25 | 44,523 | 0.059 | 1,700,420 | +| [Raspberry Pi Pico](https://www.raspberrypi.com/products/raspberry-pi-pico/) (RP2040) | Cortex-M0+ | 125 MHz | MicroPython v1.26.1 native module | — | 5,883 | 2.90 | 34,510 | 0.102 | 980,392 | +| ESP32 | Xtensa LX6 | 160 MHz | ESP-IDF v6.0.2, `-O2` | — | 6,581 | 1.756 | 56,900 | 0.068 | 1,470,000 | +| ESP32 | Xtensa LX6 | 160 MHz | ESP-IDF v6.0.2, `-O2` | `TAMP_ESP32=1` | 8,374 | 1.708 | 58,500 | 0.067 | 1,490,000 | +| ESP32-S3 | Xtensa LX7 | 160 MHz | ESP-IDF v6.0.2, `-O2` | — | 6,609 | 1.487 | 67,200 | 0.051 | 1,977,000 | +| ESP32-S3 | Xtensa LX7 | 160 MHz | ESP-IDF v6.0.2, `-O2` | `TAMP_ESP32=1` (PIE SIMD) | 8,458 | 0.255 | 392,600 | 0.048 | 2,066,000 | +| ESP32-C3 | RISC-V RV32IMC | 160 MHz | ESP-IDF v6.0.2, `-O2` | — | 7,215 | 1.413 | 70,800 | 0.040 | 2,500,000 | +| ESP32-C3 | RISC-V RV32IMC | 160 MHz | ESP-IDF v6.0.2, `-O2` | `TAMP_ESP32=1` | 10,561 | 0.961 | 104,100 | 0.036 | 2,777,800 | +| STM32H7B0 [^sram] | Cortex-M7 | 280 MHz | C, `-O3`, I+D cache | — | 6,339 | 0.518 | 192,900 | 0.015 | 6,746,000 | +| STM32H7B0 [^sram] | Cortex-M7 | 280 MHz | C, `-O3`, I+D cache | `TAMP_ARMV7EM=1` | 10,687 | 0.395 | 253,000 | 0.008 | 12,875,000 | [^sram]: The STM32H7B0's benchmark input/reference data resides in internal SRAM (the @@ -25,5 +37,12 @@ compression and output bytes per second for decompression. Reproduce with the internal flash. This makes the data-read path somewhat faster than a flash-resident workload would be. +[^codesize]: + Berkeley `size`'s `text` for `common.o + compressor.o + decompressor.o` + (plus `compressor_esp32.o`, the espidf component's platform source, for + `TAMP_ESP32=1` rows), compiled standalone with that row's CPU/opt/flags - no + linking, no `--gc-sections`. Reproduce with `tools/benchmark-code-size.sh` + (also `make benchmark-code-sizes`). + New device targets (e.g. RP2350, STM32H7) should add rows here using the same 100 KB enwik8 workload and a 10-bit window. diff --git a/devices/rp2040/CMakeLists.txt b/devices/rp2040/CMakeLists.txt index b868562f..72a68f6f 100644 --- a/devices/rp2040/CMakeLists.txt +++ b/devices/rp2040/CMakeLists.txt @@ -26,6 +26,14 @@ target_include_directories(tamp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(tamp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../common) target_link_libraries(tamp PUBLIC pico_time) +# Extra tamp compile definitions for A/B benchmark builds, e.g. +# cmake -DTAMP_BENCH_DEFINES="TAMP_FAST_DECODE_LOOP=1" (semicolon-separated +# for multiple). Empty (the portable defaults) unless explicitly set. +set(TAMP_BENCH_DEFINES "" CACHE STRING "Extra tamp compile definitions (e.g. TAMP_FAST_DECODE_LOOP=1)") +if(TAMP_BENCH_DEFINES) + target_compile_definitions(tamp PUBLIC ${TAMP_BENCH_DEFINES}) +endif() + add_executable(tamp_benchmark main.c ../common/tamp_bench.c) target_link_libraries(tamp_benchmark tamp) diff --git a/devices/rp2040/README.md b/devices/rp2040/README.md index c493ac73..3f9a70dd 100644 --- a/devices/rp2040/README.md +++ b/devices/rp2040/README.md @@ -20,6 +20,13 @@ make rp2040-device-flash # Copy UF2 to a BOOTSEL-mounted Pico (/Volumes/RP make rp2040-device-benchmark RP2040_PORT=/dev/tty.usbmodem... # Capture a run ``` +`RP2040_TAMP_OPT` adds tamp compile definitions (semicolon-separated) for A/B +builds; the default (empty) is the portable BENCHMARKS.md row: + +```bash +make rp2040-device-build RP2040_TAMP_OPT="TAMP_FAST_DECODE_LOOP=1" # fastloop row +``` + To flash: hold BOOTSEL while connecting USB, then `make rp2040-device-flash`. The Pico reboots into the benchmark, which loops forever printing `BENCH`/`INFO` lines, `PASS:`/`FAIL:` checks, and a `TAMP-DEVICE-RESULT:` sentinel each diff --git a/devices/rp2040/targets.mk b/devices/rp2040/targets.mk index e979f6b9..7342e9e6 100644 --- a/devices/rp2040/targets.mk +++ b/devices/rp2040/targets.mk @@ -4,11 +4,14 @@ # # Variables: # RP2040_PORT Serial port of the running benchmark (required for benchmark) +# RP2040_TAMP_OPT Extra tamp compile definitions (semicolon-separated), +# e.g. RP2040_TAMP_OPT="TAMP_FAST_DECODE_LOOP=1" for the +# fastloop BENCHMARKS.md row. Empty = portable defaults. .PHONY: rp2040-device-build rp2040-device-flash rp2040-device-benchmark rp2040-device-help rp2040-device-build: build/enwik8-100kb build/enwik8-100kb-v1.tamp device-vectors @[ -n "$$PICO_SDK_PATH" ] || { echo "Error: PICO_SDK_PATH is not set."; exit 1; } - cmake -B devices/rp2040/build -S devices/rp2040 + cmake -B devices/rp2040/build -S devices/rp2040 -DTAMP_BENCH_DEFINES="$(RP2040_TAMP_OPT)" $(MAKE) -C devices/rp2040/build tamp_benchmark rp2040-device-flash: rp2040-device-build diff --git a/devices/stm32h7b0/Makefile b/devices/stm32h7b0/Makefile index 5fc7d418..3d36476e 100644 --- a/devices/stm32h7b0/Makefile +++ b/devices/stm32h7b0/Makefile @@ -25,11 +25,18 @@ CMSIS_HEADERS := \ CC := arm-none-eabi-gcc TAMP_SRC := ../../tamp/_c_src +# TAMP_ARMV7EM=1: opt into the measured Cortex-M7 configuration (the core C +# sources default to portable code; see the platform tuning section in +# common.h). EXTRA_CFLAGS passes experiment defines from the command line and +# wins over earlier defines, e.g. +# make stm32h7b0-device-benchmark EXTRA_CFLAGS=-DTAMP_ARMV7EM=0 CFLAGS := -mcpu=cortex-m7 -mthumb -mfloat-abi=hard -mfpu=fpv5-d16 \ -O3 -ggdb -ffunction-sections -fdata-sections \ -Wall -Wextra \ -DSTM32H7B0xx \ - -I$(CMSIS) -I. -I../common -I$(TAMP_SRC) + -DTAMP_ARMV7EM=1 \ + -I$(CMSIS) -I. -I../common -I$(TAMP_SRC) \ + $(EXTRA_CFLAGS) # Full newlib (not nano): the harness prints %llu timings, which newlib-nano's # printf does not support. 128K of flash has plenty of room. @@ -42,6 +49,9 @@ SRCS := startup.c system.c syscalls.c main.c \ $(TAMP_SRC)/tamp/compressor.c \ $(TAMP_SRC)/tamp/decompressor.c +# #include'd by compressor.c, not compiled separately - deps only. +MATCH_VARIANTS := $(wildcard $(TAMP_SRC)/tamp/compressor_find_match_*.c) + ELF := $(BUILD)/tamp_benchmark.elf all: $(ELF) @@ -54,11 +64,18 @@ $(CMSIS)/core_cm7.h $(CMSIS)/cmsis_compiler.h $(CMSIS)/cmsis_gcc.h $(CMSIS)/cmsi @mkdir -p $(CMSIS) curl -fsSL $(CMSIS_CORE_URL)/$(notdir $@) -o $@ -$(ELF): $(SRCS) linker.ld $(CMSIS_HEADERS) Makefile +# Rebuild when the flags change (the whole firmware is one compile+link, so +# there are no per-object dependencies to catch this). +FLAGSTAMP := $(BUILD)/cflags.stamp +$(FLAGSTAMP): force + @mkdir -p $(BUILD) + @echo '$(CFLAGS)' | cmp -s - $(FLAGSTAMP) || echo '$(CFLAGS)' > $(FLAGSTAMP) + +$(ELF): $(SRCS) $(MATCH_VARIANTS) linker.ld $(CMSIS_HEADERS) Makefile $(FLAGSTAMP) $(CC) $(CFLAGS) $(SRCS) $(LDFLAGS) -o $@ arm-none-eabi-size $@ clean: rm -rf $(BUILD) -.PHONY: all clean +.PHONY: all clean force diff --git a/devices/stm32h7b0/README.md b/devices/stm32h7b0/README.md index c6366910..d2be7491 100644 --- a/devices/stm32h7b0/README.md +++ b/devices/stm32h7b0/README.md @@ -28,6 +28,17 @@ make stm32h7b0-device-benchmark # Same, plus a BENCH/INFO summary block at the e No port variable is needed; OpenOCD auto-detects the ST-Link. +Compile-time experiments can be passed through `EXTRA_CFLAGS`, e.g. +`make stm32h7b0-device-benchmark EXTRA_CFLAGS=-DTAMP_USE_EMBEDDED_MATCH=1` to +A/B the match-finder variants (the first-byte prefilter is the ARMv7E-M default; +see `devices/BENCHMARKS.md` for measured rows). + +Memory placement guidance, measured on this harness: with I+D caches enabled, +moving code to ITCM or the window buffer to DTCM changes performance by less +than 1% (the 16 KB caches already cover the hot loops and window from internal +flash/AXI SRAM). TCM placement only matters for cache-hostile configurations +such as XiP from external QSPI or windows larger than the D-cache. + The firmware configures the chip itself: LDO supply + VOS0, 280 MHz from PLL1 (25 MHz HSE, M=5 N=112 P=2), flash latency 7, I+D caches, and a 1 MHz TIM2 timebase. It prints `INFO` lines with register readbacks plus a DWT-vs-TIM2 diff --git a/devices/stm32h7b0/syscalls.c b/devices/stm32h7b0/syscalls.c index a952a0b3..3c8ec1fb 100644 --- a/devices/stm32h7b0/syscalls.c +++ b/devices/stm32h7b0/syscalls.c @@ -1,13 +1,14 @@ /* Newlib syscall stubs. stdout goes to the debugger via ARM semihosting - * (SYS_WRITE0), so the harness needs only the ST-Link - no UART wiring. The - * firmware must run under `make stm32h7b0-device-*` (OpenOCD with semihosting - * enabled); standalone, the BKPT escalates to HardFault. */ + * (SYS_WRITE, one trap per _write), so the harness needs only the ST-Link - no + * UART wiring. The firmware must run under `make stm32h7b0-device-*` (OpenOCD + * with semihosting enabled); standalone, the BKPT escalates to HardFault. */ #include #include #include #include -#define SEMIHOST_SYS_WRITE0 0x04 +#define SEMIHOST_SYS_OPEN 0x01 +#define SEMIHOST_SYS_WRITE 0x05 static uintptr_t semihost_call(uintptr_t op, uintptr_t arg) { register uintptr_t r0 __asm__("r0") = op; @@ -16,18 +17,34 @@ static uintptr_t semihost_call(uintptr_t op, uintptr_t arg) { return r0; } +/* Lazily open the debugger console (":tt") in write mode ("w" == 4) once and + * cache the handle; SYS_OPEN takes {path, mode, path_len} and returns a handle + * (or -1 on failure). */ +static int semihost_stdout(void) { + static int handle = 0; /* 0 == not yet opened; semihosting handles are >0 */ + if (handle == 0) { + static const char path[] = ":tt"; + uintptr_t args[3] = {(uintptr_t)path, 4, sizeof(path) - 1}; + handle = (int)semihost_call(SEMIHOST_SYS_OPEN, (uintptr_t)args); + } + return handle; +} + int _write(int fd, const char *buf, int len) { (void)fd; - char chunk[64]; - int remaining = len; - while (remaining > 0) { - int n = remaining < (int)sizeof(chunk) - 1 ? remaining : (int)sizeof(chunk) - 1; - for (int i = 0; i < n; i++) chunk[i] = *buf++; - chunk[n] = '\0'; - semihost_call(SEMIHOST_SYS_WRITE0, (uintptr_t)chunk); - remaining -= n; + int handle = semihost_stdout(); + if (handle == -1) { + errno = EIO; + return -1; + } + /* SYS_WRITE takes {handle, buf, len} and returns the count NOT written. */ + uintptr_t args[3] = {(uintptr_t)handle, (uintptr_t)buf, (uintptr_t)len}; + int not_written = (int)semihost_call(SEMIHOST_SYS_WRITE, (uintptr_t)args); + if (not_written == len) { + errno = EIO; + return -1; } - return len; + return len - not_written; } /* Heap bounds come from the linker script (AXI SRAM); the default newlib diff --git a/fuzz/decompressor_fuzz_case.h b/fuzz/decompressor_fuzz_case.h new file mode 100644 index 00000000..95428e50 --- /dev/null +++ b/fuzz/decompressor_fuzz_case.h @@ -0,0 +1,103 @@ +#ifndef TAMP_DECOMPRESSOR_FUZZ_CASE_H +#define TAMP_DECOMPRESSOR_FUZZ_CASE_H + +/* Single source of truth for the decompressor fuzz-corpus byte contract. + * + * Shared by the libFuzzer harness (fuzz/fuzz_decompressor.c) and the QEMU + * adversarial-replay firmware (tools/qemu-profiler/firmware/replay_main.c) so + * the corpus encoding (config byte, chunk byte, chunked feed/stall loop) cannot + * drift between the two. Pure computation only: static-const table plus one + * static-inline driver. Callers own their buffers and any post-run checks + * (libFuzzer inspects results for the sanitizers; the firmware checks canaries + * and return codes). + */ + +#include +#include +#include +#include + +#include "tamp/decompressor.h" + +/* Chunk sizes chosen around token boundaries: single bytes stress per-call + * resume, 17/31/33 straddle max classic match and the fast-loop 32-byte + * output precondition, 241 is the max RLE run, 4096 approximates "large". */ +static const uint16_t TAMP_FUZZ_CHUNK_SIZES[16] = {1, 2, 3, 4, 5, 7, 8, 16, 17, 31, 32, 33, 64, 241, 256, 4096}; + +/* Decode+drive one corpus entry against a caller-provided window (>= 1<<15 + * bytes) and output buffer (output_size bytes). Behavior is bit-identical to + * the original libFuzzer LLVMFuzzerTestOneInput; the clamps below are inert + * given the 3-bit/2-bit fields but are kept so the contract stays exact. + * Returns 0 (callers own their own pass/fail accounting). */ +static inline int tamp_fuzz_case_run(const uint8_t *data, size_t size, unsigned char *window, unsigned char *output, + size_t output_size) { + if (size < 2) return 0; + + uint8_t config_byte = data[0]; + uint8_t chunk_byte = data[1]; + data += 2; + size -= 2; + + uint8_t window_bits = 8 + (config_byte & 0x07); // 8-15 + if (window_bits > 15) window_bits = 15; + uint8_t literal_bits = 5 + ((config_byte >> 3) & 0x03); // 5-8 + if (literal_bits > 8) literal_bits = 8; + bool extended = (config_byte >> 5) & 1; + bool use_header = (config_byte >> 6) & 1; + bool custom_dictionary = (config_byte >> 7) & 1; + + size_t out_chunk = TAMP_FUZZ_CHUNK_SIZES[chunk_byte & 0x0F]; + size_t in_chunk = TAMP_FUZZ_CHUNK_SIZES[(chunk_byte >> 4) & 0x0F]; + if (out_chunk > output_size) out_chunk = output_size; + + TampDecompressor decompressor; + tamp_res res; + + if (custom_dictionary) { + // Custom-dictionary streams skip tamp_initialize_dictionary; the + // window contents are attacker-visible but must never be read or + // written out of bounds. + memset(window, 0xA5, (size_t)1 << 15); + } + + if (use_header) { + // Let the decompressor read the header from the stream + res = tamp_decompressor_init(&decompressor, NULL, window, 15); + } else { + TampConf conf = { + .window = window_bits, + .literal = literal_bits, + .use_custom_dictionary = custom_dictionary, + .extended = extended, + }; + res = tamp_decompressor_init(&decompressor, &conf, window, window_bits); + } + if (res != TAMP_OK) return 0; + + const unsigned char *remaining = data; + size_t remaining_size = size; + int stalled = 0; + + while (remaining_size > 0) { + size_t feed = remaining_size < in_chunk ? remaining_size : in_chunk; + size_t consumed = 0; + size_t written = 0; + res = tamp_decompressor_decompress(&decompressor, output, out_chunk, &written, remaining, feed, &consumed); + remaining += consumed; + remaining_size -= consumed; + + if (res < 0 && res != TAMP_INPUT_EXHAUSTED) break; + // OUTPUT_FULL with input left keeps looping (drains via resume paths); + // bail once no progress is made on either side twice in a row. + if (consumed == 0 && written == 0) { + if (++stalled >= 2) break; + } else { + stalled = 0; + } + if (res == TAMP_INPUT_EXHAUSTED && consumed == 0 && feed == remaining_size) break; + } + + return 0; +} + +#endif /* TAMP_DECOMPRESSOR_FUZZ_CASE_H */ diff --git a/fuzz/fuzz-matrix.sh b/fuzz/fuzz-matrix.sh new file mode 100755 index 00000000..88c2b909 --- /dev/null +++ b/fuzz/fuzz-matrix.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Build and run the malicious-input decompressor fuzzer across every +# meaningful compile-time configuration of the decompressor, so no +# flag-gated code path ships unfuzzed. Each config replays the shared +# corpus (regression) and then fuzzes for FUZZ_SECONDS (default 90). +# +# All configs are built serially first (builds are quick), then fuzzed with +# bounded parallelism: FUZZ_JOBS run at a time (default: CPU count via +# nproc/sysctl, override with the FUZZ_JOBS env var). Each config's fuzz +# output goes to build/fuzz_matrix/log_.txt; any crash/leak/timeout +# reproducer libFuzzer writes lands in build/fuzz_matrix/artifacts_/, +# so parallel configs never collide on disk. +# +# Usage: fuzz/fuzz-matrix.sh [FUZZ_SECONDS] +# Requires an LLVM clang with the libFuzzer runtime (brew install llvm). +set -euo pipefail +cd "$(dirname "$0")/.." + +FUZZ_SECONDS=${1:-90} +FUZZ_CC=${FUZZ_CC:-} +if [ -z "$FUZZ_CC" ]; then + for c in /opt/homebrew/opt/llvm/bin/clang /usr/local/opt/llvm/bin/clang clang; do + [ -x "$c" ] || command -v "$c" >/dev/null 2>&1 || continue + FUZZ_CC=$c + break + done +fi +BASE_FLAGS="-g -O1 -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer -Itamp/_c_src" +ESP32_FLAGS="-DTAMP_ESP32=1 -fno-strict-aliasing -fno-sanitize=alignment -Itamp/_c_src/tamp -Iespidf/tamp -Ifuzz/esp32_host" +SRC="fuzz/fuzz_decompressor.c tamp/_c_src/tamp/decompressor.c tamp/_c_src/tamp/common.c" +CORPUS=fuzz/corpus_decompressor + +# name:flags — one entry per distinct decompressor code-path configuration. +CONFIGS=( + "portable:" + # Shipping ARMV7EM profile via its own knob (NOT hand-expanded into the + # sub-flags: the profile's definition lives in common.h alone, so this + # entry keeps fuzzing whatever the profile means as it evolves). The + # profile's compressor-side match selection is inert here - only + # decompressor.c+common.c are compiled. + "v7em:-DTAMP_ARMV7EM=1" + "fastloop_only:-DTAMP_FAST_DECODE_LOOP=1" + "window_from_output_portable:-DTAMP_WINDOW_FROM_OUTPUT=1" + "no_extended:-DTAMP_EXTENDED=0" + "no_extended_fastloop:-DTAMP_EXTENDED=0 -DTAMP_FAST_DECODE_LOOP=1" + "no_memset:-DTAMP_USE_MEMSET=0" + "esp32:ESP32" + "esp32_fastloop:ESP32 -DTAMP_FAST_DECODE_LOOP=1" + "fixed_w10:-DTAMP_FIXED_WINDOW_BITS=10 -DTAMP_FIXED_LITERAL_BITS=8" + "fixed_w10_fastloop:-DTAMP_FIXED_WINDOW_BITS=10 -DTAMP_FIXED_LITERAL_BITS=8 -DTAMP_FAST_DECODE_LOOP=1 -DTAMP_WINDOW_FROM_OUTPUT=1 -DTAMP_FAST_WINDOW_COPY=1 -DTAMP_FAST_BIT_REFILL=1 -DTAMP_FAST_OUTPUT_COPY=1" +) + +default_jobs() { + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v sysctl >/dev/null 2>&1; then + sysctl -n hw.ncpu + else + echo 1 + fi +} +FUZZ_JOBS=${FUZZ_JOBS:-$(default_jobs)} +case "$FUZZ_JOBS" in + ''|*[!0-9]*) FUZZ_JOBS=1 ;; +esac +[ "$FUZZ_JOBS" -ge 1 ] || FUZZ_JOBS=1 + +mkdir -p build/fuzz_matrix + +# --------------------------------------------------------------- build phase +echo "=== building ${#CONFIGS[@]} configs" +for entry in "${CONFIGS[@]}"; do + name=${entry%%:*} + flags=${entry#*:} + extra="" + if [[ $flags == ESP32* ]]; then + extra="$ESP32_FLAGS ${flags#ESP32}" + flags="" + fi + bin=build/fuzz_matrix/fuzz_decompressor_$name + echo "=== [$name] build" + # shellcheck disable=SC2086 + "$FUZZ_CC" $BASE_FLAGS $flags $extra -o "$bin" $SRC +done + +# --------------------------------------------------------------- fuzz phase +# Each config writes only to its own corpus workdir (the first, writable, +# libFuzzer-owned positional arg) and replays the shared $CORPUS read-only +# (second positional arg) — libFuzzer never writes into it — so configs never +# race each other on disk and can safely fuzz with bounded parallelism. +run_one() { + local name="$1" + local bin="build/fuzz_matrix/fuzz_decompressor_$name" + local workdir="build/fuzz_matrix/corpus_$name" + local artifacts="build/fuzz_matrix/artifacts_$name" + local log="build/fuzz_matrix/log_$name.txt" + mkdir -p "$workdir" "$artifacts" + echo "=== [$name] corpus replay + ${FUZZ_SECONDS}s fuzz" >"$log" + local status=0 + "$bin" -max_total_time="$FUZZ_SECONDS" -print_final_stats=1 \ + -artifact_prefix="$artifacts/" \ + "$workdir" "$CORPUS" >>"$log" 2>&1 || status=$? + if [ "$status" -eq 0 ]; then + echo pass >"build/fuzz_matrix/status_$name.txt" + else + echo fail >"build/fuzz_matrix/status_$name.txt" + fi + return "$status" +} + +echo "=== fuzzing ${#CONFIGS[@]} configs, FUZZ_JOBS=$FUZZ_JOBS, ${FUZZ_SECONDS}s each" +idx=0 +total=${#CONFIGS[@]} +while [ "$idx" -lt "$total" ]; do + batch_pids=() + batch_names=() + slots=0 + while [ "$slots" -lt "$FUZZ_JOBS" ] && [ "$idx" -lt "$total" ]; do + entry=${CONFIGS[$idx]} + name=${entry%%:*} + echo "=== [$name] launched" + run_one "$name" & + batch_pids+=("$!") + batch_names+=("$name") + idx=$((idx + 1)) + slots=$((slots + 1)) + done + for k in "${!batch_pids[@]}"; do + pid=${batch_pids[$k]} + cname=${batch_names[$k]} + if wait "$pid"; then + echo "=== [$cname] done" + else + echo "!!! [$cname] FAILED (see build/fuzz_matrix/log_$cname.txt)" + fi + done +done + +# -------------------------------------------------------------------- summary +echo +echo "=== summary" +fail=0 +for entry in "${CONFIGS[@]}"; do + name=${entry%%:*} + status_file="build/fuzz_matrix/status_$name.txt" + status="unknown" + [ -f "$status_file" ] && status=$(cat "$status_file") + if [ "$status" = "pass" ]; then + echo "PASS $name" + else + fail=1 + echo "FAIL $name (log: build/fuzz_matrix/log_$name.txt, artifacts: build/fuzz_matrix/artifacts_$name/)" + fi +done +exit $fail diff --git a/fuzz/fuzz_decompressor.c b/fuzz/fuzz_decompressor.c index 53ebcdcc..b45f705e 100644 --- a/fuzz/fuzz_decompressor.c +++ b/fuzz/fuzz_decompressor.c @@ -3,63 +3,24 @@ * * This is the highest-priority target since decompressors typically * handle untrusted input. We test across all valid window/literal - * configurations, both with and without extended format. + * configurations, with and without extended format, and across + * fuzz-chosen input/output chunk sizes so the suspend/resume state + * machines (mid-token INPUT_EXHAUSTED, output-full skip_bytes / + * token_state resume, header-byte stash) run against malicious data, + * not just the happy full-buffer path. + * + * Build with the portable defaults AND with the ARMV7EM-profile flags + * (TAMP_FAST_DECODE_LOOP=1 TAMP_WINDOW_FROM_OUTPUT=1 TAMP_FAST_WINDOW_COPY=1 + * TAMP_FAST_BIT_REFILL=1 TAMP_FAST_OUTPUT_COPY=1) - the fast decode loop and + * the inline window-update variants are compiled out otherwise and would + * never be fuzzed. */ -#include "tamp/common.h" -#include "tamp/decompressor.h" +#include "decompressor_fuzz_case.h" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size < 1) return 0; - - // Use first byte to select configuration - uint8_t config_byte = data[0]; - data++; - size--; - - uint8_t window_bits = 8 + (config_byte & 0x07); // 8-15 - if (window_bits > 15) window_bits = 15; - uint8_t literal_bits = 5 + ((config_byte >> 3) & 0x03); // 5-8 - if (literal_bits > 8) literal_bits = 8; - bool extended = (config_byte >> 5) & 1; - bool use_header = (config_byte >> 6) & 1; - + // Corpus byte contract + drive loop live in decompressor_fuzz_case.h, + // shared with the QEMU replay firmware so the encoding cannot diverge. unsigned char window[1 << 15]; // Max window size - TampDecompressor decompressor; - tamp_res res; - - if (use_header) { - // Let the decompressor read the header from the stream - res = tamp_decompressor_init(&decompressor, NULL, window, 15); - } else { - TampConf conf = { - .window = window_bits, - .literal = literal_bits, - .use_custom_dictionary = 0, - .extended = extended, - }; - res = tamp_decompressor_init(&decompressor, &conf, window, window_bits); - } - if (res != TAMP_OK) return 0; - unsigned char output[4096]; - size_t input_consumed_size = 0; - size_t output_written_size = 0; - - // Decompress in chunks to exercise the output-full resume path - const unsigned char *remaining = data; - size_t remaining_size = size; - - while (remaining_size > 0) { - size_t consumed = 0; - size_t written = 0; - res = tamp_decompressor_decompress(&decompressor, output, sizeof(output), &written, remaining, remaining_size, - &consumed); - remaining += consumed; - remaining_size -= consumed; - - if (res < 0 && res != TAMP_INPUT_EXHAUSTED) break; - if (res == TAMP_INPUT_EXHAUSTED && consumed == 0) break; - } - - return 0; + return tamp_fuzz_case_run(data, size, window, output, sizeof(output)); } diff --git a/pyproject.toml b/pyproject.toml index be8e906b..8195e233 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,10 @@ convention = "numpy" "mpy_bindings/*.py" = [ "F821", # Undefined names ] +"tools/qemu-profiler/profile.py" = [ + "S603", # subprocess calls are fixed local toolchain binaries (gcc, qemu, nm) + "S607", # partial paths intentional: resolved via PATH like the Makefiles do +] "tamp/*_viper.py" = [ "F821", # Undefined names (e.g. `ptr8`) ] diff --git a/setup.py b/setup.py index 8d8c235f..2259a124 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ """Builds the Cython extensions; all other packaging configuration is in pyproject.toml.""" import os +import platform import sys from setuptools import setup @@ -52,6 +53,11 @@ def build_extensions(): if os.environ.get("TAMP_USE_EMBEDDED_MATCH", "0") == "1": print("Using embedded find_best_match implementation") define_macros.append(("TAMP_USE_EMBEDDED_MATCH", "1")) + elif platform.machine().lower() in ("x86_64", "amd64", "arm64", "aarch64"): + # The core C sources default to portable code everywhere; each build + # system opts into its platform's measured configuration (see the + # platform tuning section in tamp/_c_src/tamp/common.h). + define_macros.append(("TAMP_USE_DESKTOP_MATCH", "1")) if profile: print("Setting profiling configuration.") diff --git a/tamp/_c_src/tamp/common.c b/tamp/_c_src/tamp/common.c index e5bc0e2a..a9976247 100644 --- a/tamp/_c_src/tamp/common.c +++ b/tamp/_c_src/tamp/common.c @@ -55,35 +55,71 @@ TAMP_OPTIMIZE_SIZE int8_t tamp_compute_min_pattern_size(uint8_t window, uint8_t return 2 + (window > (10 + ((literal - 5) << 1))); } +/* Two variants, selected by TAMP_FAST_WINDOW_COPY (see common.h): the fast + * variant adds a branch that skips per-byte masking for the common case where + * the destination run does not wrap. + * + * Both handle the critical overlap case: when the destination is AHEAD of the + * source by less than match_size, a forward copy corrupts data because we + * write to positions before reading from them. + * + * Example: src=100, dst=105, match_size=8 + * - Forward copy at i=5 would read window[105], but we already overwrote it at i=0! + * - Must copy in REVERSE order (end to start) to read source bytes before overwriting. + * + * The reverse copy's destination wraps via mask; the source never needs + * wrapping (bounds are pre-validated by the caller). */ +#if TAMP_FAST_WINDOW_COPY void tamp_window_copy(unsigned char *window, uint16_t *window_pos, uint16_t window_offset, uint8_t match_size, uint16_t window_mask) { - /* Calculate distance from source to destination in circular buffer. - * src_to_dst = (dst - src) & mask gives the forward distance. */ - const uint16_t src_to_dst = (*window_pos - window_offset) & window_mask; - - /* Critical overlap case: destination is AHEAD of source and they overlap. - * When dst > src by less than match_size, a forward copy corrupts data because - * we write to positions before reading from them. - * - * Example: src=100, dst=105, match_size=8 - * - Forward copy at i=5 would read window[105], but we already overwrote it at i=0! - * - Must copy in REVERSE order (end to start) to read source bytes before overwriting. - */ + uint16_t pos = *window_pos; + /* Forward distance from source to destination in the circular buffer. */ + const uint16_t src_to_dst = (pos - window_offset) & window_mask; + + if (TAMP_UNLIKELY(src_to_dst < match_size && src_to_dst > 0)) { + /* Overlap: copy in reverse order (see above). */ + for (uint8_t i = match_size; i-- > 0;) { + window[(pos + i) & window_mask] = window[window_offset + i]; + } + pos = (pos + match_size) & window_mask; + } else if (TAMP_LIKELY((uint32_t)pos + match_size <= (uint32_t)window_mask + 1)) { + /* Common case: destination run doesn't wrap, no per-byte masking. */ + unsigned char *dst = window + pos; + const unsigned char *src = window + window_offset; + for (uint8_t i = 0; i < match_size; i++) { + dst[i] = src[i]; + } + pos = (pos + match_size) & window_mask; + } else { + for (uint8_t i = 0; i < match_size; i++) { + window[pos] = window[window_offset + i]; + pos = (pos + 1) & window_mask; + } + } + *window_pos = pos; +} +#else +void tamp_window_copy(unsigned char *window, uint16_t *window_pos, uint16_t window_offset, uint8_t match_size, + uint16_t window_mask) { + uint16_t pos = *window_pos; + /* Forward distance from source to destination in the circular buffer. */ + const uint16_t src_to_dst = (pos - window_offset) & window_mask; + if (TAMP_UNLIKELY(src_to_dst < match_size && src_to_dst > 0)) { - /* Copy in reverse order: start from last byte, work backwards to first byte. - * This ensures we read all overlapping source bytes before they're overwritten. - * Destination wraps via mask; source doesn't need wrapping (pre-validated bounds). */ + /* Overlap: copy in reverse order (see above). */ for (uint8_t i = match_size; i-- > 0;) { - window[(*window_pos + i) & window_mask] = window[window_offset + i]; + window[(pos + i) & window_mask] = window[window_offset + i]; } - *window_pos = (*window_pos + match_size) & window_mask; + pos = (pos + match_size) & window_mask; } else { for (uint8_t i = 0; i < match_size; i++) { - window[*window_pos] = window[window_offset + i]; - *window_pos = (*window_pos + 1) & window_mask; + window[pos] = window[window_offset + i]; + pos = (pos + 1) & window_mask; } } + *window_pos = pos; } +#endif /* TAMP_FAST_WINDOW_COPY */ /******************************************************************************* * Built-in I/O handler implementations diff --git a/tamp/_c_src/tamp/common.h b/tamp/_c_src/tamp/common.h index 984bd211..7fa4d6b3 100644 --- a/tamp/_c_src/tamp/common.h +++ b/tamp/_c_src/tamp/common.h @@ -69,6 +69,141 @@ extern "C" { #define TAMP_OPTIMIZE_SIZE #endif +/******************************************************************************* + * Platform performance tuning + * + * The core sources never select architecture-specific code on their own: + * every flag below defaults to the portable implementation. Build systems opt + * in to the measured configuration for their platform, mirroring TAMP_ESP32: + * + * pip/Cython (setup.py): TAMP_USE_DESKTOP_MATCH=1 on 64-bit hosts + * espidf component (Kconfig): TAMP_ESP32 (default y) + * ARMv7E-M (Cortex-M4/M7): TAMP_ARMV7EM=1 (profile flag, see below) + * + * Individual flags can still be set/overridden with -D=0/1. Measured + * numbers below are from devices/BENCHMARKS.md workloads; when enabling a + * flag on an unmeasured core, benchmark it. + ******************************************************************************/ + +/* Profile for ARMv7E-M cores (Cortex-M4/M7): little-endian with cheap + * unaligned loads. Enables the prefilter match finder and every decompressor + * fast path below (measured on STM32H7B0/Cortex-M7 vs the portable build: + * 1.31x compression, 1.92x decompression, ~5.2 KB additional flash). */ +#ifndef TAMP_ARMV7EM +#define TAMP_ARMV7EM 0 +#endif + +/* find_best_match implementation (see compressor.c). At most one of these + * may be 1 (enforced below); with none set, the portable scan is used. + * embedded: portable single-byte-first scan, the default. + * swar32: experimental 32-bit SWAR (candidate for single-issue cores; + * measured 0.93x on Cortex-M7 - opt-in only). + * desktop: 64-bit SWAR for 64-bit hosts (little-endian, cheap unaligned + * loads; measured ~2x the prefilter there). + * prefilter: first-byte prefilter (1.36x compression on Cortex-M7; + * ~0.5x on out-of-order 64-bit hosts). */ +#ifndef TAMP_USE_EMBEDDED_MATCH +#define TAMP_USE_EMBEDDED_MATCH 0 +#endif +#ifndef TAMP_USE_SWAR32_MATCH +#define TAMP_USE_SWAR32_MATCH 0 +#endif +#ifndef TAMP_USE_DESKTOP_MATCH +#define TAMP_USE_DESKTOP_MATCH 0 +#endif +#ifndef TAMP_USE_PREFILTER_MATCH +#define TAMP_USE_PREFILTER_MATCH \ + (TAMP_ARMV7EM && !TAMP_USE_EMBEDDED_MATCH && !TAMP_USE_SWAR32_MATCH && !TAMP_USE_DESKTOP_MATCH) +#endif + +/* The selections are mutually exclusive; reject conflicting configurations + * loudly rather than silently picking one. TAMP_ESP32 counts as a selection: + * the espidf platform component provides find_best_match via extern, and + * compressor.c's dispatch checks it first, so combining it with an explicit + * TAMP_USE_*_MATCH would otherwise silently drop the requested finder. */ +#if (TAMP_USE_EMBEDDED_MATCH + TAMP_USE_SWAR32_MATCH + TAMP_USE_DESKTOP_MATCH + TAMP_USE_PREFILTER_MATCH + \ + TAMP_ESP32) > 1 +#error "At most one find_best_match selection (TAMP_USE_*_MATCH / TAMP_ESP32) may be enabled" +#endif + +/* tamp_window_copy variant with a no-wrap fast path (see common.c). + * Measured: +14% decompression on Cortex-M7; -3% on Xtensa LX7; ~+160 bytes + * on Cortex-M0/M0+ where flash is the scarce resource. */ +#ifndef TAMP_FAST_WINDOW_COPY +#define TAMP_FAST_WINDOW_COPY TAMP_ARMV7EM +#endif + +/* refill_bit_buffer on locals with a single writeback (see decompressor.c). + * Measured: +5% decompression on Cortex-M7; -3% on Xtensa LX7; +324 bytes on + * Cortex-M0/M0+ (register spills on the 8-register file). */ +#ifndef TAMP_FAST_BIT_REFILL +#define TAMP_FAST_BIT_REFILL TAMP_ARMV7EM +#endif + +/* Word-at-a-time TAMP_COPY_TO_OUTPUT (see decompressor.c). Requires cheap + * unaligned 32-bit access. */ +#ifndef TAMP_FAST_OUTPUT_COPY +#define TAMP_FAST_OUTPUT_COPY TAMP_ARMV7EM +#endif + +/* Source the window update from the just-written output snapshot instead of + * calling tamp_window_copy (see decompressor.c). Removes the call plus its + * reverse-copy overlap logic from the hot path. Measured: -4% (M7) / -6.7% + * (M4) core insns/byte, but +3% on Cortex-M0+ (inlining the update costs more + * than the call on the 8-register file), so it defaults off there. */ +#ifndef TAMP_WINDOW_FROM_OUTPUT +#define TAMP_WINDOW_FROM_OUTPUT TAMP_ARMV7EM +#endif + +/* Checked-once fast inner decode loop (see decompressor.c). Under a + * per-iteration precondition (>=4 input bytes, >=32 output bytes, no pending + * skip/extended/flush state, no callback) every mid-token bounds/exhaustion + * check on the classic token path is provably dead and removed. The loop keeps + * the undecoded bits in a 64-bit reservoir local and refills 4 whole bytes at + * a time only when <=32 bits remain (~every 1-2 tokens); on loop exit the + * reservoir is written back to the 32-bit struct bit_buffer, surplus whole + * bytes are pushed back to the input cursor, and a checked refill restores the + * >=25-bit invariant the careful body relies on. The reservoir alone (measured + * against an earlier fast-loop variant that refilled the 32-bit struct + * bit_buffer per token, since removed) was worth +14.2% decompression + * throughput on STM32H7B0/M7 hardware, +4.7% AND slightly smaller code on + * RP2040/M0+ hardware despite the __aeabi_ll* helper calls for the 64-bit + * shifts, and -12.0% core insns/byte on QEMU m33 (hardware-unverified) on the + * window=10 enwik8 workload. Duplicates the classic token path (a few hundred + * bytes of flash), so it defaults off on the portable/M0+ build and on where + * the platform profile already opts into the other fast paths. */ +#ifndef TAMP_FAST_DECODE_LOOP +#define TAMP_FAST_DECODE_LOOP TAMP_ARMV7EM +#endif + +/* Compile the careful body of tamp_decompressor_decompress_cb -Os (see + * decompressor.c; GCC-only, no-op elsewhere). Only sensible with + * TAMP_FAST_DECODE_LOOP, which routes every hot token through the extracted + * -O3 fast-loop function and leaves the careful body handling buffer tails, + * resume state, and extended dispatch glue. Measured on the window=10 enwik8 + * workload: -872 B of decompressor .text on Cortex-M7 for no throughput + * change; +64 B on Cortex-M0+ (the 8-register file spills more under Os), so + * it stays off outside the ARMV7EM profile. */ +#ifndef TAMP_COMPACT_CAREFUL_BODY +#define TAMP_COMPACT_CAREFUL_BODY TAMP_ARMV7EM +#endif + +/* Compile-time-pinned stream configuration (opt-in; decompressor only). + * Most embedded deployments only ever decode streams produced with one fixed + * configuration. Defining these pins the window and/or literal bit counts at + * compile time: the decompressor folds every window_mask / window_size / bit + * shift to an immediate, and with BOTH pinned min_pattern_size becomes a + * compile-time constant too. A pinned build REJECTS (TAMP_INVALID_CONF) any + * stream whose header disagrees, so this is only safe when every stream really + * uses the pinned configuration. + * + * There is no default: leave them undefined for the normal runtime behavior + * (any valid window in [8,15] / literal in [5,8]). Valid pinned values match + * those ranges, e.g. -DTAMP_FIXED_WINDOW_BITS=10 -DTAMP_FIXED_LITERAL_BITS=8. + * They may be set independently. */ +/* #define TAMP_FIXED_WINDOW_BITS 10 */ +/* #define TAMP_FIXED_LITERAL_BITS 8 */ + /* TAMP_USE_MEMSET: Use libc memset (default: 1). * Set to 0 for environments without libc (e.g. MicroPython native modules). * When disabled, uses a volatile loop that prevents GCC from emitting a diff --git a/tamp/_c_src/tamp/compressor.c b/tamp/_c_src/tamp/compressor.c index 92a43ca7..92c81a60 100644 --- a/tamp/_c_src/tamp/compressor.c +++ b/tamp/_c_src/tamp/compressor.c @@ -78,26 +78,192 @@ inline bool tamp_compressor_full(const TampCompressor* compressor) { return compressor->input_size == sizeof(compressor->input); } +#if TAMP_USE_PREFILTER_MATCH || TAMP_USE_DESKTOP_MATCH +/* Shared helpers for the 64-bit SWAR match finders (prefilter and desktop). */ +#include // for memcpy (portable unaligned loads) + +// MSVC compatibility for count trailing zeros +#if defined(_MSC_VER) +#include +static inline int tamp_ctzll(uint64_t value) { + unsigned long index; + _BitScanForward64(&index, value); + return (int)index; +} +#else +#define tamp_ctzll(v) __builtin_ctzll(v) +#endif + +/* Detect zero bytes in a 64-bit word (classic bit manipulation trick). + * Can have false positives when a zero byte is followed by 0x01-0x7F due to + * borrow propagation; callers re-verify every hit. */ +static inline uint64_t tamp_has_zero_byte(uint64_t v) { + return (v - 0x0101010101010101ULL) & ~v & 0x8080808080808080ULL; +} + +/** + * @brief Extend a verified 2-byte candidate match at idx. + * + * Pure computation (callers update the best match): uses the pre-computed + * input_word_ext and input_bytes to avoid repeated read_input() calls. + * + * @return The match length at idx, or 0 when the best-length gate proves it + * cannot beat the current best match. + * + * NOTE: an equivalent macro form measured ~3% faster compression on + * Cortex-M7 with GCC 15 (the inline boundary shifts register save/restore + * into the per-hit path); the function form is kept for maintainability. + */ +static inline uint8_t tamp_match_extend(const unsigned char* window, uint16_t window_size_minus_1, + uint8_t max_pattern_size, const uint8_t* input_bytes, uint64_t input_word_ext, + uint16_t idx, uint8_t best_match_size) { + /* Best-length gate: a strictly longer match must also agree at offset + * best_match_size, so a single byte compare rejects most candidates early. */ + if (best_match_size && (TAMP_UNLIKELY((idx + best_match_size) > window_size_minus_1) || + TAMP_LIKELY(window[idx + best_match_size] != input_bytes[best_match_size]))) + return 0; + + uint8_t match_len = 2; + + /* Try 8-byte comparison for bytes 2-9 using pre-computed input_word_ext */ + if (max_pattern_size >= 10 && (idx + 9) <= window_size_minus_1) { + uint64_t window_word; + memcpy(&window_word, window + idx + 2, sizeof(uint64_t)); + uint64_t diff = window_word ^ input_word_ext; + if (diff != 0) return 2 + (tamp_ctzll(diff) >> 3); + match_len = 10; + } + + /* Byte-by-byte for the remainder using pre-loaded input_bytes */ + for (uint8_t i = match_len; i < max_pattern_size; i++) { + if (TAMP_UNLIKELY((idx + i) > window_size_minus_1)) break; + if (TAMP_LIKELY(window[idx + i] != input_bytes[i])) break; + match_len = i + 1; + } + return match_len; +} + +/* Update the best match with a freshly extended candidate; evaluates to + * true when the maximum possible match was found and scanning can stop. */ +#define TAMP_MATCH_UPDATE(idx, len) \ + (TAMP_UNLIKELY((len) > *match_size) && \ + (*match_size = (len), *match_index = (idx), TAMP_UNLIKELY(*match_size == max_pattern_size))) +#endif /* TAMP_USE_PREFILTER_MATCH || TAMP_USE_DESKTOP_MATCH */ + /* - * Platform-specific find_best_match implementations: + * find_best_match implementations, selected by the flags from common.h's + * platform tuning section (the default is the portable embedded scan; build + * systems opt into their platform's measured configuration): * * 1. TAMP_ESP32: External implementation in espidf/tamp/compressor_esp32.cpp * - * 2. Desktop 64-bit (x86_64, aarch64, Windows 64-bit): - * Included from compressor_find_match_desktop.c - uses bit manipulation - * and 64-bit loads for parallel match detection + * 2. TAMP_USE_PREFILTER_MATCH (via TAMP_ARMV7EM, Cortex-M4/M7): first-byte + * prefilter, defined below. + * + * 3. TAMP_USE_DESKTOP_MATCH (64-bit hosts): included from + * compressor_find_match_desktop.c - uses bit manipulation and 64-bit + * loads for parallel match detection. * - * 3. Embedded/Default (Cortex-M0/M0+, other 32-bit): - * Defined below - single-byte-first comparison, safe for all architectures + * 4. Default: defined below - portable single-byte-first comparison, safe + * for all architectures. * - * Set TAMP_USE_EMBEDDED_MATCH=1 to force the embedded implementation on desktop - * (useful for testing the embedded code path on CI). + * Implementations reachable on embedded targets are defined inline so that + * common.c/compressor.c/decompressor.c compile standalone with only the + * headers; only desktop/experimental variants live in #include'd files. */ #if TAMP_ESP32 extern void find_best_match(TampCompressor* compressor, uint16_t* match_index, uint8_t* match_size); -#elif (defined(__x86_64__) || defined(__aarch64__) || defined(_M_X64) || defined(_M_ARM64)) && !TAMP_USE_EMBEDDED_MATCH +#elif TAMP_USE_PREFILTER_MATCH +/* First-byte prefilter: SWAR-detect candidate positions holding the first + * input byte with one 64-bit load per 8 window positions, then verify the + * 2-byte seed with a 16-bit load per hit. Kept inline so compressor.c + * compiles standalone on embedded targets. + */ +/** + * @brief Find the best match for the current input buffer. + * + * First-byte prefilter: SWAR-detect candidate positions holding the first + * input byte, then verify the 2-byte seed with a 16-bit load per hit. + * + * @param[in,out] compressor TampCompressor object to perform search on. + * @param[out] match_index If match_size is 0, this value is undefined. + * @param[out] match_size Size of best found match. + */ +static inline void find_best_match(TampCompressor* compressor, uint16_t* match_index, uint8_t* match_size) { + *match_size = 0; + + if (TAMP_UNLIKELY(compressor->input_size < compressor->min_pattern_size)) return; + + const uint16_t window_size_minus_1 = WINDOW_SIZE - 1; + const uint8_t max_pattern_size = MIN(compressor->input_size, MAX_PATTERN_SIZE); + const unsigned char* window = compressor->window; + + // Pre-load input bytes into linear array to avoid repeated modular arithmetic. + // Zero-initialized: entries at max_pattern_size and beyond are read when + // assembling input_word_ext below; their values never affect the result. + uint8_t input_bytes[16] = {0}; + for (uint8_t i = 0; i < max_pattern_size && i < 16; i++) { + input_bytes[i] = read_input(i); + } + + // Little-endian format to match direct memory loads + const uint16_t first_second = input_bytes[0] | (input_bytes[1] << 8); + + // Broadcast pattern for detecting first_byte in parallel + const uint64_t first_pattern = 0x0101010101010101ULL * input_bytes[0]; + + // Pre-compute 64-bit input word for extension (bytes 2-9) + const uint64_t input_word_ext = (uint64_t)input_bytes[2] | ((uint64_t)input_bytes[3] << 8) | + ((uint64_t)input_bytes[4] << 16) | ((uint64_t)input_bytes[5] << 24) | + ((uint64_t)input_bytes[6] << 32) | ((uint64_t)input_bytes[7] << 40) | + ((uint64_t)input_bytes[8] << 48) | ((uint64_t)input_bytes[9] << 56); + + uint16_t window_index = 0; + + // Main loop: scan 8 positions per iteration with one 64-bit load. A hit + // at byte 7 verifies its pair byte via the unaligned 16-bit candidate + // load, so no overlapping second word is needed. + for (; window_index + 8 <= window_size_minus_1; window_index += 8) { + uint64_t word; + memcpy(&word, window + window_index, sizeof(uint64_t)); + + // Positions whose byte may equal the first input byte. + // Note: HAS_ZERO_BYTE can have false positives; the 16-bit candidate + // check below re-verifies every hit. + uint64_t hits = tamp_has_zero_byte(word ^ first_pattern); + + while (hits) { + int byte_pos = tamp_ctzll(hits) >> 3; + uint16_t candidate; + memcpy(&candidate, window + window_index + byte_pos, sizeof(uint16_t)); + if (TAMP_UNLIKELY(candidate == first_second)) { + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, + input_word_ext, window_index + byte_pos, *match_size); + if (TAMP_MATCH_UPDATE(window_index + byte_pos, len)) return; + } + hits &= hits - 1; // Clear lowest set bit + } + } + + // Handle remaining positions + for (; window_index < window_size_minus_1; window_index++) { + uint16_t candidate; + memcpy(&candidate, window + window_index, sizeof(uint16_t)); + if (TAMP_LIKELY(candidate != first_second)) { + continue; + } + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, input_word_ext, + window_index, *match_size); + if (TAMP_MATCH_UPDATE(window_index, len)) return; + } +} + +#elif TAMP_USE_SWAR32_MATCH +#include "compressor_find_match_swar32.c" + +#elif TAMP_USE_DESKTOP_MATCH #include "compressor_find_match_desktop.c" #else diff --git a/tamp/_c_src/tamp/compressor_find_match_desktop.c b/tamp/_c_src/tamp/compressor_find_match_desktop.c index 21ee8736..b6a1bb82 100644 --- a/tamp/_c_src/tamp/compressor_find_match_desktop.c +++ b/tamp/_c_src/tamp/compressor_find_match_desktop.c @@ -11,64 +11,11 @@ * Requirements: * - Little-endian byte order * - 64-bit compiler intrinsics (__builtin_ctzll or _BitScanForward64) + * - The shared helpers defined in compressor.c before this file is + * included: tamp_ctzll, tamp_has_zero_byte, tamp_match_extend, + * TAMP_MATCH_UPDATE. */ -#include // for memcpy (portable unaligned loads) - -// MSVC compatibility for count trailing zeros -#if defined(_MSC_VER) -#include -static inline int tamp_ctzll(uint64_t value) { - unsigned long index; - _BitScanForward64(&index, value); - return (int)index; -} -#else -#define tamp_ctzll(v) __builtin_ctzll(v) -#endif - -// Detect zero bytes in a 64-bit word (classic bit manipulation trick) -// Note: Can have false positives when a zero byte is followed by 0x01-0x7F due to borrow propagation -#define HAS_ZERO_BYTE(v) (((v)-0x0101010101010101ULL) & ~(v) & 0x8080808080808080ULL) - -// Helper macro to extend a match and update best match -// Uses pre-computed input_word_ext and input_bytes to avoid repeated read_input() calls -#define EXTEND_MATCH(idx) \ - do { \ - uint8_t match_len = 2; \ - uint8_t i = 2; \ - int extend_done = 0; \ - \ - /* Try 8-byte comparison for bytes 2-9 using pre-computed input_word_ext */ \ - if (max_pattern_size >= 10 && (idx + 9) <= window_size_minus_1) { \ - uint64_t window_word; \ - memcpy(&window_word, window + idx + 2, sizeof(uint64_t)); \ - uint64_t diff = window_word ^ input_word_ext; \ - if (diff == 0) { \ - match_len = 10; \ - i = 10; \ - } else { \ - match_len = 2 + (tamp_ctzll(diff) >> 3); \ - extend_done = 1; \ - } \ - } \ - \ - /* Byte-by-byte for remainder using pre-loaded input_bytes */ \ - if (!extend_done) { \ - for (; i < max_pattern_size; i++) { \ - if (TAMP_UNLIKELY((idx + i) > window_size_minus_1)) break; \ - if (TAMP_LIKELY(window[idx + i] != input_bytes[i])) break; \ - match_len = i + 1; \ - } \ - } \ - \ - if (TAMP_UNLIKELY(match_len > *match_size)) { \ - *match_size = match_len; \ - *match_index = idx; \ - if (TAMP_UNLIKELY(*match_size == max_pattern_size)) return; \ - } \ - } while (0) - /** * @brief Find the best match for the current input buffer. * @@ -91,7 +38,7 @@ static inline void find_best_match(TampCompressor *compressor, uint16_t *match_i // Pre-load input bytes into linear array to avoid repeated modular arithmetic. // Zero-initialized: entries at max_pattern_size and beyond are read when // assembling input_word_ext below. Their values never affect the result (the - // max_pattern_size >= 10 guard in EXTEND_MATCH sees to that), but reading + // max_pattern_size >= 10 guard in tamp_match_extend sees to that), but reading // indeterminate bytes is UB (MSan finding, -Werror=maybe-uninitialized). uint8_t input_bytes[16] = {0}; for (uint8_t i = 0; i < max_pattern_size && i < 16; i++) { @@ -122,26 +69,28 @@ static inline void find_best_match(TampCompressor *compressor, uint16_t *match_i // Find 2-byte matches in word1 (positions 0-6) // XOR with broadcast patterns to find matching bytes, then detect zeros - uint64_t first_zeros1 = HAS_ZERO_BYTE(word1 ^ first_pattern); - uint64_t second_zeros1 = HAS_ZERO_BYTE(word1 ^ second_pattern); + uint64_t first_zeros1 = tamp_has_zero_byte(word1 ^ first_pattern); + uint64_t second_zeros1 = tamp_has_zero_byte(word1 ^ second_pattern); // A 2-byte match at position i requires first_byte at i AND second_byte at i+1 uint64_t matches1 = first_zeros1 & (second_zeros1 >> 8); // Process all matches found in word1 - // Note: HAS_ZERO_BYTE can have false positives, so verify before extending + // Note: tamp_has_zero_byte can have false positives, so verify before extending while (matches1) { int byte_pos = tamp_ctzll(matches1) >> 3; uint16_t candidate; memcpy(&candidate, window + window_index + byte_pos, sizeof(uint16_t)); if (TAMP_UNLIKELY(candidate == first_second)) { - EXTEND_MATCH(window_index + byte_pos); + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, + input_word_ext, window_index + byte_pos, *match_size); + if (TAMP_MATCH_UPDATE(window_index + byte_pos, len)) return; } matches1 &= matches1 - 1; // Clear lowest set bit } // Find 2-byte matches in word2 (positions 7-13) - uint64_t first_zeros2 = HAS_ZERO_BYTE(word2 ^ first_pattern); - uint64_t second_zeros2 = HAS_ZERO_BYTE(word2 ^ second_pattern); + uint64_t first_zeros2 = tamp_has_zero_byte(word2 ^ first_pattern); + uint64_t second_zeros2 = tamp_has_zero_byte(word2 ^ second_pattern); uint64_t matches2 = first_zeros2 & (second_zeros2 >> 8); while (matches2) { @@ -149,7 +98,9 @@ static inline void find_best_match(TampCompressor *compressor, uint16_t *match_i uint16_t candidate; memcpy(&candidate, window + window_index + 7 + byte_pos, sizeof(uint16_t)); if (TAMP_UNLIKELY(candidate == first_second)) { - EXTEND_MATCH(window_index + 7 + byte_pos); + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, + input_word_ext, window_index + 7 + byte_pos, *match_size); + if (TAMP_MATCH_UPDATE(window_index + 7 + byte_pos, len)) return; } matches2 &= matches2 - 1; } @@ -162,12 +113,8 @@ static inline void find_best_match(TampCompressor *compressor, uint16_t *match_i if (TAMP_LIKELY(candidate != first_second)) { continue; } - EXTEND_MATCH(window_index); + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, input_word_ext, + window_index, *match_size); + if (TAMP_MATCH_UPDATE(window_index, len)) return; } } - -#undef EXTEND_MATCH -#undef HAS_ZERO_BYTE -#if !defined(_MSC_VER) -#undef tamp_ctzll -#endif diff --git a/tamp/_c_src/tamp/compressor_find_match_swar32.c b/tamp/_c_src/tamp/compressor_find_match_swar32.c new file mode 100644 index 00000000..128f486e --- /dev/null +++ b/tamp/_c_src/tamp/compressor_find_match_swar32.c @@ -0,0 +1,164 @@ +/** + * @file compressor_find_match_swar32.c + * @brief 32-bit SWAR find_best_match for 32-bit cores with cheap unaligned + * loads (e.g. Cortex-M7/M33, RISC-V with fast misaligned access). + * + * NOTE: This file is #include'd by compressor.c, not compiled separately. + * + * Same algorithm as compressor_find_match_desktop.c, narrowed to 32-bit + * words: two overlapping 32-bit loads check 6 window positions per iteration + * for a 2-byte match seed, and match extension compares 4 bytes at a time. + * Produces byte-identical output to the embedded and desktop implementations + * (ascending scan, strictly-longer replacement). + * + * Requirements: + * - Little-endian byte order + * - Efficient unaligned 32-bit loads (memcpy must lower to a plain load) + */ + +#include // for memcpy (portable unaligned loads) + +#define tamp_ctz(v) __builtin_ctz(v) + +/* Detect zero bytes in a 32-bit word (classic bit manipulation trick). + * Can have false positives when a zero byte is followed by 0x01-0x7F due to + * borrow propagation; callers re-verify every hit. */ +static inline uint32_t tamp_has_zero_byte32(uint32_t v) { return (v - 0x01010101u) & ~v & 0x80808080u; } + +/** + * @brief Extend a verified 2-byte candidate match at idx (32-bit variant). + * + * Pure computation (callers update the best match): uses the pre-computed + * input_word_ext and input_bytes to avoid repeated read_input() calls. + * + * @return The match length at idx. + */ +static inline uint8_t tamp_match_extend(const unsigned char *window, uint16_t window_size_minus_1, + uint8_t max_pattern_size, const uint8_t *input_bytes, uint32_t input_word_ext, + uint16_t idx) { + uint8_t match_len = 2; + + /* Try 4-byte comparison for bytes 2-5 using pre-computed input_word_ext */ + if (max_pattern_size >= 6 && (idx + 5) <= window_size_minus_1) { + uint32_t window_word; + memcpy(&window_word, window + idx + 2, sizeof(uint32_t)); + uint32_t diff = window_word ^ input_word_ext; + if (diff != 0) return 2 + (tamp_ctz(diff) >> 3); + match_len = 6; + } + + /* Byte-by-byte for the remainder using pre-loaded input_bytes */ + for (uint8_t i = match_len; i < max_pattern_size; i++) { + if (TAMP_UNLIKELY((idx + i) > window_size_minus_1)) break; + if (TAMP_LIKELY(window[idx + i] != input_bytes[i])) break; + match_len = i + 1; + } + return match_len; +} + +/* Update the best match with a freshly extended candidate; evaluates to + * true when the maximum possible match was found and scanning can stop. */ +#define TAMP_MATCH_UPDATE(idx, len) \ + (TAMP_UNLIKELY((len) > *match_size) && \ + (*match_size = (len), *match_index = (idx), TAMP_UNLIKELY(*match_size == max_pattern_size))) + +/** + * @brief Find the best match for the current input buffer. + * + * 32-bit SWAR implementation: uses bit manipulation to detect multiple 2-byte + * matches simultaneously within 32-bit words. + * + * @param[in,out] compressor TampCompressor object to perform search on. + * @param[out] match_index If match_size is 0, this value is undefined. + * @param[out] match_size Size of best found match. + */ +static inline void find_best_match(TampCompressor *compressor, uint16_t *match_index, uint8_t *match_size) { + *match_size = 0; + + if (TAMP_UNLIKELY(compressor->input_size < compressor->min_pattern_size)) return; + + const uint16_t window_size_minus_1 = WINDOW_SIZE - 1; + const uint8_t max_pattern_size = MIN(compressor->input_size, MAX_PATTERN_SIZE); + const unsigned char *window = compressor->window; + + // Pre-load input bytes into linear array to avoid repeated modular arithmetic. + // Zero-initialized: entries at max_pattern_size and beyond are read when + // assembling input_word_ext below; their values never affect the result. + uint8_t input_bytes[16] = {0}; + for (uint8_t i = 0; i < max_pattern_size && i < 16; i++) { + input_bytes[i] = read_input(i); + } + + // Little-endian format to match direct memory loads + const uint16_t first_second = input_bytes[0] | (input_bytes[1] << 8); + + // Broadcast patterns for detecting first_byte and second_byte in parallel + const uint32_t first_pattern = 0x01010101u * input_bytes[0]; + const uint32_t second_pattern = 0x01010101u * input_bytes[1]; + + // Pre-compute 32-bit input word for extension (bytes 2-5) + const uint32_t input_word_ext = (uint32_t)input_bytes[2] | ((uint32_t)input_bytes[3] << 8) | + ((uint32_t)input_bytes[4] << 16) | ((uint32_t)input_bytes[5] << 24); + + uint16_t window_index = 0; + + // Main loop: check 6 positions per iteration using two 32-bit loads + for (; window_index + 7 < window_size_minus_1; window_index += 6) { + uint32_t word1, word2; + memcpy(&word1, window + window_index, sizeof(uint32_t)); + memcpy(&word2, window + window_index + 3, sizeof(uint32_t)); + + // Find 2-byte matches in word1 (positions 0-2) + // XOR with broadcast patterns to find matching bytes, then detect zeros + uint32_t first_zeros1 = tamp_has_zero_byte32(word1 ^ first_pattern); + uint32_t second_zeros1 = tamp_has_zero_byte32(word1 ^ second_pattern); + // A 2-byte match at position i requires first_byte at i AND second_byte at i+1 + uint32_t matches1 = first_zeros1 & (second_zeros1 >> 8); + + // Process all matches found in word1 + // Note: tamp_has_zero_byte32 can have false positives, so verify before extending + while (matches1) { + int byte_pos = tamp_ctz(matches1) >> 3; + uint16_t candidate; + memcpy(&candidate, window + window_index + byte_pos, sizeof(uint16_t)); + if (TAMP_UNLIKELY(candidate == first_second)) { + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, + input_word_ext, window_index + byte_pos); + if (TAMP_MATCH_UPDATE(window_index + byte_pos, len)) return; + } + matches1 &= matches1 - 1; // Clear lowest set bit + } + + // Find 2-byte matches in word2 (positions 3-5) + uint32_t first_zeros2 = tamp_has_zero_byte32(word2 ^ first_pattern); + uint32_t second_zeros2 = tamp_has_zero_byte32(word2 ^ second_pattern); + uint32_t matches2 = first_zeros2 & (second_zeros2 >> 8); + + while (matches2) { + int byte_pos = tamp_ctz(matches2) >> 3; + uint16_t candidate; + memcpy(&candidate, window + window_index + 3 + byte_pos, sizeof(uint16_t)); + if (TAMP_UNLIKELY(candidate == first_second)) { + uint8_t len = tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, + input_word_ext, window_index + 3 + byte_pos); + if (TAMP_MATCH_UPDATE(window_index + 3 + byte_pos, len)) return; + } + matches2 &= matches2 - 1; + } + } + + // Handle remaining positions + for (; window_index < window_size_minus_1; window_index++) { + uint16_t candidate; + memcpy(&candidate, window + window_index, sizeof(uint16_t)); + if (TAMP_LIKELY(candidate != first_second)) { + continue; + } + uint8_t len = + tamp_match_extend(window, window_size_minus_1, max_pattern_size, input_bytes, input_word_ext, window_index); + if (TAMP_MATCH_UPDATE(window_index, len)) return; + } +} + +#undef TAMP_MATCH_UPDATE +#undef tamp_ctz diff --git a/tamp/_c_src/tamp/decompressor.c b/tamp/_c_src/tamp/decompressor.c index af7e207b..b0037f27 100644 --- a/tamp/_c_src/tamp/decompressor.c +++ b/tamp/_c_src/tamp/decompressor.c @@ -13,23 +13,151 @@ #else /* Copy count bytes from src to the output cursor and advance it. */ +#if TAMP_FAST_OUTPUT_COPY +/* Word-at-a-time: window and output never overlap, and the copy never writes + * past out+count, so this is safe whenever unaligned word access is cheap. */ +#define TAMP_COPY_TO_OUTPUT(out, src, count) \ + do { \ + const unsigned char* _tamp_s = (src); \ + uint8_t _tamp_n = (count); \ + while (_tamp_n >= 4) { \ + uint32_t _tamp_w; \ + __builtin_memcpy(&_tamp_w, _tamp_s, 4); \ + __builtin_memcpy((out), &_tamp_w, 4); \ + (out) += 4; \ + _tamp_s += 4; \ + _tamp_n -= 4; \ + } \ + while (_tamp_n--) { \ + *(out)++ = *_tamp_s++; \ + } \ + } while (0) +#else #define TAMP_COPY_TO_OUTPUT(out, src, count) \ do { \ for (uint8_t _tamp_i = 0; _tamp_i < (count); _tamp_i++) { \ *(out)++ = (src)[_tamp_i]; \ } \ } while (0) +#endif /* TAMP_FAST_OUTPUT_COPY */ #define TAMP_WINDOW_COPY(window, window_pos, window_offset, match_size, window_mask) \ tamp_window_copy((window), (window_pos), (window_offset), (match_size), (window_mask)) #endif /* TAMP_ESP32 */ +/* Update the circular window from a linear, non-overlapping snapshot of the + * match bytes (the copy just written to output). Only DESTINATION wrap is + * possible (match_size <= 134 << window_size, so at most one wrap); a snapshot + * source cannot overlap the destination, so no reverse-copy path is needed. + * window_pos_var is a plain uint16_t variable (not a pointer) to avoid spilling + * it to the stack across a call. Guarded so a platform component may override. */ +#ifndef TAMP_WINDOW_WRITE_FROM_OUTPUT + +#if TAMP_FAST_WINDOW_COPY +/* Word-at-a-time linear segment copy. The while(>=4)+byte-tail structure never + * writes past dst+n, so the second segment's start is never clobbered. */ +#define TAMP_WINDOW_SEG_COPY_(dst, src, n) \ + do { \ + unsigned char* _tamp_sd = (dst); \ + const unsigned char* _tamp_ss = (src); \ + uint8_t _tamp_sn = (n); \ + while (_tamp_sn >= 4) { \ + uint32_t _tamp_sw; \ + __builtin_memcpy(&_tamp_sw, _tamp_ss, 4); \ + __builtin_memcpy(_tamp_sd, &_tamp_sw, 4); \ + _tamp_sd += 4; \ + _tamp_ss += 4; \ + _tamp_sn -= 4; \ + } \ + while (_tamp_sn--) *_tamp_sd++ = *_tamp_ss++; \ + } while (0) +#else +#define TAMP_WINDOW_SEG_COPY_(dst, src, n) \ + do { \ + unsigned char* _tamp_sd = (dst); \ + const unsigned char* _tamp_ss = (src); \ + uint8_t _tamp_sn = (n); \ + for (uint8_t _tamp_si = 0; _tamp_si < _tamp_sn; _tamp_si++) _tamp_sd[_tamp_si] = _tamp_ss[_tamp_si]; \ + } while (0) +#endif /* TAMP_FAST_WINDOW_COPY */ + +#define TAMP_WINDOW_WRITE_FROM_OUTPUT(window, window_pos_var, src, match_size, window_size, window_mask) \ + do { \ + const unsigned char* _tamp_wsrc = (src); \ + uint16_t _tamp_wpos = (window_pos_var); \ + uint16_t _tamp_wrem = (uint16_t)((window_size)-_tamp_wpos); \ + uint8_t _tamp_wms = (match_size); \ + if (TAMP_LIKELY(_tamp_wms <= _tamp_wrem)) { \ + TAMP_WINDOW_SEG_COPY_((window) + _tamp_wpos, _tamp_wsrc, _tamp_wms); \ + (window_pos_var) = (uint16_t)((_tamp_wpos + _tamp_wms) & (window_mask)); \ + } else { \ + uint8_t _tamp_wr = (uint8_t)_tamp_wrem; \ + TAMP_WINDOW_SEG_COPY_((window) + _tamp_wpos, _tamp_wsrc, _tamp_wr); \ + TAMP_WINDOW_SEG_COPY_((window), _tamp_wsrc + _tamp_wr, (uint8_t)(_tamp_wms - _tamp_wr)); \ + (window_pos_var) = (uint16_t)(_tamp_wms - _tamp_wr); \ + } \ + } while (0) + +#endif /* TAMP_WINDOW_WRITE_FROM_OUTPUT */ + +#if !TAMP_WINDOW_FROM_OUTPUT && !TAMP_ESP32 +/* Out-of-line linear window update from the just-written output snapshot, for + * portable builds that keep tamp_window_copy on the resume path. The snapshot + * source never overlaps the window destination, so no reverse-copy overlap + * check and no per-byte source masking are needed. NOINLINE is mandatory: at + * -O3 GCC inlines this regardless of cost heuristics, and the inlined update + * regresses Cortex-M0+ (register pressure) - the whole point is to keep it a + * call. n and mask are packed into one word to stay within 4 register args + * (Cortex-M0+). Returns the new window_pos. */ +static uint16_t TAMP_NOINLINE tamp_window_write_from_output_fn(unsigned char* window, const unsigned char* src, + uint16_t pos, uint32_t n_and_mask) { + uint16_t mask = (uint16_t)n_and_mask; + uint8_t n = (uint8_t)(n_and_mask >> 16); + uint16_t rem = (uint16_t)((mask + 1) - pos); + if (TAMP_LIKELY(n <= rem)) { + for (uint8_t i = 0; i < n; i++) window[pos + i] = src[i]; + return (uint16_t)((pos + n) & mask); + } + uint8_t r = (uint8_t)rem; + for (uint8_t i = 0; i < r; i++) window[pos + i] = src[i]; + uint8_t tail = (uint8_t)(n - r); + for (uint8_t i = 0; i < tail; i++) window[i] = src[r + i]; + return tail; +} +#endif + #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define FLUSH 14 +/* Compile-time-pinned stream configuration (opt-in; see common.h). + * TAMP_FIXED_WINDOW_BITS / TAMP_FIXED_LITERAL_BITS let a build that only ever + * decodes one configuration fold every window/literal shift to an immediate; + * with both pinned, min_pattern_size is a compile-time constant too. The hot + * path reads the configuration exclusively through the locals initialized with + * these macros, so folding the initializer folds all downstream uses. Undefined + * => the runtime value from the decompressor state (current behavior). + * tamp_decompressor_populate_from_conf rejects any stream whose header disagrees + * with a pinned value, so these constants can be trusted unconditionally. */ +#ifdef TAMP_FIXED_WINDOW_BITS +#define TAMP_CONF_WINDOW_INIT(d) TAMP_FIXED_WINDOW_BITS +#else +#define TAMP_CONF_WINDOW_INIT(d) ((d)->conf_window) +#endif +#ifdef TAMP_FIXED_LITERAL_BITS +#define TAMP_CONF_LITERAL_INIT(d) TAMP_FIXED_LITERAL_BITS +#else +#define TAMP_CONF_LITERAL_INIT(d) ((d)->conf_literal) +#endif +#if defined(TAMP_FIXED_WINDOW_BITS) && defined(TAMP_FIXED_LITERAL_BITS) +/* Mirrors tamp_compute_min_pattern_size(window, literal). */ +#define TAMP_CONF_MIN_PATTERN_INIT(d) (2 + (TAMP_FIXED_WINDOW_BITS > (10 + ((TAMP_FIXED_LITERAL_BITS - 5) << 1)))) +#else +#define TAMP_CONF_MIN_PATTERN_INIT(d) ((d)->min_pattern_size) +#endif + #if TAMP_EXTENDED_DECOMPRESS /* Token state for extended decode suspend/resume (2 bits). * TOKEN_RLE and TOKEN_EXT_MATCH_FRESH are arranged so that: @@ -42,6 +170,14 @@ #define TOKEN_EXT_MATCH 3 /* Resume: have match_size, need window_offset */ #endif +/* token_state only exists in the struct when extended decode is compiled in; + * classic-only builds have no extended tokens, so pending state is always 0. */ +#if TAMP_EXTENDED_DECOMPRESS +#define TAMP_PENDING_TOKEN_STATE(d) ((d)->token_state) +#else +#define TAMP_PENDING_TOKEN_STATE(d) 0 +#endif + /** * Huffman lookup table indexed by 7 bits (after first "1" bit consumed). * Upper 4 bits = additional bits to consume, lower 4 bits = symbol (14 = FLUSH). @@ -111,8 +247,7 @@ static tamp_res decode_huffman(uint32_t* bit_buffer, uint8_t* bit_buffer_pos, ui * RLE format: huffman(count_high) + trailing_bits(count_low) * rle_count = (count_high << 4) + count_low + 2 */ -static tamp_res decode_rle(TampDecompressor* d, unsigned char** output, const unsigned char* output_end, - size_t* output_written_size) { +static tamp_res decode_rle(TampDecompressor* d, unsigned char** output, const unsigned char* output_end) { uint8_t rle_count; /* max 241: (14 << 4) + 15 + 2 */ uint8_t skip = d->skip_bytes; @@ -132,7 +267,7 @@ static tamp_res decode_rle(TampDecompressor* d, unsigned char** output, const un } /* Get the byte to repeat (last written byte) */ - uint16_t prev_pos = (d->window_pos - 1) & ((1u << d->conf_window) - 1); + uint16_t prev_pos = (d->window_pos - 1) & ((1u << TAMP_CONF_WINDOW_INIT(d)) - 1); uint8_t symbol = d->window[prev_pos]; /* Calculate how many to write this call */ @@ -156,12 +291,11 @@ static tamp_res decode_rle(TampDecompressor* d, unsigned char** output, const un /* Write repeated bytes to output */ TAMP_MEMSET(*output, symbol, to_write); *output += to_write; - *output_written_size += to_write; /* Update window only on first chunk (skip==0). * Write up to TAMP_RLE_MAX_WINDOW or until end of buffer (no wrap). */ if (skip == 0) { - const uint16_t window_size = 1u << d->conf_window; + const uint16_t window_size = 1u << TAMP_CONF_WINDOW_INIT(d); uint16_t remaining = window_size - d->window_pos; uint8_t window_write = MIN(MIN(rle_count, TAMP_RLE_MAX_WINDOW), remaining); /* max 8 */ for (uint8_t i = 0; i < window_write; i++) { @@ -184,9 +318,8 @@ static tamp_res decode_rle(TampDecompressor* d, unsigned char** output, const un * - TOKEN_EXT_MATCH: have match_size, need window_offset * - Output-full resume (skip > 0): have both match_size and window_offset */ -static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** output, const unsigned char* output_end, - size_t* output_written_size) { - const uint8_t conf_window = d->conf_window; +static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** output, const unsigned char* output_end) { + const uint8_t conf_window = TAMP_CONF_WINDOW_INIT(d); uint16_t window_offset; uint8_t match_size; /* max 134: (14<<3)+7 + 3 + 12 */ uint8_t skip = d->skip_bytes; @@ -210,7 +343,7 @@ static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** outpu uint8_t raw; tamp_res res = decode_huffman(&bit_buffer, &bit_buffer_pos, TAMP_LEADING_EXTENDED_MATCH_BITS, &raw); if (res != TAMP_OK) return res; - match_size = raw + d->min_pattern_size + 12; + match_size = raw + TAMP_CONF_MIN_PATTERN_INIT(d) + 12; /* Now decode window_offset */ if (TAMP_UNLIKELY(bit_buffer_pos < conf_window)) { @@ -257,7 +390,6 @@ static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** outpu /* Copy from window to output */ uint16_t src_offset = window_offset + skip; TAMP_COPY_TO_OUTPUT(*output, d->window + src_offset, to_write); - *output_written_size += to_write; /* Update window only on complete decode. * Write up to end of buffer (no wrap), matching RLE behavior. */ @@ -265,7 +397,26 @@ static tamp_res decode_extended_match(TampDecompressor* d, unsigned char** outpu uint16_t wp = d->window_pos; uint16_t remaining = window_size - wp; uint8_t window_write = (match_size < remaining) ? match_size : remaining; +#if TAMP_WINDOW_FROM_OUTPUT + if (skip == 0) { + /* Single-call complete: output's last match_size bytes are the full + * match; window_write <= remaining so this never wraps. */ + TAMP_WINDOW_WRITE_FROM_OUTPUT(d->window, wp, *output - match_size, window_write, window_size, + window_size - 1); + } else { + /* Resume completion: the match start is not in this output buffer. */ + TAMP_WINDOW_COPY(d->window, &wp, window_offset, window_write, window_size - 1); + } +#elif !TAMP_ESP32 + if (skip == 0) { + wp = tamp_window_write_from_output_fn(d->window, *output - match_size, wp, + ((uint32_t)window_write << 16) | (window_size - 1)); + } else { + TAMP_WINDOW_COPY(d->window, &wp, window_offset, window_write, window_size - 1); + } +#else TAMP_WINDOW_COPY(d->window, &wp, window_offset, window_write, window_size - 1); +#endif d->window_pos = wp; } @@ -309,6 +460,14 @@ static TAMP_OPTIMIZE_SIZE tamp_res tamp_decompressor_populate_from_conf(TampDeco if (conf_window < 8 || conf_window > 15) return TAMP_INVALID_CONF; if (conf_literal < 5 || conf_literal > 8) return TAMP_INVALID_CONF; if (conf_window > decompressor->window_bits_max) return TAMP_INVALID_CONF; +#ifdef TAMP_FIXED_WINDOW_BITS + // Build pinned to a single window configuration: reject any other stream up + // front so the hot path can treat the window bits as a compile-time constant. + if (conf_window != TAMP_FIXED_WINDOW_BITS) return TAMP_INVALID_CONF; +#endif +#ifdef TAMP_FIXED_LITERAL_BITS + if (conf_literal != TAMP_FIXED_LITERAL_BITS) return TAMP_INVALID_CONF; +#endif #if !TAMP_EXTENDED_DECOMPRESS // Reject before committing any state: marking the decompressor configured and // then returning an error would let a retrying caller decode the extended @@ -351,23 +510,266 @@ tamp_res tamp_decompressor_init(TampDecompressor* decompressor, const TampConf* * * Consumes bytes from input until bit_buffer has at least 25 bits or input is exhausted. * + * Two variants, selected by TAMP_FAST_BIT_REFILL (see common.h): the fast + * variant works on locals with a single writeback, since the unsigned char + * loads may alias anything and force a reload/store per byte otherwise. + * * NOTE: NOINLINE saves ~192 bytes on armv6m but causes ~10% decompression * speed regression. Keep this inlined for performance. */ -static inline void refill_bit_buffer(TampDecompressor* d, const unsigned char** input, const unsigned char* input_end, - size_t* input_consumed_size) { +#if TAMP_FAST_BIT_REFILL +static inline void refill_bit_buffer(TampDecompressor* d, const unsigned char** input, const unsigned char* input_end) { + const unsigned char* in = *input; + uint32_t bit_buffer = d->bit_buffer; + uint8_t bit_buffer_pos = d->bit_buffer_pos; + while (in != input_end && bit_buffer_pos <= 24) { + bit_buffer_pos += 8; + bit_buffer |= (uint32_t)*in++ << (32 - bit_buffer_pos); + } + d->bit_buffer = bit_buffer; + d->bit_buffer_pos = bit_buffer_pos; + *input = in; +} +#else +static inline void refill_bit_buffer(TampDecompressor* d, const unsigned char** input, const unsigned char* input_end) { while (*input != input_end && d->bit_buffer_pos <= 24) { d->bit_buffer_pos += 8; d->bit_buffer |= (uint32_t) * (*input) << (32 - d->bit_buffer_pos); (*input)++; - (*input_consumed_size)++; } } +#endif /* TAMP_FAST_BIT_REFILL */ + +#if TAMP_FAST_DECODE_LOOP +/* Window update for the fast-loop token body, selected by the same platform + * flags that pick the careful path's window-update strategy. */ +#if TAMP_WINDOW_FROM_OUTPUT +#define TAMP_FAST_WINDOW_UPDATE_(wp, ms, wsz) \ + TAMP_WINDOW_WRITE_FROM_OUTPUT(decompressor->window, (wp), output - (ms), (ms), (wsz), window_mask) +#elif !TAMP_ESP32 +#define TAMP_FAST_WINDOW_UPDATE_(wp, ms, wsz) \ + (wp) = tamp_window_write_from_output_fn(decompressor->window, output - (ms), (wp), \ + ((uint32_t)(uint8_t)(ms) << 16) | window_mask) +#else +#define TAMP_FAST_WINDOW_UPDATE_(wp, ms, wsz) \ + TAMP_WINDOW_COPY(decompressor->window, &(wp), window_offset, (ms), window_mask) +#endif + +/* The fast loop decodes from a 64-bit bit reservoir (see common.h, + * TAMP_FAST_DECODE_LOOP, for the measured numbers). `bb` holds valid bits in + * its top `rbits` bits; a decode reads the top 32 (`bb >> 32`). A single + * unconditional 4-byte refill fires only when <=32 valid bits remain (~every + * 1-2 tokens), instead of a per-token refill of the 32-bit struct bit_buffer. + * bb's bits below the top `rbits` are always zero + * inside the loop (consumption `bb <<=` shifts zeros in at the bottom; each + * refill ORs a chunk into exactly that zero region, contiguous with the live + * bits), which the writeback mask relies on. + * + * Writeback discipline (runs on EVERY loop exit before the struct bit_buffer is + * read again). The invariant rbits == 8*bytes_read - bits_consumed holds across + * the pushback, so the next input byte always begins exactly where the + * reservoir's valid bits end - byte alignment is automatic, no matter what + * bit_buffer_pos the loop was entered with: + * 1. while (rbits > 32) { input -= 1; rbits -= 8; } + * Un-read the surplus whole bytes (rbits now in [25,32] when the loop + * ran a token, else the untouched entry value). Only decrements the + * accounting + input cursor; bb is untouched, so the pushed-back bits + * remain sitting below the new top-`rbits` region. + * 2. bit_buffer = (top 32 bits of bb) AND a mask that clears every bit below + * the top `rbits` - those hold the just-un-read data, and the next + * refill_bit_buffer ORs new bytes into exactly that region, so they must + * be zero for the OR to stay clean. + * 3. bit_buffer_pos = rbits. + * The uncommitted-token contract is: on any structural/seam break, bb/rbits are + * left describing the exact bitstream position the careful body / decode_* + * helpers must resume from, and after the writeback the struct bit_buffer is a + * valid 32-bit-style buffer (top bit_buffer_pos bits live, low bits zero). */ + +/* Refill 4 whole bytes when the reservoir has room. Callers guarantee >=4 input + * bytes remain via the loop precondition (>=8 input covers the two per-iteration + * refills). Reads bytes MSB-first, same order as refill_bit_buffer. */ +#define TAMP_RES_REFILL() \ + do { \ + if (rbits <= 32) { \ + uint32_t _chunk = ((uint32_t)input[0] << 24) | ((uint32_t)input[1] << 16) | ((uint32_t)input[2] << 8) | \ + (uint32_t)input[3]; \ + input += 4; \ + bb |= (uint64_t)_chunk << (32 - rbits); \ + rbits += 32; \ + } \ + } while (0) + +/* Commit the reservoir back to the 32-bit struct bit_buffer (see discipline), + * then restore the fast-loop's >=25-bit invariant with a checked refill. + * + * The refill is REQUIRED, not an optimization: the loop can exit (output + * precondition) with rbits as low as ~9, so bit_buffer_pos would be < 25. The + * careful body runs immediately after a precondition exit with NO intervening + * refill and needs up to 24 bits for one classic token, so it would return + * TAMP_INPUT_EXHAUSTED prematurely (mid-buffer) without this top-up. */ +#define TAMP_RES_WRITEBACK() \ + do { \ + while (rbits > 32) { \ + input -= 1; \ + rbits -= 8; \ + } \ + uint32_t _hi = (uint32_t)(bb >> 32); \ + _hi &= (uint32_t)(0xFFFFFFFFu << (32 - rbits)); \ + decompressor->bit_buffer = _hi; \ + decompressor->bit_buffer_pos = (uint8_t)rbits; \ + refill_bit_buffer(decompressor, &input, input_end); \ + } while (0) + +/* Extended-symbol break for the reservoir plain body: consume only the + * is_literal + type-huffman bits (decode_rle/decode_extended_match re-decode + * the count/window-offset from the committed bit_buffer), set token_state, + * break. Empty in classic-only builds. */ +#if TAMP_EXTENDED_DECOMPRESS +#define TAMP_RES_EXT_BREAK_(cons, ms) \ + if (TAMP_UNLIKELY(extended_enabled && (ms) >= TAMP_RLE_SYMBOL)) { \ + bb <<= (cons); \ + rbits -= (int32_t)(cons); \ + decompressor->token_state = (uint8_t)((ms) - (TAMP_RLE_SYMBOL - 1)); \ + break; \ + } +#else +#define TAMP_RES_EXT_BREAK_(cons, ms) +#endif + +/* Fast-loop classic-token body (writes the ring), decoding from bb/rbits. + * FLUSH aligns to the next stream byte boundary the same way + * `bit_buffer_pos & ~7` does: the padding to skip equals post & 7 where + * post = rbits - consumed (post ~= -consumed_total mod 8). */ +#define TAMP_RES_TOKEN_BODY() \ + { \ + uint32_t _top = (uint32_t)(bb >> 32); \ + if (TAMP_UNLIKELY(_top >> 31)) { \ + /* Literal. */ \ + uint32_t _t = _top << 1; \ + uint8_t literal = _t >> (32 - conf_literal); \ + bb <<= (1 + conf_literal); \ + rbits -= (int32_t)(1 + conf_literal); \ + uint16_t wp = decompressor->window_pos; \ + decompressor->window_pos = (wp + 1) & window_mask; \ + decompressor->window[wp] = literal; \ + *output++ = literal; \ + } else { \ + uint32_t _t = _top << 1; /* shift out the is_literal flag */ \ + uint32_t consumed = 1; \ + uint8_t match_size; \ + if (TAMP_LIKELY((_t >> 31) == 0)) { \ + _t <<= 1; \ + consumed += 1; \ + match_size = 0; \ + } else { \ + _t <<= 1; \ + uint8_t code = HUFFMAN_TABLE[_t >> (32 - 7)]; \ + uint8_t bit_len = code >> 4; \ + _t <<= bit_len; \ + consumed += 1 + bit_len; \ + match_size = code & 0xF; \ + } \ + if (TAMP_UNLIKELY(match_size == FLUSH)) { \ + int32_t post = rbits - (int32_t)consumed; \ + uint32_t align = (uint32_t)(post & 7); \ + bb <<= (consumed + align); \ + rbits = post - (int32_t)align; \ + decompressor->last_was_flush = 1; \ + break; \ + } \ + TAMP_RES_EXT_BREAK_(consumed, match_size) \ + match_size += min_pattern_size; \ + uint16_t window_offset = _t >> (32 - conf_window); \ + const uint32_t window_size = (1u << conf_window); \ + if (TAMP_UNLIKELY((uint32_t)window_offset > window_size - (uint32_t)match_size)) { \ + /* Commit the reservoir before returning: the OOB token is left \ + * uncommitted (bb/rbits not advanced past it), so the writeback \ + * pushes back the read-ahead surplus and leaves bit_buffer / \ + * bit_buffer_pos / input consistent with the offending token. */ \ + TAMP_RES_WRITEBACK(); \ + TAMP_DECOMP_RETURN(TAMP_OOB); \ + } \ + bb <<= (consumed + conf_window); \ + rbits -= (int32_t)(consumed + conf_window); \ + TAMP_COPY_TO_OUTPUT(output, decompressor->window + window_offset, match_size); \ + uint16_t wp = decompressor->window_pos; \ + TAMP_FAST_WINDOW_UPDATE_(wp, match_size, window_size); \ + decompressor->window_pos = wp; \ + } \ + } + +/* The hot fast loop, extracted from tamp_decompressor_decompress_cb so the + * surrounding careful body (cold in fast-loop builds: buffer tails, resume + * state, extended dispatch glue) can be compiled -Os while this stays -O3. + * Called once per outer-loop iteration, so the call overhead and the per-call + * conf reloads are amortized over every token the loop consumes. Cursors are + * passed by reference and written back on every exit; returns TAMP_OK on a + * normal exit (precondition failure or structural break - the caller re-checks + * pending/flush state) or TAMP_OOB on a malicious offset. + * + * Without TAMP_COMPACT_CAREFUL_BODY the whole function is inlined back into + * the -O3 careful body: keeping it outlined there costs flash (+304 B on M0+) + * for duplicated conf loads and call glue with no Os payoff. */ +#if TAMP_COMPACT_CAREFUL_BODY +static TAMP_NOINLINE tamp_res +#else +static TAMP_ALWAYS_INLINE tamp_res +#endif +tamp_fast_decode_loop(TampDecompressor* decompressor, const unsigned char** input_p, const unsigned char* input_end, + unsigned char** output_p, const unsigned char* output_end) { + const unsigned char* input = *input_p; + unsigned char* output = *output_p; + const uint8_t conf_window = TAMP_CONF_WINDOW_INIT(decompressor); + const uint8_t conf_literal = TAMP_CONF_LITERAL_INIT(decompressor); + const uint8_t min_pattern_size = TAMP_CONF_MIN_PATTERN_INIT(decompressor); + const uint16_t window_mask = (1 << conf_window) - 1; +#if TAMP_EXTENDED_DECOMPRESS + const bool extended_enabled = decompressor->conf_extended; +#endif + tamp_res res = TAMP_OK; +/* Cursor writeback + return for the token-body macros' error paths; the + * caller's TAMP_DECOMP_RETURN derives the *_size accounting from the + * written-back cursors. */ +#define TAMP_DECOMP_RETURN(code) \ + do { \ + res = (code); \ + goto tamp_fast_done; \ + } while (0) + /* Single token per iteration: the reservoir already amortizes the + * refill to ~one per 1-2 tokens, and a two-token unroll here + * measured SLOWER on real M7 hardware (H7B0: -3.1% throughput) + * while costing ~1 KB of flash. (QEMU insn counts disagree on M4: + * unroll -4% insns there; hardware-unverified.) */ + uint64_t bb = (uint64_t)decompressor->bit_buffer << 32; + int32_t rbits = decompressor->bit_buffer_pos; + while ((size_t)(input_end - input) >= 4 && (size_t)(output_end - output) >= 32) { + TAMP_RES_REFILL(); + TAMP_RES_TOKEN_BODY() + } + TAMP_RES_WRITEBACK(); +#undef TAMP_DECOMP_RETURN +tamp_fast_done: + *input_p = input; + *output_p = output; + return res; +} + +#endif /* TAMP_FAST_DECODE_LOOP */ #if TAMP_HAS_GCC_OPTIMIZE #pragma GCC push_options #pragma GCC optimize("-fno-tree-pre") #endif +/* With TAMP_COMPACT_CAREFUL_BODY every hot token goes through + * tamp_fast_decode_loop above and the remaining body (header parsing, + * resume/tail tokens, extended dispatch glue) is compiled for size. Without + * it the careful body keeps -O3: on portable builds it IS the hot loop. */ +#if TAMP_COMPACT_CAREFUL_BODY +#define TAMP_DECOMPRESS_CB_OPT TAMP_OPTIMIZE_SIZE +#else +#define TAMP_DECOMPRESS_CB_OPT +#endif +TAMP_DECOMPRESS_CB_OPT tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigned char* output, size_t output_size, size_t* output_written_size, const unsigned char* input, size_t input_size, size_t* input_consumed_size, tamp_callback_t callback, void* user_data) { @@ -383,6 +785,20 @@ tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigne *input_consumed_size = 0; *output_written_size = 0; +/* Every exit derives input/output progress from the cursors. The hot loop + * never touches *output_written_size / *input_consumed_size: a per-token RMW + * through a size_t pointer may alias the decompressor fields (size_t == + * uint32_t on 32-bit targets), costing a load/store per iteration plus forced + * field reloads. *input_consumed_size already holds the header bytes counted + * before input_start was snapshotted. */ +#define TAMP_DECOMP_RETURN(code) \ + do { \ + *output_written_size = (size_t)(output - output_start); \ + *input_consumed_size += (size_t)(input - input_start); \ + return (code); \ + } while (0) + const unsigned char* const output_start = output; + if (TAMP_UNLIKELY(!decompressor->configured)) { // Try reading header directly from input. read_header handles // variable-length headers (1-2 bytes based on more_headers bit). @@ -420,10 +836,18 @@ tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigne decompressor->skip_bytes = 0; // Clear stale stashed_header_byte (shares union storage) } - // Cache bitfield values in local variables for faster access - const uint8_t conf_window = decompressor->conf_window; - const uint8_t conf_literal = decompressor->conf_literal; - const uint8_t min_pattern_size = decompressor->min_pattern_size; + /* Snapshot after header parsing: *input_consumed_size holds the header + * bytes, and every exit below adds (input - input_start) via + * TAMP_DECOMP_RETURN. Header-error returns above happen before this and + * account for their bytes directly. */ + const unsigned char* const input_start = input; + + // Cache bitfield values in local variables for faster access. With the + // TAMP_FIXED_* build options these initializers are compile-time constants, + // so every downstream window/literal shift and min_pattern_size use folds. + const uint8_t conf_window = TAMP_CONF_WINDOW_INIT(decompressor); + const uint8_t conf_literal = TAMP_CONF_LITERAL_INIT(decompressor); + const uint8_t min_pattern_size = TAMP_CONF_MIN_PATTERN_INIT(decompressor); const uint16_t window_mask = (1 << conf_window) - 1; #if TAMP_EXTENDED_DECOMPRESS @@ -431,71 +855,97 @@ tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigne #endif while (input != input_end || decompressor->pos_and_state) { - if (TAMP_UNLIKELY(output == output_end)) return TAMP_OUTPUT_FULL; + if (TAMP_UNLIKELY(output == output_end)) TAMP_DECOMP_RETURN(TAMP_OUTPUT_FULL); // Populate the bit buffer - refill_bit_buffer(decompressor, &input, input_end, input_consumed_size); + refill_bit_buffer(decompressor, &input, input_end); #if TAMP_EXTENDED_DECOMPRESS /* Handle extended tokens - either resuming or fresh from match_size detection below. */ if (TAMP_UNLIKELY(decompressor->token_state)) { extended_dispatch: if (decompressor->token_state == TOKEN_RLE) { - res = decode_rle(decompressor, &output, output_end, output_written_size); + res = decode_rle(decompressor, &output, output_end); } else { - res = decode_extended_match(decompressor, &output, output_end, output_written_size); + res = decode_extended_match(decompressor, &output, output_end); } if (res == TAMP_INPUT_EXHAUSTED) { uint8_t old_bit_pos = decompressor->bit_buffer_pos; - refill_bit_buffer(decompressor, &input, input_end, input_consumed_size); + refill_bit_buffer(decompressor, &input, input_end); /* If we couldn't get more bits and input is exhausted, stop. * Otherwise the loop would run forever with token_state set. */ if (decompressor->bit_buffer_pos == old_bit_pos && input == input_end) { - return TAMP_INPUT_EXHAUSTED; + TAMP_DECOMP_RETURN(TAMP_INPUT_EXHAUSTED); } continue; } - if (res != TAMP_OK) return res; + if (res != TAMP_OK) TAMP_DECOMP_RETURN(res); continue; } #endif // TAMP_EXTENDED_DECOMPRESS - if (TAMP_UNLIKELY(decompressor->bit_buffer_pos == 0)) return TAMP_INPUT_EXHAUSTED; + if (TAMP_UNLIKELY(decompressor->bit_buffer_pos == 0)) TAMP_DECOMP_RETURN(TAMP_INPUT_EXHAUSTED); + +#if TAMP_FAST_DECODE_LOOP + /* Checked-once fast inner loop. Entered only when no callback and no + * pending skip/extended/flush state; the per-iteration precondition + * (>=4 input bytes so the unguarded refill never reaches input_end, + * >=32 output bytes so no classic token can fill the buffer) makes + * every mid-token bounds/exhaustion check on the classic path dead: + * - refill reaches bit_buffer_pos >= 25 (a classic token needs <= 24) + * - literal (<= 1+conf_literal <= 9 bits) and match window offset + * (<= conf_window <= 15 bits, after 1+huffman<=9) always present + * - output has room for any single token (match_size <= 17, lit = 1) + * last_was_flush invariant: entry requires it 0; only FLUSH sets it and + * FLUSH breaks out, so no per-token clear is needed in the body. On a + * structural break (FLUSH sets last_was_flush, extended sets + * token_state) we continue the outer loop so its refill + extended + * dispatch / double-FLUSH handling own those cases; a precondition + * failure (the final tokens of the buffer) falls through to the + * careful body with clean, topped-up state - a dedicated single-token + * tail loop was measured to add ~580 bytes of flash for no meaningful + * speedup, so the careful body owns the tail. */ + if (callback == NULL && decompressor->skip_bytes == 0 && TAMP_PENDING_TOKEN_STATE(decompressor) == 0 && + decompressor->last_was_flush == 0) { + res = tamp_fast_decode_loop(decompressor, &input, input_end, &output, output_end); + if (TAMP_UNLIKELY(res != TAMP_OK)) TAMP_DECOMP_RETURN(res); + if (TAMP_PENDING_TOKEN_STATE(decompressor) || decompressor->last_was_flush) continue; + } +#endif /* TAMP_FAST_DECODE_LOOP */ // Hint that patterns are more likely than literals if (TAMP_UNLIKELY(decompressor->bit_buffer >> 31)) { // is literal if (TAMP_UNLIKELY(decompressor->last_was_flush)) decompressor->last_was_flush = 0; - if (TAMP_UNLIKELY(decompressor->bit_buffer_pos < (1 + conf_literal))) return TAMP_INPUT_EXHAUSTED; - decompressor->bit_buffer <<= 1; // shift out the is_literal flag - - // Copy literal to output - *output = decompressor->bit_buffer >> (32 - conf_literal); - decompressor->bit_buffer <<= conf_literal; + if (TAMP_UNLIKELY(decompressor->bit_buffer_pos < (1 + conf_literal))) + TAMP_DECOMP_RETURN(TAMP_INPUT_EXHAUSTED); + /* Finish all struct-field updates before the unsigned-char stores to + * output/window: char stores may alias the decompressor, so ordering + * them last avoids forced reloads of the fields. */ + uint32_t bit_buffer = decompressor->bit_buffer << 1; // shift out the is_literal flag + uint8_t literal = bit_buffer >> (32 - conf_literal); + decompressor->bit_buffer = bit_buffer << conf_literal; decompressor->bit_buffer_pos -= (1 + conf_literal); + uint16_t wp = decompressor->window_pos; + decompressor->window_pos = (wp + 1) & window_mask; - // Update window - decompressor->window[decompressor->window_pos] = *output; - decompressor->window_pos = (decompressor->window_pos + 1) & window_mask; - - output++; - (*output_written_size)++; + decompressor->window[wp] = literal; + *output++ = literal; } else { // is token; attempt a decode /* copy the bit buffers so that we can abort at any time */ uint32_t bit_buffer = decompressor->bit_buffer; uint16_t window_offset; - uint16_t window_offset_skip; uint8_t bit_buffer_pos = decompressor->bit_buffer_pos; int8_t match_size; - int8_t match_size_skip; // shift out the is_literal flag bit_buffer <<= 1; bit_buffer_pos--; uint8_t match_size_u8; - if (decode_huffman(&bit_buffer, &bit_buffer_pos, 0, &match_size_u8) != TAMP_OK) return TAMP_INPUT_EXHAUSTED; + if (decode_huffman(&bit_buffer, &bit_buffer_pos, 0, &match_size_u8) != TAMP_OK) + TAMP_DECOMP_RETURN(TAMP_INPUT_EXHAUSTED); match_size = match_size_u8; if (TAMP_UNLIKELY(match_size == FLUSH)) { @@ -528,7 +978,7 @@ tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigne if (TAMP_UNLIKELY(bit_buffer_pos < conf_window)) { // There are not enough bits to decode window offset - return TAMP_INPUT_EXHAUSTED; + TAMP_DECOMP_RETURN(TAMP_INPUT_EXHAUSTED); } match_size += min_pattern_size; window_offset = bit_buffer >> (32 - conf_window); @@ -538,47 +988,86 @@ tamp_res tamp_decompressor_decompress_cb(TampDecompressor* decompressor, unsigne // references to read past the window buffer, potentially leaking memory. // Cast to uint32_t prevents signed integer overflow. const uint32_t window_size = (1u << conf_window); - if (TAMP_UNLIKELY((uint32_t)window_offset >= window_size || - (uint32_t)window_offset + (uint32_t)match_size > window_size)) { - return TAMP_OOB; + /* window_offset < window_size by construction (conf_window-bit extraction), + * and match_size <= 30 << window_size, so the subtraction cannot underflow. */ + if (TAMP_UNLIKELY((uint32_t)window_offset > window_size - (uint32_t)match_size)) { + TAMP_DECOMP_RETURN(TAMP_OOB); } - // Apply skip_bytes - match_size_skip = match_size - decompressor->skip_bytes; - window_offset_skip = window_offset + decompressor->skip_bytes; + // Apply skip_bytes. skip is nonzero only when resuming a token that + // was cut off by a full output buffer (at most once per call), so the + // common path uses match_size/window_offset directly and never + // touches decompressor->skip_bytes (it is already 0). + uint8_t skip = decompressor->skip_bytes; + uint16_t copy_offset = window_offset; + uint8_t copy_size = match_size; + if (TAMP_UNLIKELY(skip)) { + copy_offset = window_offset + skip; + copy_size = match_size - skip; + } - // Check if we are output-buffer-limited, and if so to set skip_bytes. + // Check if we are output-buffer-limited, and if so set skip_bytes. // Next tamp_decompressor_decompress_cb we will re-decode the same // token, and skip the first skip_bytes of it. - // Otherwise, update the decompressor buffers + // Otherwise, update the decompressor buffers. size_t remaining = output_end - output; - if (TAMP_UNLIKELY((uint8_t)match_size_skip > remaining)) { - decompressor->skip_bytes += remaining; - match_size_skip = remaining; + if (TAMP_UNLIKELY(copy_size > remaining)) { + decompressor->skip_bytes = skip + remaining; + copy_size = remaining; + TAMP_COPY_TO_OUTPUT(output, decompressor->window + copy_offset, copy_size); } else { - decompressor->skip_bytes = 0; + if (TAMP_UNLIKELY(skip)) decompressor->skip_bytes = 0; decompressor->bit_buffer = bit_buffer << conf_window; decompressor->bit_buffer_pos = bit_buffer_pos - conf_window; - } - // Copy pattern to output - TAMP_COPY_TO_OUTPUT(output, decompressor->window + window_offset_skip, match_size_skip); - (*output_written_size) += match_size_skip; + TAMP_COPY_TO_OUTPUT(output, decompressor->window + copy_offset, copy_size); - if (TAMP_LIKELY(decompressor->skip_bytes == 0)) { uint16_t wp = decompressor->window_pos; +#if TAMP_WINDOW_FROM_OUTPUT + if (TAMP_LIKELY(skip == 0)) { + /* output's last match_size bytes are the full match, a linear + * non-overlapping snapshot of window[window_offset..]. */ + TAMP_WINDOW_WRITE_FROM_OUTPUT(decompressor->window, wp, output - match_size, match_size, + window_size, window_mask); + } else { + /* Resume completion: output only holds the match tail; the + * window still holds the source. */ + TAMP_WINDOW_COPY(decompressor->window, &wp, window_offset, match_size, window_mask); + } +#elif !TAMP_ESP32 + if (TAMP_LIKELY(skip == 0)) { + wp = tamp_window_write_from_output_fn(decompressor->window, output - match_size, wp, + ((uint32_t)(uint8_t)match_size << 16) | window_mask); + } else { + TAMP_WINDOW_COPY(decompressor->window, &wp, window_offset, match_size, window_mask); + } +#else TAMP_WINDOW_COPY(decompressor->window, &wp, window_offset, match_size, window_mask); +#endif decompressor->window_pos = wp; } } - if (TAMP_UNLIKELY(callback && (res = callback(user_data, *input_consumed_size, input_size)))) - return (tamp_res)res; + /* callback is rare; compute the live consumed count only when it fires + * (short-circuit keeps the callback==NULL fast path off the pointer). + * *input_consumed_size still holds only the header base here, so + * TAMP_DECOMP_RETURN's later add is not a double-count. */ + if (TAMP_UNLIKELY(callback && (res = callback(user_data, *input_consumed_size + (size_t)(input - input_start), + input_size)))) + TAMP_DECOMP_RETURN((tamp_res)res); } - return TAMP_INPUT_EXHAUSTED; + TAMP_DECOMP_RETURN(TAMP_INPUT_EXHAUSTED); +#undef TAMP_DECOMP_RETURN } #if TAMP_HAS_GCC_OPTIMIZE #pragma GCC pop_options #endif +#if TAMP_FAST_DECODE_LOOP +#undef TAMP_FAST_WINDOW_UPDATE_ +#undef TAMP_RES_REFILL +#undef TAMP_RES_WRITEBACK +#undef TAMP_RES_EXT_BREAK_ +#undef TAMP_RES_TOKEN_BODY +#endif #if TAMP_STREAM diff --git a/tools/benchmark-code-size.sh b/tools/benchmark-code-size.sh new file mode 100755 index 00000000..026b7703 --- /dev/null +++ b/tools/benchmark-code-size.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Reproduce the "Code size (B)" column in devices/BENCHMARKS.md. +# +# For each build config in that table, compiles the vendored objects +# (common.o + compressor.o + decompressor.o, plus compressor_esp32.o for +# TAMP_ESP32=1 rows) with exactly the row's documented CPU/opt/flags, and +# reports the sum of `text` (Berkeley `size`, includes .rodata) across them. +# Object-level only: no linking, no --gc-sections. +# +# Toolchains are located automatically and any that are missing are skipped +# (with a message) rather than failing the whole run: +# - arm-none-eabi-gcc must be on PATH +# - xtensa-esp-elf-gcc newest dir under ~/.espressif/tools/xtensa-esp-elf/ +# - riscv32-esp-elf-gcc newest dir under ~/.espressif/tools/riscv32-esp-elf/ +# The TAMP_ESP32=1 xtensa rows additionally need an esp-idf checkout (for +# components/xtensa//include/xtensa/config/core-isa.h, required to +# compile espidf/tamp/compressor_esp32.cpp for a real xtensa target); set +# IDF_PATH or have a checkout at ~/sdk/esp-idf or ~/esp/esp-idf. +# +# Usage: tools/benchmark-code-size.sh +set -euo pipefail +cd "$(dirname "$0")/.." + +SCRATCH=build/code-size +rm -rf "$SCRATCH" +mkdir -p "$SCRATCH" +cleanup() { rm -rf "$SCRATCH"; } +trap cleanup EXIT + +TAMP_SRC=tamp/_c_src/tamp +ESP32_CPP=espidf/tamp/compressor_esp32.cpp +ESP32_DEFINES=(-DTAMP_ESP32=1 -DTAMP_LAZY_MATCHING=1) +ESP32_INC=(-Itamp/_c_src/tamp -Iespidf/tamp -Ifuzz/esp32_host) + +# ---------------------------------------------------------------- reporting +# Compiles common.o/compressor.o/decompressor.o (and optionally +# compressor_esp32.o) into $SCRATCH/, prints the size tool's own +# per-object breakdown, and sums the `text` column across all objects. +report() { + local label="$1" size_tool="$2" + shift 2 + local out + out=$("$size_tool" "$@") + echo "=== $label ===" + echo "$out" + local total + total=$(echo "$out" | awk 'NR>1 {sum += $1} END {print sum + 0}') + echo "total text: $total" + echo +} + +compile_c() { + # compile_c + local cc="$1" out="$2" src="$3" + shift 3 + "$cc" "$@" -Itamp/_c_src -c "$src" -o "$out" +} + +# ------------------------------------------------------------ ARM (rp2040) +ARM_GCC="" +if command -v arm-none-eabi-gcc >/dev/null 2>&1; then + ARM_GCC=$(command -v arm-none-eabi-gcc) +fi +ARM_SIZE="" +if command -v arm-none-eabi-size >/dev/null 2>&1; then + ARM_SIZE=$(command -v arm-none-eabi-size) +fi + +if [ -z "$ARM_GCC" ] || [ -z "$ARM_SIZE" ]; then + echo "SKIP: arm-none-eabi-gcc/-size not found on PATH; skipping rp2040 and stm32h7b0 configs." +else + echo "Using arm-none-eabi-gcc: $ARM_GCC" + + run_arm() { + local slug="$1" label="$2" + shift 2 + local dir="$SCRATCH/$slug" + mkdir -p "$dir" + compile_c "$ARM_GCC" "$dir/common.o" "$TAMP_SRC/common.c" "$@" + compile_c "$ARM_GCC" "$dir/compressor.o" "$TAMP_SRC/compressor.c" "$@" + compile_c "$ARM_GCC" "$dir/decompressor.o" "$TAMP_SRC/decompressor.c" "$@" + report "$label" "$ARM_SIZE" "$dir/common.o" "$dir/compressor.o" "$dir/decompressor.o" + } + + run_arm rp2040_default "rp2040 default (RP2040, C, -O3)" \ + -O3 -mcpu=cortex-m0plus -mthumb + + run_arm rp2040_fastloop "rp2040 fastloop (RP2040, C, -O3, TAMP_FAST_DECODE_LOOP=1)" \ + -O3 -mcpu=cortex-m0plus -mthumb -DTAMP_FAST_DECODE_LOOP=1 + + run_arm rp2040_mpy_native "rp2040 mpy-native (RP2040, MicroPython native module)" \ + -O2 -mcpu=cortex-m0plus -mthumb -DTAMP_STREAM=0 -DTAMP_USE_MEMSET=0 + + run_arm stm32h7b0_portable "stm32h7b0 portable (Cortex-M7, C, -O3)" \ + -O3 -mcpu=cortex-m7 -mthumb -mfloat-abi=hard -mfpu=fpv5-d16 + + run_arm stm32h7b0_armv7em "stm32h7b0 armv7em (Cortex-M7, C, -O3, TAMP_ARMV7EM=1)" \ + -O3 -mcpu=cortex-m7 -mthumb -mfloat-abi=hard -mfpu=fpv5-d16 -DTAMP_ARMV7EM=1 +fi + +# --------------------------------------------------------------- xtensa/esp +find_latest_gcc() { + # $1 = glob pattern. Prints the lexicographically-last match, or nothing. + if compgen -G "$1" >/dev/null 2>&1; then + compgen -G "$1" | sort | tail -1 + fi +} + +XTENSA_GCC=$(find_latest_gcc "$HOME/.espressif/tools/xtensa-esp-elf/*/xtensa-esp-elf/bin/xtensa-esp-elf-gcc") +RISCV_GCC=$(find_latest_gcc "$HOME/.espressif/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-gcc") + +# esp-idf checkout, needed only to compile compressor_esp32.cpp for a real +# xtensa target (it pulls in xtensa/config/core-isa.h, which ships as part of +# esp-idf's xtensa component, not the bare toolchain). +IDF_DIR="" +for candidate in "${IDF_PATH:-}" "$HOME/sdk/esp-idf" "$HOME/esp/esp-idf"; do + if [ -n "$candidate" ] && [ -d "$candidate/components/xtensa" ]; then + IDF_DIR="$candidate" + break + fi +done + +if [ -z "$XTENSA_GCC" ]; then + echo "SKIP: no xtensa-esp-elf-gcc found under ~/.espressif/tools/xtensa-esp-elf/*/xtensa-esp-elf/bin/; skipping esp32 and esp32s3 configs." +else + XTENSA_BIN=$(dirname "$XTENSA_GCC") + XTENSA_SIZE="$XTENSA_BIN/xtensa-esp-elf-size" + echo "Using xtensa-esp-elf-gcc: $XTENSA_GCC" + if [ -z "$IDF_DIR" ]; then + echo "NOTE: no esp-idf checkout found (checked \$IDF_PATH, ~/sdk/esp-idf, ~/esp/esp-idf);" \ + "the TAMP_ESP32=1 xtensa rows (which compile compressor_esp32.cpp for the real xtensa" \ + "target) will be skipped. Set IDF_PATH to a checkout to include them." + else + echo "Using esp-idf checkout for xtensa/config/core-isa.h: $IDF_DIR" + fi + + run_xtensa_default() { + # run_xtensa_default