diff --git a/README.md b/README.md index aae44f48..53c1181c 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ environment manually: ``` mamba create -y -n perfecthash-dev -c conda-forge \ - python=3.12 rust cmake ninja make pkg-config clang clangxx lld llvmdev pytest + python=3.12 rust cmake ninja make pkg-config clang clangxx lld llvmdev llvm-tools pytest mamba activate perfecthash-dev ``` @@ -228,13 +228,17 @@ git clone https://github.com/tpn/perfecthash git clone https://github.com/tpn/perfecthash-keys cd perfecthash -cmake -S . -B build -G"Ninja Multi-Config" +cmake -S . -B build -G"Ninja Multi-Config" -DPERFECTHASH_BUILD_PROFILE=full cmake --build build --config Release cmake --build build --config Debug ``` Note: the default build enables `-march=native` for required SIMD intrinsics. Use `-DPERFECTHASH_ENABLE_NATIVE_ARCH=OFF` if you need a portable binary. +If you plan to exercise the LLVM JIT backend, make sure the build actually +produces `PerfectHashLLVM` by using either the `full`, +`online-rawdog-and-llvm-jit`, or `online-llvm-jit` profile. + Build profile examples: ``` diff --git a/agents/GRAPHIMPL4-NOTES.md b/agents/GRAPHIMPL4-NOTES.md new file mode 100644 index 00000000..96749a71 --- /dev/null +++ b/agents/GRAPHIMPL4-NOTES.md @@ -0,0 +1,130 @@ +# GraphImpl4 Notes + +## Branch And Commit Stack + +- Branch: `codex/graphimpl4-cpu-backend` +- Commit stack on top of `main`: + - `d47165c` Add experimental GraphImpl4 CPU solver backend + - `33c90f0` Add GraphImpl4 file-backed codegen support + - `672c172` Add solver-only GraphImpl4 benchmark support + - `97c1244` Enable GraphImpl4 JIT for assigned16/32 tables + - `a37e259` Align mamba envs with LLVM-capable builds + +## Current State + +- `GraphImpl4` exists as an experimental C++ CPU backend in + `src/PerfectHash/GraphImpl4.cpp` with template specialization over: + - assigned/table element width: `uint8_t`, `uint16_t`, `uint32_t` + - effective key width: `uint8_t`, `uint16_t`, `uint32_t` +- Solver/index path is working for: + - 32-bit keys + - downsized 64-bit keys that compact to 32 bits or less + - curated good hash set only +- File-backed create/self-test/codegen path is working for the current smoke: + - the old `--SkipTestAfterCreate` workaround is removed + - generated-project configure/build is part of the smoke now +- Persisted/reloaded GraphImpl4 tables now restore the CHM01 runtime metadata + needed by compact-key paths: + - graph implementation version + - final downsize bitmap metadata + - GraphImpl4 effective-key/inner compaction metadata +- File-backed reload coverage now includes: + - sparse 32-bit GraphImpl4 tables compiled through LLVM `JitIndex32` + - downsized 64-bit GraphImpl4 tables compiled through LLVM `JitIndex64` + - keyless non-GraphImpl downsized 64-bit tables +- Dedicated CLI/codegen coverage now exercises downsized 64-bit GraphImpl4 + output and validates the generated C/C++ header, Rust, CUDA, and Python paths. +- GraphImpl4 JIT is now enabled for: + - assigned32 tables + - assigned16 tables +- GraphImpl4 JIT remains intentionally disabled for: + - assigned8 tables, which return `PH_E_NOT_IMPLEMENTED` +- Benchmark harness no longer force-disables GraphImpl4 JIT. +- LLVM-capable env/bootstrap surfaces were updated so LLVM support is explicit: + - `dependencies.yaml` + - generated `conda/environments/*compiler-llvm.yaml` + - `scripts/install-deps/linux-mamba-env.yaml` + - `scripts/install-deps/windows-mamba-env.yaml` + - `ui/src/pages/developer/Environment.jsx` +- `tests/CMakeLists.txt` now makes `perfecthash_unit_tests` and + `perfecthash_benchmarks` depend on `PerfectHashLLVM` when that target exists, + so LLVM-specific test paths do not silently skip just because the runtime + library was never built. + +## Verified + +- Focused GraphImpl4 online tests: + + ```bash + ./build-graphimpl4/bin/perfecthash_unit_tests --gtest_filter='PerfectHashOnlineJitTests.CreateTable64AndIndex64:PerfectHashOnlineJitTests.Index64OnNonGraphImpl64BitTableUsesMetadata:GraphImpl4BitUtils.ContiguousBitmapDetection:GraphImpl4BitUtils.ComposedDownsizeMetadataUsesComposedBitmap:GraphImpl4BitUtils.ComposedDownsizeMetadataRejectsInnerContiguityMismatch:GraphImpl4BitUtils.ComposedExtractionMatchesTwoStepExtraction:GraphImpl4BitUtils.Downsized64OuterBitmapProducesIdentityInnerBitmap:PerfectHashOnlineTests.GraphImpl4Assigned8RequiresOptIn:PerfectHashOnlineTests.GraphImpl4SupportsDownsized64BitInputs:PerfectHashOnlineTests.GraphImpl4RejectsNonGoodHashes:PerfectHashOnlineTests.GraphImpl4Assigned8JitRejected:PerfectHashOnlineTests.GraphImpl4RawDogJitMatchesIndexAssigned32:PerfectHashOnlineTests.GraphImpl4RawDogJitMatchesIndexSparse32:PerfectHashOnlineTests.GraphImpl4RawDogIndex32x4MatchesIndexAssigned16:PerfectHashOnlineTests.GraphImpl4LlvmJitMatchesIndexAssigned32:PerfectHashOnlineTests.GraphImpl4LlvmJitMatchesIndexSparse32:PerfectHashOnlineTests.GraphImpl4LlvmJitMulshrolate3RXMatchesIndexAssigned32:PerfectHashOnlineTests.GraphImpl4LlvmIndex32x4MatchesIndexAssigned16:PerfectHashOnlineTests.GraphImpl4LlvmIndex64x4MatchesDownsizedIndex:PerfectHashOnlineTests.GraphImpl4FileBackedReloadPreservesSparse32Compaction:PerfectHashOnlineTests.GraphImpl4FileBackedReloadPreservesDownsized64Compaction:PerfectHashOnlineTests.NonGraphImplFileBackedReloadPreservesDownsized64Metadata:PerfectHashOnlineTests.GraphImpl4FileBackedReloadSparse32JitIndex32:PerfectHashOnlineTests.GraphImpl4FileBackedReloadDownsized64JitIndex64' + ``` + +- Focused persisted reload tests: + + ```bash + ./build-graphimpl4/bin/perfecthash_unit_tests --gtest_filter='PerfectHashOnlineTests.GraphImpl4FileBackedReloadPreservesSparse32Compaction:PerfectHashOnlineTests.GraphImpl4FileBackedReloadPreservesDownsized64Compaction:PerfectHashOnlineTests.NonGraphImplFileBackedReloadPreservesDownsized64Metadata:PerfectHashOnlineTests.GraphImpl4FileBackedReloadSparse32JitIndex32:PerfectHashOnlineTests.GraphImpl4FileBackedReloadDownsized64JitIndex64' + ``` + +- Current quick sanity slice: + + ```bash + ./build-graphimpl4/bin/perfecthash_unit_tests --gtest_filter='PerfectHashOnlineTests.GraphImpl4Assigned8JitRejected:PerfectHashOnlineTests.GraphImpl4RawDogJitMatchesIndexAssigned32:PerfectHashOnlineTests.GraphImpl4LlvmJitMatchesIndexAssigned32' + ``` + +- GraphImpl4 ctest slice: + + ```bash + ctest --test-dir build-graphimpl4 --output-on-failure -R 'graphimpl4' + ``` + +- Manual benchmark checks: + + ```bash + ./build-graphimpl4/bin/perfecthash_benchmarks --keys=256 --iterations=1 --loops=1 --graph-impl=4 --jit-backend=rawdog --no-std-map-baselines + ./build-graphimpl4/bin/perfecthash_benchmarks --keys=256 --iterations=1 --loops=1 --graph-impl=4 --jit-backend=rawdog --allow-assigned16 --no-std-map-baselines + ``` + + Observed locally: + - default path reported `Assigned element bits: 32` + - `--allow-assigned16` path reported `Assigned element bits: 16` + +## Important Caveats + +- Persisted/reloaded GraphImpl4 tables are supported for newly-created CHM01 + files that include the extended graph info metadata. +- Older experimental GraphImpl4 table files that predate the extended graph + info metadata cannot reliably reconstruct compact-key state and cannot be + distinguished from legacy pre-metadata CHM01 files on disk. +- Regenerate any experimental GraphImpl4 files produced before the persisted + metadata and downsize-contiguity fixes in this branch. They may not carry the + final composed bitmap/contiguity state needed by loaded-table JIT. +- Loaded-table JIT support is intentionally limited to the JIT compile path. + Non-JIT table compilation still requires a created table, and RawDog loaded + table JIT remains out of scope. +- Assigned8 JIT was deferred by design. It is not a guard-only problem; it + will require real LLVM/RawDog backend work. +- GraphImpl4 compare-backend / compare-isa benchmark modes are no longer + blanket-blocked, but they were not exhaustively validated in this tranche. + +## Suggested Next Pickup Points + +1. Expand GraphImpl4 benchmark coverage for: + - `--compare-backends` + - `--compare-isa` + - explicit vector-width combinations on assigned16/assigned32 tables + +2. Only if worthwhile, implement assigned8 JIT support. + Expect changes in both: + - LLVM table element typing / load paths + - RawDog code asset selection / patching + +## Practical Notes For A New Session + +- If LLVM JIT behavior looks missing, check whether `PerfectHashLLVM` was + actually built in the current tree. The env packages alone are not enough. +- LLVM-capable profiles are: + - `full` + - `online-rawdog-and-llvm-jit` + - `online-llvm-jit` +- For local work where LLVM matters, prefer one of the above profiles over an + ad hoc default build. diff --git a/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-313_cuda-none_compiler-llvm.yaml b/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-313_cuda-none_compiler-llvm.yaml index facb4e76..58dae96f 100644 --- a/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-313_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-313_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-314_cuda-none_compiler-llvm.yaml b/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-314_cuda-none_compiler-llvm.yaml index fee1542c..f35169c3 100644 --- a/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-314_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-linux-arm64_os-linux_arch-aarch64_py-314_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-124_compiler-llvm.yaml b/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-124_compiler-llvm.yaml index 05e44ddf..a2f3a53d 100644 --- a/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-124_compiler-llvm.yaml +++ b/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-124_compiler-llvm.yaml @@ -17,6 +17,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml b/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml index 5484a7ab..ae7b065d 100644 --- a/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-linux_os-linux_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-124_compiler-llvm.yaml b/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-124_compiler-llvm.yaml index a07ba928..7267d247 100644 --- a/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-124_compiler-llvm.yaml +++ b/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-124_compiler-llvm.yaml @@ -17,6 +17,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml b/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml index 301af8f6..5ca22a90 100644 --- a/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-linux_os-linux_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-macos_os-macos_arch-arm64_py-313_cuda-none_compiler-llvm.yaml b/conda/environments/dev-macos_os-macos_arch-arm64_py-313_cuda-none_compiler-llvm.yaml index a0664a2c..1cee9c77 100644 --- a/conda/environments/dev-macos_os-macos_arch-arm64_py-313_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-macos_os-macos_arch-arm64_py-313_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-macos_os-macos_arch-arm64_py-314_cuda-none_compiler-llvm.yaml b/conda/environments/dev-macos_os-macos_arch-arm64_py-314_cuda-none_compiler-llvm.yaml index b4e85141..a6198dc0 100644 --- a/conda/environments/dev-macos_os-macos_arch-arm64_py-314_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-macos_os-macos_arch-arm64_py-314_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-macos_os-macos_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml b/conda/environments/dev-macos_os-macos_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml index 4b423e5e..b60a7523 100644 --- a/conda/environments/dev-macos_os-macos_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-macos_os-macos_arch-x86_64_py-313_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/conda/environments/dev-macos_os-macos_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml b/conda/environments/dev-macos_os-macos_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml index 5d63ca0c..c08e197e 100644 --- a/conda/environments/dev-macos_os-macos_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml +++ b/conda/environments/dev-macos_os-macos_arch-x86_64_py-314_cuda-none_compiler-llvm.yaml @@ -16,6 +16,7 @@ dependencies: - libxml2-devel - lld>=15 - llvmdev>=15 +- llvm-tools>=15 - llvmlite - make - matplotlib diff --git a/dependencies.yaml b/dependencies.yaml index 417b66f3..6e921e9c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -118,6 +118,7 @@ dependencies: - clangxx>=15 - lld>=15 - llvmdev>=15 + - llvm-tools>=15 - matrix: os: macos compiler: llvm @@ -126,6 +127,7 @@ dependencies: - clangxx>=15 - lld>=15 - llvmdev>=15 + - llvm-tools>=15 - matrix: os: linux compiler: gcc diff --git a/include/PerfectHash/CompiledPerfectHash.h b/include/PerfectHash/CompiledPerfectHash.h index 5f302ce3..b58cf91e 100644 --- a/include/PerfectHash/CompiledPerfectHash.h +++ b/include/PerfectHash/CompiledPerfectHash.h @@ -40,6 +40,9 @@ RotateLeft32_C( ) { b &= 31; + if (b == 0) { + return a; + } return (a << b) | (a >> (32 - b)); } @@ -55,6 +58,9 @@ RotateRight32_C( ) { b &= 31; + if (b == 0) { + return a; + } return (a >> b) | (a << (32 - b)); } @@ -70,6 +76,9 @@ RotateLeft64_C( ) { b &= 63; + if (b == 0) { + return a; + } return (a << b) | (a >> (64 - b)); } @@ -85,6 +94,9 @@ RotateRight64_C( ) { b &= 63; + if (b == 0) { + return a; + } return (a >> b) | (a << (64 - b)); } diff --git a/include/PerfectHash/PerfectHash.h b/include/PerfectHash/PerfectHash.h index 0922b989..2dcd7e08 100644 --- a/include/PerfectHash/PerfectHash.h +++ b/include/PerfectHash/PerfectHash.h @@ -5740,6 +5740,9 @@ RotateLeft32_C( ) { b &= 31; + if (b == 0) { + return a; + } return (a << b) | (a >> (32 - b)); } @@ -5755,6 +5758,9 @@ RotateRight32_C( ) { b &= 31; + if (b == 0) { + return a; + } return (a >> b) | (a << (32 - b)); } @@ -5770,6 +5776,9 @@ RotateLeft64_C( ) { b &= 63; + if (b == 0) { + return a; + } return (a << b) | (a >> (64 - b)); } @@ -5785,6 +5794,9 @@ RotateRight64_C( ) { b &= 63; + if (b == 0) { + return a; + } return (a >> b) | (a << (64 - b)); } diff --git a/include/PerfectHash/PerfectHashErrors.h b/include/PerfectHash/PerfectHashErrors.h index 6f59e7b8..9f7a7930 100644 --- a/include/PerfectHash/PerfectHashErrors.h +++ b/include/PerfectHash/PerfectHashErrors.h @@ -732,7 +732,7 @@ Module Name: // solving: // // - Algorithm is Chm01. -// - GraphImpl is 3. +// - GraphImpl is 3 or 4. // - Number of vertices is <= 65,534 (i.e. MAX_USHORT-1). // // This provides significant performance improvements, which is why it's @@ -794,7 +794,7 @@ Module Name: // // Table Create Parameters: // -// --GraphImpl=1|2|3 [default: 3] +// --GraphImpl=1|2|3|4 [default: 3] // // Selects the backend version of the graph assignment step. Version 1 // matches the original CHM algorithm, version 2 is faster and was derived @@ -4365,7 +4365,7 @@ Module Name: // // MessageText: // -// Invalid value for --GraphImpl. Valid values: 1, 2, 3. (Default: 3.) +// Invalid value for --GraphImpl. Valid values: 1, 2, 3, 4. (Default: 3.) // #define PH_E_INVALID_GRAPH_IMPL ((HRESULT)0xE00403C1L) diff --git a/include/PerfectHash/PerfectHashOnlineJit.h b/include/PerfectHash/PerfectHashOnlineJit.h index 7e9a62ea..ec21778c 100644 --- a/include/PerfectHash/PerfectHashOnlineJit.h +++ b/include/PerfectHash/PerfectHashOnlineJit.h @@ -96,6 +96,13 @@ PhOnlineJitCreateTable32( PH_ONLINE_JIT_TABLE **TablePointer ); +// +// Creates a downsized 64-bit table. Hash functions in the curated "good" set +// are intentionally created with GraphImpl4 so assigned16/assigned32 JIT paths +// can use the compact-key backend. Other accepted hash functions keep the +// default graph implementation. +// + PH_ONLINE_JIT_API int32_t PhOnlineJitCreateTable64( diff --git a/scripts/install-deps/linux-mamba-env.yaml b/scripts/install-deps/linux-mamba-env.yaml index 6bde792f..73d6681b 100644 --- a/scripts/install-deps/linux-mamba-env.yaml +++ b/scripts/install-deps/linux-mamba-env.yaml @@ -5,6 +5,7 @@ dependencies: - cmake>=3.20 - ninja - llvmdev>=15 + - llvm-tools>=15 - nasm - clang - c-compiler @@ -14,4 +15,4 @@ dependencies: - python=3.13 - pip - llvmlite - - rust \ No newline at end of file + - rust diff --git a/scripts/install-deps/windows-mamba-env.yaml b/scripts/install-deps/windows-mamba-env.yaml index 919da524..65fe7f2d 100644 --- a/scripts/install-deps/windows-mamba-env.yaml +++ b/scripts/install-deps/windows-mamba-env.yaml @@ -5,6 +5,7 @@ dependencies: - cmake>=3.20 - ninja - nasm + - llvm-tools>=15 - llvmdev>=15 - clang - zlib @@ -12,4 +13,4 @@ dependencies: - python=3.13 - pip - llvmlite - - rust \ No newline at end of file + - rust diff --git a/src/PerfectHash/CMakeLists.txt b/src/PerfectHash/CMakeLists.txt index 2ae34d73..f918e1c8 100644 --- a/src/PerfectHash/CMakeLists.txt +++ b/src/PerfectHash/CMakeLists.txt @@ -35,6 +35,7 @@ set(Private_Header_Files "GraphImpl.h" "GraphImpl2.h" "GraphImpl3.h" + "GraphImpl4.h" "GuardedList.h" "Math.h" "PerfectHashAllocator.h" @@ -217,6 +218,7 @@ set(Source_Files "GraphImpl1.c" "GraphImpl2.c" "GraphImpl3.c" + "GraphImpl4.cpp" "GuardedList.c" "Math.c" "PerfectHashAllocator.c" @@ -234,6 +236,7 @@ set(Source_Files "PerfectHashPath.c" "PerfectHashPrimes.c" "PerfectHashTable.c" + "PerfectHashTableCompileState.c" "PerfectHashTableCreate.c" "PerfectHashTableDelete.c" "PerfectHashTableHash.c" diff --git a/src/PerfectHash/Chm01.c b/src/PerfectHash/Chm01.c index 699eb10a..8b58ab3a 100644 --- a/src/PerfectHash/Chm01.c +++ b/src/PerfectHash/Chm01.c @@ -1352,9 +1352,19 @@ Return Value: // Copy the table data over to the newly allocated buffer. // - CopyMemory(Table->TableDataBaseAddress, - Graph->Assigned, - SizeInBytes); + if (IsUsingAssigned8(Graph)) { + CopyMemory(Table->TableDataBaseAddress, + Graph->Assigned8, + SizeInBytes); + } else if (IsUsingAssigned16(Graph)) { + CopyMemory(Table->TableDataBaseAddress, + Graph->Assigned16, + SizeInBytes); + } else { + CopyMemory(Table->TableDataBaseAddress, + Graph->Assigned, + SizeInBytes); + } } diff --git a/src/PerfectHash/Chm01.h b/src/PerfectHash/Chm01.h index 3bb233d0..59df7766 100644 --- a/src/PerfectHash/Chm01.h +++ b/src/PerfectHash/Chm01.h @@ -96,6 +96,60 @@ extern PREPARE_TABLE_OUTPUT_DIRECTORY PrepareTableOutputDirectory; extern PREPARE_GRAPH_INFO PrepareGraphInfoChm01; extern PREPARE_GRAPH_INFO PrepareGraphInfoChm02; +ULONGLONG +PerfectHashComposeGraphImpl4DownsizeBitmap( + _In_ ULONGLONG OuterBitmap, + _In_ ULONG InnerBitmap + ); + +// +// N.B. ShiftedMask and TrailingZeros are meaningful only when Contiguous is +// TRUE. MetadataValid only indicates that Bitmap was non-zero; callers +// must use Bitmap/raw extraction when MetadataValid is TRUE and +// Contiguous is FALSE. +// + +VOID +PerfectHashComputeDownsizeMetadataFromBitmap( + _In_ ULONGLONG Bitmap, + _Out_ PBOOLEAN MetadataValid, + _Out_ PULONGLONG ShiftedMask, + _Out_ PBYTE TrailingZeros, + _Out_ PBOOLEAN Contiguous + ); + +#ifndef PERFECT_HASH_TABLE_JIT_INDEX32_KEY_INLINE_DEFINED +#define PERFECT_HASH_TABLE_JIT_INDEX32_KEY_INLINE_DEFINED + +FORCEINLINE +ULONG +PerfectHashTableJitIndex32Key( + _In_ PPERFECT_HASH_TABLE Table, + _In_ ULONG Key + ) +{ + ULONGLONG Value; + + if (Table->GraphImpl != 4 || + Table->GraphImpl4EffectiveKeySizeInBytes == 0 || + Table->GraphImpl4EffectiveKeySizeInBytes >= sizeof(ULONG) || + Table->GraphImpl4KeyDownsizeBitmap == 0) { + return Key; + } + + Value = (ULONGLONG)Key; + if (Table->GraphImpl4KeyDownsizeContiguous != FALSE) { + return (ULONG)( + (Value >> Table->GraphImpl4KeyDownsizeTrailingZeros) & + Table->GraphImpl4KeyDownsizeShiftedMask + ); + } + + return (ULONG)ExtractBits64(Value, Table->GraphImpl4KeyDownsizeBitmap); +} + +#endif + // // // Internal methods private to Chm01.c and Chm01Compat.c. diff --git a/src/PerfectHash/Chm01FileWorkCHeaderFile.c b/src/PerfectHash/Chm01FileWorkCHeaderFile.c index b4a0762e..103ff826 100644 --- a/src/PerfectHash/Chm01FileWorkCHeaderFile.c +++ b/src/PerfectHash/Chm01FileWorkCHeaderFile.c @@ -229,21 +229,23 @@ PrepareCHeaderFileChm01( OUTPUT_HEX64_RAW(Keys->DownsizeBitmap); OUTPUT_RAW("\n#define "); OUTPUT_STRING(Upper); - OUTPUT_RAW("_DOWNSIZE_KEY(Key) ((CPHDKEY)ExtractBits64(Key, 0x"); + OUTPUT_RAW("_DOWNSIZE_KEY(Key) ((CPHDKEY)ExtractBits64_C(Key, 0x"); OUTPUT_HEX64_RAW(Keys->DownsizeBitmap); OUTPUT_RAW("))\n"); // - // Write the left and right key rotation macros. + // Write portable 64-bit rotation macros. CompiledPerfectHash.h may + // map RotateLeft64/RotateRight64 to target-specific intrinsics; the + // generated project does not require those target flags. // OUTPUT_RAW("#define "); OUTPUT_STRING(Upper); - OUTPUT_RAW("_ROTATE_KEY_LEFT RotateLeft64\n"); + OUTPUT_RAW("_ROTATE_KEY_LEFT RotateLeft64_C\n"); OUTPUT_RAW("#define "); OUTPUT_STRING(Upper); - OUTPUT_RAW("_ROTATE_KEY_RIGHT RotateRight64\n\n"); + OUTPUT_RAW("_ROTATE_KEY_RIGHT RotateRight64_C\n\n"); } else { @@ -268,6 +270,14 @@ PrepareCHeaderFileChm01( OUTPUT_RAW("_ROTATE_KEY_RIGHT RotateRight32\n\n"); } + if (Table->GraphImpl == 4 && Table->GraphImpl4KeyDownsizeBitmap != 0) { + OUTPUT_RAW("#define "); + OUTPUT_STRING(Upper); + OUTPUT_RAW("_GRAPHIMPL4_KEY_DOWNSIZE_BITMAP 0x"); + OUTPUT_HEX_RAW(Table->GraphImpl4KeyDownsizeBitmap); + OUTPUT_RAW("\n\n"); + } + OUTPUT_RAW("#include \n"); OUTPUT_RAW("\n\n//\n// (End of preparation phase.)\n//\n\n"); diff --git a/src/PerfectHash/Chm01FileWorkCSourceDownsizedKeysFile.c b/src/PerfectHash/Chm01FileWorkCSourceDownsizedKeysFile.c index a6b928c5..ba7243dd 100644 --- a/src/PerfectHash/Chm01FileWorkCSourceDownsizedKeysFile.c +++ b/src/PerfectHash/Chm01FileWorkCSourceDownsizedKeysFile.c @@ -41,6 +41,11 @@ PrepareCSourceDownsizedKeysFileChm01( PPERFECT_HASH_PATH Path; PPERFECT_HASH_FILE File; PPERFECT_HASH_TABLE Table; + LARGE_INTEGER EndOfFile; + ULONGLONG BaseSize; + ULONGLONG KeyElementBytes; + ULONGLONG RequiredSize; + ULONGLONG AlignedSize; const ULONG Indent = 0x20202020; // @@ -65,6 +70,26 @@ PrepareCSourceDownsizedKeysFileChm01( NumberOfKeys = Keys->NumberOfKeys.QuadPart; SourceKeys = (PULONG)Keys->KeyArrayBaseAddress; + BaseSize = Context->SystemAllocationGranularity; + KeyElementBytes = 16; + RequiredSize = ( + BaseSize + + (NumberOfKeys * KeyElementBytes) + ); + AlignedSize = ALIGN_UP(RequiredSize, + Context->SystemAllocationGranularity); + + if (AlignedSize > (ULONGLONG)File->FileInfo.EndOfFile.QuadPart) { + EndOfFile.QuadPart = (LONGLONG)AlignedSize; + AcquirePerfectHashFileLockExclusive(File); + Result = File->Vtbl->Extend(File, &EndOfFile); + ReleasePerfectHashFileLockExclusive(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileExtend, Result); + return Result; + } + } + Base = (PCHAR)File->BaseAddress; Output = Base; diff --git a/src/PerfectHash/Chm01FileWorkCSourceKeysFile.c b/src/PerfectHash/Chm01FileWorkCSourceKeysFile.c index e886a1e7..fd5d4ed1 100644 --- a/src/PerfectHash/Chm01FileWorkCSourceKeysFile.c +++ b/src/PerfectHash/Chm01FileWorkCSourceKeysFile.c @@ -42,6 +42,11 @@ PrepareCSourceKeysFileChm01( PPERFECT_HASH_PATH Path; PPERFECT_HASH_FILE File; PPERFECT_HASH_TABLE Table; + LARGE_INTEGER EndOfFile; + ULONGLONG BaseSize; + ULONGLONG KeyElementBytes; + ULONGLONG RequiredSize; + ULONGLONG AlignedSize; const ULONG Indent = 0x20202020; // @@ -56,6 +61,28 @@ PrepareCSourceKeysFileChm01( Name = &Path->TableNameA; NumberOfKeys = Keys->NumberOfKeys.QuadPart; + BaseSize = Context->SystemAllocationGranularity; + KeyElementBytes = ( + Keys->OriginalKeySizeType == LongLongType ? 24 : 16 + ); + RequiredSize = ( + BaseSize + + (NumberOfKeys * KeyElementBytes) + ); + AlignedSize = ALIGN_UP(RequiredSize, + Context->SystemAllocationGranularity); + + if (AlignedSize > (ULONGLONG)File->FileInfo.EndOfFile.QuadPart) { + EndOfFile.QuadPart = (LONGLONG)AlignedSize; + AcquirePerfectHashFileLockExclusive(File); + Result = File->Vtbl->Extend(File, &EndOfFile); + ReleasePerfectHashFileLockExclusive(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileExtend, Result); + return Result; + } + } + Base = (PCHAR)File->BaseAddress; Output = Base; diff --git a/src/PerfectHash/Chm01FileWorkCSourceTableDataFile.c b/src/PerfectHash/Chm01FileWorkCSourceTableDataFile.c index 13b115e2..f9a4904c 100644 --- a/src/PerfectHash/Chm01FileWorkCSourceTableDataFile.c +++ b/src/PerfectHash/Chm01FileWorkCSourceTableDataFile.c @@ -35,11 +35,13 @@ SaveCSourceTableDataFileChm01( PCHAR Base; PCHAR Output; ULONG Value; + UCHAR Value8; USHORT Value16; ULONG Count; PULONG Long; PULONG Seed; PGRAPH Graph; + PUCHAR Source8; PULONG Source; PUSHORT Source16; ULONG NumberOfSeeds; @@ -70,6 +72,7 @@ SaveCSourceTableDataFileChm01( NumberOfElements = TotalNumberOfElements >> 1; Graph = (PGRAPH)Context->SolvedContext; NumberOfSeeds = Graph->NumberOfSeeds; + Source8 = Graph->Assigned8; Source = Graph->Assigned; Source16 = Graph->Assigned16; Output = Base = (PCHAR)File->BaseAddress; @@ -168,7 +171,33 @@ SaveCSourceTableDataFileChm01( OUTPUT_INT(TotalNumberOfElements); OUTPUT_RAW("] = {\n\n //\n // 1st half.\n //\n\n"); - if (!IsUsingAssigned16(Graph)) { + if (IsUsingAssigned8(Graph)) { + + for (Index = 0, Count = 0; Index < TotalNumberOfElements; Index++) { + + if (Count == 0) { + INDENT(); + } + + Value8 = *Source8++; + + OUTPUT_HEX(Value8); + + *Output++ = ','; + + if (++Count == 4) { + Count = 0; + *Output++ = '\n'; + } else { + *Output++ = ' '; + } + + if (Index == NumberOfElements-1) { + OUTPUT_RAW("\n //\n // 2nd half.\n //\n\n"); + } + } + + } else if (!IsUsingAssigned16(Graph)) { for (Index = 0, Count = 0; Index < TotalNumberOfElements; Index++) { diff --git a/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c b/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c index 1b7edb32..cc8ad94f 100644 --- a/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c +++ b/src/PerfectHash/Chm01FileWorkCppHeaderOnlyFile.c @@ -41,10 +41,12 @@ SaveCppHeaderOnlyFileChm01( ULONG Seed3Byte3; ULONG Seed3Byte4; PGRAPH Graph; + PUCHAR Source8; PULONG Source; PUSHORT Source16; ULONGLONG NumberOfKeys; ULONG NumberOfSeeds; + BOOLEAN UsingAssigned8; BOOLEAN UsingAssigned16; BOOLEAN Supported; HRESULT Result = S_OK; @@ -53,6 +55,12 @@ SaveCppHeaderOnlyFileChm01( PPERFECT_HASH_FILE File; PPERFECT_HASH_TABLE Table; PTABLE_INFO_ON_DISK TableInfo; + LARGE_INTEGER EndOfFile; + ULONGLONG BaseSize; + ULONGLONG KeyElementBytes; + ULONGLONG TableElementBytes; + ULONGLONG RequiredSize; + ULONGLONG AlignedSize; ULONGLONG TotalNumberOfElements; const ULONG Indent = 0x20202020; @@ -72,12 +80,37 @@ SaveCppHeaderOnlyFileChm01( NumberOfSeeds = Graph->NumberOfSeeds; Seeds = &Graph->FirstSeed; NumberOfKeys = Keys->NumberOfKeys.QuadPart; + UsingAssigned8 = IsUsingAssigned8(Graph); UsingAssigned16 = IsUsingAssigned16(Graph); Supported = ( Table->MaskFunctionId == PerfectHashAndMaskFunctionId && IsGoodPerfectHashHashFunctionId(Table->HashFunctionId) ); + BaseSize = Context->SystemAllocationGranularity * 2ULL; + TableElementBytes = 16; + KeyElementBytes = ( + Keys->OriginalKeySizeType == LongLongType ? 24 : 16 + ); + RequiredSize = ( + BaseSize + + (TotalNumberOfElements * TableElementBytes) + + (NumberOfKeys * KeyElementBytes) + ); + AlignedSize = ALIGN_UP(RequiredSize, + Context->SystemAllocationGranularity); + + if (AlignedSize > (ULONGLONG)File->FileInfo.EndOfFile.QuadPart) { + EndOfFile.QuadPart = (LONGLONG)AlignedSize; + AcquirePerfectHashFileLockExclusive(File); + Result = File->Vtbl->Extend(File, &EndOfFile); + ReleasePerfectHashFileLockExclusive(File); + if (FAILED(Result)) { + PH_ERROR(PerfectHashFileExtend, Result); + return Result; + } + } + Base = (PCHAR)File->BaseAddress; Output = Base; @@ -217,7 +250,9 @@ SaveCppHeaderOnlyFileChm01( // Write the table data array. // - if (UsingAssigned16) { + if (UsingAssigned8) { + OUTPUT_RAW("using table_data_type = std::uint8_t;\n"); + } else if (UsingAssigned16) { OUTPUT_RAW("using table_data_type = std::uint16_t;\n"); } else { OUTPUT_RAW("using table_data_type = std::uint32_t;\n"); @@ -226,7 +261,30 @@ SaveCppHeaderOnlyFileChm01( OUTPUT_RAW("inline constexpr std::array table_data = {\n"); - if (UsingAssigned16) { + Source8 = Graph->Assigned8; + + if (UsingAssigned8) { + for (Index = 0, Count = 0; + Index < TotalNumberOfElements; + Index++) { + + if (Count == 0) { + INDENT(); + } + + OUTPUT_HEX(*Source8++); + + *Output++ = ','; + + if (++Count == 4) { + Count = 0; + *Output++ = '\n'; + } else { + *Output++ = ' '; + } + } + + } else if (UsingAssigned16) { Source16 = Graph->Assigned16; for (Index = 0, Count = 0; @@ -577,4 +635,4 @@ SaveCppHeaderOnlyFileChm01( return Result; } -// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : \ No newline at end of file +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c b/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c index 6b15b745..cf71ce47 100644 --- a/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c +++ b/src/PerfectHash/Chm01FileWorkCppSourceUnityFile.c @@ -248,21 +248,23 @@ PrepareCppSourceUnityFileChm01( OUTPUT_HEX64_RAW(Keys->DownsizeBitmap); OUTPUT_RAW("\n#define "); OUTPUT_STRING(Upper); - OUTPUT_RAW("_DOWNSIZE_KEY(Key) ((CPHDKEY)ExtractBits64(Key, 0x"); + OUTPUT_RAW("_DOWNSIZE_KEY(Key) ((CPHDKEY)ExtractBits64_C(Key, 0x"); OUTPUT_HEX64_RAW(Keys->DownsizeBitmap); OUTPUT_RAW("))\n"); // - // Write the left and right key rotation macros. + // Write portable 64-bit rotation macros. CompiledPerfectHash.h may + // map RotateLeft64/RotateRight64 to target-specific intrinsics; the + // generated project does not require those target flags. // OUTPUT_RAW("#define "); OUTPUT_STRING(Upper); - OUTPUT_RAW("_ROTATE_KEY_LEFT RotateLeft64\n"); + OUTPUT_RAW("_ROTATE_KEY_LEFT RotateLeft64_C\n"); OUTPUT_RAW("#define "); OUTPUT_STRING(Upper); - OUTPUT_RAW("_ROTATE_KEY_RIGHT RotateRight64\n\n"); + OUTPUT_RAW("_ROTATE_KEY_RIGHT RotateRight64_C\n\n"); } else { @@ -457,8 +459,10 @@ SaveCppSourceUnityFileChm01( PULONG Long; PULONG Seeds; PGRAPH Graph; + UCHAR Value8; ULONG Value; USHORT Value16; + PUCHAR Source8; PULONG Source; PUSHORT Source16; ULONG NumberOfSeeds; @@ -491,6 +495,7 @@ SaveCppSourceUnityFileChm01( NumberOfElements = TotalNumberOfElements >> 1; Graph = (PGRAPH)Context->SolvedContext; NumberOfSeeds = Graph->NumberOfSeeds; + Source8 = Graph->Assigned8; Source = Graph->Assigned; Source16 = Graph->Assigned16; @@ -634,7 +639,33 @@ SaveCppSourceUnityFileChm01( OUTPUT_INT(TotalNumberOfElements); OUTPUT_RAW("] = {\n\n //\n // 1st half.\n //\n\n"); - if (!IsUsingAssigned16(Graph)) { + if (IsUsingAssigned8(Graph)) { + + for (Index = 0, Count = 0; Index < TotalNumberOfElements; Index++) { + + if (Count == 0) { + INDENT(); + } + + Value8 = *Source8++; + + OUTPUT_HEX(Value8); + + *Output++ = ','; + + if (++Count == 4) { + Count = 0; + *Output++ = '\n'; + } else { + *Output++ = ' '; + } + + if (Index == NumberOfElements-1) { + OUTPUT_RAW("\n //\n // 2nd half.\n //\n\n"); + } + } + + } else if (!IsUsingAssigned16(Graph)) { for (Index = 0, Count = 0; Index < TotalNumberOfElements; Index++) { diff --git a/src/PerfectHash/Chm01FileWorkCudaSourceFile.c b/src/PerfectHash/Chm01FileWorkCudaSourceFile.c index 6456371c..b9613146 100644 --- a/src/PerfectHash/Chm01FileWorkCudaSourceFile.c +++ b/src/PerfectHash/Chm01FileWorkCudaSourceFile.c @@ -111,10 +111,12 @@ SaveCudaSourceFileChm01( ULONG Seed3Byte3; ULONG Seed3Byte4; PGRAPH Graph; + PUCHAR Source8; PULONG Source; PUSHORT Source16; ULONGLONG NumberOfKeys; ULONG NumberOfSeeds; + BOOLEAN UsingAssigned8; BOOLEAN UsingAssigned16; BOOLEAN Supported; HRESULT Result = S_OK; @@ -142,6 +144,7 @@ SaveCudaSourceFileChm01( NumberOfSeeds = Graph->NumberOfSeeds; Seeds = &Graph->FirstSeed; NumberOfKeys = Keys->NumberOfKeys.QuadPart; + UsingAssigned8 = IsUsingAssigned8(Graph); UsingAssigned16 = IsUsingAssigned16(Graph); Supported = ( Table->MaskFunctionId == PerfectHashAndMaskFunctionId && @@ -217,7 +220,9 @@ SaveCudaSourceFileChm01( OUTPUT_RAW("using original_key_type = std::uint64_t;\n"); } - if (UsingAssigned16) { + if (UsingAssigned8) { + OUTPUT_RAW("using table_data_type = std::uint8_t;\n\n"); + } else if (UsingAssigned16) { OUTPUT_RAW("using table_data_type = std::uint16_t;\n\n"); } else { OUTPUT_RAW("using table_data_type = std::uint32_t;\n\n"); @@ -297,7 +302,30 @@ SaveCudaSourceFileChm01( OUTPUT_RAW("static const table_data_type table_data["); OUTPUT_RAW("number_of_table_elements] = {\n"); - if (UsingAssigned16) { + Source8 = Graph->Assigned8; + + if (UsingAssigned8) { + for (Index = 0, Count = 0; + Index < TotalNumberOfElements; + Index++) { + + if (Count == 0) { + INDENT(); + } + + OUTPUT_HEX(*Source8++); + + *Output++ = ','; + + if (++Count == 4) { + Count = 0; + *Output++ = '\n'; + } else { + *Output++ = ' '; + } + } + + } else if (UsingAssigned16) { Source16 = Graph->Assigned16; for (Index = 0, Count = 0; diff --git a/src/PerfectHash/Chm01FileWorkPythonFile.c b/src/PerfectHash/Chm01FileWorkPythonFile.c index 1c687726..6db4e1d6 100644 --- a/src/PerfectHash/Chm01FileWorkPythonFile.c +++ b/src/PerfectHash/Chm01FileWorkPythonFile.c @@ -41,6 +41,7 @@ SavePythonFileChm01( ULONG Seed3Byte3; ULONG Seed3Byte4; PGRAPH Graph; + PUCHAR Source8; PULONG Source; PUSHORT Source16; ULONGLONG NumberOfKeys; @@ -53,6 +54,7 @@ SavePythonFileChm01( PTABLE_INFO_ON_DISK TableInfo; ULONGLONG TotalNumberOfElements; const ULONG Indent = 0x20202020; + BOOLEAN UsingAssigned8; BOOLEAN UsingAssigned16; BOOLEAN Supported; @@ -72,6 +74,7 @@ SavePythonFileChm01( NumberOfSeeds = Graph->NumberOfSeeds; Seeds = &Graph->FirstSeed; NumberOfKeys = Keys->NumberOfKeys.QuadPart; + UsingAssigned8 = IsUsingAssigned8(Graph); UsingAssigned16 = IsUsingAssigned16(Graph); Supported = ( Table->MaskFunctionId == PerfectHashAndMaskFunctionId && @@ -191,9 +194,35 @@ SavePythonFileChm01( // Write table data. // + OUTPUT_RAW("TABLE_DATA_TYPE = int\n\n"); + OUTPUT_RAW("TABLE_DATA = [\n"); - if (UsingAssigned16) { + if (UsingAssigned8) { + + Source8 = Graph->Assigned8; + + for (Index = 0, Count = 0; + Index < TotalNumberOfElements; + Index++) { + + if (Count == 0) { + INDENT(); + } + + OUTPUT_HEX(*Source8++); + + *Output++ = ','; + + if (++Count == 4) { + Count = 0; + *Output++ = '\n'; + } else { + *Output++ = ' '; + } + } + + } else if (UsingAssigned16) { Source16 = Graph->Assigned16; @@ -509,4 +538,4 @@ SavePythonFileChm01( return Result; } -// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : \ No newline at end of file +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01FileWorkRustLibFile.c b/src/PerfectHash/Chm01FileWorkRustLibFile.c index 70a1dc27..b9ba2e76 100644 --- a/src/PerfectHash/Chm01FileWorkRustLibFile.c +++ b/src/PerfectHash/Chm01FileWorkRustLibFile.c @@ -41,10 +41,12 @@ SaveRustLibFileChm01( ULONG Seed3Byte3; ULONG Seed3Byte4; PGRAPH Graph; + PUCHAR Source8; PULONG Source; PUSHORT Source16; ULONGLONG NumberOfKeys; ULONG NumberOfSeeds; + BOOLEAN UsingAssigned8; BOOLEAN UsingAssigned16; BOOLEAN Supported; HRESULT Result = S_OK; @@ -72,6 +74,7 @@ SaveRustLibFileChm01( NumberOfSeeds = Graph->NumberOfSeeds; Seeds = &Graph->FirstSeed; NumberOfKeys = Keys->NumberOfKeys.QuadPart; + UsingAssigned8 = IsUsingAssigned8(Graph); UsingAssigned16 = IsUsingAssigned16(Graph); Supported = ( Table->MaskFunctionId == PerfectHashAndMaskFunctionId && @@ -142,7 +145,9 @@ SaveRustLibFileChm01( OUTPUT_RAW("pub type OriginalKeyType = u64;\n"); } - if (UsingAssigned16) { + if (UsingAssigned8) { + OUTPUT_RAW("pub type TableDataType = u8;\n\n"); + } else if (UsingAssigned16) { OUTPUT_RAW("pub type TableDataType = u16;\n\n"); } else { OUTPUT_RAW("pub type TableDataType = u32;\n\n"); @@ -244,7 +249,30 @@ SaveRustLibFileChm01( OUTPUT_RAW("pub const TABLE_DATA: [TableDataType; " "NUMBER_OF_TABLE_ELEMENTS] = [\n"); - if (UsingAssigned16) { + Source8 = Graph->Assigned8; + + if (UsingAssigned8) { + for (Index = 0, Count = 0; + Index < TotalNumberOfElements; + Index++) { + + if (Count == 0) { + INDENT(); + } + + OUTPUT_HEX(*Source8++); + + *Output++ = ','; + + if (++Count == 4) { + Count = 0; + *Output++ = '\n'; + } else { + *Output++ = ' '; + } + } + + } else if (UsingAssigned16) { Source16 = Graph->Assigned16; for (Index = 0, Count = 0; @@ -566,4 +594,4 @@ SaveRustLibFileChm01( return Result; } -// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : \ No newline at end of file +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/Chm01FileWorkTableFile.c b/src/PerfectHash/Chm01FileWorkTableFile.c index e87c39d0..20c815a0 100644 --- a/src/PerfectHash/Chm01FileWorkTableFile.c +++ b/src/PerfectHash/Chm01FileWorkTableFile.c @@ -30,9 +30,8 @@ SaveTableFileChm01( ) { PRTL Rtl; - PULONG Dest; PGRAPH Graph; - PULONG Source; + PVOID Source; PVOID BaseAddress; HRESULT Result = S_OK; LONGLONG SizeInBytes; @@ -49,17 +48,23 @@ SaveTableFileChm01( Rtl = Context->Rtl; Table = Context->Table; File = *Item->FilePointer; - Dest = (PULONG)File->BaseAddress; Graph = (PGRAPH)Context->SolvedContext; - Source = Graph->Assigned; TableInfoOnDisk = Table->TableInfoOnDisk; + if (IsUsingAssigned8(Graph)) { + Source = Graph->Assigned8; + } else if (IsUsingAssigned16(Graph)) { + Source = Graph->Assigned16; + } else { + Source = Graph->Assigned; + } + SizeInBytes = ( TableInfoOnDisk->NumberOfTableElements.QuadPart * TableInfoOnDisk->AssignedElementSizeInBytes ); - if (SizeInBytes != File->FileInfo.EndOfFile.QuadPart) { + if (SizeInBytes > File->FileInfo.EndOfFile.QuadPart) { PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); } @@ -68,7 +73,7 @@ SaveTableFileChm01( // backing memory map. // - CopyMemory(Dest, Source, SizeInBytes); + CopyMemory(File->BaseAddress, Source, SizeInBytes); EndOfFile.QuadPart = (LONGLONG)SizeInBytes; diff --git a/src/PerfectHash/Chm01Shared.c b/src/PerfectHash/Chm01Shared.c index e02b0d7a..8977f2db 100644 --- a/src/PerfectHash/Chm01Shared.c +++ b/src/PerfectHash/Chm01Shared.c @@ -15,6 +15,353 @@ Module Name: #include "stdafx.h" #include "Chm01.h" #include "Chm01Private.h" +#include "GraphImpl4.h" + +#define GRAPH_INFO_ON_DISK_FIELD_END(Field) ( \ + FIELD_OFFSET(GRAPH_INFO_ON_DISK, Field) + \ + RTL_FIELD_SIZE(GRAPH_INFO_ON_DISK, Field) \ +) + +// +// TABLE_INFO_ON_DISK is the common header embedded at offset zero in +// GRAPH_INFO_ON_DISK. Its SizeOfStruct field is populated with +// sizeof(GRAPH_INFO_ON_DISK), i.e. the outer CHM01 graph-info record size. +// + +#define GRAPH_INFO_ON_DISK_HAS_FIELD(GraphInfo, Field) ( \ + (GraphInfo)->TableInfoOnDisk.SizeOfStruct >= \ + GRAPH_INFO_ON_DISK_FIELD_END(Field) \ +) + +ULONGLONG +PerfectHashComposeGraphImpl4DownsizeBitmap( + _In_ ULONGLONG OuterBitmap, + _In_ ULONG InnerBitmap + ) +{ + ULONG DenseIndex; + ULONGLONG Result; + ULONGLONG Bit; + + DenseIndex = 0; + Result = 0; + + while (OuterBitmap != 0) { + Bit = OuterBitmap & (~OuterBitmap + 1ULL); + if ((InnerBitmap & (1UL << DenseIndex)) != 0) { + Result |= Bit; + } + OuterBitmap ^= Bit; + DenseIndex++; + } + + return Result; +} + +static +BYTE +DownsizeLeadingZeros64( + _In_ ULONGLONG Value + ) +{ + if (Value == 0) { + return 64; + } + +#if defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64)) + unsigned long Index; + + _BitScanReverse64(&Index, Value); + return (BYTE)(63 - Index); +#elif defined(__GNUC__) || defined(__clang__) + return (BYTE)__builtin_clzll((unsigned long long)Value); +#else + BYTE Leading = 0; + + while ((Value & (1ULL << 63)) == 0) { + Leading++; + Value <<= 1; + } + + return Leading; +#endif +} + +static +BYTE +DownsizeTrailingZeros64( + _In_ ULONGLONG Value + ) +{ + if (Value == 0) { + return 64; + } + +#if defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64)) + unsigned long Index; + + _BitScanForward64(&Index, Value); + return (BYTE)Index; +#elif defined(__GNUC__) || defined(__clang__) + return (BYTE)__builtin_ctzll((unsigned long long)Value); +#else + BYTE Trailing = 0; + + while ((Value & 1ULL) == 0) { + Trailing++; + Value >>= 1; + } + + return Trailing; +#endif +} + +static +BYTE +DownsizePopulationCount64( + _In_ ULONGLONG Value + ) +{ +#if defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64)) + return (BYTE)__popcnt64(Value); +#elif defined(__GNUC__) || defined(__clang__) + return (BYTE)__builtin_popcountll((unsigned long long)Value); +#else + BYTE PopCount = 0; + + while (Value != 0) { + PopCount++; + Value &= Value - 1ULL; + } + + return PopCount; +#endif +} + +_Use_decl_annotations_ +VOID +PerfectHashComputeDownsizeMetadataFromBitmap( + _In_ ULONGLONG Bitmap, + _Out_ PBOOLEAN MetadataValid, + _Out_ PULONGLONG ShiftedMask, + _Out_ PBYTE TrailingZeros, + _Out_ PBOOLEAN Contiguous + ) +{ + BYTE Leading; + BYTE PopCount; + BYTE LocalTrailingZeros; + ULONGLONG Mask; + ULONGLONG Shifted; + + *MetadataValid = TRUE; + *ShiftedMask = 0; + *TrailingZeros = 0; + *Contiguous = FALSE; + + if (Bitmap == 0) { + *MetadataValid = FALSE; + return; + } + + Leading = DownsizeLeadingZeros64(Bitmap); + LocalTrailingZeros = DownsizeTrailingZeros64(Bitmap); + PopCount = DownsizePopulationCount64(Bitmap); + + *TrailingZeros = LocalTrailingZeros; + + if (PopCount == 64) { + // + // All bits set is the identity extraction: (Key >> 0) & ~0. + // + *ShiftedMask = (ULONGLONG)-1; + *Contiguous = TRUE; + return; + } + + if ((Leading + LocalTrailingZeros) == 0) { + // + // MSB and LSB are both set with interior holes, so the bitmap spans + // the full 64-bit range but is still non-contiguous. Preserve the + // bitmap for raw extraction; callers must not treat ShiftedMask as + // meaningful unless DownsizeContiguous is set. TrailingZeros is zero + // because the LSB is set in this branch; it is not a meaningful shift + // amount when Contiguous is FALSE. + // + *TrailingZeros = 0; + return; + } + + Mask = (1ULL << (64 - Leading - LocalTrailingZeros)) - 1ULL; + Shifted = Bitmap >> LocalTrailingZeros; + + if (Mask == Shifted) { + *ShiftedMask = Mask; + *Contiguous = TRUE; + } +} + +static +VOID +SetTableDownsizeMetadataFromBitmap( + _In_ PPERFECT_HASH_TABLE Table, + _In_ ULONGLONG Bitmap + ) +{ + BOOLEAN MetadataValid; + + // + // Invariant: DownsizeShiftedMask is meaningful only when + // DownsizeContiguous is TRUE. If metadata is valid but non-contiguous, + // callers must use DownsizeBitmap with raw bit extraction. For GraphImpl4 + // downsized original keys, these fields describe the full composed + // original-64-bit-key to effective-key transform, not just the inner + // 32-bit compact-key bitmap; KeysBitmap->Flags.Contiguous is not the + // authority after outer and inner bitmaps have been composed. + // + + Table->DownsizeBitmap = Bitmap; + PerfectHashComputeDownsizeMetadataFromBitmap( + Bitmap, + &MetadataValid, + &Table->DownsizeShiftedMask, + &Table->DownsizeTrailingZeros, + &Table->DownsizeContiguous + ); + Table->State.DownsizeMetadataValid = MetadataValid; +} + +static +HRESULT +CaptureGraphInfoRuntimeMetadata( + _In_ PPERFECT_HASH_TABLE Table, + _Inout_ PGRAPH_INFO_ON_DISK GraphInfoOnDisk + ) +{ + BOOLEAN DownsizeMetadataValid; + + GraphInfoOnDisk->RuntimeFlags.AsULong = 0; + GraphInfoOnDisk->GraphImpl = Table->GraphImpl; + if (Table->GraphImpl != 0) { + GraphInfoOnDisk->RuntimeFlags.GraphImplVersionValid = TRUE; + } + GraphInfoOnDisk->DownsizeBitmap = 0; + GraphInfoOnDisk->DownsizeShiftedMask = 0; + GraphInfoOnDisk->DownsizeTrailingZeros = 0; + GraphInfoOnDisk->DownsizeContiguous = FALSE; + GraphInfoOnDisk->DownsizePadding[0] = 0; + GraphInfoOnDisk->DownsizePadding[1] = 0; + GraphInfoOnDisk->GraphImpl4EffectiveKeySizeInBytes = 0; + GraphInfoOnDisk->GraphImpl4KeyDownsizeBitmap = 0; + GraphInfoOnDisk->GraphImpl4KeyDownsizeShiftedMask = 0; + GraphInfoOnDisk->GraphImpl4KeyDownsizeTrailingZeros = 0; + GraphInfoOnDisk->GraphImpl4KeyDownsizeContiguous = FALSE; + GraphInfoOnDisk->GraphImpl4Padding[0] = 0; + GraphInfoOnDisk->GraphImpl4Padding[1] = 0; + + if (Table->State.DownsizeMetadataValid) { + GraphInfoOnDisk->DownsizeBitmap = Table->DownsizeBitmap; + // + // Recompute these fields from the exact bitmap being persisted. For + // downsized key sets, Table->DownsizeBitmap describes the original + // 64-bit-to-active-key transform, while the table's shifted/trailing + // fields may describe the active 32-bit key distribution. + // + PerfectHashComputeDownsizeMetadataFromBitmap( + Table->DownsizeBitmap, + &DownsizeMetadataValid, + &GraphInfoOnDisk->DownsizeShiftedMask, + &GraphInfoOnDisk->DownsizeTrailingZeros, + &GraphInfoOnDisk->DownsizeContiguous + ); + ASSERT(DownsizeMetadataValid); + if (!DownsizeMetadataValid) { + GraphInfoOnDisk->DownsizeBitmap = 0; + return PH_E_INVARIANT_CHECK_FAILED; + } + GraphInfoOnDisk->RuntimeFlags.DownsizeMetadataValid = TRUE; + } + + if (Table->GraphImpl == 4 && + Table->GraphImpl4EffectiveKeySizeInBytes != 0) { + C_ASSERT(sizeof(Table->GraphImpl4KeyDownsizeBitmap) == sizeof(ULONG)); + C_ASSERT(sizeof(Table->GraphImpl4KeyDownsizeShiftedMask) == + sizeof(ULONG)); + if (Table->GraphImpl4EffectiveKeySizeInBytes >= sizeof(ULONG)) { + ASSERT(Table->GraphImpl4KeyDownsizeBitmap == 0); + if (Table->GraphImpl4KeyDownsizeBitmap != 0) { + return PH_E_INVARIANT_CHECK_FAILED; + } + } + if (Table->GraphImpl4EffectiveKeySizeInBytes < sizeof(ULONG)) { + GraphInfoOnDisk->RuntimeFlags.GraphImpl4MetadataValid = TRUE; + } + GraphInfoOnDisk->GraphImpl4EffectiveKeySizeInBytes = + Table->GraphImpl4EffectiveKeySizeInBytes; + GraphInfoOnDisk->GraphImpl4KeyDownsizeBitmap = + Table->GraphImpl4KeyDownsizeBitmap; + GraphInfoOnDisk->GraphImpl4KeyDownsizeShiftedMask = + Table->GraphImpl4KeyDownsizeShiftedMask; + GraphInfoOnDisk->GraphImpl4KeyDownsizeTrailingZeros = + Table->GraphImpl4KeyDownsizeTrailingZeros; + GraphInfoOnDisk->GraphImpl4KeyDownsizeContiguous = + Table->GraphImpl4KeyDownsizeContiguous; + } + + return S_OK; +} + +static +VOID +InitializeGraphImpl4KeyMetadata( + _In_ PPERFECT_HASH_TABLE Table + ) +{ + ULONG Bitmap; + ULONGLONG FinalBitmap; + PPERFECT_HASH_KEYS Keys; + PPERFECT_HASH_KEYS_BITMAP KeysBitmap; + + Keys = Table->Keys; + KeysBitmap = &Keys->Stats.KeysBitmap; + Bitmap = (ULONG)KeysBitmap->Bitmap; + + Table->GraphImpl4EffectiveKeySizeInBytes = sizeof(ULONG); + Table->GraphImpl4KeyDownsizeBitmap = 0; + Table->GraphImpl4KeyDownsizeShiftedMask = 0; + Table->GraphImpl4KeyDownsizeTrailingZeros = 0; + Table->GraphImpl4KeyDownsizeContiguous = FALSE; + + if (KeysBitmap->NumberOfSetBits <= 8) { + Table->GraphImpl4EffectiveKeySizeInBytes = sizeof(BYTE); + } else if (KeysBitmap->NumberOfSetBits <= 16) { + Table->GraphImpl4EffectiveKeySizeInBytes = sizeof(USHORT); + } + + if (Table->GraphImpl4EffectiveKeySizeInBytes >= sizeof(ULONG)) { + // + // Full-width 32-bit GraphImpl4 tables use the input key as-is. Keep + // the compact-key bitmap unset so JIT Index32 wrappers only downsize + // assigned8/16 tables. + // + ASSERT(Table->GraphImpl4KeyDownsizeBitmap == 0); + return; + } + + Table->GraphImpl4KeyDownsizeBitmap = Bitmap; + Table->GraphImpl4KeyDownsizeContiguous = KeysBitmap->Flags.Contiguous; + ASSERT(KeysBitmap->ShiftedMask <= ULONG_MAX); + Table->GraphImpl4KeyDownsizeShiftedMask = (ULONG)KeysBitmap->ShiftedMask; + Table->GraphImpl4KeyDownsizeTrailingZeros = KeysBitmap->TrailingZeros; + + if (KeysWereDownsized(Keys)) { + FinalBitmap = PerfectHashComposeGraphImpl4DownsizeBitmap( + Keys->DownsizeBitmap, + Bitmap + ); + SetTableDownsizeMetadataFromBitmap(Table, FinalBitmap); + } +} PREPARE_GRAPH_INFO PrepareGraphInfoChm01; @@ -72,6 +419,7 @@ Return Value: BYTE AssignedShift; ULONG GraphImpl; ULONG NumberOfKeys; + BOOLEAN UseAssigned8; BOOLEAN UseAssigned16; USHORT NumberOfBitmaps; PGRAPH_DIMENSIONS Dim; @@ -171,6 +519,16 @@ Return Value: NumberOfEdges.QuadPart = NumberOfKeys; + UseAssigned8 = FALSE; + UseAssigned16 = FALSE; + Table->State.UsingAssigned8 = FALSE; + Table->State.UsingAssigned16 = FALSE; + Table->GraphImpl4EffectiveKeySizeInBytes = 0; + Table->GraphImpl4KeyDownsizeBitmap = 0; + Table->GraphImpl4KeyDownsizeShiftedMask = 0; + Table->GraphImpl4KeyDownsizeTrailingZeros = 0; + Table->GraphImpl4KeyDownsizeContiguous = FALSE; + // // Make sure we have at least 8 edges; this ensures the assigned array // will consume at least one cache line, which is required for our memory @@ -373,7 +731,55 @@ Return Value: // 3) Order16Index is a signed SHORT, so NumberOfKeys <= 32767 // - if ((GraphImpl == 3) && + if ((GraphImpl == 4) && + FindBestMemoryCoverage(Context)) { + Result = PH_E_NOT_IMPLEMENTED; + goto Error; + } + + if ((GraphImpl == 4) && + !IsGoodPerfectHashHashFunctionId(Table->HashFunctionId)) { + Result = PH_E_NOT_IMPLEMENTED; + goto Error; + } + + if ((GraphImpl == 4) && + (Table->Keys->KeySizeInBytes > sizeof(ULONG))) { + Result = PH_E_NOT_IMPLEMENTED; + goto Error; + } + + if (GraphImpl == 4) { + InitializeGraphImpl4KeyMetadata(Table); + + if ((TableCreateFlags.DoNotTryUseHash16Impl == FALSE) && + ((NumberOfVertices.LowPart - 1) <= 0x000000ff) && + (NumberOfEdges.LowPart <= 0x00000080) && + (NumberOfKeys <= 0x0000007f)) { + + UseAssigned8 = TRUE; + AssignedShift = 0; + Table->State.UsingAssigned8 = TRUE; + + } else if ((TableCreateFlags.DoNotTryUseHash16Impl == FALSE) && + ((NumberOfVertices.LowPart - 1) <= 0x0000ffff) && + (NumberOfEdges.LowPart <= 0x00008000) && + (NumberOfKeys <= 0x00007fff)) { + + UseAssigned16 = TRUE; + AssignedShift = ASSIGNED16_SHIFT; + Table->State.UsingAssigned16 = TRUE; + + } else { + + AssignedShift = ASSIGNED_SHIFT; + } + + Table->Vtbl->Index = PerfectHashTableIndexImpl4Chm01; + Table->Vtbl->FastIndex = NULL; + Table->Vtbl->SlowIndex = NULL; + + } else if ((GraphImpl == 3) && ((NumberOfVertices.LowPart - 1) <= 0x0000ffff) && (NumberOfEdges.LowPart <= 0x00008000) && (NumberOfKeys <= 0x00007fff) && @@ -445,7 +851,7 @@ Return Value: // Calculate the sizes required for each of the arrays. // - if (GraphImpl == 1 || GraphImpl == 2) { + if (GraphImpl == 1 || GraphImpl == 2) { EdgesSizeInBytes = ALIGN_UP_YMMWORD( RTL_ELEMENT_SIZE(GRAPH, Edges) * TotalNumberOfEdges.QuadPart @@ -471,25 +877,25 @@ Return Value: } else { - ASSERT(GraphImpl == 3); + ASSERT(GraphImpl == 3 || GraphImpl == 4); EdgesSizeInBytes = 0; NextSizeInBytes = 0; FirstSizeInBytes = 0; - if (!UseAssigned16) { + if (UseAssigned8) { VertexPairsSizeInBytes = ALIGN_UP_YMMWORD( - RTL_ELEMENT_SIZE(GRAPH, VertexPairs) * + RTL_ELEMENT_SIZE(GRAPH, Vertex8Pairs) * (ULONGLONG)NumberOfKeys ); Vertices3SizeInBytes = ALIGN_UP_YMMWORD( - RTL_ELEMENT_SIZE(GRAPH, Vertices3) * + RTL_ELEMENT_SIZE(GRAPH, Vertices83) * NumberOfVertices.QuadPart ); - } else { + } else if (UseAssigned16) { VertexPairsSizeInBytes = ALIGN_UP_YMMWORD( RTL_ELEMENT_SIZE(GRAPH, Vertex16Pairs) * @@ -501,22 +907,33 @@ Return Value: NumberOfVertices.QuadPart ); + } else { + + VertexPairsSizeInBytes = ALIGN_UP_YMMWORD( + RTL_ELEMENT_SIZE(GRAPH, VertexPairs) * + (ULONGLONG)NumberOfKeys + ); + + Vertices3SizeInBytes = ALIGN_UP_YMMWORD( + RTL_ELEMENT_SIZE(GRAPH, Vertices3) * + NumberOfVertices.QuadPart + ); } DeletedEdgesBitmapBufferSizeInBytes.QuadPart = 0; } - if (!UseAssigned16) { + if (UseAssigned8) { OrderSizeInBytes = ALIGN_UP_YMMWORD( - RTL_ELEMENT_SIZE(GRAPH, Order) * NumberOfEdges.QuadPart + RTL_ELEMENT_SIZE(GRAPH, Order8) * NumberOfEdges.QuadPart ); AssignedSizeInBytes = ALIGN_UP_YMMWORD( - RTL_ELEMENT_SIZE(GRAPH, Assigned) * NumberOfVertices.QuadPart + RTL_ELEMENT_SIZE(GRAPH, Assigned8) * NumberOfVertices.QuadPart ); - } else { + } else if (UseAssigned16) { OrderSizeInBytes = ALIGN_UP_YMMWORD( RTL_ELEMENT_SIZE(GRAPH, Order16) * NumberOfEdges.QuadPart @@ -526,6 +943,15 @@ Return Value: RTL_ELEMENT_SIZE(GRAPH, Assigned16) * NumberOfVertices.QuadPart ); + } else { + + OrderSizeInBytes = ALIGN_UP_YMMWORD( + RTL_ELEMENT_SIZE(GRAPH, Order) * NumberOfEdges.QuadPart + ); + + AssignedSizeInBytes = ALIGN_UP_YMMWORD( + RTL_ELEMENT_SIZE(GRAPH, Assigned) * NumberOfVertices.QuadPart + ); } // @@ -571,7 +997,13 @@ Return Value: // line. // - if (!UseAssigned16) { + if (GraphImpl == 4) { + + Info->NumberOfAssignedPerPageSizeInBytes = 0; + Info->NumberOfAssignedPerLargePageSizeInBytes = 0; + Info->NumberOfAssignedPerCacheLineSizeInBytes = 0; + + } else if (!UseAssigned16) { Info->NumberOfAssignedPerPageSizeInBytes = ( Info->AssignedArrayNumberOfPages * @@ -877,7 +1309,13 @@ Return Value: TableInfoOnDisk->NumberOfSeeds = ( HashRoutineNumberOfSeeds[Table->HashFunctionId] ); - TableInfoOnDisk->AssignedElementSizeInBytes = UseAssigned16 ? 2 : 4; + if (UseAssigned8) { + TableInfoOnDisk->AssignedElementSizeInBytes = 1; + } else if (UseAssigned16) { + TableInfoOnDisk->AssignedElementSizeInBytes = 2; + } else { + TableInfoOnDisk->AssignedElementSizeInBytes = 4; + } // // This will change based on masking type and whether or not the caller @@ -890,6 +1328,10 @@ Return Value: ); CopyInline(&GraphInfoOnDisk->Dimensions, Dim, sizeof(*Dim)); + Result = CaptureGraphInfoRuntimeMetadata(Table, GraphInfoOnDisk); + if (FAILED(Result)) { + return Result; + } // // Capture ratios. @@ -1111,9 +1553,14 @@ Return Value: --*/ { + PGRAPH_INFO_ON_DISK GraphInfoOnDisk; PTABLE_INFO_ON_DISK OnDisk; + BOOLEAN HasGraphImplMetadata; OnDisk = Table->TableInfoOnDisk; + GraphInfoOnDisk = CONTAINING_RECORD(OnDisk, + GRAPH_INFO_ON_DISK, + TableInfoOnDisk); Table->HashSize = OnDisk->HashSize; Table->IndexSize = OnDisk->IndexSize; @@ -1126,6 +1573,96 @@ Return Value: Table->HashModulus = OnDisk->HashModulus; Table->IndexModulus = OnDisk->IndexModulus; + HasGraphImplMetadata = ( + GRAPH_INFO_ON_DISK_HAS_FIELD(GraphInfoOnDisk, GraphImpl) && + GraphInfoOnDisk->RuntimeFlags.GraphImplVersionValid && + GraphInfoOnDisk->GraphImpl != 0 + ); + + if (HasGraphImplMetadata) { + Table->GraphImpl = GraphInfoOnDisk->GraphImpl; + } else { + // + // Legacy graph-info records predate explicit GraphImpl metadata. Use + // a sentinel instead of aliasing an in-range implementation version; + // explicitly identified GraphImpl4 records below still fail closed if + // compact-key metadata is absent. + // + Table->GraphImpl = 0; + } + + if (Table->GraphImpl != 0 && + Table->GraphImpl != 1 && + Table->GraphImpl != 2 && + Table->GraphImpl != 3 && + Table->GraphImpl != 4) { + return PH_E_INVALID_GRAPH_IMPL; + } + + // + // Restore composed downsize metadata only from the extended on-disk layout. + // Older experimental GraphImpl4 files either fail the field-size guard or + // lack DownsizeMetadataValid, so we never reinterpret their inner bitmap + // metadata as the new original-key-to-effective-key composed bitmap. + // + + if (GRAPH_INFO_ON_DISK_HAS_FIELD(GraphInfoOnDisk, DownsizeContiguous) && + GraphInfoOnDisk->RuntimeFlags.DownsizeMetadataValid) { + Table->State.DownsizeMetadataValid = TRUE; + Table->DownsizeBitmap = GraphInfoOnDisk->DownsizeBitmap; + Table->DownsizeShiftedMask = GraphInfoOnDisk->DownsizeShiftedMask; + Table->DownsizeTrailingZeros = GraphInfoOnDisk->DownsizeTrailingZeros; + Table->DownsizeContiguous = GraphInfoOnDisk->DownsizeContiguous; + } + + if (Table->GraphImpl == 4) { + if (!IsGoodPerfectHashHashFunctionId(Table->HashFunctionId)) { + return PH_E_NOT_IMPLEMENTED; + } + + if (!GRAPH_INFO_ON_DISK_HAS_FIELD(GraphInfoOnDisk, + GraphImpl4KeyDownsizeContiguous)) { + return PH_E_INVALID_GRAPH_IMPL; + } + + Table->GraphImpl4EffectiveKeySizeInBytes = + GraphInfoOnDisk->GraphImpl4EffectiveKeySizeInBytes; + + switch (Table->GraphImpl4EffectiveKeySizeInBytes) { + case sizeof(BYTE): + case sizeof(USHORT): + case sizeof(ULONG): + break; + default: + return PH_E_INVALID_KEY_SIZE; + } + + if (OnDisk->KeySizeInBytes > sizeof(ULONGLONG)) { + return PH_E_NOT_IMPLEMENTED; + } + + Table->GraphImpl4KeyDownsizeBitmap = + GraphInfoOnDisk->GraphImpl4KeyDownsizeBitmap; + Table->GraphImpl4KeyDownsizeShiftedMask = + GraphInfoOnDisk->GraphImpl4KeyDownsizeShiftedMask; + Table->GraphImpl4KeyDownsizeTrailingZeros = + GraphInfoOnDisk->GraphImpl4KeyDownsizeTrailingZeros; + Table->GraphImpl4KeyDownsizeContiguous = + GraphInfoOnDisk->GraphImpl4KeyDownsizeContiguous; + + if (Table->GraphImpl4EffectiveKeySizeInBytes >= sizeof(ULONG)) { + if (Table->GraphImpl4KeyDownsizeBitmap != 0) { + return PH_E_INVALID_GRAPH_IMPL; + } + } else if (!GraphInfoOnDisk->RuntimeFlags.GraphImpl4MetadataValid) { + return PH_E_INVALID_GRAPH_IMPL; + } + + Table->Vtbl->Index = PerfectHashTableIndexImpl4Chm01; + Table->Vtbl->FastIndex = NULL; + Table->Vtbl->SlowIndex = NULL; + } + return S_OK; } diff --git a/src/PerfectHash/ChmOnline01.c b/src/PerfectHash/ChmOnline01.c index b4d325cd..ac2b4b65 100644 --- a/src/PerfectHash/ChmOnline01.c +++ b/src/PerfectHash/ChmOnline01.c @@ -3927,7 +3927,6 @@ CompileChm01IndexJit( PTABLE_INFO_ON_DISK TableInfo; PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId; PPERFECT_HASH_KEYS Keys; - PRTL Rtl; ULONGLONG DownsizeBitmap = 0; ULONGLONG DownsizeShiftedMask = 0; BYTE DownsizeTrailingZeros = 0; @@ -3966,7 +3965,6 @@ CompileChm01IndexJit( TableInfo = Table->TableInfoOnDisk; HashFunctionId = Table->HashFunctionId; Keys = Table->Keys; - Rtl = Table->Rtl; UseAssigned16 = (Table->State.UsingAssigned16 != FALSE); KeysDownsized = (TableInfo->OriginalKeySizeInBytes > TableInfo->KeySizeInBytes); @@ -4019,14 +4017,17 @@ CompileChm01IndexJit( CompileVectorIndex64x8 = (CompileIndex64 && CompileVectorIndex32x8); if (KeysDownsized) { - ULONGLONG One; - ULONGLONG Mask; - ULONGLONG Shifted; - ULONGLONG Leading; - ULONGLONG Trailing; - ULONGLONG PopCount; - - if (Keys) { + BOOLEAN DownsizeMetadataValid; + + if (Table->GraphImpl == 4 && Table->State.DownsizeMetadataValid) { + // + // GraphImpl4 stores the composed outer+inner bitmap on the table. + // Index64 JIT extracts directly from the original 64-bit key into + // the effective key, so this takes priority over Keys->DownsizeBitmap, + // which only captures the outer key-set bitmap. + // + DownsizeBitmap = Table->DownsizeBitmap; + } else if (Keys) { DownsizeBitmap = Keys->DownsizeBitmap; } else if (Table->State.DownsizeMetadataValid) { DownsizeBitmap = Table->DownsizeBitmap; @@ -4034,26 +4035,15 @@ CompileChm01IndexJit( return PH_E_NOT_IMPLEMENTED; } - One = 1; - Leading = Rtl->LeadingZeros64(DownsizeBitmap); - Trailing = Rtl->TrailingZeros64(DownsizeBitmap); - PopCount = Rtl->PopulationCount64(DownsizeBitmap); - Mask = (One << (64 - Leading - Trailing)) - One; - - DownsizeTrailingZeros = (BYTE)Trailing; - - if (PopCount == 64) { - DownsizeContiguous = TRUE; - } else if (Leading == 0) { - DownsizeContiguous = FALSE; - } else { - Shifted = DownsizeBitmap; - Shifted >>= Trailing; - DownsizeContiguous = (Mask == Shifted); - } - - if (DownsizeContiguous) { - DownsizeShiftedMask = Mask; + PerfectHashComputeDownsizeMetadataFromBitmap( + DownsizeBitmap, + &DownsizeMetadataValid, + &DownsizeShiftedMask, + &DownsizeTrailingZeros, + &DownsizeContiguous + ); + if (!DownsizeMetadataValid) { + return PH_E_NOT_IMPLEMENTED; } } @@ -5251,6 +5241,11 @@ PerfectHashTableCompileJit( return PH_E_INVARIANT_CHECK_FAILED; } + if (Table->State.UsingAssigned8 || + Table->TableInfoOnDisk->AssignedElementSizeInBytes == 1) { + return PH_E_NOT_IMPLEMENTED; + } + if (Table->State.UsingAssigned16) { if (!ARGUMENT_PRESENT(Table->Assigned16)) { return PH_E_INVARIANT_CHECK_FAILED; @@ -5698,6 +5693,7 @@ PerfectHashTableJitInterfaceIndex32( return PH_E_NOT_IMPLEMENTED; } + Key = PerfectHashTableJitIndex32Key(Table, Key); *Index = IndexFunction(Key); return S_OK; } @@ -5735,6 +5731,12 @@ PerfectHashTableJitInterfaceIndex64( return PH_E_NOT_IMPLEMENTED; } + // + // The JIT Index64 IR performs the full 64-bit composed-bitmap extraction + // internally, including GraphImpl4 inner compact-key handling. Do not + // pre-transform Key here or GraphImpl4 keys will be compacted twice. + // + *Index = IndexFunction(Key); return S_OK; } @@ -5776,6 +5778,8 @@ PerfectHashTableJitInterfaceIndex32x2( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); IndexFunction(Key1, Key2, Index1, Index2); return S_OK; } @@ -5823,6 +5827,11 @@ PerfectHashTableJitInterfaceIndex32x4( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); + Key3 = PerfectHashTableJitIndex32Key(Table, Key3); + Key4 = PerfectHashTableJitIndex32Key(Table, Key4); + IndexFunction(Key1, Key2, Key3, @@ -5889,6 +5898,15 @@ PerfectHashTableJitInterfaceIndex32x8( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); + Key3 = PerfectHashTableJitIndex32Key(Table, Key3); + Key4 = PerfectHashTableJitIndex32Key(Table, Key4); + Key5 = PerfectHashTableJitIndex32Key(Table, Key5); + Key6 = PerfectHashTableJitIndex32Key(Table, Key6); + Key7 = PerfectHashTableJitIndex32Key(Table, Key7); + Key8 = PerfectHashTableJitIndex32Key(Table, Key8); + IndexFunction(Key1, Key2, Key3, @@ -5987,6 +6005,23 @@ PerfectHashTableJitInterfaceIndex32x16( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); + Key3 = PerfectHashTableJitIndex32Key(Table, Key3); + Key4 = PerfectHashTableJitIndex32Key(Table, Key4); + Key5 = PerfectHashTableJitIndex32Key(Table, Key5); + Key6 = PerfectHashTableJitIndex32Key(Table, Key6); + Key7 = PerfectHashTableJitIndex32Key(Table, Key7); + Key8 = PerfectHashTableJitIndex32Key(Table, Key8); + Key9 = PerfectHashTableJitIndex32Key(Table, Key9); + Key10 = PerfectHashTableJitIndex32Key(Table, Key10); + Key11 = PerfectHashTableJitIndex32Key(Table, Key11); + Key12 = PerfectHashTableJitIndex32Key(Table, Key12); + Key13 = PerfectHashTableJitIndex32Key(Table, Key13); + Key14 = PerfectHashTableJitIndex32Key(Table, Key14); + Key15 = PerfectHashTableJitIndex32Key(Table, Key15); + Key16 = PerfectHashTableJitIndex32Key(Table, Key16); + IndexFunction(Key1, Key2, Key3, @@ -6830,6 +6865,7 @@ PerfectHashTableIndexJit( return PH_E_NOT_IMPLEMENTED; } + Key = PerfectHashTableJitIndex32Key(Table, Key); *Index = IndexFunction(Key); return S_OK; } diff --git a/src/PerfectHash/ChmOnline01RawDog.c b/src/PerfectHash/ChmOnline01RawDog.c index d2d7266c..95e53385 100644 --- a/src/PerfectHash/ChmOnline01RawDog.c +++ b/src/PerfectHash/ChmOnline01RawDog.c @@ -3811,6 +3811,7 @@ RawDogTableIndexJit( return PH_E_NOT_IMPLEMENTED; } + Key = PerfectHashTableJitIndex32Key(Table, Key); *Index = IndexFunction(Key); return S_OK; } @@ -3926,6 +3927,7 @@ RawDogJitInterfaceIndex32( return PH_E_NOT_IMPLEMENTED; } + Key = PerfectHashTableJitIndex32Key(Table, Key); *Index = IndexFunction(Key); return S_OK; } @@ -4005,6 +4007,11 @@ RawDogJitInterfaceIndex32x4( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); + Key3 = PerfectHashTableJitIndex32Key(Table, Key3); + Key4 = PerfectHashTableJitIndex32Key(Table, Key4); + IndexFunction(Key1, Key2, Key3, @@ -4071,6 +4078,15 @@ RawDogJitInterfaceIndex32x8( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); + Key3 = PerfectHashTableJitIndex32Key(Table, Key3); + Key4 = PerfectHashTableJitIndex32Key(Table, Key4); + Key5 = PerfectHashTableJitIndex32Key(Table, Key5); + Key6 = PerfectHashTableJitIndex32Key(Table, Key6); + Key7 = PerfectHashTableJitIndex32Key(Table, Key7); + Key8 = PerfectHashTableJitIndex32Key(Table, Key8); + IndexFunction(Key1, Key2, Key3, @@ -4169,6 +4185,23 @@ RawDogJitInterfaceIndex32x16( return PH_E_NOT_IMPLEMENTED; } + Key1 = PerfectHashTableJitIndex32Key(Table, Key1); + Key2 = PerfectHashTableJitIndex32Key(Table, Key2); + Key3 = PerfectHashTableJitIndex32Key(Table, Key3); + Key4 = PerfectHashTableJitIndex32Key(Table, Key4); + Key5 = PerfectHashTableJitIndex32Key(Table, Key5); + Key6 = PerfectHashTableJitIndex32Key(Table, Key6); + Key7 = PerfectHashTableJitIndex32Key(Table, Key7); + Key8 = PerfectHashTableJitIndex32Key(Table, Key8); + Key9 = PerfectHashTableJitIndex32Key(Table, Key9); + Key10 = PerfectHashTableJitIndex32Key(Table, Key10); + Key11 = PerfectHashTableJitIndex32Key(Table, Key11); + Key12 = PerfectHashTableJitIndex32Key(Table, Key12); + Key13 = PerfectHashTableJitIndex32Key(Table, Key13); + Key14 = PerfectHashTableJitIndex32Key(Table, Key14); + Key15 = PerfectHashTableJitIndex32Key(Table, Key15); + Key16 = PerfectHashTableJitIndex32Key(Table, Key16); + IndexFunction(Key1, Key2, Key3, @@ -4444,13 +4477,25 @@ PerfectHashTableCompileJitRawDog( } if (!Table->Flags.Created) { - return PH_E_TABLE_NOT_CREATED; + // + // RawDog loaded-table JIT is intentionally out of scope. Return + // not-implemented for loaded tables so callers can distinguish that + // unsupported mode from a table that has neither been created nor + // loaded. + // + return Table->Flags.Loaded ? PH_E_NOT_IMPLEMENTED : + PH_E_TABLE_NOT_CREATED; } if (!ARGUMENT_PRESENT(Table->TableInfoOnDisk)) { return PH_E_INVARIANT_CHECK_FAILED; } + if (Table->State.UsingAssigned8 || + Table->TableInfoOnDisk->AssignedElementSizeInBytes == 1) { + return PH_E_NOT_IMPLEMENTED; + } + if (Table->State.UsingAssigned16) { if (!ARGUMENT_PRESENT(Table->Assigned16)) { return PH_E_INVARIANT_CHECK_FAILED; diff --git a/src/PerfectHash/CompiledPerfectHash_CHeader_RawCString.h b/src/PerfectHash/CompiledPerfectHash_CHeader_RawCString.h index 4dc516f8..4e03a340 100644 --- a/src/PerfectHash/CompiledPerfectHash_CHeader_RawCString.h +++ b/src/PerfectHash/CompiledPerfectHash_CHeader_RawCString.h @@ -51,6 +51,9 @@ const CHAR CompiledPerfectHashCHeaderRawCStr[] = " )\n" "{\n" " b &= 31;\n" + " if (b == 0) {\n" + " return a;\n" + " }\n" " return (a << b) | (a >> (32 - b));\n" "}\n" "\n" @@ -66,6 +69,9 @@ const CHAR CompiledPerfectHashCHeaderRawCStr[] = " )\n" "{\n" " b &= 31;\n" + " if (b == 0) {\n" + " return a;\n" + " }\n" " return (a >> b) | (a << (32 - b));\n" "}\n" "\n" @@ -81,6 +87,9 @@ const CHAR CompiledPerfectHashCHeaderRawCStr[] = " )\n" "{\n" " b &= 63;\n" + " if (b == 0) {\n" + " return a;\n" + " }\n" " return (a << b) | (a >> (64 - b));\n" "}\n" "\n" @@ -96,6 +105,9 @@ const CHAR CompiledPerfectHashCHeaderRawCStr[] = " )\n" "{\n" " b &= 63;\n" + " if (b == 0) {\n" + " return a;\n" + " }\n" " return (a >> b) | (a << (64 - b));\n" "}\n" "\n" diff --git a/src/PerfectHash/Graph.c b/src/PerfectHash/Graph.c index 1115a1c3..7d2a4d41 100644 --- a/src/PerfectHash/Graph.c +++ b/src/PerfectHash/Graph.c @@ -14,6 +14,7 @@ Module Name: #include "stdafx.h" #include "PerfectHashEventsPrivate.h" +#include "GraphImpl4.h" // // Forward decl. @@ -166,6 +167,9 @@ Return Value: // Override vtbl methods based on table state and table create flags. // + Graph->Flags.UsingAssigned8 = FALSE; + Graph->Flags.UsingAssigned16 = FALSE; + if (Table->State.UsingAssigned16 != FALSE) { Graph->Flags.UsingAssigned16 = TRUE; @@ -342,6 +346,8 @@ Return Value: } } + Graph->Flags.UsingAssigned8 = (Table->State.UsingAssigned8 != FALSE); + #if defined(_M_AMD64) || defined(_M_X64) // @@ -430,7 +436,6 @@ Return Value: // RELEASE(Graph->Rtl); - RELEASE(Graph->Allocator); RELEASE(Graph->Rng); RELEASE(Graph->Keys); @@ -439,13 +444,18 @@ Return Value: // if (Graph->VertexPairs != NULL) { - if (!VirtualFree(Graph->VertexPairs, - VFS(Graph->Info->VertexPairsSizeInBytes), - MEM_RELEASE)) { + if (Graph->Impl == 4) { + Graph->Allocator->Vtbl->AlignedFreePointer(Graph->Allocator, + PPV(&Graph->VertexPairs)); + } else if (!VirtualFree(Graph->VertexPairs, + VFS(Graph->Info->VertexPairsSizeInBytes), + MEM_RELEASE)) { SYS_ERROR(VirtualFree); } } + RELEASE(Graph->Allocator); + return; } @@ -4423,6 +4433,18 @@ Return Value: } break; + case 4: + Graph->Vtbl->HashKeys = GraphHashKeys4; + Graph->Vtbl->AddKeys = ( + (TableCreateFlags.HashAllKeysFirst != FALSE) ? + GraphHashKeysThenAdd4 : GraphAddKeys4 + ); + Graph->Vtbl->Verify = GraphVerify4; + Graph->Vtbl->IsAcyclic = GraphIsAcyclic4; + Graph->Vtbl->Assign = GraphAssign4; + Graph->Vtbl->RegisterSolved = GraphRegisterSolvedNoBestCoverage; + break; + default: PH_RAISE(PH_E_UNREACHABLE_CODE); break; @@ -4462,21 +4484,96 @@ Return Value: goto Error; \ } - ALLOC_ARRAY(Order, PLONG); - ALLOC_ARRAY(Assigned, PASSIGNED); +#define ALLOC_SIZED_ARRAY(Field, Type, Size) \ + if (!Graph->Field) { \ + Graph->Field = (Type)( \ + Allocator->Vtbl->AlignedMalloc( \ + Allocator, \ + (ULONG_PTR)(Size), \ + YMMWORD_ALIGNMENT \ + ) \ + ); \ + } else { \ + Graph->Field = (Type)( \ + Allocator->Vtbl->AlignedReAlloc( \ + Allocator, \ + Graph->Field, \ + (ULONG_PTR)(Size), \ + YMMWORD_ALIGNMENT \ + ) \ + ); \ + } \ + if (!Graph->Field) { \ + Result = E_OUTOFMEMORY; \ + goto Error; \ + } if (Graph->Impl == 1 || Graph->Impl == 2) { + ALLOC_ARRAY(Order, PLONG); + ALLOC_ARRAY(Assigned, PASSIGNED); ALLOC_ARRAY(Edges, PEDGE); ALLOC_ARRAY(Next, PEDGE); ALLOC_ARRAY(First, PVERTEX); - } else { + } else if (Graph->Impl == 3) { + ALLOC_ARRAY(Order, PLONG); + ALLOC_ARRAY(Assigned, PASSIGNED); ASSERT(Graph->Impl == 3); - ALLOC_ARRAY(Vertices3, PVERTEX3); + if (IsUsingAssigned16(Graph)) { + ALLOC_SIZED_ARRAY(Order16, PSHORT, Info->OrderSizeInBytes); + ALLOC_SIZED_ARRAY(Assigned16, + PASSIGNED16, + Info->AssignedSizeInBytes); + ALLOC_SIZED_ARRAY(Vertices163, + PVERTEX163, + Info->Vertices3SizeInBytes); + } else { + ALLOC_ARRAY(Vertices3, PVERTEX3); + } // // N.B. We don't do `ALLOC_ARRAY(Edges3, PEDGE3);` as it's handled by // the code block below (via VertexPairs allocation). // + } else { + SIZE_T AssignedSize; + SIZE_T OrderSize; + SIZE_T VertexPairsSize; + SIZE_T VerticesSize; + + ASSERT(Graph->Impl == 4); + + AssignedSize = (SIZE_T)Info->AssignedSizeInBytes; + OrderSize = (SIZE_T)Info->OrderSizeInBytes; + VertexPairsSize = (SIZE_T)Info->VertexPairsSizeInBytes; + VerticesSize = (SIZE_T)Info->Vertices3SizeInBytes; + + switch (Table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: + ALLOC_SIZED_ARRAY(Order8, PCHAR, OrderSize); + ALLOC_SIZED_ARRAY(Assigned8, PASSIGNED8, AssignedSize); + ALLOC_SIZED_ARRAY(Vertices83, PVERTEX83, VerticesSize); + ALLOC_SIZED_ARRAY(Vertex8Pairs, PVERTEX8_PAIR, VertexPairsSize); + break; + + case 2: + ALLOC_SIZED_ARRAY(Order16, PSHORT, OrderSize); + ALLOC_SIZED_ARRAY(Assigned16, PASSIGNED16, AssignedSize); + ALLOC_SIZED_ARRAY(Vertices163, PVERTEX163, VerticesSize); + ALLOC_SIZED_ARRAY(Vertex16Pairs, PVERTEX16_PAIR, VertexPairsSize); + break; + + case 4: + ALLOC_SIZED_ARRAY(Order, PLONG, OrderSize); + ALLOC_SIZED_ARRAY(Assigned, PASSIGNED, AssignedSize); + ALLOC_SIZED_ARRAY(Vertices3, PVERTEX3, VerticesSize); + ALLOC_SIZED_ARRAY(VertexPairs, PVERTEX_PAIR, VertexPairsSize); + break; + + default: + Result = PH_E_NOT_IMPLEMENTED; + goto Error; + } + } // @@ -4487,7 +4584,8 @@ Return Value: // above, which grow larger upon each resize event).) // - if (TableCreateFlags.HashAllKeysFirst || Graph->Impl == 3) { + if (TableCreateFlags.HashAllKeysFirst || Graph->Impl == 3 || + Graph->Impl == 4) { VertexPairsSizeInBytes = (SIZE_T)Info->VertexPairsSizeInBytes; ASSERT(VertexPairsSizeInBytes != 0); @@ -4589,6 +4687,12 @@ Return Value: // to the end and finish up. // + if (Graph->Impl == 4) { + Graph->Flags.WantsAssignedMemoryCoverage = FALSE; + Graph->Flags.WantsAssignedMemoryCoverageForKeysSubset = FALSE; + goto End; + } + if (FirstSolvedGraphWinsAndSkipMemoryCoverage(Context)) { Graph->Flags.WantsAssignedMemoryCoverage = FALSE; goto End; @@ -4790,6 +4894,7 @@ Return Value: ULONG TotalNumberOfPages; ULONG TotalNumberOfLargePages; ULONG TotalNumberOfCacheLines; + PPERFECT_HASH_TABLE Table; PPERFECT_HASH_CONTEXT Context; SIZE_T VertexPairsSizeInBytes; PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags; @@ -4807,6 +4912,7 @@ Return Value: Context = Graph->Context; Info = Graph->Info; Rtl = Context->Rtl; + Table = Context->Table; TableCreateFlags.AsULongLong = Context->Table->TableCreateFlags.AsULongLong; MAYBE_STOP_GRAPH_SOLVING(Graph); @@ -4904,19 +5010,61 @@ Return Value: Info->Name##SizeInBytes); \ } - ZERO_ARRAY(Order); - ZERO_ARRAY(Assigned); - ZERO_ARRAY(Vertices3); + if (Graph->Impl == 4) { + switch (Table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: + ZeroMemory(Graph->Order8, Info->OrderSizeInBytes); + ZeroMemory(Graph->Assigned8, Info->AssignedSizeInBytes); + ZeroMemory(Graph->Vertices83, Info->Vertices3SizeInBytes); + FillMemory(Graph->Vertex8Pairs, + Info->VertexPairsSizeInBytes, + (BYTE)~0); + Graph->Order8Index = (CHAR)Graph->NumberOfKeys; + ASSERT(Graph->Order8Index > 0); + break; - if (!IsUsingAssigned16(Graph)) { + case 2: + ZeroMemory(Graph->Order16, Info->OrderSizeInBytes); + ZeroMemory(Graph->Assigned16, Info->AssignedSizeInBytes); + ZeroMemory(Graph->Vertices163, Info->Vertices3SizeInBytes); + FillMemory(Graph->Vertex16Pairs, + Info->VertexPairsSizeInBytes, + (BYTE)~0); + Graph->Order16Index = (SHORT)Graph->NumberOfKeys; + ASSERT(Graph->Order16Index > 0); + break; + + case 4: + ZERO_ARRAY(Order); + ZERO_ARRAY(Assigned); + ZERO_ARRAY(Vertices3); + FillMemory(Graph->VertexPairs, + Info->VertexPairsSizeInBytes, + (BYTE)~0); + Graph->OrderIndex = (LONG)Graph->NumberOfKeys; + ASSERT(Graph->OrderIndex > 0); + break; + + default: + Result = PH_E_NOT_IMPLEMENTED; + goto Error; + } + } else { + ZERO_ARRAY(Order); + ZERO_ARRAY(Assigned); + ZERO_ARRAY(Vertices3); + } + + if (Graph->Impl != 4 && !IsUsingAssigned16(Graph)) { Graph->OrderIndex = (LONG)Graph->NumberOfKeys; ASSERT(Graph->OrderIndex > 0); - } else { + } else if (Graph->Impl != 4) { Graph->Order16Index = (SHORT)Graph->NumberOfKeys; ASSERT(Graph->Order16Index > 0); } - if (TableCreateFlags.HashAllKeysFirst || Graph->Impl == 3) { + if (TableCreateFlags.HashAllKeysFirst || Graph->Impl == 3 || + Graph->Impl == 4) { ASSERT(Graph->VertexPairs != NULL); @@ -4938,7 +5086,7 @@ Return Value: if (Graph->Flags.WantsWriteCombiningForVertexPairsArray && !Graph->Flags.VertexPairsArrayIsWriteCombined) { - ASSERT(!IsUsingAssigned16(Graph)); + ASSERT(!IsUsingAssigned16(Graph) || Graph->Impl == 4); // // Restore the write-combine (and read/write) page protection so @@ -4979,6 +5127,9 @@ Return Value: if (Graph->Impl == 3) { EMPTY_ARRAY(VertexPairs); + } else if (Graph->Impl == 4 && + Table->TableInfoOnDisk->AssignedElementSizeInBytes == 4) { + EMPTY_ARRAY(VertexPairs); } } @@ -5038,6 +5189,10 @@ Return Value: // graph wins" mode and have been requested to skip memory coverage. // + if (Graph->Impl == 4) { + goto End; + } + if (FirstSolvedGraphWinsAndSkipMemoryCoverage(Context)) { goto End; } diff --git a/src/PerfectHash/Graph.h b/src/PerfectHash/Graph.h index 884452e4..30768d43 100644 --- a/src/PerfectHash/Graph.h +++ b/src/PerfectHash/Graph.h @@ -478,6 +478,43 @@ CopyCoverage( // artifact of the reduced memory footprint of each solver graph.) // +typedef UCHAR EDGE8; +typedef UCHAR ASSIGNED8; +typedef ASSIGNED8 *PASSIGNED8; + +typedef UCHAR KEY8; +typedef CHAR ORDER8; +typedef UCHAR VERTEX8; +typedef UCHAR DEGREE8; + +typedef EDGE8 *PEDGE8; +typedef VERTEX8 *PVERTEX8; +typedef DEGREE8 *PDEGREE8; + +typedef union _VERTEX8_PAIR { + struct { + VERTEX8 Vertex1; + VERTEX8 Vertex2; + }; + USHORT AsUShort; +} VERTEX8_PAIR, *PVERTEX8_PAIR; +C_ASSERT(sizeof(VERTEX8_PAIR) == sizeof(USHORT)); + +typedef union _EDGE83 { + struct { + VERTEX8 Vertex1; + VERTEX8 Vertex2; + }; + VERTEX8_PAIR AsVertex8Pair; + USHORT AsUShort; +} EDGE83, *PEDGE83; + +typedef struct _VERTEX83 { + DEGREE8 Degree; + EDGE8 Edges; +} VERTEX83, *PVERTEX83; +C_ASSERT(sizeof(VERTEX83) == sizeof(USHORT)); + typedef USHORT EDGE16; typedef USHORT ASSIGNED16; typedef ASSIGNED16 *PASSIGNED16; @@ -869,6 +906,7 @@ typedef union _GRAPH_FLAGS { // active. // + ULONG UsingAssigned8:1; ULONG UsingAssigned16:1; // @@ -917,7 +955,7 @@ typedef union _GRAPH_FLAGS { // Unused bits. // - ULONG Unused:9; + ULONG Unused:8; }; LONG AsLong; ULONG AsULong; @@ -941,6 +979,7 @@ C_ASSERT(sizeof(GRAPH_FLAGS) == sizeof(ULONG)); #define WantsCuRandomHostSeeds(Graph) \ ((Graph)->Flags.WantsCuRandomHostSeeds != FALSE) #define IsGraphParanoid(Graph) ((Graph)->Flags.Paranoid != FALSE) +#define IsUsingAssigned8(Graph) ((Graph)->Flags.UsingAssigned8 != FALSE) #define IsUsingAssigned16(Graph) ((Graph)->Flags.UsingAssigned16 != FALSE) #define IsCudaDebugGraph(Graph) ((Graph)->Flags.DebugCudaChm02 != FALSE) @@ -1605,6 +1644,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _GRAPH { union { volatile LONG OrderIndex; + volatile CHAR Order8Index; volatile SHORT Order16Index; }; @@ -1672,6 +1712,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _GRAPH { _Writable_elements_(NumberOfKeys) union { PLONG Order; + PCHAR Order8; PSHORT Order16; }; @@ -1702,6 +1743,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _GRAPH { _Writable_elements_(NumberOfVertices) union { PASSIGNED Assigned; + PASSIGNED8 Assigned8; PASSIGNED16 Assigned16; }; @@ -1712,6 +1754,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _GRAPH { _Writable_elements_(NumberOfVertices) union { PVERTEX3 Vertices3; + PVERTEX83 Vertices83; PVERTEX163 Vertices163; }; @@ -1742,6 +1785,11 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _GRAPH { PVERTEX16_PAIR Vertex16Pairs; PEDGE163 Edges163; }; + + union { + PVERTEX8_PAIR Vertex8Pairs; + PEDGE83 Edges83; + }; }; // @@ -2244,8 +2292,54 @@ typedef struct _Struct_size_bytes_(Header.SizeOfStruct) _GRAPH_INFO_ON_DISK { GRAPH_DIMENSIONS Dimensions; }; + // + // Additional CHM01 runtime metadata needed to faithfully reload tables + // produced by alternate graph implementations and downsized key paths. + // + + union { + struct { + ULONG DownsizeMetadataValid:1; + // + // Non-trivial GraphImpl4 compact-key metadata is valid. Full-width + // GraphImpl4 tables still persist GraphImplVersionValid below. + // + ULONG GraphImpl4MetadataValid:1; + ULONG GraphImplVersionValid:1; + ULONG Unused:29; + }; + ULONG AsULong; + } RuntimeFlags; + + ULONG GraphImpl; + + // + // For non-GraphImpl4 tables, this is the outer 64-bit key downsize bitmap. + // For GraphImpl4 tables, this is the composed original-key-to-effective-key + // bitmap: outer 64-bit key downsize plus inner compact-key extraction. + // Callers that consume this field must preserve the GraphImpl4 distinction. + // + + ULONGLONG DownsizeBitmap; + ULONGLONG DownsizeShiftedMask; + BYTE DownsizeTrailingZeros; + BYTE DownsizeContiguous; + BYTE DownsizePadding[2]; + + ULONG GraphImpl4EffectiveKeySizeInBytes; + ULONG GraphImpl4KeyDownsizeBitmap; + ULONG GraphImpl4KeyDownsizeShiftedMask; + BYTE GraphImpl4KeyDownsizeTrailingZeros; + BYTE GraphImpl4KeyDownsizeContiguous; + BYTE GraphImpl4Padding[2]; + } GRAPH_INFO_ON_DISK; +C_ASSERT(sizeof(BOOLEAN) == sizeof(BYTE)); +C_ASSERT(sizeof(GRAPH_INFO_ON_DISK) == 296); C_ASSERT(sizeof(GRAPH_INFO_ON_DISK) <= PAGE_SIZE); +C_ASSERT(FIELD_OFFSET(GRAPH_INFO_ON_DISK, RuntimeFlags) == 248); +C_ASSERT((FIELD_OFFSET(GRAPH_INFO_ON_DISK, DownsizeBitmap) % + sizeof(ULONGLONG)) == 0); typedef GRAPH_INFO_ON_DISK *PGRAPH_INFO_ON_DISK; // diff --git a/src/PerfectHash/GraphImpl4.cpp b/src/PerfectHash/GraphImpl4.cpp new file mode 100644 index 00000000..dd6a7c27 --- /dev/null +++ b/src/PerfectHash/GraphImpl4.cpp @@ -0,0 +1,1126 @@ +/*++ + +Copyright (c) 2026 Trent Nelson + +Module Name: + + GraphImpl4.cpp + +Abstract: + + Experimental CPU graph implementation that mirrors GraphImpl3's solver but + uses a templated C++ core for storage-width and effective-key-width + specialization. The C ABI remains unchanged; only these entrypoints are + exported to the C side. + +--*/ + +#include "stdafx.h" +#include "GraphImpl4.h" +#include "PerfectHashEventsPrivate.h" + +#include +#include +#include +#include + +namespace { + +template +union hash_pair_t { + struct { + T LowPart; + T HighPart; + }; + NativeT AsPair; +}; + +using hash_pair8_t = hash_pair_t; +using hash_pair16_t = hash_pair_t; +using hash_pair32_t = hash_pair_t; + +template +FORCEINLINE +ValueType +rotate_right( + ValueType value, + ShiftType shift + ) noexcept +{ + constexpr ShiftType bits = static_cast(sizeof(ValueType) * 8); + + if (shift == 0) { + return value; + } + + shift %= bits; + return static_cast( + (value >> shift) | (value << (bits - shift)) + ); +} + +template +FORCEINLINE +ResultType +hash_key_for_id( + PERFECT_HASH_HASH_FUNCTION_ID id, + KeyType key, + PULONG seeds, + VertexType mask + ) noexcept +{ + ResultType result = {}; + VertexType vertex1 = 0; + VertexType vertex2 = 0; + VertexType downsized_key = static_cast(key); + ULONG_BYTES seed3 = {}; + ULONG seed4 = 0; + ULONG seed5 = 0; + + switch (id) { + case PerfectHashHashMultiplyShiftRFunctionId: + seed3.AsULong = seeds[2]; + vertex1 = downsized_key * seeds[0]; + vertex1 = vertex1 >> seed3.Byte1; + vertex2 = downsized_key * seeds[1]; + vertex2 = vertex2 >> seed3.Byte2; + result.LowPart = static_cast(vertex1 & mask); + result.HighPart = static_cast(vertex2 & mask); + return result; + + case PerfectHashHashMultiplyShiftRXFunctionId: + seed3.AsULong = seeds[2]; + vertex1 = downsized_key * seeds[0]; + vertex1 = vertex1 >> seed3.Byte1; + vertex2 = downsized_key * seeds[1]; + vertex2 = vertex2 >> seed3.Byte1; + result.LowPart = static_cast(vertex1); + result.HighPart = static_cast(vertex2); + return result; + + case PerfectHashHashMulshrolate1RXFunctionId: + seed3.AsULong = seeds[2]; + vertex1 = downsized_key * seeds[0]; + vertex1 = rotate_right(vertex1, seed3.Byte2); + vertex1 = vertex1 >> seed3.Byte1; + vertex2 = downsized_key * seeds[1]; + vertex2 = vertex2 >> seed3.Byte1; + result.LowPart = static_cast(vertex1); + result.HighPart = static_cast(vertex2); + return result; + + case PerfectHashHashMulshrolate2RXFunctionId: + seed3.AsULong = seeds[2]; + vertex1 = downsized_key * seeds[0]; + vertex1 = rotate_right(vertex1, seed3.Byte2); + vertex1 = vertex1 >> seed3.Byte1; + vertex2 = downsized_key * seeds[1]; + vertex2 = rotate_right(vertex2, seed3.Byte3); + vertex2 = vertex2 >> seed3.Byte1; + result.LowPart = static_cast(vertex1); + result.HighPart = static_cast(vertex2); + return result; + + case PerfectHashHashMulshrolate3RXFunctionId: + seed3.AsULong = seeds[2]; + seed4 = seeds[3]; + vertex1 = downsized_key * seeds[0]; + vertex1 = rotate_right(vertex1, seed3.Byte2); + vertex1 = vertex1 * seed4; + vertex1 = vertex1 >> seed3.Byte1; + vertex2 = downsized_key * seeds[1]; + vertex2 = rotate_right(vertex2, seed3.Byte3); + vertex2 = vertex2 >> seed3.Byte1; + result.LowPart = static_cast(vertex1); + result.HighPart = static_cast(vertex2); + return result; + + case PerfectHashHashMulshrolate4RXFunctionId: + seed3.AsULong = seeds[2]; + seed4 = seeds[3]; + seed5 = seeds[4]; + vertex1 = downsized_key * seeds[0]; + vertex1 = rotate_right(vertex1, seed3.Byte2); + vertex1 = vertex1 * seed4; + vertex1 = vertex1 >> seed3.Byte1; + vertex2 = downsized_key * seeds[1]; + vertex2 = rotate_right(vertex2, seed3.Byte3); + vertex2 = vertex2 * seed5; + vertex2 = vertex2 >> seed3.Byte1; + result.LowPart = static_cast(vertex1); + result.HighPart = static_cast(vertex2); + return result; + + default: + return result; + } +} + +template +constexpr T +empty_value() noexcept +{ + return std::numeric_limits::max(); +} + +template +constexpr bool +is_empty(T value) noexcept +{ + return value == empty_value(); +} + +template +struct storage_policy; + +template <> +struct storage_policy { + using storage_type = uint8_t; + using edge_type = uint8_t; + using degree_type = uint8_t; + using vertex_type = uint8_t; + using order_type = int8_t; + using assigned_type = uint8_t; + using pair_type = VERTEX8_PAIR; + using edge3_type = EDGE83; + using vertex3_type = VERTEX83; + using result_pair_type = hash_pair8_t; + using hash_vertex_type = uint32_t; + + static assigned_type *Assigned(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Assigned8); + } + + static assigned_type *Assigned(PPERFECT_HASH_TABLE table) noexcept + { + return reinterpret_cast(table->Assigned8); + } + + static vertex3_type *Vertices(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Vertices83); + } + + static pair_type *Pairs(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Vertex8Pairs); + } + + static edge3_type *Edges(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Edges83); + } + + static order_type *Order(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Order8); + } + + static volatile order_type &OrderIndex(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Order8Index); + } +}; + +template <> +struct storage_policy { + using storage_type = uint16_t; + using edge_type = uint16_t; + using degree_type = uint16_t; + using vertex_type = uint16_t; + using order_type = int16_t; + using assigned_type = uint16_t; + using pair_type = VERTEX16_PAIR; + using edge3_type = EDGE163; + using vertex3_type = VERTEX163; + using result_pair_type = hash_pair16_t; + using hash_vertex_type = uint32_t; + + static assigned_type *Assigned(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Assigned16); + } + + static assigned_type *Assigned(PPERFECT_HASH_TABLE table) noexcept + { + return reinterpret_cast(table->Assigned16); + } + + static vertex3_type *Vertices(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Vertices163); + } + + static pair_type *Pairs(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Vertex16Pairs); + } + + static edge3_type *Edges(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Edges163); + } + + static order_type *Order(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Order16); + } + + static volatile order_type &OrderIndex(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Order16Index); + } +}; + +template <> +struct storage_policy { + using storage_type = uint32_t; + using edge_type = uint32_t; + using degree_type = uint32_t; + using vertex_type = uint32_t; + using order_type = int32_t; + using assigned_type = uint32_t; + using pair_type = VERTEX_PAIR; + using edge3_type = EDGE3; + using vertex3_type = VERTEX3; + using result_pair_type = hash_pair32_t; + using hash_vertex_type = uint32_t; + + static assigned_type *Assigned(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Assigned); + } + + static assigned_type *Assigned(PPERFECT_HASH_TABLE table) noexcept + { + return reinterpret_cast(table->Assigned); + } + + static vertex3_type *Vertices(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Vertices3); + } + + static pair_type *Pairs(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->VertexPairs); + } + + static edge3_type *Edges(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Edges3); + } + + static order_type *Order(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->Order); + } + + static volatile order_type &OrderIndex(PGRAPH graph) noexcept + { + return reinterpret_cast(graph->OrderIndex); + } +}; + +template +inline void +set_pair( + typename StoragePolicy::pair_type &pair, + typename StoragePolicy::vertex_type low, + typename StoragePolicy::vertex_type high + ) noexcept +{ + pair.Vertex1 = low; + pair.Vertex2 = high; +} + +template +KeyType +compact_key( + PPERFECT_HASH_TABLE table, + ULONG key + ) noexcept +{ + if (table->GraphImpl4EffectiveKeySizeInBytes >= sizeof(ULONG) || + table->GraphImpl4KeyDownsizeBitmap == 0) { + return static_cast(key); + } + + uint64_t value = static_cast(key); + uint64_t result; + + if (table->GraphImpl4KeyDownsizeContiguous != FALSE) { + result = ( + value >> table->GraphImpl4KeyDownsizeTrailingZeros + ) & table->GraphImpl4KeyDownsizeShiftedMask; + } else { + result = ExtractBits64(value, table->GraphImpl4KeyDownsizeBitmap); + } + + return static_cast(result); +} + +template +void +add_edge_impl( + PGRAPH graph, + typename StoragePolicy::edge_type edge, + typename StoragePolicy::vertex_type vertex1_index, + typename StoragePolicy::vertex_type vertex2_index + ) noexcept +{ + auto *vertex1 = &StoragePolicy::Vertices(graph)[vertex1_index]; + auto *vertex2 = &StoragePolicy::Vertices(graph)[vertex2_index]; + + vertex1->Edges ^= edge; + ++vertex1->Degree; + + vertex2->Edges ^= edge; + ++vertex2->Degree; +} + +template +void +remove_vertex_impl( + PGRAPH graph, + typename StoragePolicy::vertex_type vertex_index + ) noexcept +{ + using vertex_type = typename StoragePolicy::vertex_type; + using edge_type = typename StoragePolicy::edge_type; + using order_type = typename StoragePolicy::order_type; + + auto *vertex = &StoragePolicy::Vertices(graph)[vertex_index]; + if (vertex->Degree != 1) { + return; + } + + edge_type edge = vertex->Edges; + auto *edge3 = &StoragePolicy::Edges(graph)[edge]; + + if (is_empty(edge3->Vertex1) || + is_empty(edge3->Vertex2)) { + return; + } + + auto *vertex1 = &StoragePolicy::Vertices(graph)[edge3->Vertex1]; + if (vertex1->Degree >= 1) { + vertex1->Edges ^= edge; + --vertex1->Degree; + } + + auto *vertex2 = &StoragePolicy::Vertices(graph)[edge3->Vertex2]; + if (vertex2->Degree >= 1) { + vertex2->Edges ^= edge; + --vertex2->Degree; + } + + graph->DeletedEdgeCount++; + + volatile order_type &order_index = StoragePolicy::OrderIndex(graph); + order_type next = static_cast(order_index - 1); + order_index = next; + StoragePolicy::Order(graph)[next] = static_cast(edge); +} + +template +HRESULT +hash_keys_impl( + PGRAPH graph, + ULONG number_of_keys, + PKEY keys + ) +{ + PGRAPH Graph = graph; + using result_pair_type = typename StoragePolicy::result_pair_type; + using vertex_type = typename StoragePolicy::vertex_type; + using hash_vertex_type = typename StoragePolicy::hash_vertex_type; + + auto *table = graph->Context->Table; + auto hash_mask = static_cast(table->HashMask); + auto *seeds = &graph->FirstSeed; + auto *pairs = StoragePolicy::Pairs(graph); + + DECL_GRAPH_COUNTER_LOCAL_VARS(); + + START_GRAPH_COUNTER(); + + for (ULONG index = 0; index < number_of_keys; index++) { + KeyType key = compact_key(table, keys[index]); + result_pair_type hash = hash_key_for_id( + table->HashFunctionId, + key, + seeds, + hash_mask + ); + + if (hash.LowPart == hash.HighPart) { + STOP_GRAPH_COUNTER(HashKeys); + EVENT_WRITE_GRAPH(HashKeys); + return PH_E_GRAPH_VERTEX_COLLISION_FAILURE; + } + + set_pair(pairs[index], + static_cast(hash.LowPart), + static_cast(hash.HighPart)); + } + + STOP_GRAPH_COUNTER(HashKeys); + EVENT_WRITE_GRAPH(HashKeys); + + return S_OK; +} + +template +HRESULT +add_keys_impl( + PGRAPH graph, + ULONG number_of_keys, + PKEY keys + ) +{ + PGRAPH Graph = graph; + using result_pair_type = typename StoragePolicy::result_pair_type; + using vertex_type = typename StoragePolicy::vertex_type; + using edge_type = typename StoragePolicy::edge_type; + using hash_vertex_type = typename StoragePolicy::hash_vertex_type; + + auto *table = graph->Context->Table; + auto hash_mask = static_cast(table->HashMask); + auto *seeds = &graph->FirstSeed; + auto *pairs = StoragePolicy::Pairs(graph); + + DECL_GRAPH_COUNTER_LOCAL_VARS(); + + START_GRAPH_COUNTER(); + + for (ULONG index = 0; index < number_of_keys; index++) { + KeyType key = compact_key(table, keys[index]); + result_pair_type hash = hash_key_for_id( + table->HashFunctionId, + key, + seeds, + hash_mask + ); + + if (hash.LowPart == hash.HighPart) { + STOP_GRAPH_COUNTER(AddKeys); + EVENT_WRITE_GRAPH(AddKeys); + return PH_E_GRAPH_VERTEX_COLLISION_FAILURE; + } + + set_pair(pairs[index], + static_cast(hash.LowPart), + static_cast(hash.HighPart)); + + add_edge_impl(graph, + static_cast(index), + static_cast(hash.LowPart), + static_cast(hash.HighPart)); + } + + STOP_GRAPH_COUNTER(AddKeys); + EVENT_WRITE_GRAPH(AddKeys); + + return S_OK; +} + +template +HRESULT +add_hashed_keys_impl( + PGRAPH graph, + ULONG number_of_keys + ) +{ + PGRAPH Graph = graph; + using edge_type = typename StoragePolicy::edge_type; + + auto *pairs = StoragePolicy::Pairs(graph); + + DECL_GRAPH_COUNTER_LOCAL_VARS(); + + START_GRAPH_COUNTER(); + + for (ULONG index = 0; index < number_of_keys; index++) { + auto pair = pairs[index]; + add_edge_impl(graph, + static_cast(index), + pair.Vertex1, + pair.Vertex2); + } + + STOP_GRAPH_COUNTER(AddHashedKeys); + EVENT_WRITE_GRAPH_ADD_HASHED_KEYS(); + + return S_OK; +} + +template +HRESULT +is_acyclic_impl( + PGRAPH graph + ) +{ + PGRAPH Graph = graph; + using vertex_type = typename StoragePolicy::vertex_type; + using order_type = typename StoragePolicy::order_type; + using edge_type = typename StoragePolicy::edge_type; + + DECL_GRAPH_COUNTER_LOCAL_VARS(); + + graph->Flags.Shrinking = TRUE; + StoragePolicy::OrderIndex(graph) = static_cast(graph->NumberOfKeys); + + START_GRAPH_COUNTER(); + for (ULONG vertex = 0; vertex < graph->NumberOfVertices; vertex++) { + remove_vertex_impl(graph, static_cast(vertex)); + } + STOP_GRAPH_COUNTER(IsAcyclicPhase1); + + START_GRAPH_COUNTER(); + for (order_type index = static_cast(graph->NumberOfKeys); + StoragePolicy::OrderIndex(graph) > 0 && + index > StoragePolicy::OrderIndex(graph); + NOTHING) { + + edge_type edge_index = static_cast(StoragePolicy::Order(graph)[--index]); + auto *edge3 = &StoragePolicy::Edges(graph)[edge_index]; + remove_vertex_impl(graph, edge3->Vertex1); + remove_vertex_impl(graph, edge3->Vertex2); + } + STOP_GRAPH_COUNTER(IsAcyclicPhase2); + + if (graph->DeletedEdgeCount != graph->NumberOfKeys) { + EVENT_WRITE_GRAPH_IS_ACYCLIC(); + return PH_E_GRAPH_CYCLIC_FAILURE; + } + + graph->Flags.IsAcyclic = TRUE; + EVENT_WRITE_GRAPH_IS_ACYCLIC(); + return S_OK; +} + +template +HRESULT +assign_impl( + PGRAPH graph + ) +{ + PGRAPH Graph = graph; + using vertex_type = typename StoragePolicy::vertex_type; + using assigned_type = typename StoragePolicy::assigned_type; + using order_type = typename StoragePolicy::order_type; + using edge_type = typename StoragePolicy::edge_type; + + auto *table = graph->Context->Table; + auto *assigned = StoragePolicy::Assigned(graph); + auto *order = StoragePolicy::Order(graph); + auto *edges = StoragePolicy::Edges(graph); + const assigned_type number_of_edges = + static_cast(graph->NumberOfEdges); + + DECL_GRAPH_COUNTER_LOCAL_VARS(); + + UNREFERENCED_PARAMETER(table); + + ASSERT(graph->Flags.IsAcyclic); + + EVENT_WRITE_GRAPH_ASSIGN_START(); + START_GRAPH_COUNTER(); + + for (ULONG index = 0; index < graph->NumberOfKeys; index++) { + order_type order_value = order[index]; + auto *edge = &edges[static_cast(order_value)]; + + vertex_type vertex1; + vertex_type vertex2; + + if (!TestGraphBit(VisitedVerticesBitmap, edge->Vertex1)) { + vertex1 = edge->Vertex1; + vertex2 = edge->Vertex2; + } else { + vertex1 = edge->Vertex2; + vertex2 = edge->Vertex1; + } + + assigned_type value = static_cast( + static_cast(order_value) - assigned[vertex2] + ); + if (value >= number_of_edges) { + value = static_cast(value + number_of_edges); + } + + assigned[vertex1] = value; + SetGraphBit(VisitedVerticesBitmap, vertex1); + SetGraphBit(VisitedVerticesBitmap, vertex2); + } + + STOP_GRAPH_COUNTER(Assign); + EVENT_WRITE_GRAPH_ASSIGN_STOP(); + EVENT_WRITE_GRAPH_ASSIGN_RESULT(); + + return S_OK; +} + +template +HRESULT +index_impl_with_effective_key( + PERFECT_HASH_HASH_FUNCTION_ID hash_function_id, + PULONG seeds, + typename StoragePolicy::assigned_type *assigned, + ULONG hash_mask, + ULONG index_mask, + ULONG key, + PULONG index + ) +{ + using result_pair_type = typename StoragePolicy::result_pair_type; + using assigned_type = typename StoragePolicy::assigned_type; + using hash_vertex_type = typename StoragePolicy::hash_vertex_type; + + KeyType effective_key = static_cast(key); + result_pair_type hash = hash_key_for_id( + hash_function_id, + effective_key, + seeds, + static_cast(hash_mask) + ); + + if (hash.LowPart == hash.HighPart) { + *index = 0; + return E_FAIL; + } + + assigned_type vertex1 = assigned[hash.LowPart]; + assigned_type vertex2 = assigned[hash.HighPart]; + *index = static_cast( + (static_cast(vertex1) + static_cast(vertex2)) & + index_mask + ); + return S_OK; +} + +template +HRESULT +index_impl_with_assigned( + PERFECT_HASH_HASH_FUNCTION_ID hash_function_id, + PULONG seeds, + typename StoragePolicy::assigned_type *assigned, + ULONG hash_mask, + ULONG index_mask, + PPERFECT_HASH_TABLE table, + ULONG key, + PULONG index + ) +{ + return index_impl_with_effective_key( + hash_function_id, + seeds, + assigned, + hash_mask, + index_mask, + compact_key(table, key), + index + ); +} + +template +HRESULT +index_impl( + PPERFECT_HASH_TABLE table, + ULONG key, + PULONG index + ) +{ + return index_impl_with_assigned( + table->HashFunctionId, + &table->TableInfoOnDisk->FirstSeed, + StoragePolicy::Assigned(table), + table->HashMask, + table->IndexMask, + table, + key, + index + ); +} + +template +HRESULT +index_effective_key_dispatch_impl(PPERFECT_HASH_TABLE table, + ULONG key, + PULONG index) +{ + // + // The key has already passed through the composed 64-bit-to-effective-key + // bitmap used by loaded-table Index64. Do not apply inner compact-key + // extraction again here. + // + return index_impl_with_effective_key, KeyType>( + table->HashFunctionId, + &table->TableInfoOnDisk->FirstSeed, + storage_policy::Assigned(table), + table->HashMask, + table->IndexMask, + key, + index + ); +} + +template +HRESULT +verify_impl( + PGRAPH graph + ) +{ + PGRAPH Graph = graph; + PRTL rtl; + PALLOCATOR allocator; + PPERFECT_HASH_TABLE table; + PULONG values = nullptr; + ULONG number_of_assignments; + ULONG collisions = 0; + + ASSERT(graph != nullptr); + + if (SkipGraphVerification(graph)) { + return PH_S_GRAPH_VERIFICATION_SKIPPED; + } + + table = graph->Context->Table; + rtl = graph->Context->Rtl; + allocator = graph->Allocator; + + number_of_assignments = rtl->RtlNumberOfSetBits(&graph->AssignedBitmap); + ASSERT(number_of_assignments == 0); + + values = graph->Values; + if (!values) { + values = graph->Values = static_cast( + allocator->Vtbl->Calloc( + allocator, + graph->Info->ValuesSizeInBytes, + sizeof(*graph->Values) + ) + ); + } + + if (!values) { + return E_OUTOFMEMORY; + } + + auto *keys = static_cast(table->Keys->KeyArrayBaseAddress); + auto *assigned = StoragePolicy::Assigned(graph); + auto *seeds = &graph->FirstSeed; + for (ULONG edge = 0; edge < graph->NumberOfKeys; edge++) { + ULONG key = keys[edge]; + ULONG value_index = 0; + HRESULT result = index_impl_with_assigned( + table->HashFunctionId, + seeds, + assigned, + table->HashMask, + table->IndexMask, + table, + key, + &value_index + ); + if (FAILED(result)) { + allocator->Vtbl->FreePointer(allocator, PPV(&graph->Values)); + return result; + } + + if (TestGraphBit(AssignedBitmap, value_index)) { + collisions++; + } + + SetGraphBit(AssignedBitmap, value_index); + values[value_index] = key; + } + + number_of_assignments = rtl->RtlNumberOfSetBits(&graph->AssignedBitmap); + + allocator->Vtbl->FreePointer(allocator, PPV(&graph->Values)); + + if (collisions) { + return PH_E_COLLISIONS_ENCOUNTERED_DURING_GRAPH_VERIFICATION; + } + + if (number_of_assignments != graph->NumberOfKeys) { + return PH_E_NUM_ASSIGNMENTS_NOT_EQUAL_TO_NUM_KEYS_DURING_GRAPH_VERIFICATION; + } + + return S_OK; +} + +template +HRESULT +hash_keys_dispatch_impl(PGRAPH graph, ULONG number_of_keys, PKEY keys) +{ + return hash_keys_impl, KeyType>( + graph, number_of_keys, keys + ); +} + +template +HRESULT +add_keys_dispatch_impl(PGRAPH graph, ULONG number_of_keys, PKEY keys) +{ + return add_keys_impl, KeyType>( + graph, number_of_keys, keys + ); +} + +template +HRESULT +verify_dispatch_impl(PGRAPH graph) +{ + return verify_impl, KeyType>(graph); +} + +template +HRESULT +index_dispatch_impl(PPERFECT_HASH_TABLE table, ULONG key, PULONG index) +{ + return index_impl, KeyType>(table, key, index); +} + +template +HRESULT +dispatch_storage_width( + PGRAPH graph, + Fn &&fn + ) +{ + auto *table = graph->Context->Table; + switch (table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: + return fn.template operator()(); + case 2: + return fn.template operator()(); + case 4: + return fn.template operator()(); + default: + return PH_E_NOT_IMPLEMENTED; + } +} + +template +HRESULT +dispatch_key_width( + PGRAPH graph, + Fn &&fn + ) +{ + auto *table = graph->Context->Table; + switch (table->GraphImpl4EffectiveKeySizeInBytes) { + case 1: + return dispatch_storage_width(graph, fn); + case 2: + return dispatch_storage_width(graph, fn); + case 4: + return dispatch_storage_width(graph, fn); + default: + return PH_E_NOT_IMPLEMENTED; + } +} + +template +HRESULT +dispatch_table_width( + PPERFECT_HASH_TABLE table, + Fn &&fn + ) +{ + switch (table->GraphImpl4EffectiveKeySizeInBytes) { + case 1: + switch (table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: return fn.template operator()(); + case 2: return fn.template operator()(); + case 4: return fn.template operator()(); + default: return PH_E_NOT_IMPLEMENTED; + } + case 2: + switch (table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: return fn.template operator()(); + case 2: return fn.template operator()(); + case 4: return fn.template operator()(); + default: return PH_E_NOT_IMPLEMENTED; + } + case 4: + switch (table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: return fn.template operator()(); + case 2: return fn.template operator()(); + case 4: return fn.template operator()(); + default: return PH_E_NOT_IMPLEMENTED; + } + default: + return PH_E_NOT_IMPLEMENTED; + } +} + +} // namespace + +extern "C" +HRESULT +NTAPI +GraphHashKeys4( + PGRAPH Graph, + ULONG NumberOfKeys, + PKEY Keys + ) +{ + auto dispatch = [=]() -> HRESULT { + return hash_keys_dispatch_impl( + Graph, NumberOfKeys, Keys + ); + }; + + return dispatch_key_width(Graph, dispatch); +} + +extern "C" +HRESULT +NTAPI +GraphAddKeys4( + PGRAPH Graph, + ULONG NumberOfKeys, + PKEY Keys + ) +{ + auto dispatch = [=]() -> HRESULT { + return add_keys_dispatch_impl( + Graph, NumberOfKeys, Keys + ); + }; + + return dispatch_key_width(Graph, dispatch); +} + +extern "C" +HRESULT +NTAPI +GraphHashKeysThenAdd4( + PGRAPH Graph, + ULONG NumberOfKeys, + PKEY Keys + ) +{ + HRESULT result = GraphHashKeys4(Graph, NumberOfKeys, Keys); + if (FAILED(result)) { + return result; + } + + switch (Graph->Context->Table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: + return add_hashed_keys_impl>(Graph, NumberOfKeys); + case 2: + return add_hashed_keys_impl>(Graph, NumberOfKeys); + case 4: + return add_hashed_keys_impl>(Graph, NumberOfKeys); + default: + return PH_E_NOT_IMPLEMENTED; + } +} + +extern "C" +HRESULT +NTAPI +GraphIsAcyclic4( + PGRAPH Graph + ) +{ + switch (Graph->Context->Table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: + return is_acyclic_impl>(Graph); + case 2: + return is_acyclic_impl>(Graph); + case 4: + return is_acyclic_impl>(Graph); + default: + return PH_E_NOT_IMPLEMENTED; + } +} + +extern "C" +HRESULT +NTAPI +GraphAssign4( + PGRAPH Graph + ) +{ + switch (Graph->Context->Table->TableInfoOnDisk->AssignedElementSizeInBytes) { + case 1: + return assign_impl>(Graph); + case 2: + return assign_impl>(Graph); + case 4: + return assign_impl>(Graph); + default: + return PH_E_NOT_IMPLEMENTED; + } +} + +extern "C" +HRESULT +NTAPI +GraphVerify4( + PGRAPH Graph + ) +{ + auto dispatch = [=]() -> HRESULT { + return verify_dispatch_impl(Graph); + }; + + return dispatch_key_width(Graph, dispatch); +} + +extern "C" +HRESULT +NTAPI +PerfectHashTableIndexImpl4Chm01( + PPERFECT_HASH_TABLE Table, + ULONG Key, + PULONG Index + ) +{ + // + // The table Index() entry point receives keys in the table's 32-bit + // domain. Loaded downsized-64 callers must apply the outer downsize + // bitmap first; this implementation then applies any GraphImpl4 inner + // compact-key metadata before hashing. + // + auto dispatch = [=]() -> HRESULT { + return index_dispatch_impl(Table, Key, Index); + }; + + return dispatch_table_width(Table, dispatch); +} + +extern "C" +HRESULT +NTAPI +PerfectHashTableIndexImpl4EffectiveKeyChm01( + PPERFECT_HASH_TABLE Table, + ULONG Key, + PULONG Index + ) +{ + auto dispatch = [=]() -> HRESULT { + return index_effective_key_dispatch_impl( + Table, + Key, + Index + ); + }; + + return dispatch_table_width(Table, dispatch); +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/GraphImpl4.h b/src/PerfectHash/GraphImpl4.h new file mode 100644 index 00000000..08ebe7a3 --- /dev/null +++ b/src/PerfectHash/GraphImpl4.h @@ -0,0 +1,41 @@ +/*++ + +Copyright (c) 2026 Trent Nelson + +Module Name: + + GraphImpl4.h + +Abstract: + + Private declarations for the experimental C++ graph implementation used by + the CPU solver backend. + +--*/ + +#pragma once + +#include "stdafx.h" + +EXTERN_C HRESULT NTAPI GraphAddKeys4(PGRAPH Graph, ULONG NumberOfKeys, PKEY Keys); +EXTERN_C HRESULT NTAPI GraphHashKeys4(PGRAPH Graph, ULONG NumberOfKeys, PKEY Keys); +EXTERN_C HRESULT NTAPI GraphHashKeysThenAdd4(PGRAPH Graph, + ULONG NumberOfKeys, + PKEY Keys); +EXTERN_C HRESULT NTAPI GraphIsAcyclic4(PGRAPH Graph); +EXTERN_C HRESULT NTAPI GraphAssign4(PGRAPH Graph); +EXTERN_C HRESULT NTAPI GraphVerify4(PGRAPH Graph); +EXTERN_C HRESULT NTAPI PerfectHashTableIndexImpl4Chm01(PPERFECT_HASH_TABLE Table, + ULONG Key, + PULONG Index); +// +// The Key argument must already be the GraphImpl4 effective key. For loaded +// downsized 64-bit tables, callers must pass the result of the persisted +// composed outer-bitmap ExtractBits64 step, not the original raw input key. +// +EXTERN_C HRESULT NTAPI +PerfectHashTableIndexImpl4EffectiveKeyChm01(PPERFECT_HASH_TABLE Table, + ULONG Key, + PULONG Index); + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHash.vcxproj b/src/PerfectHash/PerfectHash.vcxproj index 7fbc1bee..bf694393 100644 --- a/src/PerfectHash/PerfectHash.vcxproj +++ b/src/PerfectHash/PerfectHash.vcxproj @@ -138,6 +138,7 @@ + @@ -243,6 +244,7 @@ + @@ -272,6 +274,7 @@ + diff --git a/src/PerfectHash/PerfectHash.vcxproj.filters b/src/PerfectHash/PerfectHash.vcxproj.filters index 0b2b312a..d9d56dec 100644 --- a/src/PerfectHash/PerfectHash.vcxproj.filters +++ b/src/PerfectHash/PerfectHash.vcxproj.filters @@ -130,6 +130,9 @@ Source Files + + Source Files + Source Files @@ -343,6 +346,9 @@ Source Files + + Source Files + Source Files @@ -720,6 +726,9 @@ Private Header Files + + Private Header Files + Private Header Files diff --git a/src/PerfectHash/PerfectHashConstants.h b/src/PerfectHash/PerfectHashConstants.h index 420c0a09..1889052c 100644 --- a/src/PerfectHash/PerfectHashConstants.h +++ b/src/PerfectHash/PerfectHashConstants.h @@ -156,9 +156,17 @@ extern const STRING CTypeNames[]; extern const STRING NtTypeNames[]; // -// Hacky forward-decl of the 16-bit assigned index impl for Chm01. +// Hacky forward-decls of the specialized assigned-width index impls for Chm01. // +EXTERN_C +HRESULT +NTAPI +PerfectHashTableIndexImpl4Chm01( + _In_ PPERFECT_HASH_TABLE Table, + _In_ ULONG Key, + _Out_ PULONG Index + ); extern PERFECT_HASH_TABLE_INDEX PerfectHashTableIndex16ImplChm01; // @@ -207,7 +215,14 @@ CompletePerfectHashTableInitialization( // accordingly. // - if ((Table->State.UsingAssigned16 != FALSE) && + if ((Table->State.UsingAssigned8 != FALSE) && + (AlgorithmId == PerfectHashChm01AlgorithmId)) { + + Vtbl->Index = PerfectHashTableIndexImpl4Chm01; + Vtbl->SlowIndex = NULL; + Vtbl->FastIndex = NULL; + + } else if ((Table->State.UsingAssigned16 != FALSE) && (AlgorithmId == PerfectHashChm01AlgorithmId)) { Vtbl->Index = PerfectHashTableIndex16ImplChm01; diff --git a/src/PerfectHash/PerfectHashErrors.mc b/src/PerfectHash/PerfectHashErrors.mc index f226aafd..3daeb030 100644 --- a/src/PerfectHash/PerfectHashErrors.mc +++ b/src/PerfectHash/PerfectHashErrors.mc @@ -669,7 +669,7 @@ Table Create Flags: solving: - Algorithm is Chm01. - - GraphImpl is 3. + - GraphImpl is 3 or 4. - Number of vertices is <= 65,534 (i.e. MAX_USHORT-1). This provides significant performance improvements, which is why it's @@ -731,7 +731,7 @@ Table Compile Flags: Table Create Parameters: - --GraphImpl=1|2|3 [default: 3] + --GraphImpl=1|2|3|4 [default: 3] Selects the backend version of the graph assignment step. Version 1 matches the original CHM algorithm, version 2 is faster and was derived @@ -3979,7 +3979,7 @@ Severity=Fail Facility=ITF SymbolicName=PH_E_INVALID_GRAPH_IMPL Language=English -Invalid value for --GraphImpl. Valid values: 1, 2, 3. (Default: 3.) +Invalid value for --GraphImpl. Valid values: 1, 2, 3, 4. (Default: 3.) . MessageId=0x3c2 diff --git a/src/PerfectHash/PerfectHashOnlineCompile.c b/src/PerfectHash/PerfectHashOnlineCompile.c index 552f2c35..4585f8d7 100644 --- a/src/PerfectHash/PerfectHashOnlineCompile.c +++ b/src/PerfectHash/PerfectHashOnlineCompile.c @@ -70,6 +70,13 @@ Return Value: CompileFlags.AsULong = 0; } + if (Table->GraphImpl == 4 && + (Table->State.UsingAssigned8 || + (ARGUMENT_PRESENT(Table->TableInfoOnDisk) && + Table->TableInfoOnDisk->AssignedElementSizeInBytes == 1))) { + return PH_E_NOT_IMPLEMENTED; + } + // // Ensure the JIT path is active. // diff --git a/src/PerfectHash/PerfectHashOnlineJit.c b/src/PerfectHash/PerfectHashOnlineJit.c index 2fc0cb87..fcdd0a30 100644 --- a/src/PerfectHash/PerfectHashOnlineJit.c +++ b/src/PerfectHash/PerfectHashOnlineJit.c @@ -14,6 +14,7 @@ Module Name: --*/ #include "stdafx.h" +#include "GraphImpl4.h" #include #include @@ -29,6 +30,7 @@ struct PH_ONLINE_JIT_CONTEXT { struct PH_ONLINE_JIT_TABLE { PPERFECT_HASH_TABLE Table; PPERFECT_HASH_TABLE_JIT_INTERFACE JitInterface; + BOOLEAN Compiled; }; static @@ -210,6 +212,9 @@ PhCompileOnlineJitBackend( } CompileFlags.JitMaxIsa = (ULONG)JitMaxIsa; + if (TableInfo->OriginalKeySizeInBytes > sizeof(ULONG)) { + CompileFlags.JitIndex64 = TRUE; + } return Context->Online->Vtbl->CompileTable(Context->Online, Table->Table, @@ -382,6 +387,9 @@ PhOnlineJitCreateTable64( HRESULT Result; PERFECT_HASH_KEYS_LOAD_FLAGS KeysLoadFlags = {0}; PERFECT_HASH_TABLE_CREATE_FLAGS TableCreateFlags = {0}; + PERFECT_HASH_TABLE_CREATE_PARAMETER TableCreateParam = {0}; + PERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParams = {0}; + PPERFECT_HASH_TABLE_CREATE_PARAMETERS TableCreateParamsPointer = NULL; PERFECT_HASH_HASH_FUNCTION_ID HashFunctionId; PPERFECT_HASH_TABLE Table = NULL; PH_ONLINE_JIT_TABLE *Wrapper = NULL; @@ -408,6 +416,16 @@ PhOnlineJitCreateTable64( TableCreateFlags.Quiet = TRUE; TableCreateFlags.DoNotTryUseHash16Impl = TRUE; + if (IsGoodPerfectHashHashFunctionId(HashFunctionId)) { + TableCreateParam.Id = TableCreateParameterGraphImplId; + TableCreateParam.AsULong = 4; + + TableCreateParams.SizeOfStruct = sizeof(TableCreateParams); + TableCreateParams.NumberOfElements = 1; + TableCreateParams.Params = &TableCreateParam; + TableCreateParamsPointer = &TableCreateParams; + } + Result = Context->Online->Vtbl->CreateTableFromKeys( Context->Online, PerfectHashChm01AlgorithmId, @@ -418,7 +436,7 @@ PhOnlineJitCreateTable64( (PVOID)Keys, &KeysLoadFlags, &TableCreateFlags, - NULL, + TableCreateParamsPointer, &Table ); if (FAILED(Result)) { @@ -500,6 +518,7 @@ PhOnlineJitCompileTableEx( // PhReleaseOnlineJitInterface(Table); + Table->Compiled = FALSE; if (!IsValidPerfectHashJitMaxIsaId((PERFECT_HASH_JIT_MAX_ISA_ID)JitMaxIsa)) { return E_INVALIDARG; @@ -527,6 +546,7 @@ PhOnlineJitCompileTableEx( VectorWidth, JitMaxIsa); if (SUCCEEDED(Result)) { + Table->Compiled = TRUE; if (ARGUMENT_PRESENT(EffectiveBackend)) { *EffectiveBackend = PhOnlineJitBackendLlvmJit; } @@ -582,6 +602,7 @@ PhOnlineJitCompileTableEx( CandidateWidths[Index], JitMaxIsa); if (SUCCEEDED(Result)) { + Table->Compiled = TRUE; if (ARGUMENT_PRESENT(EffectiveBackend)) { *EffectiveBackend = PhOnlineJitBackendRawDogJit; } @@ -613,6 +634,7 @@ PhOnlineJitCompileTableEx( VectorWidth, JitMaxIsa); if (SUCCEEDED(Result)) { + Table->Compiled = TRUE; if (ARGUMENT_PRESENT(EffectiveBackend)) { *EffectiveBackend = PhOnlineJitBackendLlvmJit; } @@ -663,7 +685,9 @@ PhOnlineJitIndex64( uint32_t *Index ) { + HRESULT Result; PTABLE_INFO_ON_DISK TableInfo; + ULONGLONG DownsizeBitmap; ULONG DownsizedKey; if (!ARGUMENT_PRESENT(Table) || @@ -681,7 +705,41 @@ PhOnlineJitIndex64( TableInfo->KeySizeInBytes > sizeof(uint32_t)) { return PH_E_NOT_IMPLEMENTED; } - DownsizedKey = (ULONG)ExtractBits64((ULONGLONG)Key, Table->Table->DownsizeBitmap); + + if (Table->Compiled) { + Result = PhEnsureOnlineJitInterface(Table); + if (FAILED(Result)) { + return Result; + } + + return Table->JitInterface->Vtbl->Index64( + Table->JitInterface, + (ULONGLONG)Key, + (PULONG)Index + ); + } + + // + // The online JIT wrapper owns only the table, not the original keys object. + // Use the table metadata as the source of truth; GraphImpl4 stores a + // composed outer+inner bitmap here, whereas a keys bitmap would be outer-only. + // Non-GraphImpl4 downsized tables with captured metadata continue to use + // the regular table Index() path below. Non-downsized 64-bit tables and + // tables without captured downsize metadata are intentionally unsupported + // by Index64. + // + if (!Table->Table->State.DownsizeMetadataValid) { + return PH_E_NOT_IMPLEMENTED; + } + + DownsizeBitmap = Table->Table->DownsizeBitmap; + DownsizedKey = (ULONG)ExtractBits64((ULONGLONG)Key, DownsizeBitmap); + + if (Table->Table->GraphImpl == 4) { + return PerfectHashTableIndexImpl4EffectiveKeyChm01(Table->Table, + DownsizedKey, + (PULONG)Index); + } return Table->Table->Vtbl->Index(Table->Table, DownsizedKey, (PULONG)Index); } diff --git a/src/PerfectHash/PerfectHashTable.h b/src/PerfectHash/PerfectHashTable.h index 33d516d5..c02befa0 100644 --- a/src/PerfectHash/PerfectHashTable.h +++ b/src/PerfectHash/PerfectHashTable.h @@ -59,6 +59,13 @@ typedef union _PERFECT_HASH_TABLE_STATE { ULONG AutoResize:1; + // + // When set, indicates the 8-bit graph/assigned infrastructure is + // active. + // + + ULONG UsingAssigned8:1; + // // When set, indicates the 16-bit hash/assigned infrastructure is // active. @@ -76,7 +83,7 @@ typedef union _PERFECT_HASH_TABLE_STATE { // Unused bits. // - ULONG Unused:26; + ULONG Unused:25; }; LONG AsLong; ULONG AsULong; @@ -552,14 +559,23 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_TABLE { // // If keys were downsized during load, capture the metadata required to - // downsize 64-bit keys at runtime (e.g. for JIT Index64 routines). + // downsize 64-bit keys at runtime (e.g. for JIT Index64 routines). For + // non-GraphImpl4 tables, DownsizeBitmap is the outer 64-bit key bitmap. For + // GraphImpl4 tables, DownsizeBitmap is the composed original-key-to-effective + // key bitmap: outer 64-bit key downsize plus inner compact-key extraction. // ULONGLONG DownsizeBitmap; ULONGLONG DownsizeShiftedMask; BYTE DownsizeTrailingZeros; BYTE DownsizeContiguous; - BYTE DownsizePadding[6]; + BYTE DownsizePadding[2]; + ULONG GraphImpl4EffectiveKeySizeInBytes; + ULONG GraphImpl4KeyDownsizeBitmap; + ULONG GraphImpl4KeyDownsizeShiftedMask; + BYTE GraphImpl4KeyDownsizeTrailingZeros; + BYTE GraphImpl4KeyDownsizeContiguous; + BYTE GraphImpl4Padding[2]; // // If we're in "find best coverage" mode, a pointer to an assigned memory @@ -598,6 +614,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_TABLE { // union { + PBYTE Assigned8; PULONG Assigned; PUSHORT Assigned16; PULONG TableData; @@ -1115,6 +1132,12 @@ extern PERFECT_HASH_TABLE_GET_FILE PerfectHashTableGetFile; goto Error; \ } +HRESULT +PerfectHashTableValidateCompileState( + _In_ PPERFECT_HASH_TABLE Table, + _In_ PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags + ); + // // Declare the functions. // diff --git a/src/PerfectHash/PerfectHashTableCompile.c b/src/PerfectHash/PerfectHashTableCompile.c index 8edfac2d..6cabfb41 100644 --- a/src/PerfectHash/PerfectHashTableCompile.c +++ b/src/PerfectHash/PerfectHashTableCompile.c @@ -61,7 +61,11 @@ Return Value: PH_E_TABLE_LOCKED - The table is locked. - PH_E_TABLE_NOT_CREATED - The table has not been created. + PH_E_TABLE_NOT_CREATED - The table has neither been created nor loaded. + + PH_E_NOT_IMPLEMENTED - The table was loaded but does not satisfy the + loaded-table JIT preconditions, or the requested compile path is not + implemented. PH_E_INVALID_CPU_ARCH_ID - Invalid CPU architecture ID. @@ -118,9 +122,10 @@ Return Value: return PH_E_TABLE_LOCKED; } - if (!Table->Flags.Created) { + Result = PerfectHashTableValidateCompileState(Table, TableCompileFlags); + if (FAILED(Result)) { ReleasePerfectHashTableLockExclusive(Table); - return PH_E_TABLE_NOT_CREATED; + return Result; } if (TableCompileFlags.Jit) { diff --git a/src/PerfectHash/PerfectHashTableCompileCompat.c b/src/PerfectHash/PerfectHashTableCompileCompat.c index 20855b81..60b70af6 100644 --- a/src/PerfectHash/PerfectHashTableCompileCompat.c +++ b/src/PerfectHash/PerfectHashTableCompileCompat.c @@ -57,9 +57,11 @@ Return Value: PH_E_TABLE_LOCKED - The table is locked. - PH_E_TABLE_NOT_CREATED - The table has not been created. + PH_E_TABLE_NOT_CREATED - The table has neither been created nor loaded. - PH_E_NOT_IMPLEMENTED - Non-JIT compilation is not available. + PH_E_NOT_IMPLEMENTED - The table was loaded but does not satisfy the + loaded-table JIT preconditions, or non-JIT compilation is not + available. --*/ { @@ -80,9 +82,10 @@ Return Value: return PH_E_TABLE_LOCKED; } - if (!Table->Flags.Created) { + Result = PerfectHashTableValidateCompileState(Table, TableCompileFlags); + if (FAILED(Result)) { ReleasePerfectHashTableLockExclusive(Table); - return PH_E_TABLE_NOT_CREATED; + return Result; } if (TableCompileFlags.Jit) { @@ -101,4 +104,4 @@ Return Value: #endif // PH_WINDOWS -// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : \ No newline at end of file +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashTableCompileState.c b/src/PerfectHash/PerfectHashTableCompileState.c new file mode 100644 index 00000000..f45373f0 --- /dev/null +++ b/src/PerfectHash/PerfectHashTableCompileState.c @@ -0,0 +1,66 @@ +/*++ + +Copyright (c) 2026 Trent Nelson + +Module Name: + + PerfectHashTableCompileState.c + +Abstract: + + This module validates table state before compile dispatch. + +--*/ + +#include "stdafx.h" + +/*++ + +Routine Description: + + Validates that a table is in a state that can enter compile dispatch. A + created table may compile through the existing create-time paths. A loaded + GraphImpl4 table is admitted only when both Jit and JitBackendLlvm are set. + Loaded GraphImpl4 tables using RawDog, an unspecified backend, or + JitBackendLlvm without Jit return PH_E_NOT_IMPLEMENTED. Loaded + non-GraphImpl4 tables also return PH_E_NOT_IMPLEMENTED. Tables that are + neither created nor loaded return PH_E_TABLE_NOT_CREATED. + +--*/ + +_Use_decl_annotations_ +HRESULT +PerfectHashTableValidateCompileState( + PPERFECT_HASH_TABLE Table, + PERFECT_HASH_TABLE_COMPILE_FLAGS TableCompileFlags + ) +{ + if (Table->Flags.Created) { + return S_OK; + } + + // + // Loaded GraphImpl4 tables are admitted only when callers explicitly set + // both Jit and JitBackendLlvm. Backend-specific flags do not imply Jit. + // The IgnoreAssembly flag describes the non-JIT generated-source path, so + // it is ignored for loaded-table JIT. LLVM is currently the only backend + // admitted for loaded GraphImpl4 tables; RawDog and unspecified backends + // are rejected here before reaching backend-specific dispatch. + // + + if (Table->Flags.Loaded && Table->GraphImpl == 4) { + if (TableCompileFlags.Jit && TableCompileFlags.JitBackendLlvm) { + return S_OK; + } + + return PH_E_NOT_IMPLEMENTED; + } + + if (Table->Flags.Loaded) { + return PH_E_NOT_IMPLEMENTED; + } + + return PH_E_TABLE_NOT_CREATED; +} + +// vim:set ts=8 sw=4 sts=4 tw=80 expandtab : diff --git a/src/PerfectHash/PerfectHashTableCreate.c b/src/PerfectHash/PerfectHashTableCreate.c index 53191fd2..8c24c5d9 100644 --- a/src/PerfectHash/PerfectHashTableCreate.c +++ b/src/PerfectHash/PerfectHashTableCreate.c @@ -295,6 +295,11 @@ Return Value: Table->DownsizeShiftedMask = 0; Table->DownsizeTrailingZeros = 0; Table->DownsizeContiguous = FALSE; + Table->GraphImpl4EffectiveKeySizeInBytes = 0; + Table->GraphImpl4KeyDownsizeBitmap = 0; + Table->GraphImpl4KeyDownsizeShiftedMask = 0; + Table->GraphImpl4KeyDownsizeTrailingZeros = 0; + Table->GraphImpl4KeyDownsizeContiguous = FALSE; if (KeysWereDownsized(Keys)) { Table->DownsizeBitmap = Keys->DownsizeBitmap; @@ -522,7 +527,9 @@ Return Value: Table->Flags.Created = TRUE; Table->Flags.Loaded = FALSE; - if (Table->State.UsingAssigned16) { + if (Table->State.UsingAssigned8) { + Table->Flags.AssignedElementSizeInBits = 1; // 8 bits + } else if (Table->State.UsingAssigned16) { Table->Flags.AssignedElementSizeInBits = 2; // 16 bits } else { Table->Flags.AssignedElementSizeInBits = 4; // 32 bits @@ -1054,10 +1061,20 @@ Return Value: // Validate GraphImpl. // - if (GraphImpl != 1 && GraphImpl != 2 && GraphImpl != 3) { + if (GraphImpl != 1 && GraphImpl != 2 && GraphImpl != 3 && GraphImpl != 4) { Result = PH_E_INVALID_GRAPH_IMPL; goto Error; } + + if (GraphImpl == 4) { + if (FindBestMemoryCoverage(Context) || + !IsGoodPerfectHashHashFunctionId(Table->HashFunctionId) || + Table->Keys->KeySizeInBytes > sizeof(ULONG)) { + Result = PH_E_NOT_IMPLEMENTED; + goto Error; + } + } + Table->GraphImpl = GraphImpl; // diff --git a/src/PerfectHash/PerfectHashTableLoad.c b/src/PerfectHash/PerfectHashTableLoad.c index 4583e8e6..74359353 100644 --- a/src/PerfectHash/PerfectHashTableLoad.c +++ b/src/PerfectHash/PerfectHashTableLoad.c @@ -349,6 +349,11 @@ Return Value: Table->DownsizeShiftedMask = 0; Table->DownsizeTrailingZeros = 0; Table->DownsizeContiguous = FALSE; + Table->GraphImpl4EffectiveKeySizeInBytes = 0; + Table->GraphImpl4KeyDownsizeBitmap = 0; + Table->GraphImpl4KeyDownsizeShiftedMask = 0; + Table->GraphImpl4KeyDownsizeTrailingZeros = 0; + Table->GraphImpl4KeyDownsizeContiguous = FALSE; if (ARGUMENT_PRESENT(Keys) && KeysWereDownsized(Keys)) { Table->DownsizeBitmap = Keys->DownsizeBitmap; @@ -462,14 +467,16 @@ Return Value: Table->TableFile = File; // - // We can determine the expected file size by multipling the number of - // table elements by the key size; both of which are available in the - // :Info header. + // We can determine the expected file size by multiplying the number of + // table elements by the assigned element size; both of which are available + // in the :Info header. // ExpectedEndOfFile.QuadPart = ( NumberOfTableElements * - TableInfoOnDisk->KeySizeInBytes + (TableInfoOnDisk->AssignedElementSizeInBytes ? + TableInfoOnDisk->AssignedElementSizeInBytes : + TableInfoOnDisk->KeySizeInBytes) ); // @@ -520,6 +527,8 @@ Return Value: Table->AlgorithmId = AlgorithmId; Table->MaskFunctionId = TableInfoOnDisk->MaskFunctionId; Table->HashFunctionId = TableInfoOnDisk->HashFunctionId; + Table->State.UsingAssigned8 = + (TableInfoOnDisk->AssignedElementSizeInBytes == 1); Table->State.UsingAssigned16 = TableInfoOnDisk->Flags.UsingAssigned16; // @@ -551,6 +560,8 @@ Return Value: if (TableInfoOnDisk->AssignedElementSizeInBytes) { Table->Flags.AssignedElementSizeInBits = TableInfoOnDisk->AssignedElementSizeInBytes; + } else if (Table->State.UsingAssigned8) { + Table->Flags.AssignedElementSizeInBits = 1; // 8 bits } else if (Table->State.UsingAssigned16) { Table->Flags.AssignedElementSizeInBits = 2; // 16 bits } else { diff --git a/src/PerfectHash/PerfectHashTableTest.c b/src/PerfectHash/PerfectHashTableTest.c index 79e8df5d..0a72f4c2 100644 --- a/src/PerfectHash/PerfectHashTableTest.c +++ b/src/PerfectHash/PerfectHashTableTest.c @@ -561,7 +561,8 @@ Return Value: // just PH_RAISE() instead. // - if (Table->State.UsingAssigned16 == FALSE) { + if (Table->State.UsingAssigned16 == FALSE && + Table->State.UsingAssigned8 == FALSE) { PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a2c12fe9..602a94f0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,55 @@ find_program(CARGO_EXECUTABLE cargo) if(NOT CARGO_EXECUTABLE) set(CARGO_EXECUTABLE "") endif() +find_program(MAKE_EXECUTABLE NAMES make gmake) +if(NOT MAKE_EXECUTABLE) + set(MAKE_EXECUTABLE "") +endif() + +set(GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS 0 2 5 9 17 23 31 40 55) +string(JOIN "," GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS_CSV + ${GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS}) +string(REPLACE "," ", " GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS_CPP + "${GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS_CSV}") +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/PerfectHashGraphImpl4TestConfig.h.in + ${CMAKE_CURRENT_BINARY_DIR}/PerfectHashGraphImpl4TestConfig.h + @ONLY +) + +set(CODEGEN_TEST_CMAKE_ARGS "") +if(CMAKE_GENERATOR) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_GENERATOR=${CMAKE_GENERATOR}") +endif() +if(CMAKE_GENERATOR_PLATFORM) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}") +endif() +if(CMAKE_GENERATOR_TOOLSET) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_GENERATOR_TOOLSET=${CMAKE_GENERATOR_TOOLSET}") +endif() +if(CMAKE_MAKE_PROGRAM) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}") +endif() +if(MAKE_EXECUTABLE) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_UNIX_MAKE_PROGRAM=${MAKE_EXECUTABLE}") +endif() +if(CMAKE_TOOLCHAIN_FILE) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") +endif() +if(CMAKE_C_COMPILER) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_C_COMPILER=${CMAKE_C_COMPILER}") +endif() +if(CMAKE_CXX_COMPILER) + list(APPEND CODEGEN_TEST_CMAKE_ARGS + "-DTEST_CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") +endif() add_executable(perfecthash_unit_tests PerfectHashArgvTests.cpp @@ -30,6 +79,11 @@ target_link_libraries(perfecthash_unit_tests GTest::gtest_main ) +target_include_directories(perfecthash_unit_tests + PRIVATE + ${CMAKE_CURRENT_BINARY_DIR} +) + perfecthash_apply_common_definitions(perfecthash_unit_tests) target_compile_definitions(perfecthash_unit_tests PRIVATE PERFECTHASH_TEST_ROOT_DIR="${PERFECTHASH_ROOT_DIR}" @@ -44,6 +98,9 @@ set_target_properties(perfecthash_unit_tests PROPERTIES if(PERFECTHASH_HAS_RAWDOG_JIT) target_compile_definitions(perfecthash_unit_tests PRIVATE PH_HAS_RAWDOG_JIT) endif() +if(TARGET PerfectHashLLVM) + add_dependencies(perfecthash_unit_tests PerfectHashLLVM) +endif() add_executable(perfecthash_online_jit_invalid_env PerfectHashOnlineJitInvalidEnv.cpp @@ -80,6 +137,9 @@ set_target_properties(perfecthash_benchmarks PROPERTIES if(PERFECTHASH_HAS_RAWDOG_JIT) target_compile_definitions(perfecthash_benchmarks PRIVATE PH_HAS_RAWDOG_JIT) endif() +if(TARGET PerfectHashLLVM) + add_dependencies(perfecthash_benchmarks PerfectHashLLVM) +endif() include(GoogleTest) gtest_discover_tests(perfecthash_unit_tests @@ -102,6 +162,42 @@ add_test( ) set_tests_properties(perfecthash.fast.unit PROPERTIES LABELS fast) +if(PERFECTHASH_HAS_LLVM OR PERFECTHASH_HAS_RAWDOG_JIT) + add_test( + NAME perfecthash.bench.graphimpl4.jit + COMMAND ${CMAKE_COMMAND} -E env + LD_LIBRARY_PATH=$ + $ + --keys=256 + --iterations=1 + --loops=1 + --graph-impl=4 + --jit-backend=rawdog + --no-std-map-baselines + ) + + add_test( + NAME perfecthash.bench.graphimpl4.jit.assigned16 + COMMAND ${CMAKE_COMMAND} -E env + LD_LIBRARY_PATH=$ + $ + --keys=256 + --iterations=1 + --loops=1 + --graph-impl=4 + --jit-backend=rawdog + --allow-assigned16 + --no-std-map-baselines + ) + + set_tests_properties( + perfecthash.bench.graphimpl4.jit + perfecthash.bench.graphimpl4.jit.assigned16 + PROPERTIES + LABELS fast + ) +endif() + set(TEST_KEYS_FILE "${PERFECTHASH_ROOT_DIR}/keys/HologramWorld-31016.keys") set(TEST_KEYS_DIR "${PERFECTHASH_ROOT_DIR}/keys") set(TEST_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/cli-out") @@ -141,6 +237,7 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) add_test( NAME perfecthash.cli.codegen COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen @@ -155,6 +252,7 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) add_test( NAME perfecthash.cli.codegen.mulshrolate1rx COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-mulshrolate1rx @@ -169,6 +267,7 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) add_test( NAME perfecthash.cli.codegen.mulshrolate2rx COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-mulshrolate2rx @@ -180,9 +279,35 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) -P ${CMAKE_CURRENT_SOURCE_DIR}/run_cli_codegen_test.cmake ) + add_test( + NAME perfecthash.cli.codegen.graphimpl4 + COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} + -DTEST_EXE=$ + -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-graphimpl4 + -DTEST_PYTHON=${Python3_EXECUTABLE} + -DTEST_CARGO=${CARGO_EXECUTABLE} + -DTEST_BUILD_CONFIG=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/run_cli_graphimpl4_codegen_test.cmake + ) + + add_test( + NAME perfecthash.cli.codegen.graphimpl4-downsized64 + COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} + -DTEST_EXE=$ + -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-graphimpl4-downsized64 + -DTEST_PYTHON=${Python3_EXECUTABLE} + -DTEST_CARGO=${CARGO_EXECUTABLE} + -DTEST_BUILD_CONFIG=$ + -DTEST_GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS=${GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS_CSV} + -P ${CMAKE_CURRENT_SOURCE_DIR}/run_cli_graphimpl4_downsized64_codegen_test.cmake + ) + add_test( NAME perfecthash.cli.codegen.mulshrolate3rx COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-mulshrolate3rx @@ -197,6 +322,7 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) add_test( NAME perfecthash.cli.codegen.mulshrolate4rx COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-mulshrolate4rx @@ -211,6 +337,7 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) add_test( NAME perfecthash.cli.codegen.multiplyshiftrx COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/codegen-multiplyshiftrx @@ -247,6 +374,7 @@ if(TARGET PerfectHashCreateExe AND TARGET PerfectHashBulkCreateExe) add_test( NAME perfecthash.fast.cli.codegen COMMAND ${CMAKE_COMMAND} + ${CODEGEN_TEST_CMAKE_ARGS} -DTEST_EXE=$ -DTEST_KEYS=${TEST_KEYS_FILE} -DTEST_OUTPUT=${TEST_OUTPUT_DIR}/fast-codegen diff --git a/tests/PerfectHashGraphImpl4TestConfig.h.in b/tests/PerfectHashGraphImpl4TestConfig.h.in new file mode 100644 index 00000000..cbee1fc3 --- /dev/null +++ b/tests/PerfectHashGraphImpl4TestConfig.h.in @@ -0,0 +1,4 @@ +#pragma once + +#define PERFECTHASH_GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS \ + @GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS_CPP@ diff --git a/tests/PerfectHashOnlineJitTests.cpp b/tests/PerfectHashOnlineJitTests.cpp index 27104a0d..b7019ec6 100644 --- a/tests/PerfectHashOnlineJitTests.cpp +++ b/tests/PerfectHashOnlineJitTests.cpp @@ -106,6 +106,11 @@ TEST(PerfectHashOnlineJitTests, CreateTable64AndIndex64) { ASSERT_NE(raw_context, nullptr); context.reset(raw_context); + // + // Mulshrolate3RX is in the curated good hash set, so CreateTable64 + // intentionally selects GraphImpl4 for the compact-key JIT backend. + // + ASSERT_GE(PhOnlineJitCreateTable64(context.get(), PhOnlineJitHashMulshrolate3RX, keys.data(), @@ -115,6 +120,14 @@ TEST(PerfectHashOnlineJitTests, CreateTable64AndIndex64) { ASSERT_NE(raw_table, nullptr); table.reset(raw_table); + std::vector fallback_indexes; + fallback_indexes.reserve(keys.size()); + for (auto key : keys) { + uint32_t index = 0; + ASSERT_GE(PhOnlineJitIndex64(table.get(), key, &index), 0); + fallback_indexes.push_back(index); + } + auto result = PhOnlineJitCompileTableEx(context.get(), table.get(), PhOnlineJitBackendRawDogJit, @@ -137,9 +150,10 @@ TEST(PerfectHashOnlineJitTests, CreateTable64AndIndex64) { std::unordered_set seen; seen.reserve(keys.size()); - for (auto key : keys) { + for (size_t i = 0; i < keys.size(); ++i) { uint32_t index = 0; - ASSERT_GE(PhOnlineJitIndex64(table.get(), key, &index), 0); + ASSERT_GE(PhOnlineJitIndex64(table.get(), keys[i], &index), 0); + EXPECT_EQ(fallback_indexes[i], index); ASSERT_TRUE(seen.insert(index).second); } } @@ -217,6 +231,34 @@ TEST(PerfectHashOnlineJitTests, Index64On32BitTableIsNotImplemented) { ASSERT_EQ(PhOnlineJitIndex64(table.get(), 1ULL, &index), PH_E_NOT_IMPLEMENTED); } +TEST(PerfectHashOnlineJitTests, Index64OnNonGraphImpl64BitTableUsesMetadata) { + auto context = std::unique_ptr( + nullptr, &PhOnlineJitClose); + auto table = std::unique_ptr( + nullptr, &PhOnlineJitReleaseTable); + PH_ONLINE_JIT_CONTEXT *raw_context = nullptr; + PH_ONLINE_JIT_TABLE *raw_table = nullptr; + auto keys = MakeDownsized64Keys(64); + + ASSERT_GE(PhOnlineJitOpen(&raw_context), 0); + context.reset(raw_context); + ASSERT_GE(PhOnlineJitCreateTable64(context.get(), + PhOnlineJitHashMultiplyShiftLR, + keys.data(), + static_cast(keys.size()), + &raw_table), + 0); + table.reset(raw_table); + + std::unordered_set seen; + seen.reserve(keys.size()); + for (auto key : keys) { + uint32_t index = 0; + ASSERT_GE(PhOnlineJitIndex64(table.get(), key, &index), 0); + ASSERT_TRUE(seen.insert(index).second); + } +} + TEST(PerfectHashOnlineJitTests, CreateTable64RejectsNonDownsizedKeys) { auto context = std::unique_ptr( nullptr, &PhOnlineJitClose); diff --git a/tests/PerfectHashOnlineTests.cpp b/tests/PerfectHashOnlineTests.cpp index fe665711..f52fdee0 100644 --- a/tests/PerfectHashOnlineTests.cpp +++ b/tests/PerfectHashOnlineTests.cpp @@ -2,24 +2,118 @@ #include +#include "PerfectHashGraphImpl4TestConfig.h" + #include +#include #include +#include #include +#include +#include #include +#include #include +#include #include #include +#ifdef PH_WINDOWS +#include +#else +#include +#endif + namespace { +namespace fs = std::filesystem; + +#ifdef PH_WINDOWS +#define PH_TEST_IID(Name) IID_##Name +#else +#define PH_TEST_IID(Name) &IID_##Name +#endif + +constexpr const char *kGraphImpl4Sparse32ReloadSeedsArg = + "--Seeds=0x8A696C6B,0x5CC6B6A7,0x608"; +constexpr const char *kGraphImpl4Downsized64ReloadSeedsArg = + "--Seeds=0xDDE48099,0x20C4AF8D,0x608"; +constexpr const char *kGraphImpl3Downsized64ReloadSeedsArg = + "--Seeds=0x9F5ABF20,0x0F97985A,0x608"; +// Deterministic MultiplyShiftR/And seeds for the exact reload fixtures below. +// Each fixture has its own pinned seed triple so failures identify the key +// shape that needs regeneration. Regenerate them if the solver path, key +// generators, or hash schedule changes and the guards fire. +// Regenerate by scanning i=1.. with Seed1 = 0x01234567 + i*0x9E3779B9, +// Seed2 = 0x89ABCDEF + i*0x85EBCA6B, and Seed3 = 0x608, then use the first +// successful PerfectHashCreate candidate for the matching key generator and +// GraphImpl. + +#define ASSERT_RELOAD_FIXTURE_CREATE_SUCCEEDED(Result, SeedsArg) \ + do { \ + if ((Result) == PH_I_SOLVE_TIMEOUT_EXPIRED) { \ + GTEST_SKIP() << "reload fixture seeds timed out on this host; " \ + << "regenerate with the documented seed scan if this " \ + << "is persistent: " << (SeedsArg); \ + } \ + ASSERT_NE((Result), PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION) \ + << "reload fixture seeds did not solve; regenerate using the " \ + << "documented seed scan: " << (SeedsArg); \ + ASSERT_GE((Result), 0) \ + << "reload fixture create failed for seeds: " << (SeedsArg); \ + } while (false) + +std::vector SeedValuesFromSeedsArg(const char *seeds_arg) { + constexpr const char *prefix = "--Seeds="; + std::string text(seeds_arg); + std::vector seeds; + + if (text.rfind(prefix, 0) != 0) { + ADD_FAILURE() << "Invalid seeds arg: " << text; + return seeds; + } + + size_t start = std::string(prefix).size(); + while (start < text.size()) { + size_t comma = text.find(',', start); + std::string token = text.substr( + start, + comma == std::string::npos ? std::string::npos : comma - start); + char *end = nullptr; + unsigned long value = std::strtoul(token.c_str(), &end, 0); + if (token.empty() || end == token.c_str() || *end != '\0') { + ADD_FAILURE() << "Invalid seed token in " << text << ": " << token; + return {}; + } + seeds.push_back(static_cast(value)); + if (comma == std::string::npos) { + break; + } + start = comma + 1; + } + + return seeds; +} + struct PerfectHashTableShim { PPERFECT_HASH_TABLE_VTBL Vtbl; }; -ULONGLONG BuildDownsizeMask(const std::vector &keys) { +extern "C" { +ULONGLONG PerfectHashComposeGraphImpl4DownsizeBitmap(ULONGLONG outer_bitmap, + ULONG inner_bitmap); +VOID PerfectHashComputeDownsizeMetadataFromBitmap(ULONGLONG bitmap, + PBOOLEAN metadata_valid, + PULONGLONG shifted_mask, + PBYTE trailing_zeros, + PBOOLEAN contiguous); +} + +template +ULONGLONG BuildDownsizeMask(const std::vector &keys) { ULONGLONG mask = 0; for (auto key : keys) { - mask |= key; + mask |= static_cast(key); } return mask; } @@ -41,6 +135,228 @@ ULONG DownsizeKey(ULONGLONG key, ULONGLONG mask) { return static_cast(result); } +template +ULONGLONG BuildDownsizedKeyMask(const std::vector &keys, + ULONGLONG downsize_mask) { + ULONGLONG mask = 0; + for (auto key : keys) { + mask |= DownsizeKey(static_cast(key), downsize_mask); + } + return mask; +} + +bool IsContiguousBitmap(ULONGLONG mask) { + if (mask == 0) { + return false; + } + + while ((mask & 1ull) == 0) { + mask >>= 1; + } + + while ((mask & 1ull) != 0) { + mask >>= 1; + } + + return mask == 0; +} + +void AssertCollisionFreeIndexes(const std::vector &indexes, + size_t number_of_keys, + const char *label) { + SCOPED_TRACE(label); + ASSERT_EQ(indexes.size(), number_of_keys); + + std::unordered_set seen; + seen.reserve(number_of_keys); + for (ULONG index : indexes) { + EXPECT_LT(index, number_of_keys); + EXPECT_TRUE(seen.insert(index).second); + } + EXPECT_EQ(seen.size(), number_of_keys); +} + +PERFECT_HASH_TABLE_COMPILE_FLAGS MakeLlvmJitCompileFlags() { + PERFECT_HASH_TABLE_COMPILE_FLAGS flags = {0}; + + // + // PerfectHashOnlineCompileTable also forces Jit before table dispatch. Set + // both flags explicitly so these tests document the loaded-table LLVM + // precondition and remain equivalent if routed through Table::Compile. + // + flags.Jit = TRUE; + flags.JitBackendLlvm = TRUE; + + return flags; +} + +template +ULONGLONG ExpandSparseBits(T value, const std::vector &bit_positions) { + ULONGLONG result = 0; + for (size_t index = 0; index < bit_positions.size(); ++index) { + if ((static_cast(value) & (1ull << index)) != 0) { + result |= (1ull << bit_positions[index]); + } + } + return result; +} + +std::vector MakeSparse32Keys(size_t count) { + const std::vector bit_positions = {0, 3, 7, 12, 16, 21, 25, 29, 31}; + std::vector keys; + keys.reserve(count); + + for (ULONG value = 1; keys.size() < count; ++value) { + keys.push_back(static_cast(ExpandSparseBits(value, bit_positions))); + } + + return keys; +} + +std::vector MakeSparseDownsized64KeysWithPositions( + size_t count, + const std::vector &bit_positions) { + std::vector keys; + keys.reserve(count); + + for (ULONG value = 1; keys.size() < count; ++value) { + keys.push_back(ExpandSparseBits(value, bit_positions)); + } + + return keys; +} + +std::vector MakeSparseDownsized64Keys(size_t count) { + const std::vector bit_positions = { + PERFECTHASH_GRAPHIMPL4_DOWNSIZED64_BIT_POSITIONS + }; + return MakeSparseDownsized64KeysWithPositions(count, bit_positions); +} + +template +void WriteBinaryKeys(const fs::path &path, const std::vector &keys) { + std::ofstream file(path, std::ios::binary | std::ios::trunc); + ASSERT_TRUE(file) << path; + file.write(reinterpret_cast(keys.data()), + static_cast(keys.size() * sizeof(T))); + ASSERT_TRUE(file.good()) << path; +} + +fs::path MakeTestDirectory(const char *name) { + auto ticks = std::chrono::steady_clock::now().time_since_epoch().count(); +#ifdef PH_WINDOWS + auto process_id = _getpid(); +#else + auto process_id = getpid(); +#endif + fs::path dir = fs::temp_directory_path() / + (std::string("perfecthash_") + name + "_" + + std::to_string(static_cast(process_id)) + "_" + + std::to_string(static_cast(ticks))); + fs::remove_all(dir); + fs::create_directories(dir); + return dir; +} + +class ScopedTestDirectory { + public: + explicit ScopedTestDirectory(const char *name) : path_(MakeTestDirectory(name)) {} + + ~ScopedTestDirectory() { + std::error_code error; + fs::remove_all(path_, error); + } + + const fs::path &path() const { return path_; } + + private: + fs::path path_; +}; + +class ScopedPhKeys { + public: + explicit ScopedPhKeys(PPERFECT_HASH_KEYS value = nullptr) : value_(value) {} + ScopedPhKeys(const ScopedPhKeys &) = delete; + ScopedPhKeys &operator=(const ScopedPhKeys &) = delete; + + ~ScopedPhKeys() { + if (value_ != nullptr) { + value_->Vtbl->Release(value_); + } + } + + PPERFECT_HASH_KEYS get() const { return value_; } + + private: + PPERFECT_HASH_KEYS value_; +}; + +class ScopedPhTable { + public: + explicit ScopedPhTable(PPERFECT_HASH_TABLE value = nullptr) : value_(value) {} + ScopedPhTable(const ScopedPhTable &) = delete; + ScopedPhTable &operator=(const ScopedPhTable &) = delete; + + ~ScopedPhTable() { + if (value_ != nullptr) { + // PPERFECT_HASH_TABLE is opaque in the public C API, so table tests use + // the local vtbl-prefix shim to release table instances. + reinterpret_cast(value_)->Vtbl->Release(value_); + } + } + + PPERFECT_HASH_TABLE get() const { return value_; } + + private: + PPERFECT_HASH_TABLE value_; +}; + +class ScopedPhJit { + public: + explicit ScopedPhJit(PPERFECT_HASH_TABLE_JIT_INTERFACE value = nullptr) : + value_(value) {} + ScopedPhJit(const ScopedPhJit &) = delete; + ScopedPhJit &operator=(const ScopedPhJit &) = delete; + + ~ScopedPhJit() { + if (value_ != nullptr) { + value_->Vtbl->Release(value_); + } + } + + PPERFECT_HASH_TABLE_JIT_INTERFACE get() const { return value_; } + + private: + PPERFECT_HASH_TABLE_JIT_INTERFACE value_; +}; + +UNICODE_STRING MakeUnicodeString(std::wstring *value) { + UNICODE_STRING string = {}; + string.Length = static_cast(value->size() * sizeof(WCHAR)); + string.MaximumLength = static_cast((value->size() + 1) * sizeof(WCHAR)); + string.Buffer = const_cast(value->c_str()); + return string; +} + +fs::path FindSingleTableFile(const fs::path &output_dir) { + std::vector table_paths; + for (const auto &entry : fs::recursive_directory_iterator(output_dir)) { + if (!entry.is_regular_file()) { + continue; + } + if (entry.path().extension() == ".pht1") { + table_paths.push_back(entry.path()); + } + } + + if (table_paths.size() != 1u) { + ADD_FAILURE() << "Expected exactly one .pht1 file under " << output_dir + << ", found " << table_paths.size(); + return fs::path(); + } + return table_paths[0]; +} + std::string TrimWhitespace(std::string value) { auto not_space = [](unsigned char ch) { return !std::isspace(ch); }; auto begin = std::find_if(value.begin(), value.end(), not_space); @@ -224,6 +540,195 @@ class ScopedEnvVar { bool had_old_ = false; }; +TEST(CompiledPerfectHashPortableHelpers, ZeroRotateCountIsIdentity) { + constexpr ULONG value32 = 0x98000101ul; + constexpr ULONGLONG value64 = 0x9800010102040810ull; + + EXPECT_EQ(RotateLeft32_C(value32, 0), value32); + EXPECT_EQ(RotateRight32_C(value32, 0), value32); + EXPECT_EQ(RotateLeft32_C(value32, 32), value32); + EXPECT_EQ(RotateRight32_C(value32, 32), value32); + + EXPECT_EQ(RotateLeft64_C(value64, 0), value64); + EXPECT_EQ(RotateRight64_C(value64, 0), value64); + EXPECT_EQ(RotateLeft64_C(value64, 64), value64); + EXPECT_EQ(RotateRight64_C(value64, 64), value64); +} + +TEST(GraphImpl4BitUtils, ContiguousBitmapDetection) { + const ULONGLONG contiguous_composed_mask = 0x380ull; + const ULONGLONG high_contiguous_mask = 0xffffffffffffff00ull; + const ULONGLONG contiguous_samples[] = { + 0x080ull, + 0x100ull, + 0x180ull, + 0x380ull, + 0xffffull, + }; + + ASSERT_TRUE(IsContiguousBitmap(contiguous_composed_mask)); + ASSERT_FALSE(IsContiguousBitmap(0x81ull)); + ASSERT_TRUE(IsContiguousBitmap(high_contiguous_mask)); + ASSERT_FALSE(IsContiguousBitmap(0x8000000000000001ull)); + for (ULONGLONG sample : contiguous_samples) { + ULONG via_shift = static_cast((sample >> 7) & 0x7ull); + ULONG via_extract = DownsizeKey(sample, contiguous_composed_mask); + EXPECT_EQ(via_extract, via_shift); + } + + BOOLEAN metadata_valid = FALSE; + BOOLEAN contiguous = FALSE; + BYTE trailing_zeros = 0; + ULONGLONG shifted_mask = 0; + + PerfectHashComputeDownsizeMetadataFromBitmap(high_contiguous_mask, + &metadata_valid, + &shifted_mask, + &trailing_zeros, + &contiguous); + EXPECT_TRUE(metadata_valid); + EXPECT_TRUE(contiguous); + EXPECT_EQ(trailing_zeros, 8u); + EXPECT_EQ(shifted_mask, 0x00ffffffffffffffull); + + ULONG high_via_shift = + static_cast((0xffffffffffffff00ull >> trailing_zeros) & + shifted_mask); + ULONG high_via_extract = + DownsizeKey(0xffffffffffffff00ull, high_contiguous_mask); + EXPECT_EQ(high_via_extract, high_via_shift); + + PerfectHashComputeDownsizeMetadataFromBitmap(0x8000000000000001ull, + &metadata_valid, + &shifted_mask, + &trailing_zeros, + &contiguous); + EXPECT_TRUE(metadata_valid); + EXPECT_FALSE(contiguous); + EXPECT_EQ(shifted_mask, 0ull); +} + +TEST(GraphImpl4BitUtils, ComposedDownsizeMetadataUsesComposedBitmap) { + const ULONGLONG outer_mask = (1ull << 8) | + (1ull << 9) | + (1ull << 10) | + (1ull << 30) | + (1ull << 32); + const ULONG inner_mask = 0x7u; + const ULONGLONG composed_mask = 0x700ull; + + ASSERT_FALSE(IsContiguousBitmap(outer_mask)); + ASSERT_TRUE(IsContiguousBitmap(inner_mask)); + ASSERT_EQ(PerfectHashComposeGraphImpl4DownsizeBitmap(outer_mask, inner_mask), + composed_mask); + + BOOLEAN metadata_valid = FALSE; + BOOLEAN contiguous = FALSE; + BYTE trailing_zeros = 0; + ULONGLONG shifted_mask = 0; + + PerfectHashComputeDownsizeMetadataFromBitmap(composed_mask, + &metadata_valid, + &shifted_mask, + &trailing_zeros, + &contiguous); + EXPECT_TRUE(metadata_valid); + EXPECT_TRUE(contiguous); + EXPECT_EQ(trailing_zeros, 8u); + EXPECT_EQ(shifted_mask, 0x7ull); +} + +TEST(GraphImpl4BitUtils, + ComposedDownsizeMetadataRejectsInnerContiguityMismatch) { + const ULONGLONG outer_mask = (1ull << 8) | + (1ull << 10) | + (1ull << 30) | + (1ull << 32); + const ULONG inner_mask = 0x7u; + const ULONGLONG composed_mask = (1ull << 8) | + (1ull << 10) | + (1ull << 30); + + ASSERT_FALSE(IsContiguousBitmap(outer_mask)); + ASSERT_TRUE(IsContiguousBitmap(inner_mask)); + ASSERT_EQ(PerfectHashComposeGraphImpl4DownsizeBitmap(outer_mask, inner_mask), + composed_mask); + ASSERT_FALSE(IsContiguousBitmap(composed_mask)); + + BOOLEAN metadata_valid = FALSE; + BOOLEAN contiguous = TRUE; + BYTE trailing_zeros = 0; + ULONGLONG shifted_mask = 0; + + PerfectHashComputeDownsizeMetadataFromBitmap(composed_mask, + &metadata_valid, + &shifted_mask, + &trailing_zeros, + &contiguous); + EXPECT_TRUE(metadata_valid); + EXPECT_FALSE(contiguous); + EXPECT_EQ(shifted_mask, 0ull); + + for (ULONGLONG value = 0; value < 8; ++value) { + ULONGLONG key = 0; + if ((value & 1ull) != 0) { + key |= (1ull << 8); + } + if ((value & 2ull) != 0) { + key |= (1ull << 10); + } + if ((value & 4ull) != 0) { + key |= (1ull << 30); + } + + ULONG outer = DownsizeKey(key, outer_mask); + ULONG two_step = DownsizeKey(outer, inner_mask); + ULONG direct = DownsizeKey(key, composed_mask); + EXPECT_EQ(two_step, direct); + } +} + +TEST(GraphImpl4BitUtils, ComposedExtractionMatchesTwoStepExtraction) { + const ULONGLONG outer_mask = 0x1f00ull; + const ULONG inner_mask = 0x15u; + const ULONGLONG composed_mask = 0x1500ull; + + ASSERT_TRUE(IsContiguousBitmap(outer_mask)); + ASSERT_FALSE(IsContiguousBitmap(inner_mask)); + ASSERT_EQ(PerfectHashComposeGraphImpl4DownsizeBitmap(outer_mask, inner_mask), + composed_mask); + ASSERT_FALSE(IsContiguousBitmap(composed_mask)); + + for (ULONGLONG value = 0; value < 32; ++value) { + ULONGLONG key = value << 8; + ULONG outer = DownsizeKey(key, outer_mask); + ULONG two_step = DownsizeKey(outer, inner_mask); + ULONG direct = DownsizeKey(key, composed_mask); + EXPECT_EQ(two_step, direct); + } +} + +TEST(GraphImpl4BitUtils, Downsized64OuterBitmapProducesIdentityInnerBitmap) { + const std::vector bit_positions = {1, 3, 6, 10, 18, 24, 32, 41, 56}; + const auto keys = MakeSparseDownsized64KeysWithPositions(256, + bit_positions); + const ULONGLONG outer_mask = BuildDownsizeMask(keys); + const ULONGLONG downsized_mask = BuildDownsizedKeyMask(keys, outer_mask); + const ULONGLONG expected_mask = (1ull << bit_positions.size()) - 1ull; + + // + // Starting the raw 64-bit positions above bit 0 does not create a non-zero + // trailing shift after outer downsizing. Because the outer bitmap is the OR + // of the same key set, every packed bit has at least one source key and the + // GraphImpl4 inner bitmap is identity in the downsized64 domain. + // + + ASSERT_EQ(outer_mask & 1ull, 0ull); + EXPECT_EQ(downsized_mask, expected_mask); + EXPECT_TRUE(IsContiguousBitmap(downsized_mask)); + EXPECT_NE(downsized_mask & 1ull, 0ull); +} + class PerfectHashOnlineTests : public ::testing::Test { protected: void SetUp() override { @@ -272,7 +777,8 @@ class PerfectHashOnlineTests : public ::testing::Test { const std::vector *seeds = nullptr, ULONG maxSolveTimeInSeconds = 0, bool useSystemRng = false, - bool useRandomStartSeed = false) { + bool useRandomStartSeed = false, + bool hashAllKeysFirst = false) { PERFECT_HASH_KEYS_LOAD_FLAGS keysFlags = {0}; PERFECT_HASH_TABLE_CREATE_FLAGS tableFlags = {0}; std::vector table_params; @@ -284,6 +790,7 @@ class PerfectHashOnlineTests : public ::testing::Test { tableFlags.Quiet = TRUE; tableFlags.DoNotTryUseHash16Impl = allowAssigned16 ? FALSE : TRUE; tableFlags.RngUseRandomStartSeed = useRandomStartSeed ? TRUE : FALSE; + tableFlags.HashAllKeysFirst = hashAllKeysFirst ? TRUE : FALSE; if (graphImpl != 0) { PERFECT_HASH_TABLE_CREATE_PARAMETER param = {}; @@ -342,14 +849,52 @@ class PerfectHashOnlineTests : public ::testing::Test { PPERFECT_HASH_TABLE CreateTableFromKeys64( const std::vector &keys, PERFECT_HASH_HASH_FUNCTION_ID hashFunctionId, - bool allowAssigned16 = false) { + bool allowAssigned16 = false, + ULONG graphImpl = 0, + ULONG maxSolveTimeInSeconds = 0, + const std::vector *seeds = nullptr) { PERFECT_HASH_KEYS_LOAD_FLAGS keysFlags = {0}; PERFECT_HASH_TABLE_CREATE_FLAGS tableFlags = {0}; + std::vector table_params; + PERFECT_HASH_TABLE_CREATE_PARAMETERS tableCreateParams = {}; + PPERFECT_HASH_TABLE_CREATE_PARAMETERS tableCreateParamsPtr = nullptr; PPERFECT_HASH_TABLE table = nullptr; tableFlags.NoFileIo = TRUE; tableFlags.Quiet = TRUE; tableFlags.DoNotTryUseHash16Impl = allowAssigned16 ? FALSE : TRUE; + tableFlags.HashAllKeysFirst = FALSE; // Mirrors --DoNotHashAllKeysFirst. + + if (graphImpl != 0) { + PERFECT_HASH_TABLE_CREATE_PARAMETER param = {}; + param.Id = TableCreateParameterGraphImplId; + param.AsULong = graphImpl; + table_params.push_back(param); + } + + if (maxSolveTimeInSeconds > 0) { + PERFECT_HASH_TABLE_CREATE_PARAMETER param = {}; + param.Id = TableCreateParameterMaxSolveTimeInSecondsId; + param.AsULong = maxSolveTimeInSeconds; + table_params.push_back(param); + } + + if (seeds && !seeds->empty()) { + PERFECT_HASH_TABLE_CREATE_PARAMETER param = {}; + param.Id = TableCreateParameterSeedsId; + param.AsValueArray.Values = const_cast(seeds->data()); + param.AsValueArray.NumberOfValues = static_cast(seeds->size()); + param.AsValueArray.ValueSizeInBytes = sizeof(ULONG); + table_params.push_back(param); + } + + if (!table_params.empty()) { + tableCreateParams.SizeOfStruct = sizeof(tableCreateParams); + tableCreateParams.NumberOfElements = + static_cast(table_params.size()); + tableCreateParams.Params = table_params.data(); + tableCreateParamsPtr = &tableCreateParams; + } HRESULT result = online_->Vtbl->CreateTableFromKeys( online_, @@ -361,12 +906,267 @@ class PerfectHashOnlineTests : public ::testing::Test { const_cast(keys.data()), &keysFlags, &tableFlags, - nullptr, + tableCreateParamsPtr, &table); EXPECT_GE(result, 0); return table; } + PERFECT_HASH_TABLE_FLAGS GetTableFlags(PPERFECT_HASH_TABLE table) { + PERFECT_HASH_TABLE_FLAGS flags = {0}; + auto *shim = reinterpret_cast(table); + EXPECT_GE(shim->Vtbl->GetFlags(table, sizeof(flags), &flags), 0); + return flags; + } + + std::vector CaptureIndexes( + PPERFECT_HASH_TABLE table, + const std::vector &keys) { + std::vector indexes; + auto *shim = reinterpret_cast(table); + indexes.reserve(keys.size()); + + for (ULONG key : keys) { + ULONG index = 0; + EXPECT_GE(shim->Vtbl->Index(table, key, &index), 0); + indexes.push_back(index); + } + + return indexes; + } + + std::vector CaptureIndexes64( + PPERFECT_HASH_TABLE table, + const std::vector &keys) { + std::vector indexes; + auto *shim = reinterpret_cast(table); + const ULONGLONG mask = BuildDownsizeMask(keys); + indexes.reserve(keys.size()); + + for (ULONGLONG key : keys) { + ULONG downsized = DownsizeKey(key, mask); + ULONG index = 0; + EXPECT_GE(shim->Vtbl->Index(table, downsized, &index), 0); + indexes.push_back(index); + } + + return indexes; + } + + std::optional LoadKeyDownsizeBitmapFromHeader( + const fs::path &table_path) { + fs::path header_path = table_path; + header_path.replace_extension(".h"); + + std::ifstream file(header_path); + EXPECT_TRUE(file.is_open()) << "Failed to open " << header_path; + if (!file.is_open()) { + return std::nullopt; + } + + std::string macro_prefix = header_path.stem().string(); + std::transform(macro_prefix.begin(), + macro_prefix.end(), + macro_prefix.begin(), + [](unsigned char ch) { + return static_cast(std::toupper(ch)); + }); + + const std::string key_downsize_needle = + "#define " + macro_prefix + "_KEY_DOWNSIZE_BITMAP "; + const std::string graphimpl4_needle = + "#define " + macro_prefix + "_GRAPHIMPL4_KEY_DOWNSIZE_BITMAP "; + + std::vector lines; + std::string line; + while (std::getline(file, line)) { + lines.push_back(line); + } + + // + // Prefer the 64-bit key-downsize macro when present. Sparse 32-bit + // GraphImpl4 tables do not emit that macro, so fall back to the GraphImpl4 + // inner compact-key bitmap; the return below intentionally exits both loops. + // + for (const auto &needle : {key_downsize_needle, graphimpl4_needle}) { + for (const auto &candidate : lines) { + if (candidate.find(needle) != 0) { + continue; + } + + const std::string value = candidate.substr(needle.size()); + try { + // + // Generated hex macros are emitted with a 0x prefix by + // OUTPUT_HEX_RAW; base 0 parsing below depends on that prefix. + // + return static_cast(std::stoull(value, nullptr, 0)); + } catch (const std::exception &ex) { + ADD_FAILURE() << "Failed to parse key downsize bitmap from " + << header_path << ": " << candidate << " (" + << ex.what() << ")"; + return std::nullopt; + } + } + } + + ADD_FAILURE() << "Failed to find key downsize bitmap macro in " + << header_path; + return std::nullopt; + } + + HRESULT CreateFileBackedTableFromKeys( + const fs::path &keys_path, + const fs::path &output_dir, + ULONG key_size_in_bytes, + const std::vector &extra_args = {}, + ULONG graph_impl = 4) { + PPERFECT_HASH_CONTEXT context = nullptr; + auto createInstance = classFactory_->Vtbl->CreateInstance; + HRESULT result = createInstance( + classFactory_, + nullptr, + PH_TEST_IID(PERFECT_HASH_CONTEXT), + reinterpret_cast(&context)); + EXPECT_GE(result, 0); + EXPECT_NE(context, nullptr); + if (FAILED(result)) { + if (context != nullptr) { + context->Vtbl->Release(context); + } + return result; + } + if (context == nullptr) { + return E_POINTER; + } + + std::vector args = { + "PerfectHashCreateExe", + keys_path.string(), + output_dir.string(), + "Chm01", + "MultiplyShiftR", + "And", + "1", + "--Quiet", + "--DoNotHashAllKeysFirst", + "--MaxSolveTimeInSeconds=5", + "--KeySizeInBytes=" + std::to_string(key_size_in_bytes), + }; + + if (graph_impl != 0) { + args.push_back("--GraphImpl=" + std::to_string(graph_impl)); + } + + args.insert(args.end(), extra_args.begin(), extra_args.end()); + + std::vector argv; + argv.reserve(args.size()); + for (auto &arg : args) { + argv.push_back(arg.data()); + } + + result = context->Vtbl->TableCreateArgvA( + context, + static_cast(argv.size()), + argv.data()); + + context->Vtbl->Release(context); + return result; + } + + PPERFECT_HASH_KEYS LoadKeysFromPath( + const fs::path &keys_path, + ULONG key_size_in_bytes) { + PPERFECT_HASH_KEYS keys = nullptr; + auto createInstance = classFactory_->Vtbl->CreateInstance; + HRESULT result = createInstance( + classFactory_, + nullptr, + PH_TEST_IID(PERFECT_HASH_KEYS), + reinterpret_cast(&keys)); + EXPECT_GE(result, 0); + EXPECT_NE(keys, nullptr); + if (FAILED(result)) { + if (keys != nullptr) { + keys->Vtbl->Release(keys); + } + return nullptr; + } + if (keys == nullptr) { + return nullptr; + } + + std::wstring keys_path_w = keys_path.wstring(); + UNICODE_STRING keys_path_string = MakeUnicodeString(&keys_path_w); + PERFECT_HASH_KEYS_LOAD_FLAGS keys_load_flags = {0}; + + result = keys->Vtbl->Load(keys, + &keys_load_flags, + &keys_path_string, + key_size_in_bytes); + EXPECT_GE(result, 0); + if (FAILED(result)) { + keys->Vtbl->Release(keys); + return nullptr; + } + + return keys; + } + + PPERFECT_HASH_TABLE LoadTableFromPath( + const fs::path &table_path, + PPERFECT_HASH_KEYS keys = nullptr, + HRESULT *load_result = nullptr, + bool allow_not_implemented = false) { + PPERFECT_HASH_TABLE table = nullptr; + if (load_result != nullptr) { + *load_result = S_OK; + } + + auto createInstance = classFactory_->Vtbl->CreateInstance; + HRESULT result = createInstance( + classFactory_, + nullptr, + PH_TEST_IID(PERFECT_HASH_TABLE), + reinterpret_cast(&table)); + if (load_result != nullptr) { + *load_result = result; + } + EXPECT_GE(result, 0); + if (FAILED(result)) { + if (table != nullptr) { + auto *shim = reinterpret_cast(table); + shim->Vtbl->Release(table); + } + return nullptr; + } + if (table == nullptr) { + if (load_result != nullptr) { + *load_result = E_POINTER; + } + return nullptr; + } + + std::wstring table_path_w = table_path.wstring(); + UNICODE_STRING table_path_string = MakeUnicodeString(&table_path_w); + PERFECT_HASH_TABLE_LOAD_FLAGS load_flags = {0}; + auto *shim = reinterpret_cast(table); + result = shim->Vtbl->Load(table, &load_flags, &table_path_string, keys); + if (load_result != nullptr) { + *load_result = result; + } + if (!(allow_not_implemented && result == PH_E_NOT_IMPLEMENTED)) { + EXPECT_GE(result, 0); + } + if (FAILED(result)) { + shim->Vtbl->Release(table); + return nullptr; + } + + return table; + } + protected: PICLASSFACTORY classFactory_ = nullptr; PPERFECT_HASH_ONLINE online_ = nullptr; @@ -385,6 +1185,11 @@ TEST_F(PerfectHashOnlineTests, CreateTableFromKeysReturnsUniqueIndexes) { ASSERT_NE(table, nullptr); auto *shim = reinterpret_cast(table); + // + // The keyed and keyless reload paths consume the same persisted metadata. + // In addition to cross-checking the two paths agree, assert that the + // keyless path still maps this key set into a dense, collision-free range. + // std::unordered_set seen; seen.reserve(keys.size()); @@ -469,79 +1274,840 @@ TEST_F(PerfectHashOnlineTests, Assigned16Boundary32767Vs32768Keys) { 20); ASSERT_NE(table32768, nullptr); - auto *shim32768 = reinterpret_cast(table32768); - PERFECT_HASH_TABLE_FLAGS flags32768 = {0}; - ASSERT_GE(shim32768->Vtbl->GetFlags(table32768, sizeof(flags32768), &flags32768), 0); - EXPECT_EQ(flags32768.AssignedElementSizeInBits << 3, 32u); - EXPECT_NE(shim32768->Vtbl->SlowIndex, nullptr); - shim32768->Vtbl->Release(table32768); + auto *shim32768 = reinterpret_cast(table32768); + PERFECT_HASH_TABLE_FLAGS flags32768 = {0}; + ASSERT_GE(shim32768->Vtbl->GetFlags(table32768, sizeof(flags32768), &flags32768), 0); + EXPECT_EQ(flags32768.AssignedElementSizeInBits << 3, 32u); + EXPECT_NE(shim32768->Vtbl->SlowIndex, nullptr); + shim32768->Vtbl->Release(table32768); +} + +TEST_F(PerfectHashOnlineTests, Assigned16RequiresGraphImpl3AndOptIn) { + const PERFECT_HASH_HASH_FUNCTION_ID hashFunctionId = + PerfectHashHashMultiplyShiftRFunctionId; + const auto keys = MakePseudoRandomKeys(2048, 0xDEADBEEFu); + + PPERFECT_HASH_TABLE tableGraphImpl3 = CreateTableFromKeys( + keys, + hashFunctionId, + true, + 3, + nullptr, + 20); + ASSERT_NE(tableGraphImpl3, nullptr); + + auto *shimGraphImpl3 = reinterpret_cast(tableGraphImpl3); + PERFECT_HASH_TABLE_FLAGS flagsGraphImpl3 = {0}; + ASSERT_GE( + shimGraphImpl3->Vtbl->GetFlags( + tableGraphImpl3, + sizeof(flagsGraphImpl3), + &flagsGraphImpl3), + 0); + EXPECT_EQ(flagsGraphImpl3.AssignedElementSizeInBits << 3, 16u); + EXPECT_EQ(shimGraphImpl3->Vtbl->SlowIndex, nullptr); + shimGraphImpl3->Vtbl->Release(tableGraphImpl3); + + PPERFECT_HASH_TABLE tableOptOut = CreateTableFromKeys( + keys, + hashFunctionId, + false, + 3, + nullptr, + 20); + ASSERT_NE(tableOptOut, nullptr); + + auto *shimOptOut = reinterpret_cast(tableOptOut); + PERFECT_HASH_TABLE_FLAGS flagsOptOut = {0}; + ASSERT_GE(shimOptOut->Vtbl->GetFlags(tableOptOut, sizeof(flagsOptOut), &flagsOptOut), 0); + EXPECT_EQ(flagsOptOut.AssignedElementSizeInBits << 3, 32u); + EXPECT_NE(shimOptOut->Vtbl->SlowIndex, nullptr); + shimOptOut->Vtbl->Release(tableOptOut); + + PPERFECT_HASH_TABLE tableGraphImpl2 = CreateTableFromKeys( + keys, + hashFunctionId, + true, + 2, + nullptr, + 20); + ASSERT_NE(tableGraphImpl2, nullptr); + + auto *shimGraphImpl2 = reinterpret_cast(tableGraphImpl2); + PERFECT_HASH_TABLE_FLAGS flagsGraphImpl2 = {0}; + ASSERT_GE( + shimGraphImpl2->Vtbl->GetFlags( + tableGraphImpl2, + sizeof(flagsGraphImpl2), + &flagsGraphImpl2), + 0); + EXPECT_EQ(flagsGraphImpl2.AssignedElementSizeInBits << 3, 32u); + EXPECT_NE(shimGraphImpl2->Vtbl->SlowIndex, nullptr); + shimGraphImpl2->Vtbl->Release(tableGraphImpl2); +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4Assigned8RequiresOptIn) { + const auto keys = MakePseudoRandomKeys(8, 0x2468ACE0u); + + PPERFECT_HASH_TABLE tableGraphImpl4 = CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + true, + 4, + nullptr, + 5); + ASSERT_NE(tableGraphImpl4, nullptr); + + auto *shimGraphImpl4 = reinterpret_cast(tableGraphImpl4); + PERFECT_HASH_TABLE_FLAGS flagsGraphImpl4 = {0}; + ASSERT_GE( + shimGraphImpl4->Vtbl->GetFlags( + tableGraphImpl4, + sizeof(flagsGraphImpl4), + &flagsGraphImpl4), + 0); + EXPECT_EQ(flagsGraphImpl4.AssignedElementSizeInBits << 3, 8u); + EXPECT_EQ(shimGraphImpl4->Vtbl->SlowIndex, nullptr); + + std::unordered_set seen; + seen.reserve(keys.size()); + for (ULONG key : keys) { + ULONG index = 0; + ASSERT_GE(shimGraphImpl4->Vtbl->Index(tableGraphImpl4, key, &index), 0); + EXPECT_TRUE(seen.insert(index).second); + } + + shimGraphImpl4->Vtbl->Release(tableGraphImpl4); + + PPERFECT_HASH_TABLE tableOptOut = CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4, + nullptr, + 5); + ASSERT_NE(tableOptOut, nullptr); + + auto *shimOptOut = reinterpret_cast(tableOptOut); + PERFECT_HASH_TABLE_FLAGS flagsOptOut = {0}; + ASSERT_GE(shimOptOut->Vtbl->GetFlags(tableOptOut, sizeof(flagsOptOut), &flagsOptOut), 0); + EXPECT_EQ(flagsOptOut.AssignedElementSizeInBits << 3, 32u); + shimOptOut->Vtbl->Release(tableOptOut); +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4SupportsDownsized64BitInputs) { + const std::vector keys = { + 1ull, 3ull, 5ull, 7ull, 11ull, 13ull, 17ull, 19ull, + 23ull, 29ull, 31ull, 37ull, 41ull, 43ull, 47ull, 53ull, + }; + + PPERFECT_HASH_TABLE table = CreateTableFromKeys64( + keys, + PerfectHashHashMulshrolate3RXFunctionId, + true, + 4, + 5); + ASSERT_NE(table, nullptr); + + auto *shim = reinterpret_cast(table); + PERFECT_HASH_TABLE_FLAGS flags = {0}; + ASSERT_GE(shim->Vtbl->GetFlags(table, sizeof(flags), &flags), 0); + EXPECT_EQ(flags.AssignedElementSizeInBits << 3, 8u); + + const ULONGLONG mask = BuildDownsizeMask(keys); + std::unordered_set seen; + seen.reserve(keys.size()); + + for (auto key : keys) { + ULONG downsized = DownsizeKey(key, mask); + ULONG index = 0; + ASSERT_GE(shim->Vtbl->Index(table, downsized, &index), 0); + EXPECT_TRUE(seen.insert(index).second); + } + + shim->Vtbl->Release(table); +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4FileBackedReloadPreservesSparse32Compaction) { + const ScopedTestDirectory root("graphimpl4_sparse32_reload"); + const auto keys_path = root.path() / "GraphImpl4Sparse32.keys"; + const auto output_dir = root.path() / "out"; + const auto keys = MakeSparse32Keys(256); + + WriteBinaryKeys(keys_path, keys); + HRESULT result = CreateFileBackedTableFromKeys(keys_path, + output_dir, + sizeof(ULONG), + {kGraphImpl4Sparse32ReloadSeedsArg}); + ASSERT_RELOAD_FIXTURE_CREATE_SUCCEEDED(result, + kGraphImpl4Sparse32ReloadSeedsArg); + + fs::path table_path = FindSingleTableFile(output_dir); + ASSERT_FALSE(table_path.empty()); + + ScopedPhKeys reference_keys(LoadKeysFromPath(keys_path, sizeof(ULONG))); + ASSERT_NE(reference_keys.get(), nullptr); + + ScopedPhTable reference_table(LoadTableFromPath(table_path, + reference_keys.get())); + ASSERT_NE(reference_table.get(), nullptr); + + ScopedPhTable table(LoadTableFromPath(table_path)); + ASSERT_NE(table.get(), nullptr); + + auto *reference_shim = + reinterpret_cast(reference_table.get()); + auto *shim = reinterpret_cast(table.get()); + std::vector reference_indexes; + std::vector keyless_indexes; + reference_indexes.reserve(keys.size()); + keyless_indexes.reserve(keys.size()); + + const ULONGLONG mask = BuildDownsizeMask(keys); + const auto header_mask = LoadKeyDownsizeBitmapFromHeader(table_path); + ASSERT_TRUE(header_mask.has_value()); + ASSERT_EQ(mask, *header_mask); + ASSERT_FALSE(IsContiguousBitmap(mask)); + + // + // This sparse32 reload test is the file-backed coverage for non-identity + // GraphImpl4 inner compact-key metadata. There is no outer 64-bit downsize + // here, so the header loader falls back to _GRAPHIMPL4_KEY_DOWNSIZE_BITMAP. + // + + for (size_t index = 0; index < keys.size(); ++index) { + ULONG key = keys[index]; + ULONG expected = 0; + ULONG actual = 0; + ASSERT_GE(reference_shim->Vtbl->Index(reference_table.get(), + key, + &expected), 0); + ASSERT_GE(shim->Vtbl->Index(table.get(), key, &actual), 0); + EXPECT_EQ(expected, actual); + EXPECT_LT(expected, keys.size()); + EXPECT_LT(actual, keys.size()); + reference_indexes.push_back(expected); + keyless_indexes.push_back(actual); + } + + AssertCollisionFreeIndexes(reference_indexes, + keys.size(), + "keyed sparse32 reload"); + AssertCollisionFreeIndexes(keyless_indexes, + keys.size(), + "keyless sparse32 reload"); +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4FileBackedReloadPreservesDownsized64Compaction) { + const ScopedTestDirectory root("graphimpl4_downsized64_reload_nojit"); + const auto keys_path = root.path() / "GraphImpl4Downsized64.keys"; + const auto output_dir = root.path() / "out"; + const auto keys = MakeSparseDownsized64Keys(256); + + WriteBinaryKeys(keys_path, keys); + HRESULT result = CreateFileBackedTableFromKeys(keys_path, + output_dir, + sizeof(ULONGLONG), + {kGraphImpl4Downsized64ReloadSeedsArg}); + ASSERT_RELOAD_FIXTURE_CREATE_SUCCEEDED( + result, + kGraphImpl4Downsized64ReloadSeedsArg); + + fs::path table_path = FindSingleTableFile(output_dir); + ASSERT_FALSE(table_path.empty()); + + ScopedPhKeys loaded_keys(LoadKeysFromPath(keys_path, sizeof(ULONGLONG))); + ASSERT_NE(loaded_keys.get(), nullptr); + + HRESULT reference_load_result = S_OK; + ScopedPhTable reference_table(LoadTableFromPath(table_path, + loaded_keys.get(), + &reference_load_result)); + ASSERT_EQ(reference_load_result, S_OK); + ASSERT_NE(reference_table.get(), nullptr); + + HRESULT keyless_load_result = S_OK; + ScopedPhTable table(LoadTableFromPath(table_path, + nullptr, + &keyless_load_result)); + ASSERT_EQ(keyless_load_result, S_OK); + ASSERT_NE(table.get(), nullptr); + + auto *reference_shim = + reinterpret_cast(reference_table.get()); + auto *shim = reinterpret_cast(table.get()); + const ULONGLONG mask = BuildDownsizeMask(keys); + const auto create_seeds = + SeedValuesFromSeedsArg(kGraphImpl4Downsized64ReloadSeedsArg); + ASSERT_FALSE(create_seeds.empty()); + ScopedPhTable created_table(CreateTableFromKeys64( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4, + 5, + &create_seeds)); + ASSERT_NE(created_table.get(), nullptr); + auto *created_shim = + reinterpret_cast(created_table.get()); + std::vector reference_indexes; + std::vector keyless_indexes; + std::vector created_indexes; + reference_indexes.reserve(keys.size()); + keyless_indexes.reserve(keys.size()); + created_indexes.reserve(keys.size()); + + const auto header_mask = LoadKeyDownsizeBitmapFromHeader(table_path); + ASSERT_TRUE(header_mask.has_value()); + ASSERT_EQ(mask, *header_mask); + ASSERT_FALSE(IsContiguousBitmap(mask)); + + // + // The loaded-table Index() API is 32-bit. For downsized-64 GraphImpl4 + // tables, callers pass the outer-bitmap result rather than the raw 64-bit + // key; the GraphImpl4 vtbl then applies the inner compact-key step. + // The in-memory table anchors reload correctness to creation-time state so a + // shared keyed/keyless reload bug cannot make this test pass by agreement. + // + + for (size_t index = 0; index < keys.size(); ++index) { + ULONGLONG key = keys[index]; + ULONG created = 0; + ULONG expected = 0; + ULONG actual = 0; + ULONG downsized = DownsizeKey(key, mask); + + ASSERT_GE(created_shim->Vtbl->Index(created_table.get(), + downsized, + &created), 0); + ASSERT_GE(reference_shim->Vtbl->Index(reference_table.get(), + downsized, + &expected), 0); + ASSERT_GE(shim->Vtbl->Index(table.get(), downsized, &actual), 0); + EXPECT_EQ(created, expected); + EXPECT_EQ(created, actual); + EXPECT_EQ(expected, actual); + EXPECT_LT(created, keys.size()); + EXPECT_LT(expected, keys.size()); + EXPECT_LT(actual, keys.size()); + created_indexes.push_back(created); + reference_indexes.push_back(expected); + keyless_indexes.push_back(actual); + } + + AssertCollisionFreeIndexes(created_indexes, + keys.size(), + "created downsized64 reference"); + AssertCollisionFreeIndexes(reference_indexes, + keys.size(), + "keyed downsized64 reload"); + AssertCollisionFreeIndexes(keyless_indexes, + keys.size(), + "keyless downsized64 reload"); +} + +TEST_F(PerfectHashOnlineTests, NonGraphImplFileBackedReloadPreservesDownsized64Metadata) { + const ScopedTestDirectory root("graphimpl3_downsized64_reload_nojit"); + const auto keys_path = root.path() / "GraphImpl3Downsized64.keys"; + const auto output_dir = root.path() / "out"; + const auto keys = MakeSparseDownsized64Keys(128); + + WriteBinaryKeys(keys_path, keys); + HRESULT result = CreateFileBackedTableFromKeys(keys_path, + output_dir, + sizeof(ULONGLONG), + {kGraphImpl3Downsized64ReloadSeedsArg}, + 3); + ASSERT_RELOAD_FIXTURE_CREATE_SUCCEEDED( + result, + kGraphImpl3Downsized64ReloadSeedsArg); + + fs::path table_path = FindSingleTableFile(output_dir); + ASSERT_FALSE(table_path.empty()); + + ScopedPhKeys loaded_keys(LoadKeysFromPath(keys_path, sizeof(ULONGLONG))); + ASSERT_NE(loaded_keys.get(), nullptr); + + ScopedPhTable reference_table(LoadTableFromPath(table_path, + loaded_keys.get())); + ASSERT_NE(reference_table.get(), nullptr); + + ScopedPhTable table(LoadTableFromPath(table_path)); + ASSERT_NE(table.get(), nullptr); + + auto *reference_shim = + reinterpret_cast(reference_table.get()); + auto *shim = reinterpret_cast(table.get()); + const ULONGLONG mask = BuildDownsizeMask(keys); + const auto create_seeds = + SeedValuesFromSeedsArg(kGraphImpl3Downsized64ReloadSeedsArg); + ASSERT_FALSE(create_seeds.empty()); + ScopedPhTable created_table(CreateTableFromKeys64( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 3, + 5, + &create_seeds)); + ASSERT_NE(created_table.get(), nullptr); + auto *created_shim = + reinterpret_cast(created_table.get()); + const auto header_mask = LoadKeyDownsizeBitmapFromHeader(table_path); + ASSERT_TRUE(header_mask.has_value()); + ASSERT_EQ(mask, *header_mask); + ASSERT_FALSE(IsContiguousBitmap(mask)); + + std::unordered_set created_seen; + std::unordered_set seen; + created_seen.reserve(keys.size()); + seen.reserve(keys.size()); + for (ULONGLONG key : keys) { + ULONG created = 0; + ULONG expected = 0; + ULONG actual = 0; + ULONG downsized = DownsizeKey(key, mask); + + ASSERT_GE(created_shim->Vtbl->Index(created_table.get(), + downsized, + &created), 0); + ASSERT_GE(reference_shim->Vtbl->Index(reference_table.get(), + downsized, + &expected), 0); + ASSERT_GE(shim->Vtbl->Index(table.get(), downsized, &actual), 0); + EXPECT_EQ(created, expected); + EXPECT_EQ(created, actual); + EXPECT_EQ(expected, actual); + EXPECT_LT(created, keys.size()); + EXPECT_LT(expected, keys.size()); + EXPECT_LT(actual, keys.size()); + EXPECT_TRUE(created_seen.insert(created).second); + EXPECT_TRUE(seen.insert(actual).second); + } + + EXPECT_EQ(created_seen.size(), keys.size()); + EXPECT_EQ(seen.size(), keys.size()); +} + +#if defined(PH_HAS_LLVM) +TEST_F(PerfectHashOnlineTests, GraphImpl4FileBackedReloadSparse32JitIndex32) { + const ScopedTestDirectory root("graphimpl4_sparse32_reload_jit"); + const auto keys_path = root.path() / "GraphImpl4Sparse32.keys"; + const auto output_dir = root.path() / "out"; + const auto keys = MakeSparse32Keys(256); + + WriteBinaryKeys(keys_path, keys); + HRESULT result = CreateFileBackedTableFromKeys(keys_path, + output_dir, + sizeof(ULONG), + {kGraphImpl4Sparse32ReloadSeedsArg}); + ASSERT_RELOAD_FIXTURE_CREATE_SUCCEEDED(result, + kGraphImpl4Sparse32ReloadSeedsArg); + + fs::path table_path = FindSingleTableFile(output_dir); + ASSERT_FALSE(table_path.empty()); + + ScopedPhTable table(LoadTableFromPath(table_path)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compile_flags = MakeLlvmJitCompileFlags(); + result = online_->Vtbl->CompileTable(online_, + table.get(), + &compile_flags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM backend not found on this host."; + } + // + // Backend availability is the only skip case. Any other compile failure is + // specific to loaded sparse32 GraphImpl4 JIT state and should fail loudly. + // + ASSERT_EQ(result, S_OK) + << "LLVM compile of keyless sparse32 table failed: 0x" + << std::hex << result; + + std::vector indexes; + indexes.reserve(keys.size()); + for (size_t index = 0; index < keys.size(); ++index) { + ULONG key = keys[index]; + ULONG actual = 0; + + ASSERT_GE(shim->Vtbl->Index(table.get(), key, &actual), 0); + ASSERT_LT(index, expected.size()); + EXPECT_EQ(expected[index], actual); + indexes.push_back(actual); + } + + AssertCollisionFreeIndexes(indexes, + keys.size(), + "keyless sparse32 reload jit"); +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4FileBackedReloadDownsized64JitIndex64) { + const ScopedTestDirectory root("graphimpl4_downsized64_reload"); + const auto keys_path = root.path() / "GraphImpl4Downsized64.keys"; + const auto output_dir = root.path() / "out"; + const auto keys = MakeSparseDownsized64Keys(256); + + WriteBinaryKeys(keys_path, keys); + HRESULT result = CreateFileBackedTableFromKeys(keys_path, + output_dir, + sizeof(ULONGLONG), + {kGraphImpl4Downsized64ReloadSeedsArg}); + ASSERT_RELOAD_FIXTURE_CREATE_SUCCEEDED( + result, + kGraphImpl4Downsized64ReloadSeedsArg); + + fs::path table_path = FindSingleTableFile(output_dir); + ASSERT_FALSE(table_path.empty()); + + ScopedPhTable table(LoadTableFromPath(table_path)); + ASSERT_NE(table.get(), nullptr); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compile_flags = MakeLlvmJitCompileFlags(); + compile_flags.JitIndex64 = TRUE; + + result = online_->Vtbl->CompileTable(online_, + table.get(), + &compile_flags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM backend not found on this host."; + } + ASSERT_EQ(result, S_OK) + << "LLVM compile of keyless loaded table failed: 0x" + << std::hex << result; + + PPERFECT_HASH_TABLE_JIT_INTERFACE raw_jit = nullptr; + auto *shim = reinterpret_cast(table.get()); + result = shim->Vtbl->QueryInterface( + table.get(), + PH_TEST_IID(PERFECT_HASH_TABLE_JIT_INTERFACE), + reinterpret_cast(&raw_jit)); + ScopedPhJit jit(raw_jit); + ASSERT_GE(result, 0); + ASSERT_NE(jit.get(), nullptr); + + const ULONGLONG mask = BuildDownsizeMask(keys); + const ULONGLONG downsized_mask = BuildDownsizedKeyMask(keys, mask); + + ASSERT_FALSE(IsContiguousBitmap(mask)); + ASSERT_TRUE(IsContiguousBitmap(downsized_mask)); + + std::vector keyless_indexes; + keyless_indexes.reserve(keys.size()); + std::unordered_set keyless_seen; + keyless_seen.reserve(keys.size()); + + for (size_t index = 0; index < keys.size(); ++index) { + ULONGLONG key = keys[index]; + ULONG reloaded = 0; + ULONG actual = 0; + ULONG downsized = DownsizeKey(key, mask); + + // + // Index() receives the already-outer-downsized 32-bit key here. Comparing + // it with Index64() validates the outer-downsize plus inner-compact + // decomposition used by loaded-table JIT. + // + ASSERT_GE(shim->Vtbl->Index(table.get(), downsized, &reloaded), 0); + ASSERT_GE(jit.get()->Vtbl->Index64(jit.get(), key, &actual), 0); + EXPECT_EQ(reloaded, actual); + EXPECT_LT(reloaded, keys.size()); + EXPECT_LT(actual, keys.size()); + EXPECT_TRUE(keyless_seen.insert(actual).second); + keyless_indexes.push_back(actual); + } + EXPECT_EQ(keyless_seen.size(), keys.size()); + + ScopedPhKeys loaded_keys(LoadKeysFromPath(keys_path, sizeof(ULONGLONG))); + ASSERT_NE(loaded_keys.get(), nullptr); + const auto header_mask = LoadKeyDownsizeBitmapFromHeader(table_path); + ASSERT_TRUE(header_mask.has_value()); + ASSERT_EQ(mask, *header_mask); + + ScopedPhTable keyed_table(LoadTableFromPath(table_path, loaded_keys.get())); + ASSERT_NE(keyed_table.get(), nullptr); + + result = online_->Vtbl->CompileTable(online_, + keyed_table.get(), + &compile_flags); + // + // The keyless compile above already proved LLVM availability. A failure here + // is keyed-load specific, even if it reports backend availability, and should + // fail loudly rather than skip. + // + ASSERT_EQ(result, S_OK) + << "LLVM compile of keyed-loaded table failed: 0x" + << std::hex << result; + + PPERFECT_HASH_TABLE_JIT_INTERFACE raw_keyed_jit = nullptr; + auto *keyed_shim = + reinterpret_cast(keyed_table.get()); + result = keyed_shim->Vtbl->QueryInterface( + keyed_table.get(), + PH_TEST_IID(PERFECT_HASH_TABLE_JIT_INTERFACE), + reinterpret_cast(&raw_keyed_jit)); + ScopedPhJit keyed_jit(raw_keyed_jit); + ASSERT_GE(result, 0); + ASSERT_NE(keyed_jit.get(), nullptr); + + std::unordered_set keyed_seen; + keyed_seen.reserve(keys.size()); + + for (size_t index = 0; index < keys.size(); ++index) { + ULONGLONG key = keys[index]; + ULONG reloaded = 0; + ULONG actual = 0; + ULONG downsized = DownsizeKey(key, mask); + + ASSERT_GE(keyed_shim->Vtbl->Index(keyed_table.get(), + downsized, + &reloaded), 0); + ASSERT_GE(keyed_jit.get()->Vtbl->Index64(keyed_jit.get(), + key, + &actual), 0); + ASSERT_LT(index, keyless_indexes.size()); + EXPECT_EQ(keyless_indexes[index], reloaded); + EXPECT_EQ(keyless_indexes[index], actual); + EXPECT_LT(reloaded, keys.size()); + EXPECT_LT(actual, keys.size()); + EXPECT_TRUE(keyed_seen.insert(actual).second); + } + EXPECT_EQ(keyed_seen.size(), keys.size()); +} +#endif + +TEST_F(PerfectHashOnlineTests, GraphImpl4RejectsNonGoodHashes) { + const auto keys = MakePseudoRandomKeys(64, 0x13579BDFu); + PERFECT_HASH_KEYS_LOAD_FLAGS keysFlags = {0}; + PERFECT_HASH_TABLE_CREATE_FLAGS tableFlags = {0}; + PERFECT_HASH_TABLE_CREATE_PARAMETER param = {}; + PERFECT_HASH_TABLE_CREATE_PARAMETERS params = {}; + PPERFECT_HASH_TABLE table = nullptr; + + tableFlags.NoFileIo = TRUE; + tableFlags.Quiet = TRUE; + tableFlags.DoNotTryUseHash16Impl = FALSE; + + param.Id = TableCreateParameterGraphImplId; + param.AsULong = 4; + params.SizeOfStruct = sizeof(params); + params.NumberOfElements = 1; + params.Params = ¶m; + + HRESULT result = online_->Vtbl->CreateTableFromKeys( + online_, + PerfectHashChm01AlgorithmId, + PerfectHashHashJenkinsFunctionId, + PerfectHashAndMaskFunctionId, + sizeof(ULONG), + static_cast(keys.size()), + const_cast(keys.data()), + &keysFlags, + &tableFlags, + ¶ms, + &table); + + EXPECT_EQ(result, PH_E_NOT_IMPLEMENTED); + EXPECT_EQ(table, nullptr); +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4Assigned8JitRejected) { + const auto keys = MakePseudoRandomKeys(8, 0xD00DFEEDu); + + PPERFECT_HASH_TABLE table = CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + true, + 4, + nullptr, + 5); + ASSERT_NE(table, nullptr); + + auto *shim = reinterpret_cast(table); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 8u); + + bool tested_backend = false; + +#if defined(PH_HAS_LLVM) + { + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = {0}; + compileFlags.JitBackendLlvm = TRUE; + EXPECT_EQ(online_->Vtbl->CompileTable(online_, table, &compileFlags), + PH_E_NOT_IMPLEMENTED); + tested_backend = true; + } +#endif + +#if defined(PH_HAS_RAWDOG_JIT) + { + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = {0}; + compileFlags.JitBackendRawDog = TRUE; + EXPECT_EQ(online_->Vtbl->CompileTable(online_, table, &compileFlags), + PH_E_NOT_IMPLEMENTED); + tested_backend = true; + } +#endif + + if (!tested_backend) { + shim->Vtbl->Release(table); + GTEST_SKIP() << "No JIT backend enabled for this build."; + } + + shim->Vtbl->Release(table); +} + +#if defined(PH_HAS_RAWDOG_JIT) +TEST_F(PerfectHashOnlineTests, GraphImpl4RawDogJitMatchesIndexAssigned32) { + const auto keys = MakePseudoRandomKeys(64, 0x31415926u); + + ScopedPhTable table(CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4, + nullptr, + 5)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 32u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = {0}; + compileFlags.JitBackendRawDog = TRUE; + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_NOT_IMPLEMENTED) { + GTEST_SKIP() << "RawDog GraphImpl4 scalar JIT unavailable on this host."; + } + ASSERT_GE(result, 0); + + for (size_t i = 0; i < keys.size(); ++i) { + ULONG index = 0; + ASSERT_GE(shim->Vtbl->Index(table.get(), keys[i], &index), 0); + EXPECT_EQ(expected[i], index); + } } -TEST_F(PerfectHashOnlineTests, Assigned16RequiresGraphImpl3AndOptIn) { - const PERFECT_HASH_HASH_FUNCTION_ID hashFunctionId = - PerfectHashHashMultiplyShiftRFunctionId; - const auto keys = MakePseudoRandomKeys(2048, 0xDEADBEEFu); +TEST_F(PerfectHashOnlineTests, GraphImpl4RawDogJitMatchesIndexSparse32) { + const auto keys = MakeSparse32Keys(256); - PPERFECT_HASH_TABLE tableGraphImpl3 = CreateTableFromKeys( + ScopedPhTable table(CreateTableFromKeys( keys, - hashFunctionId, - true, - 3, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4, nullptr, - 20); - ASSERT_NE(tableGraphImpl3, nullptr); + 5)); + ASSERT_NE(table.get(), nullptr); - auto *shimGraphImpl3 = reinterpret_cast(tableGraphImpl3); - PERFECT_HASH_TABLE_FLAGS flagsGraphImpl3 = {0}; - ASSERT_GE( - shimGraphImpl3->Vtbl->GetFlags( - tableGraphImpl3, - sizeof(flagsGraphImpl3), - &flagsGraphImpl3), - 0); - EXPECT_EQ(flagsGraphImpl3.AssignedElementSizeInBits << 3, 16u); - EXPECT_EQ(shimGraphImpl3->Vtbl->SlowIndex, nullptr); - shimGraphImpl3->Vtbl->Release(tableGraphImpl3); + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 32u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); - PPERFECT_HASH_TABLE tableOptOut = CreateTableFromKeys( - keys, - hashFunctionId, - false, - 3, - nullptr, - 20); - ASSERT_NE(tableOptOut, nullptr); + const auto expected = CaptureIndexes(table.get(), keys); - auto *shimOptOut = reinterpret_cast(tableOptOut); - PERFECT_HASH_TABLE_FLAGS flagsOptOut = {0}; - ASSERT_GE(shimOptOut->Vtbl->GetFlags(tableOptOut, sizeof(flagsOptOut), &flagsOptOut), 0); - EXPECT_EQ(flagsOptOut.AssignedElementSizeInBits << 3, 32u); - EXPECT_NE(shimOptOut->Vtbl->SlowIndex, nullptr); - shimOptOut->Vtbl->Release(tableOptOut); + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = {0}; + compileFlags.JitBackendRawDog = TRUE; - PPERFECT_HASH_TABLE tableGraphImpl2 = CreateTableFromKeys( + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_NOT_IMPLEMENTED) { + GTEST_SKIP() << "RawDog GraphImpl4 scalar JIT unavailable on this host."; + } + ASSERT_GE(result, 0); + + for (size_t i = 0; i < keys.size(); ++i) { + ULONG index = 0; + ASSERT_GE(shim->Vtbl->Index(table.get(), keys[i], &index), 0); + EXPECT_EQ(expected[i], index); + } +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4RawDogIndex32x4MatchesIndexAssigned16) { + const auto keys = MakeSparse32Keys(256); + + ScopedPhTable table(CreateTableFromKeys( keys, - hashFunctionId, + PerfectHashHashMultiplyShiftRFunctionId, true, - 2, + 4, nullptr, - 20); - ASSERT_NE(tableGraphImpl2, nullptr); + 5)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 16u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = {0}; + compileFlags.JitBackendRawDog = TRUE; + compileFlags.JitIndex32x4 = TRUE; + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_NOT_IMPLEMENTED) { + GTEST_SKIP() << "RawDog GraphImpl4 Index32x4 unavailable on this host."; + } + ASSERT_GE(result, 0); + + PPERFECT_HASH_TABLE_JIT_INTERFACE raw_jit = nullptr; + result = shim->Vtbl->QueryInterface( + table.get(), +#ifdef PH_WINDOWS + IID_PERFECT_HASH_TABLE_JIT_INTERFACE, +#else + &IID_PERFECT_HASH_TABLE_JIT_INTERFACE, +#endif + reinterpret_cast(&raw_jit)); + ScopedPhJit jit(raw_jit); + ASSERT_GE(result, 0); + ASSERT_NE(jit.get(), nullptr); + + PERFECT_HASH_TABLE_JIT_INFO info = {0}; + result = jit.get()->Vtbl->GetInfo(jit.get(), &info); + ASSERT_GE(result, 0); + EXPECT_TRUE(info.Flags.Index32x4Compiled); + + for (size_t i = 0; i < keys.size(); i += 4) { + ULONG index1 = 0; + ULONG index2 = 0; + ULONG index3 = 0; + ULONG index4 = 0; + + result = jit.get()->Vtbl->Index32x4(jit.get(), + keys[i], + keys[i + 1], + keys[i + 2], + keys[i + 3], + &index1, + &index2, + &index3, + &index4); + ASSERT_GE(result, 0); + + EXPECT_EQ(expected[i], index1); + EXPECT_EQ(expected[i + 1], index2); + EXPECT_EQ(expected[i + 2], index3); + EXPECT_EQ(expected[i + 3], index4); + } - auto *shimGraphImpl2 = reinterpret_cast(tableGraphImpl2); - PERFECT_HASH_TABLE_FLAGS flagsGraphImpl2 = {0}; - ASSERT_GE( - shimGraphImpl2->Vtbl->GetFlags( - tableGraphImpl2, - sizeof(flagsGraphImpl2), - &flagsGraphImpl2), - 0); - EXPECT_EQ(flagsGraphImpl2.AssignedElementSizeInBits << 3, 32u); - EXPECT_NE(shimGraphImpl2->Vtbl->SlowIndex, nullptr); - shimGraphImpl2->Vtbl->Release(tableGraphImpl2); } -#if defined(PH_HAS_RAWDOG_JIT) TEST_F(PerfectHashOnlineTests, RawDogJitIndexMatchesSlowIndex) { const std::vector keys = { 1, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, @@ -3814,6 +5380,264 @@ TEST_F(PerfectHashOnlineTests, #endif #if defined(PH_HAS_LLVM) +TEST_F(PerfectHashOnlineTests, GraphImpl4LlvmJitMatchesIndexAssigned32) { + const auto keys = MakePseudoRandomKeys(64, 0xC001D00Du); + + ScopedPhTable table(CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4, + nullptr, + 5)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 32u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = MakeLlvmJitCompileFlags(); + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM GraphImpl4 scalar JIT backend not found on this host."; + } + ASSERT_EQ(result, S_OK); + + for (size_t i = 0; i < keys.size(); ++i) { + ULONG index = 0; + ASSERT_GE(shim->Vtbl->Index(table.get(), keys[i], &index), 0); + EXPECT_EQ(expected[i], index); + } +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4LlvmJitMatchesIndexSparse32) { + const auto keys = MakeSparse32Keys(256); + + ScopedPhTable table(CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4, + nullptr, + 5)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 32u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = MakeLlvmJitCompileFlags(); + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM GraphImpl4 sparse32 scalar JIT backend not found " + << "on this host."; + } + ASSERT_EQ(result, S_OK); + + for (size_t i = 0; i < keys.size(); ++i) { + ULONG index = 0; + ASSERT_GE(shim->Vtbl->Index(table.get(), keys[i], &index), 0); + EXPECT_EQ(expected[i], index); + } +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4LlvmJitMulshrolate3RXMatchesIndexAssigned32) { + const auto keys = MakePseudoRandomKeys(64, 0xB16B00B5u); + + ScopedPhTable table(CreateTableFromKeys( + keys, + PerfectHashHashMulshrolate3RXFunctionId, + false, + 4, + nullptr, + 5)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 32u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = MakeLlvmJitCompileFlags(); + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM GraphImpl4 Mulshrolate3RX JIT backend not found on this host."; + } + ASSERT_EQ(result, S_OK); + + for (size_t i = 0; i < keys.size(); ++i) { + ULONG index = 0; + ASSERT_GE(shim->Vtbl->Index(table.get(), keys[i], &index), 0); + EXPECT_EQ(expected[i], index); + } +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4LlvmIndex32x4MatchesIndexAssigned16) { + const auto keys = MakeSparse32Keys(256); + + ScopedPhTable table(CreateTableFromKeys( + keys, + PerfectHashHashMultiplyShiftRFunctionId, + true, + 4, + nullptr, + 5)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 16u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = MakeLlvmJitCompileFlags(); + compileFlags.JitVectorIndex32x4 = TRUE; + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM GraphImpl4 Index32x4 backend not found on this host."; + } + ASSERT_EQ(result, S_OK); + + PPERFECT_HASH_TABLE_JIT_INTERFACE raw_jit = nullptr; + result = shim->Vtbl->QueryInterface( + table.get(), +#ifdef PH_WINDOWS + IID_PERFECT_HASH_TABLE_JIT_INTERFACE, +#else + &IID_PERFECT_HASH_TABLE_JIT_INTERFACE, +#endif + reinterpret_cast(&raw_jit)); + ScopedPhJit jit(raw_jit); + ASSERT_GE(result, 0); + ASSERT_NE(jit.get(), nullptr); + + PERFECT_HASH_TABLE_JIT_INFO info = {0}; + result = jit.get()->Vtbl->GetInfo(jit.get(), &info); + ASSERT_GE(result, 0); + EXPECT_TRUE(info.Flags.Index32x4Vector); + + for (size_t i = 0; i < keys.size(); i += 4) { + ULONG index1 = 0; + ULONG index2 = 0; + ULONG index3 = 0; + ULONG index4 = 0; + + result = jit.get()->Vtbl->Index32x4(jit.get(), + keys[i], + keys[i + 1], + keys[i + 2], + keys[i + 3], + &index1, + &index2, + &index3, + &index4); + ASSERT_GE(result, 0); + + EXPECT_EQ(expected[i], index1); + EXPECT_EQ(expected[i + 1], index2); + EXPECT_EQ(expected[i + 2], index3); + EXPECT_EQ(expected[i + 3], index4); + } +} + +TEST_F(PerfectHashOnlineTests, GraphImpl4LlvmIndex64x4MatchesDownsizedIndex) { + const std::vector keys = { + 1ull, 3ull, 5ull, 7ull, 11ull, 13ull, 17ull, 19ull, + 23ull, 29ull, 31ull, 37ull, 41ull, 43ull, 47ull, 53ull, + 59ull, 61ull, 67ull, 71ull, 73ull, 79ull, 83ull, 89ull, + 97ull, 101ull, 103ull, 107ull, 109ull, 113ull, 127ull, 131ull, + }; + + ScopedPhTable table(CreateTableFromKeys64(keys, + PerfectHashHashMultiplyShiftRFunctionId, + false, + 4)); + ASSERT_NE(table.get(), nullptr); + + auto *shim = reinterpret_cast(table.get()); + PERFECT_HASH_TABLE_FLAGS flags = GetTableFlags(table.get()); + ASSERT_EQ(flags.AssignedElementSizeInBits << 3, 32u); + ASSERT_EQ(shim->Vtbl->SlowIndex, nullptr); + + const auto expected = CaptureIndexes64(table.get(), keys); + + PERFECT_HASH_TABLE_COMPILE_FLAGS compileFlags = MakeLlvmJitCompileFlags(); + compileFlags.JitIndex64 = TRUE; + compileFlags.JitIndex32x4 = TRUE; + + HRESULT result = online_->Vtbl->CompileTable(online_, + table.get(), + &compileFlags); + if (result == PH_E_LLVM_BACKEND_NOT_FOUND) { + GTEST_SKIP() << "LLVM GraphImpl4 Index64 JIT backend not found on this host."; + } + ASSERT_EQ(result, S_OK); + + PPERFECT_HASH_TABLE_JIT_INTERFACE raw_jit = nullptr; + result = shim->Vtbl->QueryInterface( + table.get(), +#ifdef PH_WINDOWS + IID_PERFECT_HASH_TABLE_JIT_INTERFACE, +#else + &IID_PERFECT_HASH_TABLE_JIT_INTERFACE, +#endif + reinterpret_cast(&raw_jit)); + ScopedPhJit jit(raw_jit); + ASSERT_GE(result, 0); + ASSERT_NE(jit.get(), nullptr); + + for (size_t i = 0; i < keys.size(); ++i) { + ULONG index = 0; + result = jit.get()->Vtbl->Index64(jit.get(), keys[i], &index); + ASSERT_GE(result, 0); + EXPECT_EQ(expected[i], index); + } + + for (size_t i = 0; i < keys.size(); i += 4) { + ULONG index1 = 0; + ULONG index2 = 0; + ULONG index3 = 0; + ULONG index4 = 0; + + result = jit.get()->Vtbl->Index64x4(jit.get(), + keys[i], + keys[i + 1], + keys[i + 2], + keys[i + 3], + &index1, + &index2, + &index3, + &index4); + ASSERT_GE(result, 0); + + EXPECT_EQ(expected[i], index1); + EXPECT_EQ(expected[i + 1], index2); + EXPECT_EQ(expected[i + 2], index3); + EXPECT_EQ(expected[i + 3], index4); + } +} + TEST_F(PerfectHashOnlineTests, JitIndexMatchesSlowIndex) { const std::vector keys = { 1, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, diff --git a/tests/nested_cmake_args.cmake b/tests/nested_cmake_args.cmake new file mode 100644 index 00000000..480b4b11 --- /dev/null +++ b/tests/nested_cmake_args.cmake @@ -0,0 +1,45 @@ +set(nested_cmake_compiler_args "") +if(DEFINED TEST_CMAKE_TOOLCHAIN_FILE AND NOT TEST_CMAKE_TOOLCHAIN_FILE STREQUAL "") + list(APPEND nested_cmake_compiler_args + "-DCMAKE_TOOLCHAIN_FILE=${TEST_CMAKE_TOOLCHAIN_FILE}") +endif() +if(DEFINED TEST_CMAKE_C_COMPILER AND NOT TEST_CMAKE_C_COMPILER STREQUAL "") + list(APPEND nested_cmake_compiler_args + "-DCMAKE_C_COMPILER=${TEST_CMAKE_C_COMPILER}") +endif() +if(DEFINED TEST_CMAKE_CXX_COMPILER AND NOT TEST_CMAKE_CXX_COMPILER STREQUAL "") + list(APPEND nested_cmake_compiler_args + "-DCMAKE_CXX_COMPILER=${TEST_CMAKE_CXX_COMPILER}") +endif() + +# +# Keep the inherited generator make program separate from the Unix Makefiles +# fallback make program. Some tests run one nested configure with the parent +# generator and a different nested configure with Unix Makefiles; these argument +# lists must not be concatenated into one CMake invocation. +# +set(nested_cmake_unix_make_args "") +if(DEFINED TEST_CMAKE_UNIX_MAKE_PROGRAM AND + NOT TEST_CMAKE_UNIX_MAKE_PROGRAM STREQUAL "") + list(APPEND nested_cmake_unix_make_args + "-DCMAKE_MAKE_PROGRAM=${TEST_CMAKE_UNIX_MAKE_PROGRAM}") +endif() + +set(nested_cmake_generator_args "") +if(DEFINED TEST_CMAKE_GENERATOR AND NOT TEST_CMAKE_GENERATOR STREQUAL "") + list(APPEND nested_cmake_generator_args -G "${TEST_CMAKE_GENERATOR}") +endif() +if(DEFINED TEST_CMAKE_GENERATOR_PLATFORM AND + NOT TEST_CMAKE_GENERATOR_PLATFORM STREQUAL "") + list(APPEND nested_cmake_generator_args + -A "${TEST_CMAKE_GENERATOR_PLATFORM}") +endif() +if(DEFINED TEST_CMAKE_GENERATOR_TOOLSET AND + NOT TEST_CMAKE_GENERATOR_TOOLSET STREQUAL "") + list(APPEND nested_cmake_generator_args + -T "${TEST_CMAKE_GENERATOR_TOOLSET}") +endif() +if(DEFINED TEST_CMAKE_MAKE_PROGRAM AND NOT TEST_CMAKE_MAKE_PROGRAM STREQUAL "") + list(APPEND nested_cmake_generator_args + "-DCMAKE_MAKE_PROGRAM=${TEST_CMAKE_MAKE_PROGRAM}") +endif() diff --git a/tests/run_cli_codegen_test.cmake b/tests/run_cli_codegen_test.cmake index 61d593d1..01756f04 100644 --- a/tests/run_cli_codegen_test.cmake +++ b/tests/run_cli_codegen_test.cmake @@ -15,6 +15,8 @@ if(NOT DEFINED TEST_CARGO) set(TEST_CARGO "") endif() +include("${CMAKE_CURRENT_LIST_DIR}/nested_cmake_args.cmake") + function(require_file path) if(NOT EXISTS "${path}") message(FATAL_ERROR "Expected file not found: ${path}") @@ -42,6 +44,11 @@ endif() if(DEFINED TEST_FLAGS) string(REPLACE "|" ";" flags_list "${TEST_FLAGS}") endif() +if(DEFINED TEST_SKIP_CREATE_RESULTS) + string(REPLACE "|" ";" skip_create_results "${TEST_SKIP_CREATE_RESULTS}") +else() + set(skip_create_results "") +endif() file(REMOVE_RECURSE "${test_output_native}") file(MAKE_DIRECTORY "${test_output_native}") @@ -54,6 +61,12 @@ execute_process( ) if(NOT result EQUAL 0) + if(NOT skip_create_results STREQUAL "" AND + "${result}" IN_LIST skip_create_results) + message(STATUS "SKIPPED: create command returned ${result}") + set(TEST_CODEGEN_SKIPPED TRUE) + return() + endif() message(STATUS "stdout: ${stdout}") message(STATUS "stderr: ${stderr}") message(FATAL_ERROR "Command failed with exit code ${result}") @@ -71,8 +84,13 @@ get_filename_component(gen_dir "${gen_cmake}" DIRECTORY) set(build_dir "${test_output_native}/_build") file(MAKE_DIRECTORY "${build_dir}") +set(configure_command "${CMAKE_COMMAND}" -S "${gen_dir}" -B "${build_dir}") +list(APPEND configure_command + ${nested_cmake_generator_args} + ${nested_cmake_compiler_args}) + execute_process( - COMMAND "${CMAKE_COMMAND}" -S "${gen_dir}" -B "${build_dir}" + COMMAND ${configure_command} RESULT_VARIABLE result OUTPUT_VARIABLE stdout ERROR_VARIABLE stderr @@ -122,6 +140,9 @@ require_file("${gen_dir}/${table_name}.h") require_file("${gen_dir}/${table_name}.cpp") require_file("${gen_dir}/${table_name}.def") require_file("${gen_dir}/${table_name}.pht1") +# Windows alternate data stream checks live in the GraphImpl4-specific smoke +# tests behind CMAKE_HOST_WIN32; the shared codegen smoke only requires files +# that exist on every host filesystem. require_file("${gen_dir}/${table_name}.sln") require_file("${gen_dir}/${table_name}_StdAfx.c") require_file("${gen_dir}/${table_name}_StdAfx.h") @@ -153,6 +174,16 @@ require_glob("${gen_dir}/*_Test.mk" "Test makefile fragment") require_glob("${gen_dir}/*_BenchmarkFull.mk" "BenchmarkFull makefile fragment") require_glob("${gen_dir}/*_BenchmarkIndex.mk" "BenchmarkIndex makefile fragment") +file(READ "${gen_dir}/${table_name}_Python.py" generated_python) +string(FIND "${generated_python}" "TABLE_DATA_TYPE =" python_table_data_type_pos) +string(FIND "${generated_python}" "TABLE_DATA = [" python_table_data_pos) +if(python_table_data_type_pos LESS 0 OR python_table_data_pos LESS 0) + message(FATAL_ERROR "Expected TABLE_DATA_TYPE and TABLE_DATA in generated Python file") +endif() +if(NOT python_table_data_type_pos LESS python_table_data_pos) + message(FATAL_ERROR "Expected TABLE_DATA_TYPE before TABLE_DATA in generated Python file") +endif() + if(NOT CMAKE_HOST_WIN32) set(make_build_dir "${test_output_native}/_build_make") file(MAKE_DIRECTORY "${make_build_dir}") @@ -165,6 +196,8 @@ if(NOT CMAKE_HOST_WIN32) execute_process( COMMAND "${CMAKE_COMMAND}" -S "${gen_dir}" -B "${make_build_dir}" -G "Unix Makefiles" ${make_config_args} + ${nested_cmake_compiler_args} + ${nested_cmake_unix_make_args} RESULT_VARIABLE result OUTPUT_VARIABLE stdout ERROR_VARIABLE stderr @@ -366,4 +399,4 @@ if(EXISTS "${gen_dir}/Cargo.toml") message(STATUS "stderr: ${stderr}") message(FATAL_ERROR "Cargo rust_bench failed with exit code ${result}") endif() -endif() \ No newline at end of file +endif() diff --git a/tests/run_cli_graphimpl4_codegen_test.cmake b/tests/run_cli_graphimpl4_codegen_test.cmake new file mode 100644 index 00000000..71a31e0a --- /dev/null +++ b/tests/run_cli_graphimpl4_codegen_test.cmake @@ -0,0 +1,171 @@ +if(NOT DEFINED TEST_EXE) + message(FATAL_ERROR "TEST_EXE is required") +endif() +if(NOT DEFINED TEST_OUTPUT) + message(FATAL_ERROR "TEST_OUTPUT is required") +endif() +if(NOT DEFINED TEST_PYTHON) + message(FATAL_ERROR "TEST_PYTHON is required") +endif() + +if(NOT DEFINED TEST_CARGO) + set(TEST_CARGO "") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/nested_cmake_args.cmake") + +function(require_file path) + if(NOT EXISTS "${path}") + message(FATAL_ERROR "Expected file not found: ${path}") + endif() +endfunction() + +get_filename_component(test_output_parent "${TEST_OUTPUT}" DIRECTORY) +file(MAKE_DIRECTORY "${test_output_parent}") + +set(graphimpl4_keys "${test_output_parent}/graphimpl4.keys") +set(generator_script "${test_output_parent}/generate_graphimpl4_keys.py") +file(WRITE "${generator_script}" "import struct\n") +file(APPEND "${generator_script}" "from pathlib import Path\n") +file(APPEND "${generator_script}" "path = Path(r'''${graphimpl4_keys}''')\n") +file(APPEND "${generator_script}" "keys = [0x2468ACE1,0x2468ACE3,0x2468ACE5,0x2468ACE7,0x2468ACEB,0x2468ACED,0x2468ACF1,0x2468ACF3]\n") +file(APPEND "${generator_script}" "with path.open('wb') as f:\n") +file(APPEND "${generator_script}" " for key in keys:\n") +file(APPEND "${generator_script}" " f.write(struct.pack('