From db7f7aabafb6664ff88e19b134742e756edf28ee Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 09:08:22 +1000 Subject: [PATCH 01/24] test(stack): cover empty text order pg rejection --- .../v3-matrix/matrix-live-pg.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts b/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts index c29b739f..4886da68 100644 --- a/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts @@ -113,6 +113,9 @@ const ordDomains = domains.filter( const storageDomains = domains.filter( ([, spec]) => proofKindFor(spec.indexes) === 'storage', ) +const textOrderDomains = domains.filter( + ([t]) => t === 'eql_v3.text_ord_ore' || t === 'eql_v3.text_ord', +) type Row = { id: number } @@ -278,6 +281,23 @@ describeLivePg( expect(rows.map((r) => r.id)).toEqual([idA]) }) + it.each( + textOrderDomains, + )('%s: encrypted empty string is rejected by the Postgres domain', async (eqlType) => { + const col = slug(eqlType) + const column = (table as unknown as Record)[col] as never + const encrypted = unwrapResult( + await client.encrypt('', { + table: table as never, + column, + }), + ) + + await expect( + sql.unsafe(`SELECT $1::${eqlType}`, [sql.json(encrypted as never)]), + ).rejects.toThrow(/violates check constraint/) + }) + it.each( matchDomains, )('%s: match_term/bloom_filter selects row B (containing "ada"), not row A', async (eqlType) => { From f769cad21784e833f182fad786f9ec64fb605077 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 09:19:39 +1000 Subject: [PATCH 02/24] test(stack): avoid empty text search pg seed --- .../__tests__/v3-matrix/matrix-live-pg.test.ts | 15 +++++++++------ packages/stack/__tests__/v3-matrix/matrix.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts b/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts index 4886da68..d997bfe2 100644 --- a/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts @@ -113,8 +113,11 @@ const ordDomains = domains.filter( const storageDomains = domains.filter( ([, spec]) => proofKindFor(spec.indexes) === 'storage', ) -const textOrderDomains = domains.filter( - ([t]) => t === 'eql_v3.text_ord_ore' || t === 'eql_v3.text_ord', +const textOreDomains = domains.filter( + ([t]) => + t === 'eql_v3.text_ord_ore' || + t === 'eql_v3.text_ord' || + t === 'eql_v3.text_search', ) type Row = { id: number } @@ -227,9 +230,9 @@ beforeAll(async () => { ), ) } - // text_match/text_search: query a substring of row B's sample. Row A's - // shared `TEXT_S[0]` is `''` — a degenerate containment target — so the - // match proof targets row B instead of the usual row A. + // text_match/text_search: query a substring of whichever seeded sample + // contains "ada". `text_match` still uses the shared TEXT_S with an empty row + // A, while `text_search` uses non-empty TEXT_ORE_S to satisfy its `ob` check. for (const [t] of matchDomains) { matchTerms[slug(t)] = unwrapResult( await termClient.encryptQuery( @@ -282,7 +285,7 @@ describeLivePg( }) it.each( - textOrderDomains, + textOreDomains, )('%s: encrypted empty string is rejected by the Postgres domain', async (eqlType) => { const col = slug(eqlType) const column = (table as unknown as Record)[col] as never diff --git a/packages/stack/__tests__/v3-matrix/matrix.test.ts b/packages/stack/__tests__/v3-matrix/matrix.test.ts index c2bdf320..36da2381 100644 --- a/packages/stack/__tests__/v3-matrix/matrix.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix.test.ts @@ -33,4 +33,12 @@ describe('eql_v3 type-driven domain matrix (runtime)', () => { indexes: spec.indexes, }) }) + + it.each([ + 'eql_v3.text_ord_ore', + 'eql_v3.text_ord', + 'eql_v3.text_search', + ] as const)('%s uses non-empty positive samples for live Postgres inserts', (eqlType) => { + expect(V3_MATRIX[eqlType].samples[0]).not.toBe('') + }) }) From 3c050b8a24195c9bc8130b969d25acdd20d1bc02 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 11:50:19 +1000 Subject: [PATCH 03/24] fix(stack): address EQL v3 types-module review follow-ups Resolves pending review feedback from #541 (merged). Seven findings: - test: restore the live ORE proof for text order domains. text_ord / text_ord_ore carry both hm (unique) and ob (ore), so the single-kind classifier ran only the eq_term/hmac_256 proof and silently skipped ord_term/ore_block_256 -- a wrong-valued ob would pass the matrix green. Run both proofs; build the ob term with queryType:'orderAndRange' since equality resolves to hm on these domains. - schema: extract a shared resolveMatchOpts() merge+clone helper used by both the v2 and v3 freeTextSearch builders, giving v2 the clone-on-write protection it lacked (it stored caller opts by reference). - eql/v3: derive EncryptedTextSearchColumn.build()'s unique/ore blocks via indexesForCapabilities (override only match) so the emitted index shape cannot drift from the shared capability mapping. - encryption/v3: precompute row reconstructors per schema table at typedClient construction and return a DecryptionError failure for unknown tables, so table.build()'s duplicate-column throw can no longer escape the decrypt Result contract as a promise rejection. - eql/v3: alias V3DecryptedModel to V3ModelInput (character-identical) to stop the input/output model shapes silently drifting. - ci: add src/schema/match-defaults.ts to the fta-v3 path scope -- it shapes every emitted v3 match block but sat outside the gate. - ci: re-baseline FTA --score-cap 72 -> 69 after the eql/v3 file split and refresh the spec doc's stale 71.08 baseline claim. --- .github/workflows/fta-v3.yml | 5 ++ .../2026-07-02-stryker-v3-ci-gate-design.md | 6 +- .../v3-matrix/matrix-live-pg.test.ts | 76 ++++++++++++------- packages/stack/package.json | 2 +- packages/stack/src/encryption/v3.ts | 37 ++++++++- packages/stack/src/eql/v3/columns.ts | 37 +++++---- packages/stack/src/eql/v3/table.ts | 19 +++-- packages/stack/src/schema/index.ts | 17 ++--- packages/stack/src/schema/match-defaults.ts | 22 ++++++ 9 files changed, 148 insertions(+), 73 deletions(-) diff --git a/.github/workflows/fta-v3.yml b/.github/workflows/fta-v3.yml index 541ffc33..15b2c1f9 100644 --- a/.github/workflows/fta-v3.yml +++ b/.github/workflows/fta-v3.yml @@ -12,6 +12,10 @@ on: - 'main' paths: - 'packages/stack/src/eql/v3/**' + # Shared match-index defaults live outside src/eql/v3 but shape every + # emitted v3 match block (load-bearing `k`/`m` ciphertext params), so edits + # here must trigger the v3 gate too. + - 'packages/stack/src/schema/match-defaults.ts' - 'packages/stack/package.json' - '.github/workflows/fta-v3.yml' pull_request: @@ -19,6 +23,7 @@ on: - "**" paths: - 'packages/stack/src/eql/v3/**' + - 'packages/stack/src/schema/match-defaults.ts' - 'packages/stack/package.json' - '.github/workflows/fta-v3.yml' diff --git a/docs/superpowers/specs/2026-07-02-stryker-v3-ci-gate-design.md b/docs/superpowers/specs/2026-07-02-stryker-v3-ci-gate-design.md index a149cdad..82eb2f41 100644 --- a/docs/superpowers/specs/2026-07-02-stryker-v3-ci-gate-design.md +++ b/docs/superpowers/specs/2026-07-02-stryker-v3-ci-gate-design.md @@ -147,8 +147,10 @@ Implementation therefore includes a **baseline step**: locally. 2. Record the reported mutation score for `eql/v3`. 3. Set `thresholds.break` just **below** the measured score (a small buffer, the - way FTA sets `--score-cap 72` against a current 71.08). This ensures the - current state passes while any regression that lowers the score fails the gate. + way FTA sets `--score-cap 69` against a current worst-file score of 68.00 — + re-baselined from the pre-split monolith's 71.08/72 after `eql/v3` was split + into per-file modules). This ensures the current state passes while any + regression that lowers the score fails the gate. 4. Set `high`/`low` to reasonable display bands (do not affect pass/fail). If the measured baseline is very low (tests are weak), surface that to the user diff --git a/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts b/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts index d997bfe2..5b485c6e 100644 --- a/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix-live-pg.test.ts @@ -22,12 +22,14 @@ * and not the other. Dispatch mirrors the priority `resolveIndexType` itself * uses (match > unique > ore > none): * - match (text_match, text_search): `eql_v3.match_term` + `bloom_filter` - * - eq (*_eq domains): `eql_v3.eq_term` + `hmac_256` - * - ord (*_ord / *_ord_ore domains): `eql_v3.ord_term` + `ore_block_256`, - * queried with `queryType:'equality'` — the exact path Part A fixed. Most - * ord-tier domains (all but text) have no `eq_term` at all in the real - * `eql_v3` SQL (verified against the fixture), so this is not a stylistic - * choice: it is the only equality path that exists for them. + * - eq (any `unique`/`hm` domain): `eql_v3.eq_term` + `hmac_256` + * - ord (any `ore`/`ob` domain): `eql_v3.ord_term` + `ore_block_256`. + * Pure-ORE domains (numeric/date `*_ord`/`*_ord_ore`) are queried with + * `queryType:'equality'` — the equality-via-ORE path Part A fixed — because + * `ob` is their only index and they have no `eq_term` at all in the real + * `eql_v3` SQL (verified against the fixture). Text order domains carry BOTH + * `hm` and `ob`, so they run the eq proof AND the ord proof; their `ob` term + * is built with `queryType:'orderAndRange'` (equality would resolve to `hm`). * - storage (no index): no query is possible; proves the ciphertext, cast to * THIS SPECIFIC Postgres domain type, survives a real INSERT/SELECT and * still decrypts — the one thing the FFI-only round-trip can't show. @@ -84,34 +86,45 @@ const columns = Object.fromEntries( const table = encryptedTable(TABLE_NAME, columns as never) /** - * The one proof each domain's configured indexes call for — mirrors the - * priority `resolveIndexType`/`inferIndexType` themselves use: match wins over - * unique wins over ore. `text_search` carries all three but gets the match - * proof (its distinguishing, richest capability); the plain `*_eq` domains get - * the eq proof; every `*_ord`/`*_ord_ore` domain (including the text ones, - * which also have an `eq_term` but are queried the same way as their - * non-text siblings for consistency) gets the equality-via-ORE proof. + * Which proofs a domain's configured indexes call for. Unlike a single-kind + * classifier these lists are NOT mutually exclusive — a domain runs EVERY proof + * its indexes support: + * + * - eq (`hm`): every domain carrying a `unique` index → `eq_term`/`hmac_256`. + * - ord (`ob`): every domain carrying an `ore` index → `ord_term`/`ore_block_256`. + * - match (`bf`): every domain carrying a `match` index → `match_term`/`bloom_filter`. + * - storage: a domain with NO index — only the ciphertext round-trip proof. + * + * Text order domains (`text_ord`/`text_ord_ore`) carry BOTH `unique` and `ore`, + * so they appear in `eqDomains` AND `ordDomains` and run both proofs — a + * wrong-valued `ob` would otherwise slip through an eq-only check (text equality + * is HMAC-based, so `queryType:'equality'` on them resolves to `hm`, never `ob`; + * their ord term is built with `queryType:'orderAndRange'` below). `text_search` + * also carries all three indexes but is deliberately exercised by the match + * proof ALONE — its distinguishing, richest capability and the one canonical + * example per tier — so it is excluded from the eq/ord lists via `!match`. */ -type ProofKind = 'match' | 'eq' | 'ord' | 'storage' -function proofKindFor(indexes: DomainSpec['indexes']): ProofKind { - const idx = indexes ?? {} - if (idx.match) return 'match' - if (idx.unique) return 'eq' - if (idx.ore) return 'ord' - return 'storage' -} +const hasIndex = ( + indexes: DomainSpec['indexes'], + key: 'unique' | 'ore' | 'match', +): boolean => Boolean((indexes ?? {})[key]) -const matchDomains = domains.filter( - ([, spec]) => proofKindFor(spec.indexes) === 'match', +const matchDomains = domains.filter(([, spec]) => + hasIndex(spec.indexes, 'match'), ) const eqDomains = domains.filter( - ([, spec]) => proofKindFor(spec.indexes) === 'eq', + ([, spec]) => + hasIndex(spec.indexes, 'unique') && !hasIndex(spec.indexes, 'match'), ) const ordDomains = domains.filter( - ([, spec]) => proofKindFor(spec.indexes) === 'ord', + ([, spec]) => + hasIndex(spec.indexes, 'ore') && !hasIndex(spec.indexes, 'match'), ) const storageDomains = domains.filter( - ([, spec]) => proofKindFor(spec.indexes) === 'storage', + ([, spec]) => + !hasIndex(spec.indexes, 'unique') && + !hasIndex(spec.indexes, 'ore') && + !hasIndex(spec.indexes, 'match'), ) const textOreDomains = domains.filter( ([t]) => @@ -219,13 +232,22 @@ beforeAll(async () => { ) } for (const [t, spec] of ordDomains) { + // Pure-ORE domains (numeric/date) answer equality via ORE, so + // `queryType:'equality'` resolves to the `ob` term — the exact + // equality-via-ORE path Part A fixed. Text order domains ALSO carry `hm`, + // where `equality` resolves to HMAC by the shared `unique > … > ore` + // priority; force `orderAndRange` there so the term still carries the `ob` + // this proof extracts with `ore_block_256`. + const queryType = hasIndex(spec.indexes, 'unique') + ? 'orderAndRange' + : 'equality' ordTerms[slug(t)] = unwrapResult( await termClient.encryptQuery( spec.samples[0] as never, { table, column: columnRef(t), - queryType: 'equality', + queryType, } as never, ), ) diff --git a/packages/stack/package.json b/packages/stack/package.json index c4bcacc6..83de07f1 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -217,7 +217,7 @@ "prebuild": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", "build": "tsup", "dev": "tsup --watch", - "analyze:complexity": "fta src/eql/v3 --score-cap 72", + "analyze:complexity": "fta src/eql/v3 --score-cap 69", "db:eql-v3:install": "tsx scripts/install-eql-v3.ts", "test": "vitest run", "test:types": "vitest --run --typecheck.only", diff --git a/packages/stack/src/encryption/v3.ts b/packages/stack/src/encryption/v3.ts index 9b84dff2..c91bb17c 100644 --- a/packages/stack/src/encryption/v3.ts +++ b/packages/stack/src/encryption/v3.ts @@ -9,7 +9,7 @@ import type { V3EncryptedModel, V3ModelInput, } from '@/eql/v3' -import type { EncryptionError } from '@/errors' +import { type EncryptionError, EncryptionErrorTypes } from '@/errors' import type { LockContextInput } from '@/identity' import type { BulkDecryptPayload, @@ -172,8 +172,34 @@ function rowReconstructor( */ export function typedClient( client: EncryptionClient, - ..._schemas: S + ...schemas: S ): TypedEncryptionClient { + // Precompute one row reconstructor per schema table at construction. This runs + // each table's `build()` — which throws on duplicate DB column names — ONCE, + // here, off the Result-returning decrypt path. `decryptModel`/ + // `bulkDecryptModels` therefore never call `build()` (whose throw would surface + // as a promise rejection and break their `Promise>` contract) and no + // longer rebuild the row-invariant config on every call. + const reconstructors = new Map< + AnyV3Table, + (row: Record) => Record + >() + for (const table of schemas) { + reconstructors.set(table, rowReconstructor(table)) + } + + // A table not among the schemas this client was initialized with (only + // reachable by bypassing the `Table extends S[number]` type constraint) has no + // precomputed reconstructor. Return a Result failure rather than building one + // inline, which could throw and reject the Result-shaped decrypt promise. + const unknownTableFailure: { failure: EncryptionError } = { + failure: { + type: EncryptionErrorTypes.DecryptionError, + message: + '[eql/v3]: decryptModel received a table this client was not initialized with — pass the same table object(s) given to EncryptionV3/typedClient', + }, + } + return { encrypt: (plaintext, opts) => client.encrypt(plaintext as never, opts as never), @@ -185,16 +211,19 @@ export function typedClient( client.bulkEncryptModels(input as never, table as never) as never, decrypt: (encrypted) => client.decrypt(encrypted), decryptModel: async (input, table, lockContext) => { + const reconstruct = reconstructors.get(table) + if (!reconstruct) return unknownTableFailure as never const op = client.decryptModel(input as never) const result = await (lockContext ? op.withLockContext(lockContext) : op) if (result.failure) return result as never - return { data: rowReconstructor(table)(result.data) } as never + return { data: reconstruct(result.data) } as never }, bulkDecryptModels: async (input, table, lockContext) => { + const reconstruct = reconstructors.get(table) + if (!reconstruct) return unknownTableFailure as never const op = client.bulkDecryptModels(input as never) const result = await (lockContext ? op.withLockContext(lockContext) : op) if (result.failure) return result as never - const reconstruct = rowReconstructor(table) return { data: result.data.map((row) => reconstruct(row as Record), diff --git a/packages/stack/src/eql/v3/columns.ts b/packages/stack/src/eql/v3/columns.ts index 71503645..0e0c265c 100644 --- a/packages/stack/src/eql/v3/columns.ts +++ b/packages/stack/src/eql/v3/columns.ts @@ -3,6 +3,7 @@ import { type BuiltMatchIndexOpts, cloneMatchOpts, defaultMatchOpts, + resolveMatchOpts, } from '@/schema/match-defaults' /** @@ -451,33 +452,29 @@ export class EncryptedTextSearchColumn extends EncryptedV3Column< * on for this type. Merge semantics mirror v2's `opts?.x ?? default`. */ freeTextSearch(opts?: MatchIndexOpts): this { - // A fresh defaults object per call supplies the `?? ` fallbacks, so no - // nested default object is ever shared into `this.matchOpts` by reference. - const defaults = defaultMatchOpts() - // Clone-on-write: deep-copy the nested tokenizer / token_filters when - // storing them so a caller mutating their own opts object between - // freeTextSearch(opts) and build() cannot leak into the emitted config. - this.matchOpts = cloneMatchOpts({ - tokenizer: opts?.tokenizer ?? defaults.tokenizer, - token_filters: opts?.token_filters ?? defaults.token_filters, - k: opts?.k ?? defaults.k, - m: opts?.m ?? defaults.m, - include_original: opts?.include_original ?? defaults.include_original, - }) + // Shared merge+clone (schema/match-defaults) — one source of truth with the + // v2 `freeTextSearch()` builder. `resolveMatchOpts` merges each key over the + // per-call defaults and deep-clones, so a caller mutating their own opts + // object between `freeTextSearch(opts)` and `build()` cannot leak into the + // emitted config (clone-on-write). + this.matchOpts = resolveMatchOpts(opts) return this } /** Emit the encrypt-config column. Byte-identical to a v2 equality+order+match column. */ override build(): ColumnSchema { - // Deep-clone the match block so the returned config NEVER aliases this - // builder's internal `matchOpts` (or any caller-supplied opts merged into - // it). A caller mutating the returned object cannot corrupt this builder's - // state or another column's defaults. + // Derive `cast_as` + the `unique`/`ore` blocks from the shared + // capability→index mapping (this domain's TEXT_SEARCH capabilities produce + // exactly `{ unique, ore, match }`), then override ONLY `match` with this + // builder's tuned opts. Hand-writing the unique/ore shape here would let it + // silently drift from `indexesForCapabilities` — the exact divergence class + // behind the text_ord `hm`-index bug. Deep-clone the match block so the + // returned config never aliases this builder's internal `matchOpts`. + const base = super.build() return { - cast_as: 'string', + ...base, indexes: { - unique: { token_filters: [] }, - ore: {}, + ...base.indexes, match: cloneMatchOpts(this.matchOpts), }, } diff --git a/packages/stack/src/eql/v3/table.ts b/packages/stack/src/eql/v3/table.ts index c5b3f069..8ea14497 100644 --- a/packages/stack/src/eql/v3/table.ts +++ b/packages/stack/src/eql/v3/table.ts @@ -211,11 +211,14 @@ export type V3EncryptedModel = { : T[K] } -/** The decrypted result model: schema columns become their plaintext type, others pass through. */ -export type V3DecryptedModel
= { - [K in keyof T]: K extends keyof InferPlaintext
- ? null extends T[K] - ? InferPlaintext
[K] | null - : InferPlaintext
[K] - : T[K] -} +/** + * The decrypted result model: schema columns become their plaintext type, others + * pass through. Structurally identical to {@link V3ModelInput} — decrypt yields + * the same plaintext shape encrypt accepts — so it is aliased rather than + * re-declared to keep the input and output shapes from silently drifting when + * one copy is edited. The distinct name is kept for call-site readability. + */ +export type V3DecryptedModel
= V3ModelInput< + Table, + T +> diff --git a/packages/stack/src/schema/index.ts b/packages/stack/src/schema/index.ts index 2b2842a9..4eb7a66a 100644 --- a/packages/stack/src/schema/index.ts +++ b/packages/stack/src/schema/index.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import type { BuildableTable, Encrypted } from '@/types' -import { defaultMatchOpts } from './match-defaults' +import { resolveMatchOpts } from './match-defaults' // ------------------------ // Zod schemas @@ -365,16 +365,11 @@ export class EncryptedColumn { * ``` */ freeTextSearch(opts?: MatchIndexOpts) { - // Shared defaults (schema/match-defaults) — one source of truth with the - // EQL v3 domain builders. The factory returns fresh nested objects. - const defaults = defaultMatchOpts() - this.indexesValue.match = { - tokenizer: opts?.tokenizer ?? defaults.tokenizer, - token_filters: opts?.token_filters ?? defaults.token_filters, - k: opts?.k ?? defaults.k, - m: opts?.m ?? defaults.m, - include_original: opts?.include_original ?? defaults.include_original, - } + // Shared merge+clone (schema/match-defaults) — one source of truth with the + // EQL v3 domain builders. `resolveMatchOpts` deep-clones, so a caller + // mutating their own `opts` (or its nested tokenizer/token_filters) after + // this call cannot leak into the stored schema. + this.indexesValue.match = resolveMatchOpts(opts) return this } diff --git a/packages/stack/src/schema/match-defaults.ts b/packages/stack/src/schema/match-defaults.ts index e1a4090b..e7518590 100644 --- a/packages/stack/src/schema/match-defaults.ts +++ b/packages/stack/src/schema/match-defaults.ts @@ -52,3 +52,25 @@ export function cloneMatchOpts(opts: BuiltMatchIndexOpts): BuiltMatchIndexOpts { token_filters: opts.token_filters.map((f) => ({ ...f })), } } + +/** + * Resolve user-supplied `freeTextSearch(opts)` input into a fully-built match + * block: each provided key replaces its default, omitted keys keep the default + * (`opts?.x ?? default.x`). The single source of truth for that five-field merge + * shared by the v2 `freeTextSearch()` builder and the v3 domain builders. + * + * The result is deep-cloned ({@link cloneMatchOpts}) so a caller mutating their + * own `opts` object (or its nested `tokenizer`/`token_filters`) after this call + * can never leak into the stored builder state or the emitted config — clone-on- + * write for both builders, not just v3. + */ +export function resolveMatchOpts(opts?: MatchIndexOpts): BuiltMatchIndexOpts { + const defaults = defaultMatchOpts() + return cloneMatchOpts({ + tokenizer: opts?.tokenizer ?? defaults.tokenizer, + token_filters: opts?.token_filters ?? defaults.token_filters, + k: opts?.k ?? defaults.k, + m: opts?.m ?? defaults.m, + include_original: opts?.include_original ?? defaults.include_original, + }) +} From 2cc1144ba44aa2b0bfcea318c44bd06f2cc02148 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 15:00:08 +1000 Subject: [PATCH 04/24] docs(stack): design for EQL v3 Drizzle concrete-type support --- ...06-eql-v3-drizzle-concrete-types-design.md | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md diff --git a/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md b/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md new file mode 100644 index 00000000..5995194d --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md @@ -0,0 +1,286 @@ +# EQL v3 Drizzle support — concrete-type authoring — design + +Status: proposed +Date: 2026-07-06 +Branch: `feat/eql-v3-text-search-schema` + +## 1. Goal + +Add EQL v3 support to the Drizzle integration inside `@cipherstash/stack`, mirroring +the existing v2 Drizzle integration (`@cipherstash/stack/drizzle`) but re-shaped +around the v3 **concrete-type** model. + +The defining change of v3: **the concrete type defines the search capabilities.** +A column declared `eql_v3.int4_eq` supports equality; `eql_v3.int4_ord` supports +order/range (and equality via ORE); `eql_v3.text_search` supports equality, order, +and free-text match. There is no separate `.equality()` / `.orderAndRange()` / +`.freeTextSearch()` index configuration as in v2 — everything the adapter needs +(SQL column domain, `cast_as`, and which operators are legal) is **derived from the +concrete type**, which already carries that metadata in +`@cipherstash/stack/eql/v3`. + +Success = a developer can declare a Drizzle `pgTable` with encrypted v3 columns, +feed the same schema to `EncryptionV3`, and run equality / range / free-text / +ordering queries through Drizzle using capability-checked operators that emit the +correct `eql_v3` term-function SQL. + +## 2. Background: why v3 is different from v2 + +Confirmed against this branch's SQL bundle +(`packages/stack/__tests__/fixtures/eql-v3/cipherstash-encrypt-v3.sql`) and the live +pg tests (`packages/stack/__tests__/schema-v3-pg.test.ts`). + +**Column type.** v2 uses a single composite type `eql_v2_encrypted` for every +encrypted column. v3 uses **one `CREATE DOMAIN … AS jsonb` per concrete type** — +`eql_v3.int4_ord`, `eql_v3.text_eq`, `eql_v3.text_search`, `eql_v3.bool`, … There is +no single catch-all `eql_v3_encrypted`. + +**Wire form.** v2 writes a composite literal `("…")`. v3 domains are plain jsonb, so +the value is inserted as plain JSON cast to the domain (`$1::eql_v3.int4_ord`). + +**Query form.** v2 compares encrypted payloads directly (native `=` on +`eql_v2_encrypted`, or `eql_v2.gt/lt/like/order_by(...)` convenience functions). v3 +has **no convenience operators**; it compares **extracted index terms** on both +sides: + +| Capability | v3 SQL | +| --- | --- | +| equality (HMAC) | `eql_v3.eq_term(col) = eql_v3.hmac_256($::jsonb)` | +| equality (ORE, numeric/date order domains) | `eql_v3.ord_term(col) = eql_v3.ore_block_256($::jsonb)` | +| order/range | `eql_v3.ord_term(col) />= eql_v3.ore_block_256($::jsonb)` | +| free-text match | `eql_v3.match_term(col) @> eql_v3.bloom_filter($::jsonb)` | +| order by | `ORDER BY eql_v3.ord_term(col)` | + +The column-side extractor (`eq_term`/`ord_term`/`match_term`) takes the column +domain (its stored value has ciphertext `c`, so it passes the domain CHECK). The +search-side constructor (`hmac_256`/`ore_block_256`/`bloom_filter`) pulls the index +field straight out of the query-term jsonb with no domain coercion — this is why a +query term (which has no ciphertext) must be cast to `::jsonb`, never to the domain. + +**Concrete types already exist.** `@cipherstash/stack/eql/v3` ships the full +concrete-type system (`packages/stack/src/eql/v3/{columns,types,table,index}.ts`): +- `types.(name)` factories for all 35 shipped domains. +- Each `EncryptedV3Column` carries `getEqlType()` (`eql_v3.int4_ord`), `castAs` + (`'string' | 'number' | 'boolean' | 'date'`), `getQueryCapabilities()` + (`{ equality, orderAndRange, freeTextSearch }`), and `build()` → + `{ cast_as, indexes: { unique?, ore?, match? } }`. +- `encryptedTable(name, columns)` builds the schema `EncryptionV3` consumes. +- `EncryptionV3(...).encryptQuery(value, { table, column, queryType })` produces the + query term; the FFI's `resolveIndexType` picks the index (including equality-via-ORE + for numeric/date order domains). + +This adapter **reuses** that system verbatim — it never re-declares domain or +capability data. + +## 3. Scope + +### In scope +- A new Drizzle v3 module in `@cipherstash/stack`, exported at + `@cipherstash/stack/eql/v3/drizzle` (nested under the existing `eql/v3` namespace). +- A Drizzle-native `types` namespace with the **identical PascalCase factory names** + as `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.Int4Ord`, `types.Bool`, + … all 35 shipped domains), each returning a Drizzle `customType` column. +- Plain-jsonb codec (`toDriver` / `fromDriver`). +- Schema extraction: Drizzle table → v3 `encryptedTable` for `EncryptionV3`. +- `createEncryptionOperatorsV3(client)` — capability-checked async operators: + `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `between`, `notBetween`, `like`, `ilike`, + `notIlike`, `inArray`, `notInArray`, `asc`, `desc`, `and`, `or`, plus the + pass-through non-encrypting operators (`isNull`, `isNotNull`, `not`, `exists`, + `notExists`). +- Detection: recover the concrete builder from a processed Drizzle column. +- A `.changeset/` entry (minor `@cipherstash/stack`). + +### Out of scope (explicit gaps, not silent omissions) +- **JSON / `ste_vec`** operators (`jsonbPathQueryFirst`, `jsonbGet`, + `jsonbPathExists`). No v3 JSON column builder ships yet (`eql_v3.json` / + `ste_vec` have no `types.*` factory), so these are simply absent from the v3 + operator surface — there is no v3 column they could target. Deferred until a v3 + JSON builder exists. +- **int8 / bigint** domains — intentionally absent from `eql/v3` pending lossless + FFI round-tripping. This adapter mirrors that: no int8 factory. +- **`like`/`ilike` pattern semantics.** v3's only text-match SQL is bloom-filter + containment (`match_term @> bloom_filter`), which is **token/substring matching, + not SQL `LIKE` patterns**. `like`/`ilike`/`notIlike` all map to containment + (`notIlike` = `NOT (…)`); wildcards in the argument are not interpreted. Documented + on each operator. +- **No changes to the v2 Drizzle module** (`@cipherstash/stack/drizzle`) or the + standalone `@cipherstash/drizzle` package. Zero v2 regression surface. +- **No dialect-seam refactor of the v2 `operators.ts`.** The v3 operators are a + self-contained module; the SQL-dialect seam (below) lives only inside v3. + +## 4. Architecture & location + +``` +packages/stack/src/eql/v3/drizzle/ + index.ts // barrel: types, createEncryptionOperatorsV3, extractEncryptionSchemaV3, errors + types.ts // Drizzle `types` namespace (PascalCase factories → customType columns) + column.ts // customType wrapper + detection + config stash + codec.ts // plain-jsonb toDriver / fromDriver + sql-dialect.ts // v3 term-function SQL emission (local seam) + operators.ts // createEncryptionOperatorsV3 + schema-extraction.ts // Drizzle table → eql/v3 encryptedTable +``` + +Package wiring: add `"./eql/v3/drizzle"` to `packages/stack/package.json` `exports` +and to the tsup entry list, following the existing `./eql/v3` and `./drizzle` +entries. + +The v3 module depends inward on `@/eql/v3` (concrete types + `encryptedTable`) and +the base `EncryptionClient` (`@/encryption`) — the same client the v2 Drizzle +operators use. It does **not** depend on the v2 Drizzle module. + +## 5. Components + +### 5.1 `types` namespace + column wrapper (`types.ts`, `column.ts`, `codec.ts`) + +`types.TextSearch(name)` (and the 34 siblings) returns a Drizzle column built via +`customType`. Each factory: + +1. Constructs the corresponding `eql/v3` builder — `v3.types.TextSearch(name)` — the + **single source of truth** for domain / `cast_as` / capabilities. No metadata is + re-declared here; this file is a name→delegate map. +2. Builds a `customType<{ data: Plaintext; driverData: string | null }>` whose + `dataType()` returns `builder.getEqlType()` (e.g. `eql_v3.int4_ord`), and whose + `toDriver`/`fromDriver` are the plain-jsonb codec. +3. Stashes the `eql/v3` builder on the column (`_eqlv3Column`) **and** in a + module-global map keyed by column name — mirroring v2's dual registration — so + detection and schema-extraction can recover it after `pgTable` strips custom + props. + +The decrypted TypeScript type is `PlaintextForColumn` (string / +number / boolean / Date), so `users.age` decrypts to `number`, `users.createdAt` to +`Date`, etc. + +**Codec** (`codec.ts`), proven by the `eql-v3-drizzle-adapter` branch: +- `toDriver`: `null`/`undefined` → SQL `NULL` (JS `null`). The v3 domains + `CHECK jsonb_typeof(VALUE) = 'object'`, so a JSONB `null` literal would fail the + domain; SQL NULL is accepted. Otherwise `JSON.stringify(value)`. +- `fromDriver`: pass `null` through; return already-parsed objects as-is (the + postgres driver may hand back a parsed jsonb object); else `JSON.parse`. + +**Detection** (`column.ts`): `isEqlV3Column(column)` / `getEqlV3Column(name, column)` +check `_eqlv3Column`, falling back to the name-keyed map, and validate the +`dataType()` string is one of the known `eql_v3.*` domains (source: iterate the +`eql/v3` `types` factories once at module load to build the domain set — no +hand-maintained string list). + +### 5.2 SQL dialect (`sql-dialect.ts`) + +A small object that emits the v3 term-function SQL, keyed off which index the column +exposes. Gating and dialect both read `builder.build().indexes` — the authoritative +index set: + +- `indexes.unique` present → equality via `eql_v3.eq_term(col) = eql_v3.hmac_256($::jsonb)`. +- `indexes.ore` present → order/range via `eql_v3.ord_term(col) eql_v3.ore_block_256($::jsonb)`; + order-by via `eql_v3.ord_term(col)`; **and** equality via ORE when `unique` is + absent (`eql_v3.ord_term(col) = eql_v3.ore_block_256($::jsonb)`). +- `indexes.match` present → free-text via `eql_v3.match_term(col) @> eql_v3.bloom_filter($::jsonb)`. + +Query terms are bound params already wrapped with `bindIfParam` by the caller, then +cast `::jsonb` inside the constructor call. Constructor names are pinned to this +branch's bundle: `hmac_256`, `ore_block_256`, `bloom_filter` (note: **not** the +`ore_block_u64_8_256` spelling from the older adapter branch — that predates this +bundle). + +### 5.3 Operators (`operators.ts`) + +`createEncryptionOperatorsV3(client: EncryptionClient)` returns the async-operator +object, same ergonomics as v2 (`await ops.eq(users.email, 'x')`, `ops.and(...)` / +`ops.or(...)` batch encryption, `ops.asc(...)` / `ops.desc(...)`, +`ops.inArray(...)`). Per operator: + +| Operator | Requires index | `queryType` sent to `encryptQuery` | Emitted SQL | +| --- | --- | --- | --- | +| `eq` / `ne` | `unique` or `ore` | `equality` | `unique` → `eq_term = hmac_256`; else `ord_term =/<> ore_block_256` | +| `gt`/`gte`/`lt`/`lte` | `ore` | `orderAndRange` | `ord_term />= ore_block_256` | +| `between` / `notBetween` | `ore` | `orderAndRange` | `ord_term >= ore_block_256($min) AND ord_term <= ore_block_256($max)`, NOT-wrapped for `notBetween` | +| `like`/`ilike`/`notIlike` | `match` | `freeTextSearch` | `match_term @> bloom_filter`, NOT-wrapped for `notIlike` | +| `inArray`/`notInArray` | `unique` or `ore` | `equality` | OR of `eq` terms / AND of `ne` terms | +| `asc` / `desc` | `ore` | — | `ORDER BY eql_v3.ord_term(col)` | +| `isNull`/`isNotNull`/`not`/`exists`/`notExists` | — | — | pass-through Drizzle operators | + +Term encryption reuses the column's recovered `eql/v3` builder + its owning +`encryptedTable` (rebuilt via schema-extraction and cached per table, exactly as v2 +caches the ProtectTable): `client.encryptQuery(value, { table, column, queryType })`. + +**Error handling — no silent fallback.** In v2, an operator on a non-encrypted +column falls through to the plain Drizzle operator. In v3 that is impossible: a v3 +domain column has no plaintext form, and using the wrong operator emits SQL the +domain CHECK rejects at runtime. So the operators **throw `EncryptionOperatorError`** +(reusing the v2 error class name) when a column lacks the required index — e.g. +`ops.gt` on a `types.TextEq` column (equality-only, no `ore`), or `ops.ilike` on a +column without `match`. The message names the column, operator, and the missing +capability. A non-v3 column passed to a v3 operator also throws (it can't be a v3 +domain), rather than silently degrading. + +### 5.4 Schema extraction (`schema-extraction.ts`) + +`extractEncryptionSchemaV3(drizzleTable)` iterates the Drizzle columns, recovers each +stashed `eql/v3` builder via detection, and returns +`encryptedTable(tableName, { : builder, … })` — ready for +`EncryptionV3({ schemas: [users] })`. Throws if the table has no v3 columns (mirrors +v2's `extractEncryptionSchema`). Operators call this internally to resolve the +`encryptedTable` for a column's parent table, caching per table name. + +## 6. Data flow (per query) + +``` +types.Int4Ord('age') // authoring + → customType dataType() = 'eql_v3.int4_ord', stash eql/v3 builder +pgTable('users', { age }) // Drizzle table + +extractEncryptionSchemaV3(users) // → encryptedTable('users', { age: }) +EncryptionV3({ schemas: [users-schema] }) // typed client + +await ops.gte(users.age, 30) + → recover builder (ore index present ✓) + → client.encryptQuery(30, { table, column, queryType: 'orderAndRange' }) // term + → sql`eql_v3.ord_term(${users.age}) >= eql_v3.ore_block_256(${term}::jsonb)` +``` + +## 7. Testing + +Mirror `packages/drizzle/__tests__` structure, plus the v3 live-pg pattern from +`packages/stack/__tests__/schema-v3-pg.test.ts`. Location: +`packages/stack/__tests__/drizzle-v3/` (or alongside the existing +`packages/stack/__tests__/drizzle-*.test.ts` files, matching whatever the v2 drizzle +tests in stack already use). + +**Unit (no DB):** +- `types.*` produce the correct `dataType()` domain string for a representative set + across every scalar family and capability tier. +- Codec round-trip (object ⇄ jsonb string; null ⇄ SQL NULL). +- `extractEncryptionSchemaV3` rebuilds the expected `encryptedTable` (`build()` + output equals the directly-authored `eql/v3` table's). +- Operator SQL emission via a mock client + `PgDialect`: assert exact + `eql_v3.eq_term/ord_term/match_term` + `hmac_256/ore_block_256/bloom_filter` + strings and the equality-via-ORE branch on an order-only numeric column. +- Gating errors: `gt` on equality-only, `ilike` on non-match column, and a non-v3 + column passed into a v3 operator. + +**Live pg** (gated by `LIVE_EQL_V3_PG_ENABLED`, install via `installEqlV3IfNeeded`): +- One `pgTable` with a representative column per tier: `types.TextSearch('email')`, + `types.Int4Ord('age')`, `types.TextEq('nickname')`, `types.Bool('active')`. +- Seed with `EncryptionV3(...).bulkEncryptModels` (or raw insert with + `$::eql_v3.` casts), then real Drizzle `select().where(await ops.eq(...))`, + `ops.gte(...)`, `ops.between(...)`, `ops.ilike(...)`, `orderBy(ops.asc(...))`; + decrypt; assert the selected rows. Use `test_run_id` isolation like the v2 suite. + +## 8. Open implementation details (resolved during build, not blocking) + +- Exact test directory name / whether to gate live tests behind the same env var the + stack v3 pg tests use (`LIVE_EQL_V3_PG_ENABLED`) — adopt the existing helper. +- Whether `between` on a text order domain (which has both `unique` and `ore`) needs + any special handling — it uses `ore`, same as numeric; verify against a live row. +- Confirm the postgres driver path Drizzle uses binds `$::jsonb` correctly for the + term param (the pg tests use `sql.json(...)`; Drizzle's `bindIfParam` + `::jsonb` + cast is the equivalent — assert in a live round-trip). + +## 9. Non-goals / follow-ups (tracked, not in this milestone) + +- v3 JSON / `ste_vec` operators once a v3 JSON column builder ships. +- int8 / bigint once the FFI round-trips losslessly. +- Mirroring into the standalone `@cipherstash/drizzle` package (only if it must ship + v3 independently of `@cipherstash/stack`). +- drizzle-kit migration generation for v3 domains (the v2 `generate-eql-migration` + path emits `eql_v2_encrypted`; a v3 equivalent is a separate effort). From 9d0cabeafcba9091e83144f54ad5522afad5cab4 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 12:12:32 +1000 Subject: [PATCH 05/24] test(stack): pin timestamp time-of-day preservation + dedupe date-like casts Follow-ups from PR review (non-blocking): - catalog: give v3 timestamp domains a non-midnight sample set (TS_S) so the live matrix actually detects a cast_as 'timestamp' -> 'date' truncation regression; midnight-only DATE_S survived truncation silently. Add a runnable matrix.test.ts guard so the detection power can't be reverted unnoticed. - schema: document the 'timestamp' cast_as on both dataType() JSDoc @param lists (and restore the dropped 'text' on EncryptedColumn), noting 'timestamp' preserves time-of-day where 'date' truncates. - columns/encryption: extract DATE_LIKE_CASTS as the single source of truth for the date-like set, shared by the type-level PlaintextFromKind and the runtime rowReconstructor (previously hand-synced in two places). --- packages/stack/__tests__/v3-matrix/catalog.ts | 17 ++++++++++---- .../stack/__tests__/v3-matrix/matrix.test.ts | 23 +++++++++++++++++++ packages/stack/src/encryption/v3.ts | 5 +++- packages/stack/src/eql/v3/columns.ts | 21 ++++++++++++++--- packages/stack/src/schema/index.ts | 4 ++-- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/packages/stack/__tests__/v3-matrix/catalog.ts b/packages/stack/__tests__/v3-matrix/catalog.ts index fa5e3232..bb62ed6f 100644 --- a/packages/stack/__tests__/v3-matrix/catalog.ts +++ b/packages/stack/__tests__/v3-matrix/catalog.ts @@ -228,6 +228,15 @@ const DATE_S = [ new Date('2026-07-01T00:00:00.000Z'), new Date('1970-01-01T00:00:00.000Z'), ] as const +// Timestamp domains (`cast_as: 'timestamp'`) preserve the full instant, unlike +// `date` which truncates to midnight. These samples deliberately carry a +// NON-ZERO time-of-day so the live round-trip actually detects a regression +// that truncates to the day boundary — reusing the midnight-only `DATE_S` here +// would let such a regression pass silently (see matrix.test.ts). +const TS_S = [ + new Date('2026-07-01T12:34:56.000Z'), + new Date('1970-01-01T23:59:59.000Z'), +] as const // Every number domain rejects these via the global encrypt guard. const NUM_ERR = [ Number.NaN, @@ -258,10 +267,10 @@ export const V3_MATRIX = { 'eql_v3.date_ord_ore': { builder: types.DateOrdOre, ColumnClass: EncryptedDateOrdOreColumn, castAs: 'date', capabilities: ORD, indexes: ORE_IDX, samples: DATE_S }, 'eql_v3.date_ord': { builder: types.DateOrd, ColumnClass: EncryptedDateOrdColumn, castAs: 'date', capabilities: ORD, indexes: ORE_IDX, samples: DATE_S }, // timestamp - 'eql_v3.timestamp': { builder: types.Timestamp, ColumnClass: EncryptedTimestampColumn, castAs: 'timestamp', capabilities: STORAGE, indexes: NONE, samples: DATE_S }, - 'eql_v3.timestamp_eq': { builder: types.TimestampEq, ColumnClass: EncryptedTimestampEqColumn, castAs: 'timestamp', capabilities: EQ, indexes: UNIQUE_IDX, samples: DATE_S }, - 'eql_v3.timestamp_ord_ore': { builder: types.TimestampOrdOre, ColumnClass: EncryptedTimestampOrdOreColumn, castAs: 'timestamp', capabilities: ORD, indexes: ORE_IDX, samples: DATE_S }, - 'eql_v3.timestamp_ord': { builder: types.TimestampOrd, ColumnClass: EncryptedTimestampOrdColumn, castAs: 'timestamp', capabilities: ORD, indexes: ORE_IDX, samples: DATE_S }, + 'eql_v3.timestamp': { builder: types.Timestamp, ColumnClass: EncryptedTimestampColumn, castAs: 'timestamp', capabilities: STORAGE, indexes: NONE, samples: TS_S }, + 'eql_v3.timestamp_eq': { builder: types.TimestampEq, ColumnClass: EncryptedTimestampEqColumn, castAs: 'timestamp', capabilities: EQ, indexes: UNIQUE_IDX, samples: TS_S }, + 'eql_v3.timestamp_ord_ore': { builder: types.TimestampOrdOre, ColumnClass: EncryptedTimestampOrdOreColumn, castAs: 'timestamp', capabilities: ORD, indexes: ORE_IDX, samples: TS_S }, + 'eql_v3.timestamp_ord': { builder: types.TimestampOrd, ColumnClass: EncryptedTimestampOrdColumn, castAs: 'timestamp', capabilities: ORD, indexes: ORE_IDX, samples: TS_S }, // numeric 'eql_v3.numeric': { builder: types.Numeric, ColumnClass: EncryptedNumericColumn, castAs: 'number', capabilities: STORAGE, indexes: NONE, samples: NUMERIC_S, errorSamples: NUM_ERR }, 'eql_v3.numeric_eq': { builder: types.NumericEq, ColumnClass: EncryptedNumericEqColumn, castAs: 'number', capabilities: EQ, indexes: UNIQUE_IDX, samples: NUMERIC_S, errorSamples: NUM_ERR }, diff --git a/packages/stack/__tests__/v3-matrix/matrix.test.ts b/packages/stack/__tests__/v3-matrix/matrix.test.ts index 36da2381..cbecb2a0 100644 --- a/packages/stack/__tests__/v3-matrix/matrix.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix.test.ts @@ -41,4 +41,27 @@ describe('eql_v3 type-driven domain matrix (runtime)', () => { ] as const)('%s uses non-empty positive samples for live Postgres inserts', (eqlType) => { expect(V3_MATRIX[eqlType].samples[0]).not.toBe('') }) + + // `timestamp` domains set `cast_as: 'timestamp'` (not `'date'`) precisely to + // preserve the time-of-day. The live matrix can only PROVE that preservation + // if its samples actually carry a non-zero time-of-day: a regression back to + // `'date'` truncation collapses the instant to midnight, so midnight-only + // samples (`DATE_S`) would survive truncation and pass silently. This guards + // the live suite's truncation-detection power at build time — flip the + // timestamp rows back to `DATE_S` and this fails. + it.each([ + 'eql_v3.timestamp', + 'eql_v3.timestamp_eq', + 'eql_v3.timestamp_ord_ore', + 'eql_v3.timestamp_ord', + ] as const)('%s carries a sample with a non-zero time-of-day', (eqlType) => { + const samples = V3_MATRIX[eqlType].samples as readonly Date[] + const hasTimeOfDay = (d: Date) => + d.getUTCHours() + + d.getUTCMinutes() + + d.getUTCSeconds() + + d.getUTCMilliseconds() > + 0 + expect(samples.some(hasTimeOfDay)).toBe(true) + }) }) diff --git a/packages/stack/src/encryption/v3.ts b/packages/stack/src/encryption/v3.ts index c91bb17c..5dcb4f3b 100644 --- a/packages/stack/src/encryption/v3.ts +++ b/packages/stack/src/encryption/v3.ts @@ -9,6 +9,7 @@ import type { V3EncryptedModel, V3ModelInput, } from '@/eql/v3' +import { DATE_LIKE_CASTS } from '@/eql/v3/columns' import { type EncryptionError, EncryptionErrorTypes } from '@/errors' import type { LockContextInput } from '@/identity' import type { @@ -146,7 +147,9 @@ function rowReconstructor( const dateProperties = Object.entries(propToDb) .filter(([, dbName]) => { const castAs = columns[dbName]?.cast_as - return castAs === 'date' || castAs === 'timestamp' + // Date-like casts share one source of truth with the type-level + // reconstruction (`PlaintextFromKind`) — see `DATE_LIKE_CASTS`. + return (DATE_LIKE_CASTS as readonly string[]).includes(castAs as string) }) .map(([property]) => property) diff --git a/packages/stack/src/eql/v3/columns.ts b/packages/stack/src/eql/v3/columns.ts index 0e0c265c..caf0fbd7 100644 --- a/packages/stack/src/eql/v3/columns.ts +++ b/packages/stack/src/eql/v3/columns.ts @@ -22,6 +22,22 @@ export type QueryCapabilities = Readonly<{ freeTextSearch: boolean }> +/** + * The `cast_as` kinds whose decrypted plaintext reconstructs to a JS `Date`. + * + * SINGLE SOURCE OF TRUTH for the date-like set. Both the type-level + * {@link PlaintextFromKind} and the runtime `rowReconstructor` (encryption/v3.ts) + * derive their "reconstructs to `Date`" decision from this array, so the next + * `Date`-backed cast is added in exactly one place — never hand-synced across a + * type and a runtime guard that could silently drift. + * + * (`timestamp` reconstructs to `Date` just like `date`, but its `cast_as` tells + * the FFI not to truncate the time-of-day.) + */ +export const DATE_LIKE_CASTS = ['date', 'timestamp'] as const +/** A `cast_as` kind that reconstructs to `Date` — see {@link DATE_LIKE_CASTS}. */ +export type DateLikeCast = (typeof DATE_LIKE_CASTS)[number] + /** The plaintext (TypeScript) kind a v3 domain decrypts to. A subset of the * SDK `CastAs` enum, restricted to the scalar kinds v3 domains actually use. */ type PlaintextKind = @@ -29,8 +45,7 @@ type PlaintextKind = | 'number' | 'bigint' | 'boolean' - | 'date' - | 'timestamp' + | DateLikeCast /** * The full, literal definition of a v3 domain. This is the LOAD-BEARING type: @@ -666,7 +681,7 @@ type PlaintextFromKind = K extends 'string' ? bigint : K extends 'boolean' ? boolean - : K extends 'date' | 'timestamp' + : K extends DateLikeCast ? Date : never diff --git a/packages/stack/src/schema/index.ts b/packages/stack/src/schema/index.ts index 4eb7a66a..dd1973e2 100644 --- a/packages/stack/src/schema/index.ts +++ b/packages/stack/src/schema/index.ts @@ -224,7 +224,7 @@ export class EncryptedField { * a different type so the encryption layer knows how to encode the plaintext * before encrypting. * - * @param castAs - The plaintext data type: `'string'`, `'number'`, `'boolean'`, `'date'`, `'text'`, `'bigint'`, or `'json'`. + * @param castAs - The plaintext data type: `'string'`, `'number'`, `'boolean'`, `'date'`, `'timestamp'`, `'text'`, `'bigint'`, or `'json'`. Use `'timestamp'` (not `'date'`) to preserve time-of-day — `'date'` truncates to midnight. * @returns This `EncryptedField` instance for method chaining. * * @example @@ -273,7 +273,7 @@ export class EncryptedColumn { * a different type so the encryption layer knows how to encode the plaintext * before encrypting. * - * @param castAs - The plaintext data type: `'string'`, `'number'`, `'boolean'`, `'date'`, `'bigint'`, or `'json'`. + * @param castAs - The plaintext data type: `'string'`, `'number'`, `'boolean'`, `'date'`, `'timestamp'`, `'text'`, `'bigint'`, or `'json'`. Use `'timestamp'` (not `'date'`) to preserve time-of-day — `'date'` truncates to midnight. * @returns This `EncryptedColumn` instance for method chaining. * * @example From 5c9906e850439c4c28ffb34478d7db0cca727474 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 15:55:48 +1000 Subject: [PATCH 06/24] feat(stack): v3 drizzle plain-jsonb codec --- .../stack/__tests__/drizzle-v3/codec.test.ts | 27 +++++++++++++++++++ packages/stack/src/eql/v3/drizzle/codec.ts | 23 ++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 packages/stack/__tests__/drizzle-v3/codec.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/codec.ts diff --git a/packages/stack/__tests__/drizzle-v3/codec.test.ts b/packages/stack/__tests__/drizzle-v3/codec.test.ts new file mode 100644 index 00000000..9dba2899 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/codec.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest" +import { v3FromDriver, v3ToDriver } from "@/eql/v3/drizzle/codec" + +describe("v3 codec", () => { + it("serialises an object to a jsonb string", () => { + expect(v3ToDriver({ v: 1, c: "ct" })).toBe('{"v":1,"c":"ct"}') + }) + + it("maps null/undefined to SQL NULL (JS null), never the JSON null literal", () => { + expect(v3ToDriver(null)).toBeNull() + expect(v3ToDriver(undefined)).toBeNull() + }) + + it("parses a jsonb string back to an object", () => { + expect(v3FromDriver('{"v":1,"c":"ct"}')).toEqual({ v: 1, c: "ct" }) + }) + + it("passes an already-parsed object through unchanged", () => { + const obj = { v: 1 } + expect(v3FromDriver(obj)).toBe(obj) + }) + + it("passes null/undefined through on read", () => { + expect(v3FromDriver(null)).toBeNull() + expect(v3FromDriver(undefined)).toBeUndefined() + }) +}) diff --git a/packages/stack/src/eql/v3/drizzle/codec.ts b/packages/stack/src/eql/v3/drizzle/codec.ts new file mode 100644 index 00000000..3fd79d67 --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/codec.ts @@ -0,0 +1,23 @@ +/** + * v3 columns are `CREATE DOMAIN ... AS jsonb`, so they serialise as plain jsonb, + * distinct from v2's composite-literal parser. + */ + +export function v3ToDriver(value: TData): string | null { + if (value === null || value === undefined) { + return null + } + return JSON.stringify(value) +} + +export function v3FromDriver( + value: string | object | null | undefined, +): TData { + if (value === null || value === undefined) { + return value as TData + } + if (typeof value === "object") { + return value as TData + } + return JSON.parse(value) as TData +} From c68f55306eec6d3017f2cf372355737436bfe576 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 15:57:15 +1000 Subject: [PATCH 07/24] feat(stack): v3 drizzle column wrapper and detection --- .../stack/__tests__/drizzle-v3/column.test.ts | 41 +++++++++++ packages/stack/src/eql/v3/drizzle/column.ts | 69 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 packages/stack/__tests__/drizzle-v3/column.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/column.ts diff --git a/packages/stack/__tests__/drizzle-v3/column.test.ts b/packages/stack/__tests__/drizzle-v3/column.test.ts new file mode 100644 index 00000000..9fa62b3c --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/column.test.ts @@ -0,0 +1,41 @@ +import { pgTable } from "drizzle-orm/pg-core" +import { describe, expect, it } from "vitest" +import { types as v3Types } from "@/eql/v3" +import { + EQL_V3_DOMAINS, + getEqlV3Column, + isEqlV3Column, + makeEqlV3Column, +} from "@/eql/v3/drizzle/column" + +describe("makeEqlV3Column", () => { + it("sets dataType() to the concrete eql_v3 domain", () => { + const col = makeEqlV3Column(v3Types.Int4Ord("age")) + // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals in test + expect((col as any).config.customTypeParams.dataType()).toBe( + "eql_v3.int4_ord", + ) + }) + + it("recovers the stashed builder before and after pgTable processing", () => { + const col = makeEqlV3Column(v3Types.TextEq("nickname")) + expect(isEqlV3Column(col)).toBe(true) + expect(getEqlV3Column("nickname", col)?.getEqlType()).toBe( + "eql_v3.text_eq", + ) + + const t = pgTable("users", { nickname: col }) + expect(getEqlV3Column("nickname", t.nickname)?.getEqlType()).toBe( + "eql_v3.text_eq", + ) + }) + + it("EQL_V3_DOMAINS contains every concrete domain", () => { + const all = Object.values(v3Types).map((f) => f("x").getEqlType()) + for (const domain of all) expect(EQL_V3_DOMAINS.has(domain)).toBe(true) + }) + + it("isEqlV3Column is false for a plain value", () => { + expect(isEqlV3Column({})).toBe(false) + }) +}) diff --git a/packages/stack/src/eql/v3/drizzle/column.ts b/packages/stack/src/eql/v3/drizzle/column.ts new file mode 100644 index 00000000..8d9ca187 --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/column.ts @@ -0,0 +1,69 @@ +import { customType } from "drizzle-orm/pg-core" +import { + type AnyEncryptedV3Column, + type PlaintextForColumn, + types as v3Types, +} from "@/eql/v3" +import { v3FromDriver, v3ToDriver } from "./codec.js" + +/** Every concrete `eql_v3.` string, derived from the eql/v3 factories. */ +export const EQL_V3_DOMAINS: ReadonlySet = new Set( + Object.values(v3Types).map((factory) => factory("__probe__").getEqlType()), +) + +/** + * Mirrors v2's bare-name lookup so a builder can be recovered after pgTable + * creates a fresh column object. Same-named columns with different v3 domains in + * one module collide; table-qualified keys need Drizzle table context that is + * not available when the builder is created. + */ +const columnBuilderMap = new Map() + +export function makeEqlV3Column(builder: C) { + type TData = PlaintextForColumn + const domain = builder.getEqlType() + const name = builder.getName() + + const column = customType<{ data: TData; driverData: string | null }>({ + dataType() { + return domain + }, + toDriver(value: TData): string | null { + return v3ToDriver(value) + }, + fromDriver(value: string | object | null | undefined): TData { + return v3FromDriver(value) + }, + })(name) + + columnBuilderMap.set(name, builder) + // biome-ignore lint/suspicious/noExplicitAny: Drizzle columns don't expose custom props + ;(column as any)._eqlv3Column = builder + return column +} + +export function getEqlV3Column( + columnName: string, + column: unknown, +): AnyEncryptedV3Column | undefined { + if (column && typeof column === "object") { + // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals + const columnAny = column as any + if (columnAny._eqlv3Column) { + return columnAny._eqlv3Column as AnyEncryptedV3Column + } + const lookupName = (columnAny.name as string | undefined) ?? columnName + return columnBuilderMap.get(lookupName) + } + return undefined +} + +export function isEqlV3Column(column: unknown): boolean { + if (!column || typeof column !== "object") return false + // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals + const columnAny = column as any + if (columnAny._eqlv3Column) return true + const dt = columnAny.dataType + const domain = typeof dt === "function" ? dt() : (columnAny.sqlName ?? dt) + return typeof domain === "string" && EQL_V3_DOMAINS.has(domain) +} From ae7a20f6d6ed6c15aef87eb344bf0514f5c9b2ce Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 15:58:28 +1000 Subject: [PATCH 08/24] feat(stack): v3 drizzle types namespace --- .../__tests__/drizzle-v3/types.test-d.ts | 21 +++++++++++++++++++ .../stack/__tests__/drizzle-v3/types.test.ts | 21 +++++++++++++++++++ packages/stack/src/eql/v3/drizzle/types.ts | 19 +++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 packages/stack/__tests__/drizzle-v3/types.test-d.ts create mode 100644 packages/stack/__tests__/drizzle-v3/types.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/types.ts diff --git a/packages/stack/__tests__/drizzle-v3/types.test-d.ts b/packages/stack/__tests__/drizzle-v3/types.test-d.ts new file mode 100644 index 00000000..0e25c9d7 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/types.test-d.ts @@ -0,0 +1,21 @@ +import { describe, expectTypeOf, it } from "vitest" +import { types as v3Types } from "@/eql/v3" +import { types } from "@/eql/v3/drizzle/types" + +describe("v3 drizzle types - type-level", () => { + it("exposes exactly the same factory keys as @/eql/v3 types", () => { + expectTypeOf().toEqualTypeOf() + }) + + it("columns infer their concrete plaintext type via the data type slot", () => { + const age = types.Int4Ord("age") + const created = types.Timestamptz("created_at") + const flag = types.Bool("flag") + const nick = types.TextEq("nickname") + + expectTypeOf(age._.data).toEqualTypeOf() + expectTypeOf(created._.data).toEqualTypeOf() + expectTypeOf(flag._.data).toEqualTypeOf() + expectTypeOf(nick._.data).toEqualTypeOf() + }) +}) diff --git a/packages/stack/__tests__/drizzle-v3/types.test.ts b/packages/stack/__tests__/drizzle-v3/types.test.ts new file mode 100644 index 00000000..716a7fb6 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/types.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest" +import { types as v3Types } from "@/eql/v3" +import { getEqlV3Column } from "@/eql/v3/drizzle/column" +import { types } from "@/eql/v3/drizzle/types" + +describe("v3 drizzle types namespace", () => { + it("exposes the same factory names as @/eql/v3 types", () => { + expect(Object.keys(types).sort()).toEqual(Object.keys(v3Types).sort()) + }) + + it("each factory produces a column carrying the matching concrete builder", () => { + const email = types.TextSearch("email") + expect(getEqlV3Column("email", email)?.getEqlType()).toBe( + "eql_v3.text_search", + ) + const age = types.Int4Ord("age") + expect(getEqlV3Column("age", age)?.getEqlType()).toBe("eql_v3.int4_ord") + const active = types.Bool("active") + expect(getEqlV3Column("active", active)?.getEqlType()).toBe("eql_v3.bool") + }) +}) diff --git a/packages/stack/src/eql/v3/drizzle/types.ts b/packages/stack/src/eql/v3/drizzle/types.ts new file mode 100644 index 00000000..1f125bbe --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/types.ts @@ -0,0 +1,19 @@ +import { types as v3Types } from "@/eql/v3" +import { makeEqlV3Column } from "./column.js" + +type V3Types = typeof v3Types + +/** + * Drizzle-native mirror of `@/eql/v3`'s `types` namespace. Each PascalCase + * factory returns a Drizzle column wrapping the matching concrete v3 builder. + */ +export const types = Object.fromEntries( + Object.entries(v3Types).map(([name, factory]) => [ + name, + (columnName: string) => makeEqlV3Column(factory(columnName)), + ]), +) as { + [K in keyof V3Types]: ( + name: string, + ) => ReturnType>> +} From e840b599e5bee6e6b3c4680bc2ffb7e3436e8310 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 15:59:14 +1000 Subject: [PATCH 09/24] feat(stack): v3 drizzle schema extraction --- .../drizzle-v3/schema-extraction.test.ts | 28 +++++++++++++++ .../src/eql/v3/drizzle/schema-extraction.ts | 34 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/schema-extraction.ts diff --git a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts new file mode 100644 index 00000000..f636cf40 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts @@ -0,0 +1,28 @@ +import { integer, pgTable } from "drizzle-orm/pg-core" +import { describe, expect, it } from "vitest" +import { encryptedTable, types as v3Types } from "@/eql/v3" +import { extractEncryptionSchemaV3 } from "@/eql/v3/drizzle/schema-extraction" +import { types } from "@/eql/v3/drizzle/types" + +describe("extractEncryptionSchemaV3", () => { + it("rebuilds an equivalent eql/v3 encryptedTable from a drizzle table", () => { + const users = pgTable("users", { + id: integer().primaryKey(), + email: types.TextSearch("email"), + age: types.Int4Ord("age"), + }) + + const extracted = extractEncryptionSchemaV3(users) + const authored = encryptedTable("users", { + email: v3Types.TextSearch("email"), + age: v3Types.Int4Ord("age"), + }) + + expect(extracted.build()).toEqual(authored.build()) + }) + + it("throws when the table has no encrypted v3 columns", () => { + const plain = pgTable("plain", { id: integer() }) + expect(() => extractEncryptionSchemaV3(plain)).toThrow(/no encrypted v3/i) + }) +}) diff --git a/packages/stack/src/eql/v3/drizzle/schema-extraction.ts b/packages/stack/src/eql/v3/drizzle/schema-extraction.ts new file mode 100644 index 00000000..4f622e07 --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/schema-extraction.ts @@ -0,0 +1,34 @@ +import type { PgTable } from "drizzle-orm/pg-core" +import { + type AnyEncryptedV3Column, + type AnyV3Table, + encryptedTable, +} from "@/eql/v3" +import { getEqlV3Column } from "./column.js" + +export function extractEncryptionSchemaV3(table: PgTable): AnyV3Table { + // biome-ignore lint/suspicious/noExplicitAny: drizzle stores table metadata on symbols + const tableName = (table as any)[Symbol.for("drizzle:Name")] as + | string + | undefined + if (!tableName) { + throw new Error( + "Unable to read table name from Drizzle table. Use a table created with pgTable().", + ) + } + + const columns: Record = {} + for (const [property, column] of Object.entries(table)) { + if (typeof column !== "object" || column === null) continue + const builder = getEqlV3Column(property, column) + if (builder) columns[property] = builder + } + + if (Object.keys(columns).length === 0) { + throw new Error( + `No encrypted v3 columns found in table "${tableName}". Declare columns with the v3 drizzle \`types\` namespace.`, + ) + } + + return encryptedTable(tableName, columns) +} From 18fa4dbe7c549629cbd02e50b2030794f3246894 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:00:11 +1000 Subject: [PATCH 10/24] feat(stack): v3 drizzle sql dialect --- .../__tests__/drizzle-v3/sql-dialect.test.ts | 53 +++++++++++++++++++ .../stack/src/eql/v3/drizzle/sql-dialect.ts | 36 +++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/sql-dialect.ts diff --git a/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts b/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts new file mode 100644 index 00000000..ba876b05 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts @@ -0,0 +1,53 @@ +import { sql } from "drizzle-orm" +import { PgDialect } from "drizzle-orm/pg-core" +import { describe, expect, it } from "vitest" +import { v3Dialect } from "@/eql/v3/drizzle/sql-dialect" + +const dialect = new PgDialect() +const render = (s: ReturnType) => dialect.sqlToQuery(s).sql +const col = sql`"users"."x"` +const enc = sql`${'{"v":"t"}'}` + +describe("v3Dialect", () => { + it("equality via HMAC", () => { + expect(render(v3Dialect.equality("eq", col, enc, false))).toBe( + 'eql_v3.eq_term("users"."x") = eql_v3.hmac_256($1::jsonb)', + ) + }) + + it("inequality via HMAC", () => { + expect(render(v3Dialect.equality("ne", col, enc, false))).toBe( + 'eql_v3.eq_term("users"."x") <> eql_v3.hmac_256($1::jsonb)', + ) + }) + + it("equality via ORE (order-only numeric/date domains)", () => { + expect(render(v3Dialect.equality("eq", col, enc, true))).toBe( + 'eql_v3.ord_term("users"."x") = eql_v3.ore_block_256($1::jsonb)', + ) + }) + + it("comparison via ORE", () => { + expect(render(v3Dialect.comparison("gte", col, enc))).toBe( + 'eql_v3.ord_term("users"."x") >= eql_v3.ore_block_256($1::jsonb)', + ) + }) + + it("range via ORE", () => { + const lo = sql`${'{"v":"lo"}'}` + const hi = sql`${'{"v":"hi"}'}` + expect(render(v3Dialect.range(col, lo, hi))).toBe( + 'eql_v3.ord_term("users"."x") >= eql_v3.ore_block_256($1::jsonb) AND eql_v3.ord_term("users"."x") <= eql_v3.ore_block_256($2::jsonb)', + ) + }) + + it("match via bloom filter containment", () => { + expect(render(v3Dialect.match(col, enc))).toBe( + 'eql_v3.match_term("users"."x") @> eql_v3.bloom_filter($1::jsonb)', + ) + }) + + it("orderBy extracts the ord term", () => { + expect(render(v3Dialect.orderBy(col))).toBe('eql_v3.ord_term("users"."x")') + }) +}) diff --git a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts new file mode 100644 index 00000000..251f1fbc --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts @@ -0,0 +1,36 @@ +import { type SQL, sql } from "drizzle-orm" + +export type EqualityOp = "eq" | "ne" +export type ComparisonOp = "gt" | "gte" | "lt" | "lte" + +const ORD_SYMBOL: Record = { + gt: ">", + gte: ">=", + lt: "<", + lte: "<=", +} + +export const v3Dialect = { + equality(op: EqualityOp, left: SQL, enc: SQL, viaOre: boolean): SQL { + const cmp = op === "eq" ? sql.raw("=") : sql.raw("<>") + return viaOre + ? sql`eql_v3.ord_term(${left}) ${cmp} eql_v3.ore_block_256(${enc}::jsonb)` + : sql`eql_v3.eq_term(${left}) ${cmp} eql_v3.hmac_256(${enc}::jsonb)` + }, + + comparison(op: ComparisonOp, left: SQL, enc: SQL): SQL { + return sql`eql_v3.ord_term(${left}) ${sql.raw(ORD_SYMBOL[op])} eql_v3.ore_block_256(${enc}::jsonb)` + }, + + range(left: SQL, min: SQL, max: SQL): SQL { + return sql`eql_v3.ord_term(${left}) >= eql_v3.ore_block_256(${min}::jsonb) AND eql_v3.ord_term(${left}) <= eql_v3.ore_block_256(${max}::jsonb)` + }, + + match(left: SQL, enc: SQL): SQL { + return sql`eql_v3.match_term(${left}) @> eql_v3.bloom_filter(${enc}::jsonb)` + }, + + orderBy(left: SQL): SQL { + return sql`eql_v3.ord_term(${left})` + }, +} From cd224073ce09211f31060b73dc82e759d018679f Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:03:17 +1000 Subject: [PATCH 11/24] feat(stack): v3 drizzle capability-checked operators --- .../__tests__/drizzle-v3/operators.test.ts | 210 +++++++++++++++ packages/stack/src/eql/v3/drizzle/codec.ts | 2 +- packages/stack/src/eql/v3/drizzle/column.ts | 16 +- .../stack/src/eql/v3/drizzle/operators.ts | 246 ++++++++++++++++++ .../src/eql/v3/drizzle/schema-extraction.ts | 12 +- .../stack/src/eql/v3/drizzle/sql-dialect.ts | 16 +- packages/stack/src/eql/v3/drizzle/types.ts | 4 +- 7 files changed, 481 insertions(+), 25 deletions(-) create mode 100644 packages/stack/__tests__/drizzle-v3/operators.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/operators.ts diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts new file mode 100644 index 00000000..66d95f4a --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -0,0 +1,210 @@ +import { sql } from "drizzle-orm" +import { PgDialect, integer, pgTable } from "drizzle-orm/pg-core" +import { describe, expect, it, vi } from "vitest" +import type { EncryptionClient } from "@/encryption" +import { + createEncryptionOperatorsV3, + EncryptionOperatorError, +} from "@/eql/v3/drizzle/operators" +import { types } from "@/eql/v3/drizzle/types" + +const TERM = { c: "ct", v: 1 } +const TERM_JSON = JSON.stringify(TERM) + +function setup() { + const encryptQuery = vi.fn(async (_value: unknown, _opts?: unknown) => ({ + data: TERM, + })) + const client = { encryptQuery } as unknown as EncryptionClient + const ops = createEncryptionOperatorsV3(client) + const dialect = new PgDialect() + const render = (s: unknown) => dialect.sqlToQuery(s as ReturnType) + return { ops, encryptQuery, render } +} + +const users = pgTable("users", { + id: integer().primaryKey(), + email: types.TextSearch("email"), + nickname: types.TextEq("nickname"), + age: types.Int4Ord("age"), + flag: types.Bool("flag"), +}) + +describe("createEncryptionOperatorsV3 - equality", () => { + it("eq on a hmac column emits eq_term = hmac_256, binds the JSON term, sends equality", async () => { + const { ops, encryptQuery, render } = setup() + const q = render(await ops.eq(users.nickname, "ada")) + expect(q.sql).toContain( + 'eql_v3.eq_term("users"."nickname") = eql_v3.hmac_256($1::jsonb)', + ) + expect(q.params[0]).toBe(TERM_JSON) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: "equality", + }) + }) + + it("ne on a hmac column emits eq_term <> hmac_256", async () => { + const { ops, render } = setup() + const q = render(await ops.ne(users.nickname, "ada")) + expect(q.sql).toContain( + 'eql_v3.eq_term("users"."nickname") <> eql_v3.hmac_256($1::jsonb)', + ) + }) + + it("eq on an order-only numeric column emits ord_term = ore_block_256", async () => { + const { ops, render } = setup() + const q = render(await ops.eq(users.age, 37)) + expect(q.sql).toContain( + 'eql_v3.ord_term("users"."age") = eql_v3.ore_block_256($1::jsonb)', + ) + }) + + it("ne on an order-only numeric column emits ord_term <> ore_block_256", async () => { + const { ops, render } = setup() + const q = render(await ops.ne(users.age, 37)) + expect(q.sql).toContain( + 'eql_v3.ord_term("users"."age") <> eql_v3.ore_block_256($1::jsonb)', + ) + }) +}) + +describe("createEncryptionOperatorsV3 - comparison & range", () => { + it.each([ + ["gt", ">"], + ["gte", ">="], + ["lt", "<"], + ["lte", "<="], + ] as const)("%s emits ord_term %s ore_block_256", async (op, sym) => { + const { ops, encryptQuery, render } = setup() + // biome-ignore lint/suspicious/noExplicitAny: dynamic operator dispatch in test + const q = render(await (ops as any)[op](users.age, 30)) + expect(q.sql).toContain( + `eql_v3.ord_term("users"."age") ${sym} eql_v3.ore_block_256($1::jsonb)`, + ) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: "orderAndRange", + }) + }) + + it("between emits a bounded ord_term range with two ORE bounds", async () => { + const { ops, render } = setup() + const q = render(await ops.between(users.age, 20, 40)) + expect(q.sql).toContain( + 'eql_v3.ord_term("users"."age") >= eql_v3.ore_block_256($1::jsonb)', + ) + expect(q.sql).toContain( + 'eql_v3.ord_term("users"."age") <= eql_v3.ore_block_256($2::jsonb)', + ) + }) + + it("notBetween wraps the range in NOT (...)", async () => { + const { ops, render } = setup() + const q = render(await ops.notBetween(users.age, 20, 40)) + expect(q.sql).toMatch(/^not \(/i) + expect(q.sql).toContain(">= eql_v3.ore_block_256(") + expect(q.sql).toContain("<= eql_v3.ore_block_256(") + }) +}) + +describe("createEncryptionOperatorsV3 - free-text match", () => { + it("like emits match_term @> bloom_filter with freeTextSearch", async () => { + const { ops, encryptQuery, render } = setup() + const q = render(await ops.like(users.email, "example.com")) + expect(q.sql).toContain( + 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter($1::jsonb)', + ) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: "freeTextSearch", + }) + }) + + it("ilike emits match_term @> bloom_filter", async () => { + const { ops, render } = setup() + const q = render(await ops.ilike(users.email, "example.com")) + expect(q.sql).toContain( + 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter($1::jsonb)', + ) + }) + + it("notIlike wraps the match in NOT (...)", async () => { + const { ops, render } = setup() + const q = render(await ops.notIlike(users.email, "example.com")) + expect(q.sql).toMatch(/^not \(/i) + expect(q.sql).toContain( + 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter(', + ) + }) +}) + +describe("createEncryptionOperatorsV3 - array, ordering, combinators", () => { + it("inArray ORs one eq term per value; empty array is false", async () => { + const { ops, render } = setup() + const q = render(await ops.inArray(users.nickname, ["ada", "grace"])) + expect(q.sql).toContain(" or ") + expect((q.sql.match(/eql_v3\.eq_term/g) ?? []).length).toBe(2) + const empty = render(await ops.inArray(users.nickname, [])) + expect(empty.sql).toBe("false") + }) + + it("notInArray ANDs one ne term per value; empty array is true", async () => { + const { ops, render } = setup() + const q = render(await ops.notInArray(users.nickname, ["ada", "grace"])) + expect(q.sql).toContain(" and ") + const empty = render(await ops.notInArray(users.nickname, [])) + expect(empty.sql).toBe("true") + }) + + it("asc / desc extract the ord term", () => { + const { ops, render } = setup() + const ascq = render(ops.asc(users.age)) + expect(ascq.sql).toContain('eql_v3.ord_term("users"."age")') + expect(ascq.sql.toLowerCase()).toContain("asc") + const descq = render(ops.desc(users.age)) + expect(descq.sql.toLowerCase()).toContain("desc") + }) + + it("and / or await and combine v3 conditions", async () => { + const { ops, render } = setup() + const q = render( + await ops.and(ops.eq(users.nickname, "ada"), ops.gte(users.age, 30)), + ) + expect(q.sql).toContain('eql_v3.eq_term("users"."nickname")') + expect(q.sql).toContain('eql_v3.ord_term("users"."age")') + expect(q.sql).toContain(" and ") + }) +}) + +describe("createEncryptionOperatorsV3 - gating errors", () => { + it("gt on an equality-only column throws", async () => { + const { ops } = setup() + await expect(ops.gt(users.nickname, "ada")).rejects.toBeInstanceOf( + EncryptionOperatorError, + ) + }) + + it("ilike on a column without match throws", async () => { + const { ops } = setup() + await expect(ops.ilike(users.nickname, "ada")).rejects.toBeInstanceOf( + EncryptionOperatorError, + ) + }) + + it("eq on a storage-only column throws", async () => { + const { ops } = setup() + await expect(ops.eq(users.flag, true)).rejects.toBeInstanceOf( + EncryptionOperatorError, + ) + }) + + it("asc on a non-ore column throws synchronously", () => { + const { ops } = setup() + expect(() => ops.asc(users.nickname)).toThrow(EncryptionOperatorError) + }) + + it("a non-v3 column throws", async () => { + const { ops } = setup() + await expect(ops.eq(users.id, 1)).rejects.toBeInstanceOf( + EncryptionOperatorError, + ) + }) +}) diff --git a/packages/stack/src/eql/v3/drizzle/codec.ts b/packages/stack/src/eql/v3/drizzle/codec.ts index 3fd79d67..86d7e55a 100644 --- a/packages/stack/src/eql/v3/drizzle/codec.ts +++ b/packages/stack/src/eql/v3/drizzle/codec.ts @@ -16,7 +16,7 @@ export function v3FromDriver( if (value === null || value === undefined) { return value as TData } - if (typeof value === "object") { + if (typeof value === 'object') { return value as TData } return JSON.parse(value) as TData diff --git a/packages/stack/src/eql/v3/drizzle/column.ts b/packages/stack/src/eql/v3/drizzle/column.ts index 8d9ca187..0b357ff8 100644 --- a/packages/stack/src/eql/v3/drizzle/column.ts +++ b/packages/stack/src/eql/v3/drizzle/column.ts @@ -1,14 +1,14 @@ -import { customType } from "drizzle-orm/pg-core" +import { customType } from 'drizzle-orm/pg-core' import { type AnyEncryptedV3Column, type PlaintextForColumn, types as v3Types, -} from "@/eql/v3" -import { v3FromDriver, v3ToDriver } from "./codec.js" +} from '@/eql/v3' +import { v3FromDriver, v3ToDriver } from './codec.js' /** Every concrete `eql_v3.` string, derived from the eql/v3 factories. */ export const EQL_V3_DOMAINS: ReadonlySet = new Set( - Object.values(v3Types).map((factory) => factory("__probe__").getEqlType()), + Object.values(v3Types).map((factory) => factory('__probe__').getEqlType()), ) /** @@ -46,7 +46,7 @@ export function getEqlV3Column( columnName: string, column: unknown, ): AnyEncryptedV3Column | undefined { - if (column && typeof column === "object") { + if (column && typeof column === 'object') { // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals const columnAny = column as any if (columnAny._eqlv3Column) { @@ -59,11 +59,11 @@ export function getEqlV3Column( } export function isEqlV3Column(column: unknown): boolean { - if (!column || typeof column !== "object") return false + if (!column || typeof column !== 'object') return false // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals const columnAny = column as any if (columnAny._eqlv3Column) return true const dt = columnAny.dataType - const domain = typeof dt === "function" ? dt() : (columnAny.sqlName ?? dt) - return typeof domain === "string" && EQL_V3_DOMAINS.has(domain) + const domain = typeof dt === 'function' ? dt() : (columnAny.sqlName ?? dt) + return typeof domain === 'string' && EQL_V3_DOMAINS.has(domain) } diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts new file mode 100644 index 00000000..c5369383 --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -0,0 +1,246 @@ +import { + and, + asc, + desc, + exists, + isNotNull, + isNull, + not, + notExists, + or, + type SQL, + type SQLWrapper, + sql, +} from 'drizzle-orm' +import type { PgTable } from 'drizzle-orm/pg-core' +import type { EncryptionClient } from '@/encryption' +import type { AnyEncryptedV3Column, AnyV3Table } from '@/eql/v3' +import type { ColumnSchema } from '@/schema' +import type { BuildableQueryColumn, Plaintext, QueryTypeName } from '@/types' +import { getEqlV3Column } from './column.js' +import { extractEncryptionSchemaV3 } from './schema-extraction.js' +import { type ComparisonOp, type EqualityOp, v3Dialect } from './sql-dialect.js' + +export class EncryptionOperatorError extends Error { + constructor( + message: string, + public readonly context?: { + columnName?: string + tableName?: string + operator?: string + }, + ) { + super(message) + this.name = 'EncryptionOperatorError' + } +} + +interface ColumnContext { + builder: AnyEncryptedV3Column + table: AnyV3Table + indexes: ColumnSchema['indexes'] + columnName: string + tableName: string +} + +export function createEncryptionOperatorsV3(client: EncryptionClient) { + const tableCache = new Map() + + function drizzleTableOf(column: SQLWrapper): PgTable | undefined { + // biome-ignore lint/suspicious/noExplicitAny: drizzle column internals + return (column as any).table as PgTable | undefined + } + + function resolveContext(column: SQLWrapper, operator: string): ColumnContext { + // biome-ignore lint/suspicious/noExplicitAny: drizzle column internals + const columnName = ((column as any).name as string | undefined) ?? 'unknown' + const builder = getEqlV3Column(columnName, column) + if (!builder) { + throw new EncryptionOperatorError( + `Operator "${operator}" requires an encrypted v3 column, but "${columnName}" is not one.`, + { columnName, operator }, + ) + } + + const drizzleTable = drizzleTableOf(column) + const drizzleTableSymbols = drizzleTable as + | Record + | undefined + const tableName = + drizzleTableSymbols?.[Symbol.for('drizzle:Name')] ?? 'unknown' + + let table = tableCache.get(tableName) + if (!table && drizzleTable) { + table = extractEncryptionSchemaV3(drizzleTable) + tableCache.set(tableName, table) + } + if (!table) { + throw new EncryptionOperatorError( + `Unable to resolve the encrypted table for column "${columnName}".`, + { columnName, operator }, + ) + } + + return { + builder, + table, + indexes: builder.build().indexes, + columnName, + tableName, + } + } + + function requireIndex( + ctx: ColumnContext, + index: 'unique' | 'ore' | 'match', + operator: string, + capability: string, + ): void { + if (!ctx.indexes[index]) { + throw new EncryptionOperatorError( + `Operator "${operator}" requires ${capability} on column "${ctx.columnName}" (eql_v3 domain ${ctx.builder.getEqlType()} does not support it).`, + { columnName: ctx.columnName, tableName: ctx.tableName, operator }, + ) + } + } + + async function encryptTerm( + ctx: ColumnContext, + value: unknown, + queryType: QueryTypeName, + ): Promise { + const result = await client.encryptQuery(value as Plaintext, { + table: ctx.table, + column: ctx.builder as BuildableQueryColumn, + queryType, + }) + if (result.failure) { + throw new EncryptionOperatorError( + `Failed to encrypt query term for "${ctx.columnName}": ${result.failure.message}`, + { columnName: ctx.columnName, tableName: ctx.tableName }, + ) + } + return sql`${JSON.stringify(result.data)}` + } + + const colSql = (column: SQLWrapper): SQL => sql`${column}` + + async function equality( + op: EqualityOp, + left: SQLWrapper, + right: unknown, + ): Promise { + const ctx = resolveContext(left, op) + if (!ctx.indexes.unique && !ctx.indexes.ore) { + throw new EncryptionOperatorError( + `Operator "${op}" requires equality on column "${ctx.columnName}".`, + { columnName: ctx.columnName, tableName: ctx.tableName, operator: op }, + ) + } + const viaOre = !ctx.indexes.unique + const enc = await encryptTerm(ctx, right, 'equality') + return v3Dialect.equality(op, colSql(left), enc, viaOre) + } + + async function comparison( + op: ComparisonOp, + left: SQLWrapper, + right: unknown, + ): Promise { + const ctx = resolveContext(left, op) + requireIndex(ctx, 'ore', op, 'order/range') + const enc = await encryptTerm(ctx, right, 'orderAndRange') + return v3Dialect.comparison(op, colSql(left), enc) + } + + async function range( + left: SQLWrapper, + min: unknown, + max: unknown, + negate: boolean, + operator: string, + ): Promise { + const ctx = resolveContext(left, operator) + requireIndex(ctx, 'ore', operator, 'order/range') + const encMin = await encryptTerm(ctx, min, 'orderAndRange') + const encMax = await encryptTerm(ctx, max, 'orderAndRange') + const condition = v3Dialect.range(colSql(left), encMin, encMax) + return negate ? sql`NOT (${condition})` : condition + } + + async function match( + left: SQLWrapper, + right: unknown, + negate: boolean, + operator: string, + ): Promise { + const ctx = resolveContext(left, operator) + requireIndex(ctx, 'match', operator, 'free-text search') + const enc = await encryptTerm(ctx, right, 'freeTextSearch') + const condition = v3Dialect.match(colSql(left), enc) + return negate ? sql`NOT (${condition})` : condition + } + + async function inArrayOp( + left: SQLWrapper, + values: unknown[], + negate: boolean, + _operator: string, + ): Promise { + const conditions = await Promise.all( + values.map((v) => equality(negate ? 'ne' : 'eq', left, v)), + ) + if (conditions.length === 0) return negate ? sql`true` : sql`false` + const combined = negate ? and(...conditions) : or(...conditions) + return combined ?? (negate ? sql`true` : sql`false`) + } + + function orderTerm(column: SQLWrapper, operator: string): SQL { + const ctx = resolveContext(column, operator) + requireIndex(ctx, 'ore', operator, 'order/range') + return v3Dialect.orderBy(colSql(column)) + } + + async function combine( + joiner: typeof and, + empty: SQL, + conditions: (SQL | SQLWrapper | Promise | undefined)[], + ): Promise { + const present = conditions.filter( + (c): c is SQL | SQLWrapper | Promise => c !== undefined, + ) + const resolved = await Promise.all(present) + return joiner(...resolved) ?? empty + } + + return { + eq: (l: SQLWrapper, r: unknown) => equality('eq', l, r), + ne: (l: SQLWrapper, r: unknown) => equality('ne', l, r), + gt: (l: SQLWrapper, r: unknown) => comparison('gt', l, r), + gte: (l: SQLWrapper, r: unknown) => comparison('gte', l, r), + lt: (l: SQLWrapper, r: unknown) => comparison('lt', l, r), + lte: (l: SQLWrapper, r: unknown) => comparison('lte', l, r), + between: (l: SQLWrapper, min: unknown, max: unknown) => + range(l, min, max, false, 'between'), + notBetween: (l: SQLWrapper, min: unknown, max: unknown) => + range(l, min, max, true, 'notBetween'), + like: (l: SQLWrapper, r: unknown) => match(l, r, false, 'like'), + ilike: (l: SQLWrapper, r: unknown) => match(l, r, false, 'ilike'), + notIlike: (l: SQLWrapper, r: unknown) => match(l, r, true, 'notIlike'), + inArray: (l: SQLWrapper, values: unknown[]) => + inArrayOp(l, values, false, 'inArray'), + notInArray: (l: SQLWrapper, values: unknown[]) => + inArrayOp(l, values, true, 'notInArray'), + asc: (c: SQLWrapper) => asc(orderTerm(c, 'asc')), + desc: (c: SQLWrapper) => desc(orderTerm(c, 'desc')), + and: (...conds: (SQL | SQLWrapper | Promise | undefined)[]) => + combine(and, sql`true`, conds), + or: (...conds: (SQL | SQLWrapper | Promise | undefined)[]) => + combine(or, sql`false`, conds), + isNull, + isNotNull, + not, + exists, + notExists, + } +} diff --git a/packages/stack/src/eql/v3/drizzle/schema-extraction.ts b/packages/stack/src/eql/v3/drizzle/schema-extraction.ts index 4f622e07..75f66c74 100644 --- a/packages/stack/src/eql/v3/drizzle/schema-extraction.ts +++ b/packages/stack/src/eql/v3/drizzle/schema-extraction.ts @@ -1,25 +1,25 @@ -import type { PgTable } from "drizzle-orm/pg-core" +import type { PgTable } from 'drizzle-orm/pg-core' import { type AnyEncryptedV3Column, type AnyV3Table, encryptedTable, -} from "@/eql/v3" -import { getEqlV3Column } from "./column.js" +} from '@/eql/v3' +import { getEqlV3Column } from './column.js' export function extractEncryptionSchemaV3(table: PgTable): AnyV3Table { // biome-ignore lint/suspicious/noExplicitAny: drizzle stores table metadata on symbols - const tableName = (table as any)[Symbol.for("drizzle:Name")] as + const tableName = (table as any)[Symbol.for('drizzle:Name')] as | string | undefined if (!tableName) { throw new Error( - "Unable to read table name from Drizzle table. Use a table created with pgTable().", + 'Unable to read table name from Drizzle table. Use a table created with pgTable().', ) } const columns: Record = {} for (const [property, column] of Object.entries(table)) { - if (typeof column !== "object" || column === null) continue + if (typeof column !== 'object' || column === null) continue const builder = getEqlV3Column(property, column) if (builder) columns[property] = builder } diff --git a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts index 251f1fbc..153ca5b8 100644 --- a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts +++ b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts @@ -1,18 +1,18 @@ -import { type SQL, sql } from "drizzle-orm" +import { type SQL, sql } from 'drizzle-orm' -export type EqualityOp = "eq" | "ne" -export type ComparisonOp = "gt" | "gte" | "lt" | "lte" +export type EqualityOp = 'eq' | 'ne' +export type ComparisonOp = 'gt' | 'gte' | 'lt' | 'lte' const ORD_SYMBOL: Record = { - gt: ">", - gte: ">=", - lt: "<", - lte: "<=", + gt: '>', + gte: '>=', + lt: '<', + lte: '<=', } export const v3Dialect = { equality(op: EqualityOp, left: SQL, enc: SQL, viaOre: boolean): SQL { - const cmp = op === "eq" ? sql.raw("=") : sql.raw("<>") + const cmp = op === 'eq' ? sql.raw('=') : sql.raw('<>') return viaOre ? sql`eql_v3.ord_term(${left}) ${cmp} eql_v3.ore_block_256(${enc}::jsonb)` : sql`eql_v3.eq_term(${left}) ${cmp} eql_v3.hmac_256(${enc}::jsonb)` diff --git a/packages/stack/src/eql/v3/drizzle/types.ts b/packages/stack/src/eql/v3/drizzle/types.ts index 1f125bbe..b52fe83f 100644 --- a/packages/stack/src/eql/v3/drizzle/types.ts +++ b/packages/stack/src/eql/v3/drizzle/types.ts @@ -1,5 +1,5 @@ -import { types as v3Types } from "@/eql/v3" -import { makeEqlV3Column } from "./column.js" +import { types as v3Types } from '@/eql/v3' +import { makeEqlV3Column } from './column.js' type V3Types = typeof v3Types From 2473bcc8527e60d137e1fcf21a9a574877a0942b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:06:12 +1000 Subject: [PATCH 12/24] feat(stack): export @cipherstash/stack/eql/v3/drizzle + changeset --- .changeset/eql-v3-drizzle.md | 12 ++++++++++++ packages/stack/__tests__/drizzle-v3/exports.test.ts | 12 ++++++++++++ packages/stack/package.json | 13 +++++++++++++ packages/stack/src/eql/v3/drizzle/index.ts | 8 ++++++++ packages/stack/tsup.config.ts | 1 + 5 files changed, 46 insertions(+) create mode 100644 .changeset/eql-v3-drizzle.md create mode 100644 packages/stack/__tests__/drizzle-v3/exports.test.ts create mode 100644 packages/stack/src/eql/v3/drizzle/index.ts diff --git a/.changeset/eql-v3-drizzle.md b/.changeset/eql-v3-drizzle.md new file mode 100644 index 00000000..70700eae --- /dev/null +++ b/.changeset/eql-v3-drizzle.md @@ -0,0 +1,12 @@ +--- +"@cipherstash/stack": minor +--- + +Add EQL v3 Drizzle support at `@cipherstash/stack/eql/v3/drizzle`. A Drizzle-native +`types` namespace (same PascalCase names as `@cipherstash/stack/eql/v3`) declares +encrypted columns whose Postgres type is the concrete `eql_v3.`; the concrete +type drives the legal query operators. `createEncryptionOperatorsV3` provides +capability-checked `eq`/`ne`/`gt`/`gte`/`lt`/`lte`/`between`/`like`/`ilike`/`inArray`/ +`asc`/`desc`/`and`/`or` that emit `eql_v3` term-function SQL, and +`extractEncryptionSchemaV3` rebuilds the schema for `EncryptionV3`. The existing v2 +`@cipherstash/stack/drizzle` integration is unchanged. diff --git a/packages/stack/__tests__/drizzle-v3/exports.test.ts b/packages/stack/__tests__/drizzle-v3/exports.test.ts new file mode 100644 index 00000000..d000f7da --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/exports.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest" +import * as barrel from "@/eql/v3/drizzle" + +describe("v3 drizzle barrel", () => { + it("exports the public surface", () => { + expect(typeof barrel.createEncryptionOperatorsV3).toBe("function") + expect(typeof barrel.extractEncryptionSchemaV3).toBe("function") + expect(typeof barrel.EncryptionOperatorError).toBe("function") + expect(typeof barrel.types.TextSearch).toBe("function") + expect(barrel.types.Int4Ord("age")).toBeDefined() + }) +}) diff --git a/packages/stack/package.json b/packages/stack/package.json index 83de07f1..38ea644f 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -54,6 +54,9 @@ "eql/v3": [ "./dist/eql/v3/index.d.ts" ], + "eql/v3/drizzle": [ + "./dist/eql/v3/drizzle/index.d.ts" + ], "v3": [ "./dist/encryption/v3.d.ts" ], @@ -135,6 +138,16 @@ "default": "./dist/eql/v3/index.cjs" } }, + "./eql/v3/drizzle": { + "import": { + "types": "./dist/eql/v3/drizzle/index.d.ts", + "default": "./dist/eql/v3/drizzle/index.js" + }, + "require": { + "types": "./dist/eql/v3/drizzle/index.d.cts", + "default": "./dist/eql/v3/drizzle/index.cjs" + } + }, "./v3": { "import": { "types": "./dist/encryption/v3.d.ts", diff --git a/packages/stack/src/eql/v3/drizzle/index.ts b/packages/stack/src/eql/v3/drizzle/index.ts new file mode 100644 index 00000000..7daecab0 --- /dev/null +++ b/packages/stack/src/eql/v3/drizzle/index.ts @@ -0,0 +1,8 @@ +export { getEqlV3Column, isEqlV3Column, makeEqlV3Column } from './column.js' +export { v3FromDriver, v3ToDriver } from './codec.js' +export { + createEncryptionOperatorsV3, + EncryptionOperatorError, +} from './operators.js' +export { extractEncryptionSchemaV3 } from './schema-extraction.js' +export { types } from './types.js' diff --git a/packages/stack/tsup.config.ts b/packages/stack/tsup.config.ts index 2380c8c4..0a342ae0 100644 --- a/packages/stack/tsup.config.ts +++ b/packages/stack/tsup.config.ts @@ -16,6 +16,7 @@ export default defineConfig([ 'src/secrets/index.ts', 'src/schema/index.ts', 'src/eql/v3/index.ts', + 'src/eql/v3/drizzle/index.ts', 'src/drizzle/index.ts', 'src/dynamodb/index.ts', 'src/supabase/index.ts', From 6e8415aec4e31c22195cc72b0859d7cff74ed8ab Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:07:38 +1000 Subject: [PATCH 13/24] test(stack): v3 drizzle live postgres integration --- .../drizzle-v3/operators-live-pg.test.ts | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts new file mode 100644 index 00000000..4c24564c --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -0,0 +1,133 @@ +import 'dotenv/config' +import { and, eq as drizzleEq, type SQL } from 'drizzle-orm' +import { drizzle } from 'drizzle-orm/postgres-js' +import { integer, pgTable, text } from 'drizzle-orm/pg-core' +import postgres from 'postgres' +import { afterAll, beforeAll, expect, it } from 'vitest' +import { EncryptionV3 } from '@/encryption/v3' +import { + createEncryptionOperatorsV3, + extractEncryptionSchemaV3, + types, +} from '@/eql/v3/drizzle' +import { installEqlV3IfNeeded } from '../helpers/eql-v3' +import { describeLivePg, LIVE_EQL_V3_PG_ENABLED } from '../helpers/live-gate' + +const url = process.env.DATABASE_URL +const sqlClient = LIVE_EQL_V3_PG_ENABLED + ? postgres(url as string, { prepare: false }) + : (undefined as unknown as postgres.Sql) + +const usersTable = pgTable('protect_ci_v3_drizzle', { + id: integer('id').primaryKey().generatedAlwaysAsIdentity(), + email: types.TextSearch('email'), + nickname: types.TextEq('nickname'), + age: types.Int4Ord('age'), + testRunId: text('test_run_id').notNull(), +}) + +const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` +const schema = extractEncryptionSchemaV3(usersTable) + +// biome-ignore lint/suspicious/noExplicitAny: test client typing +let client: any +// biome-ignore lint/suspicious/noExplicitAny: test operator typing +let ops: any +// biome-ignore lint/suspicious/noExplicitAny: drizzle db typing +let db: any + +beforeAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + await installEqlV3IfNeeded(sqlClient) + client = await EncryptionV3({ schemas: [schema] }) + ops = createEncryptionOperatorsV3(client) + db = drizzle({ client: sqlClient }) + + await sqlClient` + CREATE TABLE IF NOT EXISTS protect_ci_v3_drizzle ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + email eql_v3.text_search NOT NULL, + nickname eql_v3.text_eq NOT NULL, + age eql_v3.int4_ord NOT NULL, + test_run_id TEXT NOT NULL + ) + ` + + const seed = [ + { email: 'ada@example.com', nickname: 'ada', age: 37 }, + { email: 'grace@example.com', nickname: 'grace', age: 30 }, + { email: 'alan@example.net', nickname: 'alan', age: 42 }, + ] + for (const row of seed) { + const enc = unwrap(await client.encryptModel({ ...row }, schema)) + await sqlClient` + INSERT INTO protect_ci_v3_drizzle (email, nickname, age, test_run_id) + VALUES ( + ${sqlClient.json(enc.email)}::eql_v3.text_search, + ${sqlClient.json(enc.nickname)}::eql_v3.text_eq, + ${sqlClient.json(enc.age)}::eql_v3.int4_ord, + ${RUN} + ) + ` + } +}, 60000) + +afterAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + await sqlClient`DELETE FROM protect_ci_v3_drizzle WHERE test_run_id = ${RUN}` + await sqlClient.end() +}, 30000) + +// biome-ignore lint/suspicious/noExplicitAny: result unwrap +function unwrap(r: any) { + if (r.failure) throw new Error(r.failure.message) + return r.data +} + +describeLivePg('v3 drizzle operators (live pg)', () => { + const scoped = (cond: SQL) => and(drizzleEq(usersTable.testRunId, RUN), cond) + + it('eq selects the exact nickname row', async () => { + const rows = await db + .select({ id: usersTable.id, age: usersTable.age }) + .from(usersTable) + .where(scoped(await ops.eq(usersTable.nickname, 'grace'))) + const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + expect(decrypted.map((r: { age: number }) => r.age)).toEqual([30]) + }, 30000) + + it('gte selects rows at or above the bound', async () => { + const rows = await db + .select({ id: usersTable.id }) + .from(usersTable) + .where(scoped(await ops.gte(usersTable.age, 37))) + expect(rows.length).toBe(2) + }, 30000) + + it('between selects the inclusive range', async () => { + const rows = await db + .select({ id: usersTable.id }) + .from(usersTable) + .where(scoped(await ops.between(usersTable.age, 30, 37))) + expect(rows.length).toBe(2) + }, 30000) + + it('ilike matches on the free-text email column', async () => { + const rows = await db + .select({ id: usersTable.id }) + .from(usersTable) + .where(scoped(await ops.ilike(usersTable.email, 'example.com'))) + expect(rows.length).toBe(2) + }, 30000) + + it('orderBy asc returns ascending ages', async () => { + const rows = await db + .select({ age: usersTable.age }) + .from(usersTable) + .where(scoped(await ops.gte(usersTable.age, 0))) + .orderBy(ops.asc(usersTable.age)) + const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + const ages = decrypted.map((r: { age: number }) => r.age) + expect(ages).toEqual([...ages].sort((a, b) => a - b)) + }, 30000) +}) From 1f4dee61fb0102e0b11ba3f625a8b8d5bfc3af26 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:08:44 +1000 Subject: [PATCH 14/24] chore(stack): format v3 drizzle tests --- .../stack/__tests__/drizzle-v3/codec.test.ts | 20 +-- .../stack/__tests__/drizzle-v3/column.test.ts | 36 +++-- .../__tests__/drizzle-v3/exports.test.ts | 18 +-- .../drizzle-v3/operators-live-pg.test.ts | 2 +- .../__tests__/drizzle-v3/operators.test.ts | 124 +++++++++--------- .../drizzle-v3/schema-extraction.test.ts | 30 ++--- .../__tests__/drizzle-v3/sql-dialect.test.ts | 32 ++--- .../__tests__/drizzle-v3/types.test-d.ts | 20 +-- .../stack/__tests__/drizzle-v3/types.test.ts | 28 ++-- packages/stack/src/eql/v3/drizzle/index.ts | 2 +- 10 files changed, 155 insertions(+), 157 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/codec.test.ts b/packages/stack/__tests__/drizzle-v3/codec.test.ts index 9dba2899..db50fea7 100644 --- a/packages/stack/__tests__/drizzle-v3/codec.test.ts +++ b/packages/stack/__tests__/drizzle-v3/codec.test.ts @@ -1,26 +1,26 @@ -import { describe, expect, it } from "vitest" -import { v3FromDriver, v3ToDriver } from "@/eql/v3/drizzle/codec" +import { describe, expect, it } from 'vitest' +import { v3FromDriver, v3ToDriver } from '@/eql/v3/drizzle/codec' -describe("v3 codec", () => { - it("serialises an object to a jsonb string", () => { - expect(v3ToDriver({ v: 1, c: "ct" })).toBe('{"v":1,"c":"ct"}') +describe('v3 codec', () => { + it('serialises an object to a jsonb string', () => { + expect(v3ToDriver({ v: 1, c: 'ct' })).toBe('{"v":1,"c":"ct"}') }) - it("maps null/undefined to SQL NULL (JS null), never the JSON null literal", () => { + it('maps null/undefined to SQL NULL (JS null), never the JSON null literal', () => { expect(v3ToDriver(null)).toBeNull() expect(v3ToDriver(undefined)).toBeNull() }) - it("parses a jsonb string back to an object", () => { - expect(v3FromDriver('{"v":1,"c":"ct"}')).toEqual({ v: 1, c: "ct" }) + it('parses a jsonb string back to an object', () => { + expect(v3FromDriver('{"v":1,"c":"ct"}')).toEqual({ v: 1, c: 'ct' }) }) - it("passes an already-parsed object through unchanged", () => { + it('passes an already-parsed object through unchanged', () => { const obj = { v: 1 } expect(v3FromDriver(obj)).toBe(obj) }) - it("passes null/undefined through on read", () => { + it('passes null/undefined through on read', () => { expect(v3FromDriver(null)).toBeNull() expect(v3FromDriver(undefined)).toBeUndefined() }) diff --git a/packages/stack/__tests__/drizzle-v3/column.test.ts b/packages/stack/__tests__/drizzle-v3/column.test.ts index 9fa62b3c..9a759474 100644 --- a/packages/stack/__tests__/drizzle-v3/column.test.ts +++ b/packages/stack/__tests__/drizzle-v3/column.test.ts @@ -1,41 +1,39 @@ -import { pgTable } from "drizzle-orm/pg-core" -import { describe, expect, it } from "vitest" -import { types as v3Types } from "@/eql/v3" +import { pgTable } from 'drizzle-orm/pg-core' +import { describe, expect, it } from 'vitest' +import { types as v3Types } from '@/eql/v3' import { EQL_V3_DOMAINS, getEqlV3Column, isEqlV3Column, makeEqlV3Column, -} from "@/eql/v3/drizzle/column" +} from '@/eql/v3/drizzle/column' -describe("makeEqlV3Column", () => { - it("sets dataType() to the concrete eql_v3 domain", () => { - const col = makeEqlV3Column(v3Types.Int4Ord("age")) +describe('makeEqlV3Column', () => { + it('sets dataType() to the concrete eql_v3 domain', () => { + const col = makeEqlV3Column(v3Types.Int4Ord('age')) // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals in test expect((col as any).config.customTypeParams.dataType()).toBe( - "eql_v3.int4_ord", + 'eql_v3.int4_ord', ) }) - it("recovers the stashed builder before and after pgTable processing", () => { - const col = makeEqlV3Column(v3Types.TextEq("nickname")) + it('recovers the stashed builder before and after pgTable processing', () => { + const col = makeEqlV3Column(v3Types.TextEq('nickname')) expect(isEqlV3Column(col)).toBe(true) - expect(getEqlV3Column("nickname", col)?.getEqlType()).toBe( - "eql_v3.text_eq", - ) + expect(getEqlV3Column('nickname', col)?.getEqlType()).toBe('eql_v3.text_eq') - const t = pgTable("users", { nickname: col }) - expect(getEqlV3Column("nickname", t.nickname)?.getEqlType()).toBe( - "eql_v3.text_eq", + const t = pgTable('users', { nickname: col }) + expect(getEqlV3Column('nickname', t.nickname)?.getEqlType()).toBe( + 'eql_v3.text_eq', ) }) - it("EQL_V3_DOMAINS contains every concrete domain", () => { - const all = Object.values(v3Types).map((f) => f("x").getEqlType()) + it('EQL_V3_DOMAINS contains every concrete domain', () => { + const all = Object.values(v3Types).map((f) => f('x').getEqlType()) for (const domain of all) expect(EQL_V3_DOMAINS.has(domain)).toBe(true) }) - it("isEqlV3Column is false for a plain value", () => { + it('isEqlV3Column is false for a plain value', () => { expect(isEqlV3Column({})).toBe(false) }) }) diff --git a/packages/stack/__tests__/drizzle-v3/exports.test.ts b/packages/stack/__tests__/drizzle-v3/exports.test.ts index d000f7da..52351cfa 100644 --- a/packages/stack/__tests__/drizzle-v3/exports.test.ts +++ b/packages/stack/__tests__/drizzle-v3/exports.test.ts @@ -1,12 +1,12 @@ -import { describe, expect, it } from "vitest" -import * as barrel from "@/eql/v3/drizzle" +import { describe, expect, it } from 'vitest' +import * as barrel from '@/eql/v3/drizzle' -describe("v3 drizzle barrel", () => { - it("exports the public surface", () => { - expect(typeof barrel.createEncryptionOperatorsV3).toBe("function") - expect(typeof barrel.extractEncryptionSchemaV3).toBe("function") - expect(typeof barrel.EncryptionOperatorError).toBe("function") - expect(typeof barrel.types.TextSearch).toBe("function") - expect(barrel.types.Int4Ord("age")).toBeDefined() +describe('v3 drizzle barrel', () => { + it('exports the public surface', () => { + expect(typeof barrel.createEncryptionOperatorsV3).toBe('function') + expect(typeof barrel.extractEncryptionSchemaV3).toBe('function') + expect(typeof barrel.EncryptionOperatorError).toBe('function') + expect(typeof barrel.types.TextSearch).toBe('function') + expect(barrel.types.Int4Ord('age')).toBeDefined() }) }) diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts index 4c24564c..65c48263 100644 --- a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -1,7 +1,7 @@ import 'dotenv/config' import { and, eq as drizzleEq, type SQL } from 'drizzle-orm' -import { drizzle } from 'drizzle-orm/postgres-js' import { integer, pgTable, text } from 'drizzle-orm/pg-core' +import { drizzle } from 'drizzle-orm/postgres-js' import postgres from 'postgres' import { afterAll, beforeAll, expect, it } from 'vitest' import { EncryptionV3 } from '@/encryption/v3' diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 66d95f4a..1adcec61 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -1,14 +1,14 @@ -import { sql } from "drizzle-orm" -import { PgDialect, integer, pgTable } from "drizzle-orm/pg-core" -import { describe, expect, it, vi } from "vitest" -import type { EncryptionClient } from "@/encryption" +import type { sql } from 'drizzle-orm' +import { integer, PgDialect, pgTable } from 'drizzle-orm/pg-core' +import { describe, expect, it, vi } from 'vitest' +import type { EncryptionClient } from '@/encryption' import { createEncryptionOperatorsV3, EncryptionOperatorError, -} from "@/eql/v3/drizzle/operators" -import { types } from "@/eql/v3/drizzle/types" +} from '@/eql/v3/drizzle/operators' +import { types } from '@/eql/v3/drizzle/types' -const TERM = { c: "ct", v: 1 } +const TERM = { c: 'ct', v: 1 } const TERM_JSON = JSON.stringify(TERM) function setup() { @@ -22,36 +22,36 @@ function setup() { return { ops, encryptQuery, render } } -const users = pgTable("users", { +const users = pgTable('users', { id: integer().primaryKey(), - email: types.TextSearch("email"), - nickname: types.TextEq("nickname"), - age: types.Int4Ord("age"), - flag: types.Bool("flag"), + email: types.TextSearch('email'), + nickname: types.TextEq('nickname'), + age: types.Int4Ord('age'), + flag: types.Bool('flag'), }) -describe("createEncryptionOperatorsV3 - equality", () => { - it("eq on a hmac column emits eq_term = hmac_256, binds the JSON term, sends equality", async () => { +describe('createEncryptionOperatorsV3 - equality', () => { + it('eq on a hmac column emits eq_term = hmac_256, binds the JSON term, sends equality', async () => { const { ops, encryptQuery, render } = setup() - const q = render(await ops.eq(users.nickname, "ada")) + const q = render(await ops.eq(users.nickname, 'ada')) expect(q.sql).toContain( 'eql_v3.eq_term("users"."nickname") = eql_v3.hmac_256($1::jsonb)', ) expect(q.params[0]).toBe(TERM_JSON) expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: "equality", + queryType: 'equality', }) }) - it("ne on a hmac column emits eq_term <> hmac_256", async () => { + it('ne on a hmac column emits eq_term <> hmac_256', async () => { const { ops, render } = setup() - const q = render(await ops.ne(users.nickname, "ada")) + const q = render(await ops.ne(users.nickname, 'ada')) expect(q.sql).toContain( 'eql_v3.eq_term("users"."nickname") <> eql_v3.hmac_256($1::jsonb)', ) }) - it("eq on an order-only numeric column emits ord_term = ore_block_256", async () => { + it('eq on an order-only numeric column emits ord_term = ore_block_256', async () => { const { ops, render } = setup() const q = render(await ops.eq(users.age, 37)) expect(q.sql).toContain( @@ -59,7 +59,7 @@ describe("createEncryptionOperatorsV3 - equality", () => { ) }) - it("ne on an order-only numeric column emits ord_term <> ore_block_256", async () => { + it('ne on an order-only numeric column emits ord_term <> ore_block_256', async () => { const { ops, render } = setup() const q = render(await ops.ne(users.age, 37)) expect(q.sql).toContain( @@ -68,13 +68,13 @@ describe("createEncryptionOperatorsV3 - equality", () => { }) }) -describe("createEncryptionOperatorsV3 - comparison & range", () => { +describe('createEncryptionOperatorsV3 - comparison & range', () => { it.each([ - ["gt", ">"], - ["gte", ">="], - ["lt", "<"], - ["lte", "<="], - ] as const)("%s emits ord_term %s ore_block_256", async (op, sym) => { + ['gt', '>'], + ['gte', '>='], + ['lt', '<'], + ['lte', '<='], + ] as const)('%s emits ord_term %s ore_block_256', async (op, sym) => { const { ops, encryptQuery, render } = setup() // biome-ignore lint/suspicious/noExplicitAny: dynamic operator dispatch in test const q = render(await (ops as any)[op](users.age, 30)) @@ -82,11 +82,11 @@ describe("createEncryptionOperatorsV3 - comparison & range", () => { `eql_v3.ord_term("users"."age") ${sym} eql_v3.ore_block_256($1::jsonb)`, ) expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: "orderAndRange", + queryType: 'orderAndRange', }) }) - it("between emits a bounded ord_term range with two ORE bounds", async () => { + it('between emits a bounded ord_term range with two ORE bounds', async () => { const { ops, render } = setup() const q = render(await ops.between(users.age, 20, 40)) expect(q.sql).toContain( @@ -97,38 +97,38 @@ describe("createEncryptionOperatorsV3 - comparison & range", () => { ) }) - it("notBetween wraps the range in NOT (...)", async () => { + it('notBetween wraps the range in NOT (...)', async () => { const { ops, render } = setup() const q = render(await ops.notBetween(users.age, 20, 40)) expect(q.sql).toMatch(/^not \(/i) - expect(q.sql).toContain(">= eql_v3.ore_block_256(") - expect(q.sql).toContain("<= eql_v3.ore_block_256(") + expect(q.sql).toContain('>= eql_v3.ore_block_256(') + expect(q.sql).toContain('<= eql_v3.ore_block_256(') }) }) -describe("createEncryptionOperatorsV3 - free-text match", () => { - it("like emits match_term @> bloom_filter with freeTextSearch", async () => { +describe('createEncryptionOperatorsV3 - free-text match', () => { + it('like emits match_term @> bloom_filter with freeTextSearch', async () => { const { ops, encryptQuery, render } = setup() - const q = render(await ops.like(users.email, "example.com")) + const q = render(await ops.like(users.email, 'example.com')) expect(q.sql).toContain( 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter($1::jsonb)', ) expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: "freeTextSearch", + queryType: 'freeTextSearch', }) }) - it("ilike emits match_term @> bloom_filter", async () => { + it('ilike emits match_term @> bloom_filter', async () => { const { ops, render } = setup() - const q = render(await ops.ilike(users.email, "example.com")) + const q = render(await ops.ilike(users.email, 'example.com')) expect(q.sql).toContain( 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter($1::jsonb)', ) }) - it("notIlike wraps the match in NOT (...)", async () => { + it('notIlike wraps the match in NOT (...)', async () => { const { ops, render } = setup() - const q = render(await ops.notIlike(users.email, "example.com")) + const q = render(await ops.notIlike(users.email, 'example.com')) expect(q.sql).toMatch(/^not \(/i) expect(q.sql).toContain( 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter(', @@ -136,72 +136,72 @@ describe("createEncryptionOperatorsV3 - free-text match", () => { }) }) -describe("createEncryptionOperatorsV3 - array, ordering, combinators", () => { - it("inArray ORs one eq term per value; empty array is false", async () => { +describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { + it('inArray ORs one eq term per value; empty array is false', async () => { const { ops, render } = setup() - const q = render(await ops.inArray(users.nickname, ["ada", "grace"])) - expect(q.sql).toContain(" or ") + const q = render(await ops.inArray(users.nickname, ['ada', 'grace'])) + expect(q.sql).toContain(' or ') expect((q.sql.match(/eql_v3\.eq_term/g) ?? []).length).toBe(2) const empty = render(await ops.inArray(users.nickname, [])) - expect(empty.sql).toBe("false") + expect(empty.sql).toBe('false') }) - it("notInArray ANDs one ne term per value; empty array is true", async () => { + it('notInArray ANDs one ne term per value; empty array is true', async () => { const { ops, render } = setup() - const q = render(await ops.notInArray(users.nickname, ["ada", "grace"])) - expect(q.sql).toContain(" and ") + const q = render(await ops.notInArray(users.nickname, ['ada', 'grace'])) + expect(q.sql).toContain(' and ') const empty = render(await ops.notInArray(users.nickname, [])) - expect(empty.sql).toBe("true") + expect(empty.sql).toBe('true') }) - it("asc / desc extract the ord term", () => { + it('asc / desc extract the ord term', () => { const { ops, render } = setup() const ascq = render(ops.asc(users.age)) expect(ascq.sql).toContain('eql_v3.ord_term("users"."age")') - expect(ascq.sql.toLowerCase()).toContain("asc") + expect(ascq.sql.toLowerCase()).toContain('asc') const descq = render(ops.desc(users.age)) - expect(descq.sql.toLowerCase()).toContain("desc") + expect(descq.sql.toLowerCase()).toContain('desc') }) - it("and / or await and combine v3 conditions", async () => { + it('and / or await and combine v3 conditions', async () => { const { ops, render } = setup() const q = render( - await ops.and(ops.eq(users.nickname, "ada"), ops.gte(users.age, 30)), + await ops.and(ops.eq(users.nickname, 'ada'), ops.gte(users.age, 30)), ) expect(q.sql).toContain('eql_v3.eq_term("users"."nickname")') expect(q.sql).toContain('eql_v3.ord_term("users"."age")') - expect(q.sql).toContain(" and ") + expect(q.sql).toContain(' and ') }) }) -describe("createEncryptionOperatorsV3 - gating errors", () => { - it("gt on an equality-only column throws", async () => { +describe('createEncryptionOperatorsV3 - gating errors', () => { + it('gt on an equality-only column throws', async () => { const { ops } = setup() - await expect(ops.gt(users.nickname, "ada")).rejects.toBeInstanceOf( + await expect(ops.gt(users.nickname, 'ada')).rejects.toBeInstanceOf( EncryptionOperatorError, ) }) - it("ilike on a column without match throws", async () => { + it('ilike on a column without match throws', async () => { const { ops } = setup() - await expect(ops.ilike(users.nickname, "ada")).rejects.toBeInstanceOf( + await expect(ops.ilike(users.nickname, 'ada')).rejects.toBeInstanceOf( EncryptionOperatorError, ) }) - it("eq on a storage-only column throws", async () => { + it('eq on a storage-only column throws', async () => { const { ops } = setup() await expect(ops.eq(users.flag, true)).rejects.toBeInstanceOf( EncryptionOperatorError, ) }) - it("asc on a non-ore column throws synchronously", () => { + it('asc on a non-ore column throws synchronously', () => { const { ops } = setup() expect(() => ops.asc(users.nickname)).toThrow(EncryptionOperatorError) }) - it("a non-v3 column throws", async () => { + it('a non-v3 column throws', async () => { const { ops } = setup() await expect(ops.eq(users.id, 1)).rejects.toBeInstanceOf( EncryptionOperatorError, diff --git a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts index f636cf40..a538371a 100644 --- a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts +++ b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts @@ -1,28 +1,28 @@ -import { integer, pgTable } from "drizzle-orm/pg-core" -import { describe, expect, it } from "vitest" -import { encryptedTable, types as v3Types } from "@/eql/v3" -import { extractEncryptionSchemaV3 } from "@/eql/v3/drizzle/schema-extraction" -import { types } from "@/eql/v3/drizzle/types" +import { integer, pgTable } from 'drizzle-orm/pg-core' +import { describe, expect, it } from 'vitest' +import { encryptedTable, types as v3Types } from '@/eql/v3' +import { extractEncryptionSchemaV3 } from '@/eql/v3/drizzle/schema-extraction' +import { types } from '@/eql/v3/drizzle/types' -describe("extractEncryptionSchemaV3", () => { - it("rebuilds an equivalent eql/v3 encryptedTable from a drizzle table", () => { - const users = pgTable("users", { +describe('extractEncryptionSchemaV3', () => { + it('rebuilds an equivalent eql/v3 encryptedTable from a drizzle table', () => { + const users = pgTable('users', { id: integer().primaryKey(), - email: types.TextSearch("email"), - age: types.Int4Ord("age"), + email: types.TextSearch('email'), + age: types.Int4Ord('age'), }) const extracted = extractEncryptionSchemaV3(users) - const authored = encryptedTable("users", { - email: v3Types.TextSearch("email"), - age: v3Types.Int4Ord("age"), + const authored = encryptedTable('users', { + email: v3Types.TextSearch('email'), + age: v3Types.Int4Ord('age'), }) expect(extracted.build()).toEqual(authored.build()) }) - it("throws when the table has no encrypted v3 columns", () => { - const plain = pgTable("plain", { id: integer() }) + it('throws when the table has no encrypted v3 columns', () => { + const plain = pgTable('plain', { id: integer() }) expect(() => extractEncryptionSchemaV3(plain)).toThrow(/no encrypted v3/i) }) }) diff --git a/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts b/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts index ba876b05..0ef7da07 100644 --- a/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts +++ b/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts @@ -1,39 +1,39 @@ -import { sql } from "drizzle-orm" -import { PgDialect } from "drizzle-orm/pg-core" -import { describe, expect, it } from "vitest" -import { v3Dialect } from "@/eql/v3/drizzle/sql-dialect" +import { sql } from 'drizzle-orm' +import { PgDialect } from 'drizzle-orm/pg-core' +import { describe, expect, it } from 'vitest' +import { v3Dialect } from '@/eql/v3/drizzle/sql-dialect' const dialect = new PgDialect() const render = (s: ReturnType) => dialect.sqlToQuery(s).sql const col = sql`"users"."x"` const enc = sql`${'{"v":"t"}'}` -describe("v3Dialect", () => { - it("equality via HMAC", () => { - expect(render(v3Dialect.equality("eq", col, enc, false))).toBe( +describe('v3Dialect', () => { + it('equality via HMAC', () => { + expect(render(v3Dialect.equality('eq', col, enc, false))).toBe( 'eql_v3.eq_term("users"."x") = eql_v3.hmac_256($1::jsonb)', ) }) - it("inequality via HMAC", () => { - expect(render(v3Dialect.equality("ne", col, enc, false))).toBe( + it('inequality via HMAC', () => { + expect(render(v3Dialect.equality('ne', col, enc, false))).toBe( 'eql_v3.eq_term("users"."x") <> eql_v3.hmac_256($1::jsonb)', ) }) - it("equality via ORE (order-only numeric/date domains)", () => { - expect(render(v3Dialect.equality("eq", col, enc, true))).toBe( + it('equality via ORE (order-only numeric/date domains)', () => { + expect(render(v3Dialect.equality('eq', col, enc, true))).toBe( 'eql_v3.ord_term("users"."x") = eql_v3.ore_block_256($1::jsonb)', ) }) - it("comparison via ORE", () => { - expect(render(v3Dialect.comparison("gte", col, enc))).toBe( + it('comparison via ORE', () => { + expect(render(v3Dialect.comparison('gte', col, enc))).toBe( 'eql_v3.ord_term("users"."x") >= eql_v3.ore_block_256($1::jsonb)', ) }) - it("range via ORE", () => { + it('range via ORE', () => { const lo = sql`${'{"v":"lo"}'}` const hi = sql`${'{"v":"hi"}'}` expect(render(v3Dialect.range(col, lo, hi))).toBe( @@ -41,13 +41,13 @@ describe("v3Dialect", () => { ) }) - it("match via bloom filter containment", () => { + it('match via bloom filter containment', () => { expect(render(v3Dialect.match(col, enc))).toBe( 'eql_v3.match_term("users"."x") @> eql_v3.bloom_filter($1::jsonb)', ) }) - it("orderBy extracts the ord term", () => { + it('orderBy extracts the ord term', () => { expect(render(v3Dialect.orderBy(col))).toBe('eql_v3.ord_term("users"."x")') }) }) diff --git a/packages/stack/__tests__/drizzle-v3/types.test-d.ts b/packages/stack/__tests__/drizzle-v3/types.test-d.ts index 0e25c9d7..2967f57d 100644 --- a/packages/stack/__tests__/drizzle-v3/types.test-d.ts +++ b/packages/stack/__tests__/drizzle-v3/types.test-d.ts @@ -1,17 +1,17 @@ -import { describe, expectTypeOf, it } from "vitest" -import { types as v3Types } from "@/eql/v3" -import { types } from "@/eql/v3/drizzle/types" +import { describe, expectTypeOf, it } from 'vitest' +import type { types as v3Types } from '@/eql/v3' +import { types } from '@/eql/v3/drizzle/types' -describe("v3 drizzle types - type-level", () => { - it("exposes exactly the same factory keys as @/eql/v3 types", () => { +describe('v3 drizzle types - type-level', () => { + it('exposes exactly the same factory keys as @/eql/v3 types', () => { expectTypeOf().toEqualTypeOf() }) - it("columns infer their concrete plaintext type via the data type slot", () => { - const age = types.Int4Ord("age") - const created = types.Timestamptz("created_at") - const flag = types.Bool("flag") - const nick = types.TextEq("nickname") + it('columns infer their concrete plaintext type via the data type slot', () => { + const age = types.Int4Ord('age') + const created = types.Timestamptz('created_at') + const flag = types.Bool('flag') + const nick = types.TextEq('nickname') expectTypeOf(age._.data).toEqualTypeOf() expectTypeOf(created._.data).toEqualTypeOf() diff --git a/packages/stack/__tests__/drizzle-v3/types.test.ts b/packages/stack/__tests__/drizzle-v3/types.test.ts index 716a7fb6..4c8086f7 100644 --- a/packages/stack/__tests__/drizzle-v3/types.test.ts +++ b/packages/stack/__tests__/drizzle-v3/types.test.ts @@ -1,21 +1,21 @@ -import { describe, expect, it } from "vitest" -import { types as v3Types } from "@/eql/v3" -import { getEqlV3Column } from "@/eql/v3/drizzle/column" -import { types } from "@/eql/v3/drizzle/types" +import { describe, expect, it } from 'vitest' +import { types as v3Types } from '@/eql/v3' +import { getEqlV3Column } from '@/eql/v3/drizzle/column' +import { types } from '@/eql/v3/drizzle/types' -describe("v3 drizzle types namespace", () => { - it("exposes the same factory names as @/eql/v3 types", () => { +describe('v3 drizzle types namespace', () => { + it('exposes the same factory names as @/eql/v3 types', () => { expect(Object.keys(types).sort()).toEqual(Object.keys(v3Types).sort()) }) - it("each factory produces a column carrying the matching concrete builder", () => { - const email = types.TextSearch("email") - expect(getEqlV3Column("email", email)?.getEqlType()).toBe( - "eql_v3.text_search", + it('each factory produces a column carrying the matching concrete builder', () => { + const email = types.TextSearch('email') + expect(getEqlV3Column('email', email)?.getEqlType()).toBe( + 'eql_v3.text_search', ) - const age = types.Int4Ord("age") - expect(getEqlV3Column("age", age)?.getEqlType()).toBe("eql_v3.int4_ord") - const active = types.Bool("active") - expect(getEqlV3Column("active", active)?.getEqlType()).toBe("eql_v3.bool") + const age = types.Int4Ord('age') + expect(getEqlV3Column('age', age)?.getEqlType()).toBe('eql_v3.int4_ord') + const active = types.Bool('active') + expect(getEqlV3Column('active', active)?.getEqlType()).toBe('eql_v3.bool') }) }) diff --git a/packages/stack/src/eql/v3/drizzle/index.ts b/packages/stack/src/eql/v3/drizzle/index.ts index 7daecab0..0eaf9830 100644 --- a/packages/stack/src/eql/v3/drizzle/index.ts +++ b/packages/stack/src/eql/v3/drizzle/index.ts @@ -1,5 +1,5 @@ -export { getEqlV3Column, isEqlV3Column, makeEqlV3Column } from './column.js' export { v3FromDriver, v3ToDriver } from './codec.js' +export { getEqlV3Column, isEqlV3Column, makeEqlV3Column } from './column.js' export { createEncryptionOperatorsV3, EncryptionOperatorError, From 17cd5d9a9cb647620e95efe921716293efb25d48 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:22:12 +1000 Subject: [PATCH 15/24] fix(stack): bind v3 drizzle columns without name collisions --- .../__tests__/drizzle-v3/operators.test.ts | 16 +++++ .../drizzle-v3/schema-extraction.test.ts | 23 +++++++ packages/stack/src/eql/v3/drizzle/column.ts | 69 +++++++++++++------ 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 1adcec61..fe12062b 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -66,6 +66,22 @@ describe('createEncryptionOperatorsV3 - equality', () => { 'eql_v3.ord_term("users"."age") <> eql_v3.ore_block_256($1::jsonb)', ) }) + + it('same-named columns on different tables use their own equality capability', async () => { + const accounts = pgTable('accounts', { + email: types.TextEq('email'), + }) + pgTable('metrics', { + email: types.Int4Ord('email'), + }) + const { ops, render } = setup() + + const q = render(await ops.eq(accounts.email, 'ada@example.com')) + + expect(q.sql).toContain( + 'eql_v3.eq_term("accounts"."email") = eql_v3.hmac_256($1::jsonb)', + ) + }) }) describe('createEncryptionOperatorsV3 - comparison & range', () => { diff --git a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts index a538371a..384f986f 100644 --- a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts +++ b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts @@ -21,6 +21,29 @@ describe('extractEncryptionSchemaV3', () => { expect(extracted.build()).toEqual(authored.build()) }) + it('keeps same-named columns on different tables bound to their own v3 domains', () => { + const accounts = pgTable('accounts', { + email: types.TextEq('email'), + }) + const metrics = pgTable('metrics', { + email: types.Int4Ord('email'), + }) + + const accountsSchema = extractEncryptionSchemaV3(accounts) + const metricsSchema = extractEncryptionSchemaV3(metrics) + + expect(accountsSchema.build()).toEqual( + encryptedTable('accounts', { + email: v3Types.TextEq('email'), + }).build(), + ) + expect(metricsSchema.build()).toEqual( + encryptedTable('metrics', { + email: v3Types.Int4Ord('email'), + }).build(), + ) + }) + it('throws when the table has no encrypted v3 columns', () => { const plain = pgTable('plain', { id: integer() }) expect(() => extractEncryptionSchemaV3(plain)).toThrow(/no encrypted v3/i) diff --git a/packages/stack/src/eql/v3/drizzle/column.ts b/packages/stack/src/eql/v3/drizzle/column.ts index 0b357ff8..3fa8d9f9 100644 --- a/packages/stack/src/eql/v3/drizzle/column.ts +++ b/packages/stack/src/eql/v3/drizzle/column.ts @@ -12,12 +12,46 @@ export const EQL_V3_DOMAINS: ReadonlySet = new Set( ) /** - * Mirrors v2's bare-name lookup so a builder can be recovered after pgTable - * creates a fresh column object. Same-named columns with different v3 domains in - * one module collide; table-qualified keys need Drizzle table context that is - * not available when the builder is created. + * Drizzle copies `config.customTypeParams` from the builder into the processed + * PgColumn. Stashing the v3 builder there keeps recovery tied to the concrete + * column instance instead of a module-global name lookup. */ -const columnBuilderMap = new Map() +const EQL_V3_COLUMN_PARAM = Symbol.for('cipherstash:eqlv3Column') +const EQL_V3_COLUMN_LEGACY_PARAM = '_eqlv3Column' + +type EqlV3ColumnCarrier = Record & { + [EQL_V3_COLUMN_LEGACY_PARAM]?: AnyEncryptedV3Column +} + +function readBuilder( + carrier: EqlV3ColumnCarrier | undefined, +): AnyEncryptedV3Column | undefined { + return ( + (carrier?.[EQL_V3_COLUMN_PARAM] as AnyEncryptedV3Column | undefined) ?? + carrier?.[EQL_V3_COLUMN_LEGACY_PARAM] + ) +} + +function writeBuilder( + carrier: EqlV3ColumnCarrier | undefined, + builder: AnyEncryptedV3Column, +): void { + if (!carrier) return + carrier[EQL_V3_COLUMN_PARAM] = builder + carrier[EQL_V3_COLUMN_LEGACY_PARAM] = builder +} + +function getCarrier(column: unknown): EqlV3ColumnCarrier | undefined { + if (!column || typeof column !== 'object') return undefined + + const direct = column as EqlV3ColumnCarrier + if (readBuilder(direct)) return direct + + const maybeWithConfig = column as { + config?: { customTypeParams?: EqlV3ColumnCarrier } + } + return maybeWithConfig.config?.customTypeParams +} export function makeEqlV3Column(builder: C) { type TData = PlaintextForColumn @@ -36,9 +70,8 @@ export function makeEqlV3Column(builder: C) { }, })(name) - columnBuilderMap.set(name, builder) - // biome-ignore lint/suspicious/noExplicitAny: Drizzle columns don't expose custom props - ;(column as any)._eqlv3Column = builder + writeBuilder(getCarrier(column), builder) + writeBuilder(column as unknown as EqlV3ColumnCarrier, builder) return column } @@ -46,23 +79,17 @@ export function getEqlV3Column( columnName: string, column: unknown, ): AnyEncryptedV3Column | undefined { - if (column && typeof column === 'object') { - // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals - const columnAny = column as any - if (columnAny._eqlv3Column) { - return columnAny._eqlv3Column as AnyEncryptedV3Column - } - const lookupName = (columnAny.name as string | undefined) ?? columnName - return columnBuilderMap.get(lookupName) - } - return undefined + void columnName + return readBuilder(getCarrier(column)) } export function isEqlV3Column(column: unknown): boolean { if (!column || typeof column !== 'object') return false - // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals - const columnAny = column as any - if (columnAny._eqlv3Column) return true + if (readBuilder(getCarrier(column))) return true + const columnAny = column as { + dataType?: unknown + sqlName?: unknown + } const dt = columnAny.dataType const domain = typeof dt === 'function' ? dt() : (columnAny.sqlName ?? dt) return typeof domain === 'string' && EQL_V3_DOMAINS.has(domain) From a133072c124813ef0c28d79d66413d668e4405e2 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:28:38 +1000 Subject: [PATCH 16/24] fix(stack): detect v3 drizzle columns via getSQLType --- packages/stack/__tests__/drizzle-v3/column.test.ts | 8 ++++++++ packages/stack/src/eql/v3/drizzle/column.ts | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/packages/stack/__tests__/drizzle-v3/column.test.ts b/packages/stack/__tests__/drizzle-v3/column.test.ts index 9a759474..ff823405 100644 --- a/packages/stack/__tests__/drizzle-v3/column.test.ts +++ b/packages/stack/__tests__/drizzle-v3/column.test.ts @@ -36,4 +36,12 @@ describe('makeEqlV3Column', () => { it('isEqlV3Column is false for a plain value', () => { expect(isEqlV3Column({})).toBe(false) }) + + it('recognises v3 columns by the Drizzle getSQLType API', () => { + expect( + isEqlV3Column({ + getSQLType: () => 'eql_v3.text_eq', + }), + ).toBe(true) + }) }) diff --git a/packages/stack/src/eql/v3/drizzle/column.ts b/packages/stack/src/eql/v3/drizzle/column.ts index 3fa8d9f9..3db8148b 100644 --- a/packages/stack/src/eql/v3/drizzle/column.ts +++ b/packages/stack/src/eql/v3/drizzle/column.ts @@ -87,9 +87,15 @@ export function isEqlV3Column(column: unknown): boolean { if (!column || typeof column !== 'object') return false if (readBuilder(getCarrier(column))) return true const columnAny = column as { + getSQLType?: () => unknown dataType?: unknown sqlName?: unknown } + const sqlType = + typeof columnAny.getSQLType === 'function' + ? columnAny.getSQLType() + : undefined + if (typeof sqlType === 'string' && EQL_V3_DOMAINS.has(sqlType)) return true const dt = columnAny.dataType const domain = typeof dt === 'function' ? dt() : (columnAny.sqlName ?? dt) return typeof domain === 'string' && EQL_V3_DOMAINS.has(domain) From 32b68781a9ff03cb173395b09adb0c6a3a7249e2 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 16:48:07 +1000 Subject: [PATCH 17/24] fix(stack): align v3 drizzle detection and ore equality --- .../stack/__tests__/drizzle-v3/column.test.ts | 9 ++++ .../__tests__/drizzle-v3/operators.test.ts | 5 +- packages/stack/src/eql/v3/drizzle/column.ts | 50 +++++++++++++------ .../stack/src/eql/v3/drizzle/operators.ts | 6 ++- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/column.test.ts b/packages/stack/__tests__/drizzle-v3/column.test.ts index ff823405..f78c633a 100644 --- a/packages/stack/__tests__/drizzle-v3/column.test.ts +++ b/packages/stack/__tests__/drizzle-v3/column.test.ts @@ -44,4 +44,13 @@ describe('makeEqlV3Column', () => { }), ).toBe(true) }) + + it('recovers a v3 builder for columns recognised by getSQLType', () => { + const builder = getEqlV3Column('nickname', { + getSQLType: () => 'eql_v3.text_eq', + }) + + expect(builder?.getName()).toBe('nickname') + expect(builder?.getEqlType()).toBe('eql_v3.text_eq') + }) }) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index fe12062b..d75e4a04 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -52,11 +52,14 @@ describe('createEncryptionOperatorsV3 - equality', () => { }) it('eq on an order-only numeric column emits ord_term = ore_block_256', async () => { - const { ops, render } = setup() + const { ops, encryptQuery, render } = setup() const q = render(await ops.eq(users.age, 37)) expect(q.sql).toContain( 'eql_v3.ord_term("users"."age") = eql_v3.ore_block_256($1::jsonb)', ) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: 'orderAndRange', + }) }) it('ne on an order-only numeric column emits ord_term <> ore_block_256', async () => { diff --git a/packages/stack/src/eql/v3/drizzle/column.ts b/packages/stack/src/eql/v3/drizzle/column.ts index 3db8148b..6727e447 100644 --- a/packages/stack/src/eql/v3/drizzle/column.ts +++ b/packages/stack/src/eql/v3/drizzle/column.ts @@ -11,6 +11,16 @@ export const EQL_V3_DOMAINS: ReadonlySet = new Set( Object.values(v3Types).map((factory) => factory('__probe__').getEqlType()), ) +const buildersByDomain: ReadonlyMap< + string, + (name: string) => AnyEncryptedV3Column +> = new Map( + Object.values(v3Types).map((factory) => [ + factory('__probe__').getEqlType(), + factory, + ]), +) + /** * Drizzle copies `config.customTypeParams` from the builder into the processed * PgColumn. Stashing the v3 builder there keeps recovery tied to the concrete @@ -53,6 +63,23 @@ function getCarrier(column: unknown): EqlV3ColumnCarrier | undefined { return maybeWithConfig.config?.customTypeParams } +function getSqlType(column: unknown): string | undefined { + if (!column || typeof column !== 'object') return undefined + const columnAny = column as { + getSQLType?: () => unknown + dataType?: unknown + sqlName?: unknown + } + const sqlType = + typeof columnAny.getSQLType === 'function' + ? columnAny.getSQLType() + : undefined + if (typeof sqlType === 'string') return sqlType + const dt = columnAny.dataType + const domain = typeof dt === 'function' ? dt() : (columnAny.sqlName ?? dt) + return typeof domain === 'string' ? domain : undefined +} + export function makeEqlV3Column(builder: C) { type TData = PlaintextForColumn const domain = builder.getEqlType() @@ -79,24 +106,17 @@ export function getEqlV3Column( columnName: string, column: unknown, ): AnyEncryptedV3Column | undefined { - void columnName - return readBuilder(getCarrier(column)) + const stashed = readBuilder(getCarrier(column)) + if (stashed) return stashed + + const sqlType = getSqlType(column) + const builderFactory = sqlType ? buildersByDomain.get(sqlType) : undefined + return builderFactory?.(columnName) } export function isEqlV3Column(column: unknown): boolean { if (!column || typeof column !== 'object') return false if (readBuilder(getCarrier(column))) return true - const columnAny = column as { - getSQLType?: () => unknown - dataType?: unknown - sqlName?: unknown - } - const sqlType = - typeof columnAny.getSQLType === 'function' - ? columnAny.getSQLType() - : undefined - if (typeof sqlType === 'string' && EQL_V3_DOMAINS.has(sqlType)) return true - const dt = columnAny.dataType - const domain = typeof dt === 'function' ? dt() : (columnAny.sqlName ?? dt) - return typeof domain === 'string' && EQL_V3_DOMAINS.has(domain) + const sqlType = getSqlType(column) + return typeof sqlType === 'string' && EQL_V3_DOMAINS.has(sqlType) } diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index c5369383..2a7ec1f5 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -138,7 +138,11 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { ) } const viaOre = !ctx.indexes.unique - const enc = await encryptTerm(ctx, right, 'equality') + const enc = await encryptTerm( + ctx, + right, + viaOre ? 'orderAndRange' : 'equality', + ) return v3Dialect.equality(op, colSql(left), enc, viaOre) } From 671d31a8e9de78705de222873c02b825610537ba Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 18:56:46 +1000 Subject: [PATCH 18/24] fix(stack): address v3 drizzle review follow-ups --- ...06-eql-v3-drizzle-concrete-types-design.md | 4 +- .../stack/__tests__/drizzle-v3/column.test.ts | 30 +++- .../drizzle-v3/operators-live-pg.test.ts | 57 ++++++ .../__tests__/drizzle-v3/operators.test.ts | 170 +++++++++++++++++- .../drizzle-v3/schema-extraction.test.ts | 52 ++++++ .../__tests__/drizzle-v3/types.test-d.ts | 26 +++ .../stack/__tests__/drizzle-v3/types.test.ts | 17 +- .../stack/src/eql/v3/drizzle/operators.ts | 49 +++-- .../src/eql/v3/drizzle/schema-extraction.ts | 6 +- 9 files changed, 379 insertions(+), 32 deletions(-) diff --git a/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md b/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md index 5995194d..19b95c62 100644 --- a/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md +++ b/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md @@ -110,7 +110,7 @@ capability data. ## 4. Architecture & location -``` +```text packages/stack/src/eql/v3/drizzle/ index.ts // barrel: types, createEncryptionOperatorsV3, extractEncryptionSchemaV3, errors types.ts // Drizzle `types` namespace (PascalCase factories → customType columns) @@ -224,7 +224,7 @@ v2's `extractEncryptionSchema`). Operators call this internally to resolve the ## 6. Data flow (per query) -``` +```text types.Int4Ord('age') // authoring → customType dataType() = 'eql_v3.int4_ord', stash eql/v3 builder pgTable('users', { age }) // Drizzle table diff --git a/packages/stack/__tests__/drizzle-v3/column.test.ts b/packages/stack/__tests__/drizzle-v3/column.test.ts index f78c633a..5efd7885 100644 --- a/packages/stack/__tests__/drizzle-v3/column.test.ts +++ b/packages/stack/__tests__/drizzle-v3/column.test.ts @@ -7,14 +7,16 @@ import { isEqlV3Column, makeEqlV3Column, } from '@/eql/v3/drizzle/column' +import { typedEntries, V3_MATRIX } from '../v3-matrix/catalog' + +const slug = (eqlType: string) => eqlType.replace('eql_v3.', '') describe('makeEqlV3Column', () => { it('sets dataType() to the concrete eql_v3 domain', () => { const col = makeEqlV3Column(v3Types.Int4Ord('age')) - // biome-ignore lint/suspicious/noExplicitAny: reading drizzle internals in test - expect((col as any).config.customTypeParams.dataType()).toBe( - 'eql_v3.int4_ord', - ) + const table = pgTable('users', { age: col }) + + expect(table.age.getSQLType()).toBe('eql_v3.int4_ord') }) it('recovers the stashed builder before and after pgTable processing', () => { @@ -53,4 +55,24 @@ describe('makeEqlV3Column', () => { expect(builder?.getName()).toBe('nickname') expect(builder?.getEqlType()).toBe('eql_v3.text_eq') }) + + it.each( + typedEntries(V3_MATRIX), + )('%s round-trips through makeEqlV3Column and pgTable', (eqlType, spec) => { + const columnName = slug(eqlType) + const builder = spec.builder(columnName) + const column = makeEqlV3Column(builder) + + expect(builder.getEqlType()).toBe(eqlType) + expect(builder).toBeInstanceOf(spec.ColumnClass) + expect(isEqlV3Column(column)).toBe(true) + expect(getEqlV3Column(columnName, column)?.getEqlType()).toBe(eqlType) + + const table = pgTable('users', { [columnName]: column } as never) + const pgColumn = (table as Record)[ + columnName + ] + expect(pgColumn?.getSQLType()).toBe(eqlType) + expect(getEqlV3Column(columnName, pgColumn)?.getEqlType()).toBe(eqlType) + }) }) diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts index 65c48263..71bc5f2d 100644 --- a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -112,6 +112,15 @@ describeLivePg('v3 drizzle operators (live pg)', () => { expect(rows.length).toBe(2) }, 30000) + it('notBetween excludes the inclusive range', async () => { + const rows = await db + .select({ id: usersTable.id, age: usersTable.age }) + .from(usersTable) + .where(scoped(await ops.notBetween(usersTable.age, 30, 37))) + const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + expect(decrypted.map((r: { age: number }) => r.age).sort()).toEqual([42]) + }, 30000) + it('ilike matches on the free-text email column', async () => { const rows = await db .select({ id: usersTable.id }) @@ -120,6 +129,28 @@ describeLivePg('v3 drizzle operators (live pg)', () => { expect(rows.length).toBe(2) }, 30000) + it('inArray selects the exact nickname rows', async () => { + const rows = await db + .select({ id: usersTable.id, nickname: usersTable.nickname }) + .from(usersTable) + .where(scoped(await ops.inArray(usersTable.nickname, ['ada', 'grace']))) + const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + expect( + decrypted.map((r: { nickname: string }) => r.nickname).sort(), + ).toEqual(['ada', 'grace']) + }, 30000) + + it('notInArray excludes the listed nickname rows', async () => { + const rows = await db + .select({ id: usersTable.id, nickname: usersTable.nickname }) + .from(usersTable) + .where(scoped(await ops.notInArray(usersTable.nickname, ['grace']))) + const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + expect( + decrypted.map((r: { nickname: string }) => r.nickname).sort(), + ).toEqual(['ada', 'alan']) + }, 30000) + it('orderBy asc returns ascending ages', async () => { const rows = await db .select({ age: usersTable.age }) @@ -130,4 +161,30 @@ describeLivePg('v3 drizzle operators (live pg)', () => { const ages = decrypted.map((r: { age: number }) => r.age) expect(ages).toEqual([...ages].sort((a, b) => a - b)) }, 30000) + + it('orderBy desc returns descending ages', async () => { + const rows = await db + .select({ age: usersTable.age }) + .from(usersTable) + .where(scoped(await ops.gte(usersTable.age, 0))) + .orderBy(ops.desc(usersTable.age)) + const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + const ages = decrypted.map((r: { age: number }) => r.age) + expect(ages).toEqual([...ages].sort((a, b) => b - a)) + }, 30000) + + it('or mixes encrypted and plain predicates', async () => { + const rows = await db + .select({ id: usersTable.id }) + .from(usersTable) + .where( + scoped( + await ops.or( + ops.eq(usersTable.nickname, 'grace'), + drizzleEq(usersTable.testRunId, RUN), + ), + ), + ) + expect(rows.length).toBe(3) + }, 30000) }) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index d75e4a04..8842a1c7 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -1,4 +1,14 @@ -import type { sql } from 'drizzle-orm' +import { + eq as drizzleEq, + exists, + isNotNull, + isNull, + not, + notExists, + type SQL, + type SQLWrapper, + sql, +} from 'drizzle-orm' import { integer, PgDialect, pgTable } from 'drizzle-orm/pg-core' import { describe, expect, it, vi } from 'vitest' import type { EncryptionClient } from '@/encryption' @@ -6,19 +16,26 @@ import { createEncryptionOperatorsV3, EncryptionOperatorError, } from '@/eql/v3/drizzle/operators' +import { extractEncryptionSchemaV3 } from '@/eql/v3/drizzle/schema-extraction' import { types } from '@/eql/v3/drizzle/types' const TERM = { c: 'ct', v: 1 } const TERM_JSON = JSON.stringify(TERM) -function setup() { - const encryptQuery = vi.fn(async (_value: unknown, _opts?: unknown) => ({ +type EncryptQueryResult = Promise< + { data: typeof TERM } | { failure: { message: string } } +> + +function setup( + encryptQueryImpl: (...args: unknown[]) => EncryptQueryResult = async () => ({ data: TERM, - })) + }), +) { + const encryptQuery = vi.fn(encryptQueryImpl) const client = { encryptQuery } as unknown as EncryptionClient const ops = createEncryptionOperatorsV3(client) const dialect = new PgDialect() - const render = (s: unknown) => dialect.sqlToQuery(s as ReturnType) + const render = (s: unknown) => dialect.sqlToQuery(s as SQL) return { ops, encryptQuery, render } } @@ -26,7 +43,16 @@ const users = pgTable('users', { id: integer().primaryKey(), email: types.TextSearch('email'), nickname: types.TextEq('nickname'), + textMatch: types.TextMatch('text_match'), + textOrd: types.TextOrd('text_ord'), + textOrdOre: types.TextOrdOre('text_ord_ore'), + int2Age: types.Int2Ord('int2_age'), age: types.Int4Ord('age'), + createdOn: types.DateOrd('created_on'), + createdAt: types.TimestamptzOrd('created_at'), + amount: types.NumericOrd('amount'), + score4: types.Float4Ord('score4'), + score8: types.Float8Ord('score8'), flag: types.Bool('flag'), }) @@ -70,6 +96,23 @@ describe('createEncryptionOperatorsV3 - equality', () => { ) }) + it('eq on text order domains still uses HMAC equality', async () => { + const { ops, encryptQuery, render } = setup() + + const textOrd = render(await ops.eq(users.textOrd, 'ada')) + expect(textOrd.sql).toContain( + 'eql_v3.eq_term("users"."text_ord") = eql_v3.hmac_256($1::jsonb)', + ) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: 'equality', + }) + + const textOrdOre = render(await ops.eq(users.textOrdOre, 'grace')) + expect(textOrdOre.sql).toContain( + 'eql_v3.eq_term("users"."text_ord_ore") = eql_v3.hmac_256($1::jsonb)', + ) + }) + it('same-named columns on different tables use their own equality capability', async () => { const accounts = pgTable('accounts', { email: types.TextEq('email'), @@ -85,6 +128,23 @@ describe('createEncryptionOperatorsV3 - equality', () => { 'eql_v3.eq_term("accounts"."email") = eql_v3.hmac_256($1::jsonb)', ) }) + + it('does not reuse a cached extracted schema across distinct pgTable objects with the same SQL name', async () => { + const first = pgTable('shared', { + email: types.TextEq('email'), + }) + const second = pgTable('shared', { + age: types.Int4Ord('age'), + }) + const { ops, encryptQuery } = setup() + + await ops.eq(first.email, 'ada@example.com') + await ops.eq(second.age, 37) + + expect(encryptQuery.mock.calls[1]?.[1]?.table.build()).toEqual( + extractEncryptionSchemaV3(second).build(), + ) + }) }) describe('createEncryptionOperatorsV3 - comparison & range', () => { @@ -105,6 +165,23 @@ describe('createEncryptionOperatorsV3 - comparison & range', () => { }) }) + it.each([ + ['int2_age', users.int2Age, 21], + ['created_on', users.createdOn, new Date('2026-07-01T00:00:00.000Z')], + ['created_at', users.createdAt, new Date('2026-07-01T00:00:00.000Z')], + ['amount', users.amount, 123.45], + ['score4', users.score4, 12.5], + ['score8', users.score8, 12.5], + ] as const)('%s eq on an order-only column uses ORE equality', async (_label, column, value) => { + const { ops, encryptQuery, render } = setup() + const q = render(await ops.eq(column, value)) + expect(q.sql).toContain('eql_v3.ord_term(') + expect(q.sql).toContain('eql_v3.ore_block_256($1::jsonb)') + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: 'orderAndRange', + }) + }) + it('between emits a bounded ord_term range with two ORE bounds', async () => { const { ops, render } = setup() const q = render(await ops.between(users.age, 20, 40)) @@ -137,6 +214,14 @@ describe('createEncryptionOperatorsV3 - free-text match', () => { }) }) + it('like on a text_match column uses the same match term path', async () => { + const { ops, render } = setup() + const q = render(await ops.like(users.textMatch, 'engineer')) + expect(q.sql).toContain( + 'eql_v3.match_term("users"."text_match") @> eql_v3.bloom_filter($1::jsonb)', + ) + }) + it('ilike emits match_term @> bloom_filter', async () => { const { ops, render } = setup() const q = render(await ops.ilike(users.email, 'example.com')) @@ -165,6 +250,16 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { expect(empty.sql).toBe('false') }) + it('inArray on an ORE column uses ORE equality for each term', async () => { + const { ops, encryptQuery, render } = setup() + const q = render(await ops.inArray(users.age, [30, 42])) + expect(q.sql).toContain(' or ') + expect((q.sql.match(/eql_v3\.ord_term/g) ?? []).length).toBe(2) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: 'orderAndRange', + }) + }) + it('notInArray ANDs one ne term per value; empty array is true', async () => { const { ops, render } = setup() const q = render(await ops.notInArray(users.nickname, ['ada', 'grace'])) @@ -173,6 +268,13 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { expect(empty.sql).toBe('true') }) + it('notInArray on an ORE column uses ORE inequality for each term', async () => { + const { ops, render } = setup() + const q = render(await ops.notInArray(users.age, [30, 42])) + expect(q.sql).toContain(' and ') + expect((q.sql.match(/eql_v3\.ord_term/g) ?? []).length).toBe(2) + }) + it('asc / desc extract the ord term', () => { const { ops, render } = setup() const ascq = render(ops.asc(users.age)) @@ -182,18 +284,58 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { expect(descq.sql.toLowerCase()).toContain('desc') }) - it('and / or await and combine v3 conditions', async () => { + it('and ignores undefined conditions and keeps the encrypted predicates', async () => { const { ops, render } = setup() const q = render( - await ops.and(ops.eq(users.nickname, 'ada'), ops.gte(users.age, 30)), + await ops.and( + undefined, + ops.eq(users.nickname, 'ada'), + undefined, + ops.gte(users.age, 30), + ), ) expect(q.sql).toContain('eql_v3.eq_term("users"."nickname")') expect(q.sql).toContain('eql_v3.ord_term("users"."age")') expect(q.sql).toContain(' and ') }) + + it('empty and/or resolve to their neutral predicates', async () => { + const { ops, render } = setup() + expect(render(await ops.and()).sql).toBe('true') + expect(render(await ops.or()).sql).toBe('false') + }) + + it('or combines encrypted and plain predicates', async () => { + const { ops, render } = setup() + const q = render( + await ops.or(ops.eq(users.nickname, 'ada'), drizzleEq(users.id, 1)), + ) + expect(q.sql).toContain(' or ') + expect(q.sql).toContain('eql_v3.eq_term("users"."nickname")') + expect(q.sql).toContain('"users"."id" = $2') + }) + + it('exports the passthrough Drizzle operators unchanged', () => { + const { ops } = setup() + expect(ops.isNull).toBe(isNull) + expect(ops.isNotNull).toBe(isNotNull) + expect(ops.not).toBe(not) + expect(ops.exists).toBe(exists) + expect(ops.notExists).toBe(notExists) + }) }) describe('createEncryptionOperatorsV3 - gating errors', () => { + it('wraps encryptQuery failures with operator context', async () => { + const { ops } = setup(async () => ({ + failure: { message: 'bad query term' }, + })) + + await expect(ops.eq(users.nickname, 'ada')).rejects.toMatchObject({ + name: 'EncryptionOperatorError', + }) + }) + it('gt on an equality-only column throws', async () => { const { ops } = setup() await expect(ops.gt(users.nickname, 'ada')).rejects.toBeInstanceOf( @@ -226,4 +368,18 @@ describe('createEncryptionOperatorsV3 - gating errors', () => { EncryptionOperatorError, ) }) + + it('does not treat SQLWrapper objects with column-shaped fields as Drizzle columns', async () => { + const { ops } = setup() + const spoof = { + name: 'nickname', + table: users, + getSQL: () => sql`"users"."nickname"`, + } as unknown as SQLWrapper + + await expect(ops.eq(spoof, 'ada')).rejects.toMatchObject({ + name: 'EncryptionOperatorError', + context: { columnName: 'unknown', operator: 'eq' }, + }) + }) }) diff --git a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts index 384f986f..bea97013 100644 --- a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts +++ b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts @@ -5,6 +5,28 @@ import { extractEncryptionSchemaV3 } from '@/eql/v3/drizzle/schema-extraction' import { types } from '@/eql/v3/drizzle/types' describe('extractEncryptionSchemaV3', () => { + it('rebuilds an equivalent eql/v3 encryptedTable from every drizzle v3 factory', () => { + const drizzleColumns = Object.fromEntries( + Object.entries(types).map(([factoryName, factory]) => [ + factoryName, + factory(factoryName), + ]), + ) + const authoredColumns = Object.fromEntries( + Object.entries(v3Types).map(([factoryName, factory]) => [ + factoryName, + factory(factoryName), + ]), + ) + + const extracted = extractEncryptionSchemaV3( + pgTable('users', drizzleColumns as never), + ) + const authored = encryptedTable('users', authoredColumns as never) + + expect(extracted.build()).toEqual(authored.build()) + }) + it('rebuilds an equivalent eql/v3 encryptedTable from a drizzle table', () => { const users = pgTable('users', { id: integer().primaryKey(), @@ -44,6 +66,36 @@ describe('extractEncryptionSchemaV3', () => { ) }) + it('uses the JS property key while preserving distinct SQL column names', () => { + const users = pgTable('users', { + createdOn: types.Date('created_on'), + emailAddress: types.TextEq('email_address'), + }) + + expect(extractEncryptionSchemaV3(users).build()).toEqual( + encryptedTable('users', { + createdOn: v3Types.Date('created_on'), + emailAddress: v3Types.TextEq('email_address'), + }).build(), + ) + }) + + it('uses the Drizzle column name when rebuilding fallback SQL-type columns', () => { + const table = { + [Symbol.for('drizzle:Name')]: 'users', + createdOn: { + name: 'created_on', + getSQLType: () => 'eql_v3.date', + }, + } + + expect(extractEncryptionSchemaV3(table as never).build()).toEqual( + encryptedTable('users', { + createdOn: v3Types.Date('created_on'), + }).build(), + ) + }) + it('throws when the table has no encrypted v3 columns', () => { const plain = pgTable('plain', { id: integer() }) expect(() => extractEncryptionSchemaV3(plain)).toThrow(/no encrypted v3/i) diff --git a/packages/stack/__tests__/drizzle-v3/types.test-d.ts b/packages/stack/__tests__/drizzle-v3/types.test-d.ts index 2967f57d..ba57f118 100644 --- a/packages/stack/__tests__/drizzle-v3/types.test-d.ts +++ b/packages/stack/__tests__/drizzle-v3/types.test-d.ts @@ -9,13 +9,39 @@ describe('v3 drizzle types - type-level', () => { it('columns infer their concrete plaintext type via the data type slot', () => { const age = types.Int4Ord('age') + const ageEq = types.Int4Eq('age_eq') + const int2 = types.Int2Ord('int2') + const date = types.DateOrd('created_at') + const dateOre = types.DateOrdOre('created_at_ore') const created = types.Timestamptz('created_at') + const tsOrd = types.TimestamptzOrd('created_at_ord') + const numOre = types.NumericOrdOre('amount_ore') const flag = types.Bool('flag') const nick = types.TextEq('nickname') + const text = types.Text('text') + const match = types.TextMatch('bio') + const textOrd = types.TextOrd('text_ord') + const textOrdOre = types.TextOrdOre('text_ord_ore') + const search = types.TextSearch('search') + const float4Eq = types.Float4Eq('float4_eq') + const float8Ord = types.Float8Ord('float8_ord') expectTypeOf(age._.data).toEqualTypeOf() + expectTypeOf(ageEq._.data).toEqualTypeOf() + expectTypeOf(int2._.data).toEqualTypeOf() + expectTypeOf(date._.data).toEqualTypeOf() + expectTypeOf(dateOre._.data).toEqualTypeOf() expectTypeOf(created._.data).toEqualTypeOf() + expectTypeOf(tsOrd._.data).toEqualTypeOf() + expectTypeOf(numOre._.data).toEqualTypeOf() expectTypeOf(flag._.data).toEqualTypeOf() expectTypeOf(nick._.data).toEqualTypeOf() + expectTypeOf(text._.data).toEqualTypeOf() + expectTypeOf(match._.data).toEqualTypeOf() + expectTypeOf(textOrd._.data).toEqualTypeOf() + expectTypeOf(textOrdOre._.data).toEqualTypeOf() + expectTypeOf(search._.data).toEqualTypeOf() + expectTypeOf(float4Eq._.data).toEqualTypeOf() + expectTypeOf(float8Ord._.data).toEqualTypeOf() }) }) diff --git a/packages/stack/__tests__/drizzle-v3/types.test.ts b/packages/stack/__tests__/drizzle-v3/types.test.ts index 4c8086f7..c143fc17 100644 --- a/packages/stack/__tests__/drizzle-v3/types.test.ts +++ b/packages/stack/__tests__/drizzle-v3/types.test.ts @@ -8,14 +8,15 @@ describe('v3 drizzle types namespace', () => { expect(Object.keys(types).sort()).toEqual(Object.keys(v3Types).sort()) }) - it('each factory produces a column carrying the matching concrete builder', () => { - const email = types.TextSearch('email') - expect(getEqlV3Column('email', email)?.getEqlType()).toBe( - 'eql_v3.text_search', + it.each( + Object.entries(types), + )('%s mirrors the authoring DSL and recovers the concrete eql type', (factoryName, factory) => { + const drizzleColumn = factory(factoryName) + const authoredColumn = + v3Types[factoryName as keyof typeof v3Types](factoryName) + + expect(getEqlV3Column(factoryName, drizzleColumn)?.getEqlType()).toBe( + authoredColumn.getEqlType(), ) - const age = types.Int4Ord('age') - expect(getEqlV3Column('age', age)?.getEqlType()).toBe('eql_v3.int4_ord') - const active = types.Bool('active') - expect(getEqlV3Column('active', active)?.getEqlType()).toBe('eql_v3.bool') }) }) diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index 2a7ec1f5..ac3f6555 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -1,8 +1,10 @@ import { and, asc, + Column, desc, exists, + is, isNotNull, isNull, not, @@ -21,6 +23,8 @@ import { getEqlV3Column } from './column.js' import { extractEncryptionSchemaV3 } from './schema-extraction.js' import { type ComparisonOp, type EqualityOp, v3Dialect } from './sql-dialect.js' +const MAX_IN_ARRAY_CONCURRENCY = 4 + export class EncryptionOperatorError extends Error { constructor( message: string, @@ -43,17 +47,40 @@ interface ColumnContext { tableName: string } +async function mapWithConcurrency( + values: T[], + limit: number, + mapper: (value: T) => Promise, +): Promise { + const results = new Array(values.length) + let next = 0 + + async function worker(): Promise { + while (true) { + const index = next + next += 1 + if (index >= values.length) return + results[index] = await mapper(values[index]) + } + } + + await Promise.all( + Array.from({ length: Math.min(limit, values.length) }, () => worker()), + ) + return results +} + export function createEncryptionOperatorsV3(client: EncryptionClient) { - const tableCache = new Map() + const tableCache = new WeakMap() function drizzleTableOf(column: SQLWrapper): PgTable | undefined { - // biome-ignore lint/suspicious/noExplicitAny: drizzle column internals - return (column as any).table as PgTable | undefined + return is(column, Column) + ? (column.table as PgTable | undefined) + : undefined } function resolveContext(column: SQLWrapper, operator: string): ColumnContext { - // biome-ignore lint/suspicious/noExplicitAny: drizzle column internals - const columnName = ((column as any).name as string | undefined) ?? 'unknown' + const columnName = is(column, Column) ? column.name : 'unknown' const builder = getEqlV3Column(columnName, column) if (!builder) { throw new EncryptionOperatorError( @@ -69,10 +96,10 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { const tableName = drizzleTableSymbols?.[Symbol.for('drizzle:Name')] ?? 'unknown' - let table = tableCache.get(tableName) + let table = drizzleTable ? tableCache.get(drizzleTable) : undefined if (!table && drizzleTable) { table = extractEncryptionSchemaV3(drizzleTable) - tableCache.set(tableName, table) + tableCache.set(drizzleTable, table) } if (!table) { throw new EncryptionOperatorError( @@ -191,10 +218,12 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { negate: boolean, _operator: string, ): Promise { - const conditions = await Promise.all( - values.map((v) => equality(negate ? 'ne' : 'eq', left, v)), + if (values.length === 0) return negate ? sql`true` : sql`false` + const conditions = await mapWithConcurrency( + values, + MAX_IN_ARRAY_CONCURRENCY, + (v) => equality(negate ? 'ne' : 'eq', left, v), ) - if (conditions.length === 0) return negate ? sql`true` : sql`false` const combined = negate ? and(...conditions) : or(...conditions) return combined ?? (negate ? sql`true` : sql`false`) } diff --git a/packages/stack/src/eql/v3/drizzle/schema-extraction.ts b/packages/stack/src/eql/v3/drizzle/schema-extraction.ts index 75f66c74..d7df04d6 100644 --- a/packages/stack/src/eql/v3/drizzle/schema-extraction.ts +++ b/packages/stack/src/eql/v3/drizzle/schema-extraction.ts @@ -20,7 +20,11 @@ export function extractEncryptionSchemaV3(table: PgTable): AnyV3Table { const columns: Record = {} for (const [property, column] of Object.entries(table)) { if (typeof column !== 'object' || column === null) continue - const builder = getEqlV3Column(property, column) + const columnName = + 'name' in column && typeof column.name === 'string' + ? column.name + : property + const builder = getEqlV3Column(columnName, column) if (builder) columns[property] = builder } From f524d8ea59789c6be57b42ea67aa0b66ebb1dc81 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 6 Jul 2026 20:56:35 +1000 Subject: [PATCH 19/24] test(stack): seed v3 drizzle live test through drizzle --- .../drizzle-v3/operators-live-pg.test.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts index 71bc5f2d..90e300c8 100644 --- a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -60,15 +60,12 @@ beforeAll(async () => { ] for (const row of seed) { const enc = unwrap(await client.encryptModel({ ...row }, schema)) - await sqlClient` - INSERT INTO protect_ci_v3_drizzle (email, nickname, age, test_run_id) - VALUES ( - ${sqlClient.json(enc.email)}::eql_v3.text_search, - ${sqlClient.json(enc.nickname)}::eql_v3.text_eq, - ${sqlClient.json(enc.age)}::eql_v3.int4_ord, - ${RUN} - ) - ` + await db.insert(usersTable).values({ + email: enc.email, + nickname: enc.nickname, + age: enc.age, + testRunId: RUN, + }) } }, 60000) From 82a2f232404bc170b3dfcdf43742677b61a8147f Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 7 Jul 2026 08:59:14 +1000 Subject: [PATCH 20/24] Align EQL v3 Drizzle integration --- .changeset/eql-v3-drizzle.md | 11 +- .changeset/eql-v3-typed-schema.md | 4 +- docs/query-api-walkthrough.md | 2 +- ...06-eql-v3-drizzle-concrete-types-design.md | 18 +- packages/stack/__tests__/cjs-require.test.ts | 19 ++ .../stack/__tests__/drizzle-v3/column.test.ts | 4 +- .../__tests__/drizzle-v3/exports.test.ts | 2 +- .../drizzle-v3/operators-live-pg.test.ts | 8 +- .../__tests__/drizzle-v3/operators.test.ts | 233 ++++++++---------- .../drizzle-v3/schema-extraction.test.ts | 8 +- .../__tests__/drizzle-v3/sql-dialect.test.ts | 24 +- .../__tests__/drizzle-v3/types.test-d.ts | 37 ++- .../stack/__tests__/schema-v3-client.test.ts | 12 +- .../__tests__/v3-matrix/matrix-bulk.test.ts | 4 +- .../__tests__/v3-matrix/matrix-live.test.ts | 4 +- .../stack/src/eql/v3/drizzle/operators.ts | 158 ++++++++---- .../stack/src/eql/v3/drizzle/sql-dialect.ts | 23 +- 17 files changed, 324 insertions(+), 247 deletions(-) diff --git a/.changeset/eql-v3-drizzle.md b/.changeset/eql-v3-drizzle.md index 70700eae..d184c671 100644 --- a/.changeset/eql-v3-drizzle.md +++ b/.changeset/eql-v3-drizzle.md @@ -4,9 +4,14 @@ Add EQL v3 Drizzle support at `@cipherstash/stack/eql/v3/drizzle`. A Drizzle-native `types` namespace (same PascalCase names as `@cipherstash/stack/eql/v3`) declares -encrypted columns whose Postgres type is the concrete `eql_v3.`; the concrete +encrypted columns whose Postgres type is the semantic `eql_v3.`; the concrete type drives the legal query operators. `createEncryptionOperatorsV3` provides -capability-checked `eq`/`ne`/`gt`/`gte`/`lt`/`lte`/`between`/`like`/`ilike`/`inArray`/ -`asc`/`desc`/`and`/`or` that emit `eql_v3` term-function SQL, and +capability-checked `eq`/`ne`/`gt`/`gte`/`lt`/`lte`/`between`/`contains`/`inArray`/ +`asc`/`desc`/`and`/`or` that emit the latest two-argument `eql_v3` SQL functions with +full-envelope operands, and `extractEncryptionSchemaV3` rebuilds the schema for `EncryptionV3`. The existing v2 `@cipherstash/stack/drizzle` integration is unchanged. + +The v3 text-search helper is `contains`; obsolete `like`/`ilike` helpers are not +exposed because v3 free-text search is token containment rather than SQL wildcard +matching. diff --git a/.changeset/eql-v3-typed-schema.md b/.changeset/eql-v3-typed-schema.md index a805d285..29373734 100644 --- a/.changeset/eql-v3-typed-schema.md +++ b/.changeset/eql-v3-typed-schema.md @@ -2,6 +2,6 @@ '@cipherstash/stack': minor --- -Add EQL v3 schema builders for all generated SQL domains under `@cipherstash/stack/eql/v3`, exposed as the `types` namespace (one member per EQL v3 domain, e.g. `types.TextEq` / `types.Int4Ord` / `types.Timestamptz`), including explicit query capability metadata (`getQueryCapabilities()` / `isQueryable()`) and v3 table support in model encryption helpers (`encryptModel` / `bulkEncryptModels`). +Add EQL v3 schema builders for all generated SQL domains under `@cipherstash/stack/eql/v3`, exposed as the `types` namespace (one member per EQL v3 domain, e.g. `types.TextEq` / `types.IntegerOrd` / `types.Timestamp`), including explicit query capability metadata (`getQueryCapabilities()` / `isQueryable()`) and v3 table support in model encryption helpers (`encryptModel` / `bulkEncryptModels`). -Also widen the accepted plaintext input type for `encrypt` / `encryptQuery` to include `Date` and `bigint` (via the new `Plaintext` type), so v3 `date` / `timestamptz` / `int8` domains can be encrypted and queried with their natural JavaScript values. +Also widen the accepted plaintext input type for `encrypt` / `encryptQuery` to include `Date` and `bigint` (via the new `Plaintext` type), so v3 `date` / `timestamp` / `bigint` domains can be encrypted and queried with their natural JavaScript values. diff --git a/docs/query-api-walkthrough.md b/docs/query-api-walkthrough.md index 8559a2ed..e582481e 100644 --- a/docs/query-api-walkthrough.md +++ b/docs/query-api-walkthrough.md @@ -73,5 +73,5 @@ flowchart LR - **Client init:** `EncryptionClient.init()` (`encryption/index.ts:81`) calls FFI `newClient()` once; the returned `Client` handle is passed into every `encryptQuery` call. - **`cipherstashclient`** = the CipherStash Client **Rust SDK**, compiled via Neon into the platform `.node` binary inside `@cipherstash/protect-ffi`. It performs the actual crypto and talks to ZeroKMS. - **Result shape:** `EncryptedQueryResult` (`types.ts:175`); shaped by `formatEncryptedResult(..., returnType)` (`eql` vs raw). -- **Version:** `package.json` pins `@cipherstash/protect-ffi@0.24.0` (installed tree observed at `0.23.0` — confirm before relying on it). +- **Version:** `package.json` pins `@cipherstash/protect-ffi@0.27.0` for the current EQL v3 semantic-domain bundle. - `packages/protect/src/ffi/*` mirrors this flow under the older `protect` package name. diff --git a/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md b/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md index 19b95c62..5ee27120 100644 --- a/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md +++ b/docs/superpowers/specs/2026-07-06-eql-v3-drizzle-concrete-types-design.md @@ -11,7 +11,7 @@ the existing v2 Drizzle integration (`@cipherstash/stack/drizzle`) but re-shaped around the v3 **concrete-type** model. The defining change of v3: **the concrete type defines the search capabilities.** -A column declared `eql_v3.int4_eq` supports equality; `eql_v3.int4_ord` supports +A column declared `eql_v3.integer_eq` supports equality; `eql_v3.integer_ord` supports order/range (and equality via ORE); `eql_v3.text_search` supports equality, order, and free-text match. There is no separate `.equality()` / `.orderAndRange()` / `.freeTextSearch()` index configuration as in v2 — everything the adapter needs @@ -32,11 +32,11 @@ pg tests (`packages/stack/__tests__/schema-v3-pg.test.ts`). **Column type.** v2 uses a single composite type `eql_v2_encrypted` for every encrypted column. v3 uses **one `CREATE DOMAIN … AS jsonb` per concrete type** — -`eql_v3.int4_ord`, `eql_v3.text_eq`, `eql_v3.text_search`, `eql_v3.bool`, … There is +`eql_v3.integer_ord`, `eql_v3.text_eq`, `eql_v3.text_search`, `eql_v3.boolean`, … There is no single catch-all `eql_v3_encrypted`. **Wire form.** v2 writes a composite literal `("…")`. v3 domains are plain jsonb, so -the value is inserted as plain JSON cast to the domain (`$1::eql_v3.int4_ord`). +the value is inserted as plain JSON cast to the domain (`$1::eql_v3.integer_ord`). **Query form.** v2 compares encrypted payloads directly (native `=` on `eql_v2_encrypted`, or `eql_v2.gt/lt/like/order_by(...)` convenience functions). v3 @@ -60,7 +60,7 @@ query term (which has no ciphertext) must be cast to `::jsonb`, never to the dom **Concrete types already exist.** `@cipherstash/stack/eql/v3` ships the full concrete-type system (`packages/stack/src/eql/v3/{columns,types,table,index}.ts`): - `types.(name)` factories for all 35 shipped domains. -- Each `EncryptedV3Column` carries `getEqlType()` (`eql_v3.int4_ord`), `castAs` +- Each `EncryptedV3Column` carries `getEqlType()` (`eql_v3.integer_ord`), `castAs` (`'string' | 'number' | 'boolean' | 'date'`), `getQueryCapabilities()` (`{ equality, orderAndRange, freeTextSearch }`), and `build()` → `{ cast_as, indexes: { unique?, ore?, match? } }`. @@ -78,7 +78,7 @@ capability data. - A new Drizzle v3 module in `@cipherstash/stack`, exported at `@cipherstash/stack/eql/v3/drizzle` (nested under the existing `eql/v3` namespace). - A Drizzle-native `types` namespace with the **identical PascalCase factory names** - as `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.Int4Ord`, `types.Bool`, + as `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Boolean`, … all 35 shipped domains), each returning a Drizzle `customType` column. - Plain-jsonb codec (`toDriver` / `fromDriver`). - Schema extraction: Drizzle table → v3 `encryptedTable` for `EncryptionV3`. @@ -140,7 +140,7 @@ operators use. It does **not** depend on the v2 Drizzle module. **single source of truth** for domain / `cast_as` / capabilities. No metadata is re-declared here; this file is a name→delegate map. 2. Builds a `customType<{ data: Plaintext; driverData: string | null }>` whose - `dataType()` returns `builder.getEqlType()` (e.g. `eql_v3.int4_ord`), and whose + `dataType()` returns `builder.getEqlType()` (e.g. `eql_v3.integer_ord`), and whose `toDriver`/`fromDriver` are the plain-jsonb codec. 3. Stashes the `eql/v3` builder on the column (`_eqlv3Column`) **and** in a module-global map keyed by column name — mirroring v2's dual registration — so @@ -225,8 +225,8 @@ v2's `extractEncryptionSchema`). Operators call this internally to resolve the ## 6. Data flow (per query) ```text -types.Int4Ord('age') // authoring - → customType dataType() = 'eql_v3.int4_ord', stash eql/v3 builder +types.IntegerOrd('age') // authoring + → customType dataType() = 'eql_v3.integer_ord', stash eql/v3 builder pgTable('users', { age }) // Drizzle table extractEncryptionSchemaV3(users) // → encryptedTable('users', { age: }) @@ -260,7 +260,7 @@ tests in stack already use). **Live pg** (gated by `LIVE_EQL_V3_PG_ENABLED`, install via `installEqlV3IfNeeded`): - One `pgTable` with a representative column per tier: `types.TextSearch('email')`, - `types.Int4Ord('age')`, `types.TextEq('nickname')`, `types.Bool('active')`. + `types.IntegerOrd('age')`, `types.TextEq('nickname')`, `types.Boolean('active')`. - Seed with `EncryptionV3(...).bulkEncryptModels` (or raw insert with `$::eql_v3.` casts), then real Drizzle `select().where(await ops.eq(...))`, `ops.gte(...)`, `ops.between(...)`, `ops.ilike(...)`, `orderBy(ops.asc(...))`; diff --git a/packages/stack/__tests__/cjs-require.test.ts b/packages/stack/__tests__/cjs-require.test.ts index 0b530fd5..b1b7c0ba 100644 --- a/packages/stack/__tests__/cjs-require.test.ts +++ b/packages/stack/__tests__/cjs-require.test.ts @@ -84,6 +84,25 @@ describe('CJS consumers can require the built bundles', () => { expect(cjsEntries).toContain('dist/index.cjs') expect(cjsEntries).toContain('dist/encryption/index.cjs') expect(cjsEntries).toContain('dist/eql/v3/index.cjs') + expect(cjsEntries).toContain('dist/eql/v3/drizzle/index.cjs') + }) + + it('exposes the drizzle integration from the CJS bundle', () => { + const drizzleBundle = path.join( + distDir, + 'eql', + 'v3', + 'drizzle', + 'index.cjs', + ) + const script = [ + `const drizzle = require(${JSON.stringify(drizzleBundle)})`, + `if (typeof drizzle.createEncryptionOperatorsV3 !== 'function') { throw new Error('missing drizzle CJS export: createEncryptionOperatorsV3') }`, + `if (typeof drizzle.EncryptionOperatorError !== 'function') { throw new Error('missing drizzle CJS export: EncryptionOperatorError') }`, + `if (typeof drizzle.types !== 'object' || drizzle.types === null) { throw new Error('missing drizzle CJS export: types namespace') }`, + `if (typeof drizzle.types.IntegerOrd !== 'function') { throw new Error('missing drizzle types.IntegerOrd CJS export') }`, + ].join('\n') + execFileSync(process.execPath, ['-e', script]) }) it('exposes the v3 `types` namespace + table API from the CJS bundle', () => { diff --git a/packages/stack/__tests__/drizzle-v3/column.test.ts b/packages/stack/__tests__/drizzle-v3/column.test.ts index 5efd7885..ee914d8d 100644 --- a/packages/stack/__tests__/drizzle-v3/column.test.ts +++ b/packages/stack/__tests__/drizzle-v3/column.test.ts @@ -13,10 +13,10 @@ const slug = (eqlType: string) => eqlType.replace('eql_v3.', '') describe('makeEqlV3Column', () => { it('sets dataType() to the concrete eql_v3 domain', () => { - const col = makeEqlV3Column(v3Types.Int4Ord('age')) + const col = makeEqlV3Column(v3Types.IntegerOrd('age')) const table = pgTable('users', { age: col }) - expect(table.age.getSQLType()).toBe('eql_v3.int4_ord') + expect(table.age.getSQLType()).toBe('eql_v3.integer_ord') }) it('recovers the stashed builder before and after pgTable processing', () => { diff --git a/packages/stack/__tests__/drizzle-v3/exports.test.ts b/packages/stack/__tests__/drizzle-v3/exports.test.ts index 52351cfa..541c06fa 100644 --- a/packages/stack/__tests__/drizzle-v3/exports.test.ts +++ b/packages/stack/__tests__/drizzle-v3/exports.test.ts @@ -7,6 +7,6 @@ describe('v3 drizzle barrel', () => { expect(typeof barrel.extractEncryptionSchemaV3).toBe('function') expect(typeof barrel.EncryptionOperatorError).toBe('function') expect(typeof barrel.types.TextSearch).toBe('function') - expect(barrel.types.Int4Ord('age')).toBeDefined() + expect(barrel.types.IntegerOrd('age')).toBeDefined() }) }) diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts index 90e300c8..9cc65a89 100644 --- a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -22,7 +22,7 @@ const usersTable = pgTable('protect_ci_v3_drizzle', { id: integer('id').primaryKey().generatedAlwaysAsIdentity(), email: types.TextSearch('email'), nickname: types.TextEq('nickname'), - age: types.Int4Ord('age'), + age: types.IntegerOrd('age'), testRunId: text('test_run_id').notNull(), }) @@ -48,7 +48,7 @@ beforeAll(async () => { id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, email eql_v3.text_search NOT NULL, nickname eql_v3.text_eq NOT NULL, - age eql_v3.int4_ord NOT NULL, + age eql_v3.integer_ord NOT NULL, test_run_id TEXT NOT NULL ) ` @@ -118,11 +118,11 @@ describeLivePg('v3 drizzle operators (live pg)', () => { expect(decrypted.map((r: { age: number }) => r.age).sort()).toEqual([42]) }, 30000) - it('ilike matches on the free-text email column', async () => { + it('contains matches on the free-text email column', async () => { const rows = await db .select({ id: usersTable.id }) .from(usersTable) - .where(scoped(await ops.ilike(usersTable.email, 'example.com'))) + .where(scoped(await ops.contains(usersTable.email, 'example.com'))) expect(rows.length).toBe(2) }, 30000) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 8842a1c7..61ce6249 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -21,22 +21,34 @@ import { types } from '@/eql/v3/drizzle/types' const TERM = { c: 'ct', v: 1 } const TERM_JSON = JSON.stringify(TERM) +const lockContext = { identityClaim: 'user-123' } +const audit = { metadata: { actor: 'test' } } -type EncryptQueryResult = Promise< +type EncryptResult = Promise< { data: typeof TERM } | { failure: { message: string } } > +function chainable(result: EncryptResult) { + const op = result as EncryptResult & { + withLockContext: ReturnType + audit: ReturnType + } + op.withLockContext = vi.fn(() => op) + op.audit = vi.fn(() => op) + return op +} + function setup( - encryptQueryImpl: (...args: unknown[]) => EncryptQueryResult = async () => ({ + encryptImpl: (...args: unknown[]) => EncryptResult = async () => ({ data: TERM, }), ) { - const encryptQuery = vi.fn(encryptQueryImpl) - const client = { encryptQuery } as unknown as EncryptionClient - const ops = createEncryptionOperatorsV3(client) + const encrypt = vi.fn((...args: unknown[]) => chainable(encryptImpl(...args))) + const client = { encrypt } as unknown as EncryptionClient + const ops = createEncryptionOperatorsV3(client, { lockContext, audit }) const dialect = new PgDialect() const render = (s: unknown) => dialect.sqlToQuery(s as SQL) - return { ops, encryptQuery, render } + return { ops, encrypt, render } } const users = pgTable('users', { @@ -46,70 +58,54 @@ const users = pgTable('users', { textMatch: types.TextMatch('text_match'), textOrd: types.TextOrd('text_ord'), textOrdOre: types.TextOrdOre('text_ord_ore'), - int2Age: types.Int2Ord('int2_age'), - age: types.Int4Ord('age'), + int2Age: types.SmallintOrd('int2_age'), + age: types.IntegerOrd('age'), createdOn: types.DateOrd('created_on'), - createdAt: types.TimestamptzOrd('created_at'), + createdAt: types.TimestampOrd('created_at'), amount: types.NumericOrd('amount'), - score4: types.Float4Ord('score4'), - score8: types.Float8Ord('score8'), - flag: types.Bool('flag'), + score4: types.RealOrd('score4'), + score8: types.DoubleOrd('score8'), + flag: types.Boolean('flag'), }) describe('createEncryptionOperatorsV3 - equality', () => { - it('eq on a hmac column emits eq_term = hmac_256, binds the JSON term, sends equality', async () => { - const { ops, encryptQuery, render } = setup() + it('eq emits the latest two-arg eql_v3.eq with a full-envelope operand', async () => { + const { ops, encrypt, render } = setup() const q = render(await ops.eq(users.nickname, 'ada')) - expect(q.sql).toContain( - 'eql_v3.eq_term("users"."nickname") = eql_v3.hmac_256($1::jsonb)', - ) + expect(q.sql).toContain('eql_v3.eq("users"."nickname", $1::jsonb)') expect(q.params[0]).toBe(TERM_JSON) - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'equality', - }) + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('nickname') }) - it('ne on a hmac column emits eq_term <> hmac_256', async () => { + it('ne emits eql_v3.neq', async () => { const { ops, render } = setup() const q = render(await ops.ne(users.nickname, 'ada')) - expect(q.sql).toContain( - 'eql_v3.eq_term("users"."nickname") <> eql_v3.hmac_256($1::jsonb)', - ) + expect(q.sql).toContain('eql_v3.neq("users"."nickname", $1::jsonb)') }) - it('eq on an order-only numeric column emits ord_term = ore_block_256', async () => { - const { ops, encryptQuery, render } = setup() + it('eq on an order-only numeric column uses the same two-arg equality function', async () => { + const { ops, encrypt, render } = setup() const q = render(await ops.eq(users.age, 37)) - expect(q.sql).toContain( - 'eql_v3.ord_term("users"."age") = eql_v3.ore_block_256($1::jsonb)', - ) - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'orderAndRange', - }) + expect(q.sql).toContain('eql_v3.eq("users"."age", $1::jsonb)') + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('age') }) - it('ne on an order-only numeric column emits ord_term <> ore_block_256', async () => { + it('ne on an order-only numeric column emits eql_v3.neq', async () => { const { ops, render } = setup() const q = render(await ops.ne(users.age, 37)) - expect(q.sql).toContain( - 'eql_v3.ord_term("users"."age") <> eql_v3.ore_block_256($1::jsonb)', - ) + expect(q.sql).toContain('eql_v3.neq("users"."age", $1::jsonb)') }) - it('eq on text order domains still uses HMAC equality', async () => { - const { ops, encryptQuery, render } = setup() + it('eq on text order domains uses the same two-arg equality function', async () => { + const { ops, encrypt, render } = setup() const textOrd = render(await ops.eq(users.textOrd, 'ada')) - expect(textOrd.sql).toContain( - 'eql_v3.eq_term("users"."text_ord") = eql_v3.hmac_256($1::jsonb)', - ) - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'equality', - }) + expect(textOrd.sql).toContain('eql_v3.eq("users"."text_ord", $1::jsonb)') + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('text_ord') const textOrdOre = render(await ops.eq(users.textOrdOre, 'grace')) expect(textOrdOre.sql).toContain( - 'eql_v3.eq_term("users"."text_ord_ore") = eql_v3.hmac_256($1::jsonb)', + 'eql_v3.eq("users"."text_ord_ore", $1::jsonb)', ) }) @@ -118,15 +114,13 @@ describe('createEncryptionOperatorsV3 - equality', () => { email: types.TextEq('email'), }) pgTable('metrics', { - email: types.Int4Ord('email'), + email: types.IntegerOrd('email'), }) const { ops, render } = setup() const q = render(await ops.eq(accounts.email, 'ada@example.com')) - expect(q.sql).toContain( - 'eql_v3.eq_term("accounts"."email") = eql_v3.hmac_256($1::jsonb)', - ) + expect(q.sql).toContain('eql_v3.eq("accounts"."email", $1::jsonb)') }) it('does not reuse a cached extracted schema across distinct pgTable objects with the same SQL name', async () => { @@ -134,35 +128,39 @@ describe('createEncryptionOperatorsV3 - equality', () => { email: types.TextEq('email'), }) const second = pgTable('shared', { - age: types.Int4Ord('age'), + age: types.IntegerOrd('age'), }) - const { ops, encryptQuery } = setup() + const { ops, encrypt } = setup() await ops.eq(first.email, 'ada@example.com') await ops.eq(second.age, 37) - expect(encryptQuery.mock.calls[1]?.[1]?.table.build()).toEqual( + expect(encrypt.mock.calls[1]?.[1]?.table.build()).toEqual( extractEncryptionSchemaV3(second).build(), ) }) + + it('passes default lock context and audit to operand encryption', async () => { + const { ops, encrypt } = setup() + await ops.eq(users.nickname, 'ada') + const op = encrypt.mock.results[0]?.value + expect(op.withLockContext).toHaveBeenCalledWith(lockContext) + expect(op.audit).toHaveBeenCalledWith(audit) + }) }) describe('createEncryptionOperatorsV3 - comparison & range', () => { it.each([ - ['gt', '>'], - ['gte', '>='], - ['lt', '<'], - ['lte', '<='], - ] as const)('%s emits ord_term %s ore_block_256', async (op, sym) => { - const { ops, encryptQuery, render } = setup() + ['gt', 'eql_v3.gt'], + ['gte', 'eql_v3.gte'], + ['lt', 'eql_v3.lt'], + ['lte', 'eql_v3.lte'], + ] as const)('%s emits %s', async (op, fn) => { + const { ops, encrypt, render } = setup() // biome-ignore lint/suspicious/noExplicitAny: dynamic operator dispatch in test const q = render(await (ops as any)[op](users.age, 30)) - expect(q.sql).toContain( - `eql_v3.ord_term("users"."age") ${sym} eql_v3.ore_block_256($1::jsonb)`, - ) - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'orderAndRange', - }) + expect(q.sql).toContain(`${fn}("users"."age", $1::jsonb)`) + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('age') }) it.each([ @@ -173,106 +171,87 @@ describe('createEncryptionOperatorsV3 - comparison & range', () => { ['score4', users.score4, 12.5], ['score8', users.score8, 12.5], ] as const)('%s eq on an order-only column uses ORE equality', async (_label, column, value) => { - const { ops, encryptQuery, render } = setup() + const { ops, encrypt, render } = setup() const q = render(await ops.eq(column, value)) - expect(q.sql).toContain('eql_v3.ord_term(') - expect(q.sql).toContain('eql_v3.ore_block_256($1::jsonb)') - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'orderAndRange', - }) + expect(q.sql).toContain('eql_v3.eq(') + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe(column.name) }) - it('between emits a bounded ord_term range with two ORE bounds', async () => { + it('between emits a bounded range with two full-envelope operands', async () => { const { ops, render } = setup() const q = render(await ops.between(users.age, 20, 40)) - expect(q.sql).toContain( - 'eql_v3.ord_term("users"."age") >= eql_v3.ore_block_256($1::jsonb)', - ) - expect(q.sql).toContain( - 'eql_v3.ord_term("users"."age") <= eql_v3.ore_block_256($2::jsonb)', - ) + expect(q.sql).toContain('eql_v3.gte("users"."age", $1::jsonb)') + expect(q.sql).toContain('eql_v3.lte("users"."age", $2::jsonb)') }) it('notBetween wraps the range in NOT (...)', async () => { const { ops, render } = setup() const q = render(await ops.notBetween(users.age, 20, 40)) expect(q.sql).toMatch(/^not \(/i) - expect(q.sql).toContain('>= eql_v3.ore_block_256(') - expect(q.sql).toContain('<= eql_v3.ore_block_256(') + expect(q.sql).toContain('eql_v3.gte(') + expect(q.sql).toContain('eql_v3.lte(') }) }) describe('createEncryptionOperatorsV3 - free-text match', () => { - it('like emits match_term @> bloom_filter with freeTextSearch', async () => { - const { ops, encryptQuery, render } = setup() - const q = render(await ops.like(users.email, 'example.com')) - expect(q.sql).toContain( - 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter($1::jsonb)', - ) - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'freeTextSearch', - }) + it('contains emits latest eql_v3.contains with a full-envelope operand', async () => { + const { ops, encrypt, render } = setup() + const q = render(await ops.contains(users.email, 'example.com')) + expect(q.sql).toContain('eql_v3.contains("users"."email", $1::jsonb)') + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('email') }) - it('like on a text_match column uses the same match term path', async () => { + it('contains on a text_match column uses the same path', async () => { const { ops, render } = setup() - const q = render(await ops.like(users.textMatch, 'engineer')) - expect(q.sql).toContain( - 'eql_v3.match_term("users"."text_match") @> eql_v3.bloom_filter($1::jsonb)', - ) + const q = render(await ops.contains(users.textMatch, 'engineer')) + expect(q.sql).toContain('eql_v3.contains("users"."text_match", $1::jsonb)') }) - it('ilike emits match_term @> bloom_filter', async () => { + it('negation is expressed through the passthrough Drizzle not operator', async () => { const { ops, render } = setup() - const q = render(await ops.ilike(users.email, 'example.com')) - expect(q.sql).toContain( - 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter($1::jsonb)', - ) + const q = render(ops.not(await ops.contains(users.email, 'example.com'))) + expect(q.sql).toMatch(/^not /i) + expect(q.sql).toContain('eql_v3.contains("users"."email", $1::jsonb)') }) - it('notIlike wraps the match in NOT (...)', async () => { - const { ops, render } = setup() - const q = render(await ops.notIlike(users.email, 'example.com')) - expect(q.sql).toMatch(/^not \(/i) - expect(q.sql).toContain( - 'eql_v3.match_term("users"."email") @> eql_v3.bloom_filter(', - ) + it('does not expose obsolete like/ilike helpers', () => { + const { ops } = setup() + expect('like' in ops).toBe(false) + expect('ilike' in ops).toBe(false) + expect('notIlike' in ops).toBe(false) }) }) describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { - it('inArray ORs one eq term per value; empty array is false', async () => { + it('inArray ORs one eq term per value; empty array throws', async () => { const { ops, render } = setup() const q = render(await ops.inArray(users.nickname, ['ada', 'grace'])) expect(q.sql).toContain(' or ') - expect((q.sql.match(/eql_v3\.eq_term/g) ?? []).length).toBe(2) - const empty = render(await ops.inArray(users.nickname, [])) - expect(empty.sql).toBe('false') + expect((q.sql.match(/eql_v3\.eq/g) ?? []).length).toBe(2) + await expect(ops.inArray(users.nickname, [])).rejects.toThrow(/non-empty/) }) it('inArray on an ORE column uses ORE equality for each term', async () => { - const { ops, encryptQuery, render } = setup() + const { ops, render } = setup() const q = render(await ops.inArray(users.age, [30, 42])) expect(q.sql).toContain(' or ') - expect((q.sql.match(/eql_v3\.ord_term/g) ?? []).length).toBe(2) - expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ - queryType: 'orderAndRange', - }) + expect((q.sql.match(/eql_v3\.eq/g) ?? []).length).toBe(2) }) - it('notInArray ANDs one ne term per value; empty array is true', async () => { + it('notInArray ANDs one ne term per value; empty array throws', async () => { const { ops, render } = setup() const q = render(await ops.notInArray(users.nickname, ['ada', 'grace'])) expect(q.sql).toContain(' and ') - const empty = render(await ops.notInArray(users.nickname, [])) - expect(empty.sql).toBe('true') + await expect(ops.notInArray(users.nickname, [])).rejects.toThrow( + /non-empty/, + ) }) it('notInArray on an ORE column uses ORE inequality for each term', async () => { const { ops, render } = setup() const q = render(await ops.notInArray(users.age, [30, 42])) expect(q.sql).toContain(' and ') - expect((q.sql.match(/eql_v3\.ord_term/g) ?? []).length).toBe(2) + expect((q.sql.match(/eql_v3\.neq/g) ?? []).length).toBe(2) }) it('asc / desc extract the ord term', () => { @@ -294,8 +273,8 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { ops.gte(users.age, 30), ), ) - expect(q.sql).toContain('eql_v3.eq_term("users"."nickname")') - expect(q.sql).toContain('eql_v3.ord_term("users"."age")') + expect(q.sql).toContain('eql_v3.eq("users"."nickname"') + expect(q.sql).toContain('eql_v3.gte("users"."age"') expect(q.sql).toContain(' and ') }) @@ -311,7 +290,7 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { await ops.or(ops.eq(users.nickname, 'ada'), drizzleEq(users.id, 1)), ) expect(q.sql).toContain(' or ') - expect(q.sql).toContain('eql_v3.eq_term("users"."nickname")') + expect(q.sql).toContain('eql_v3.eq("users"."nickname"') expect(q.sql).toContain('"users"."id" = $2') }) @@ -326,7 +305,7 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { }) describe('createEncryptionOperatorsV3 - gating errors', () => { - it('wraps encryptQuery failures with operator context', async () => { + it('wraps encryption failures with operator context', async () => { const { ops } = setup(async () => ({ failure: { message: 'bad query term' }, })) @@ -343,13 +322,19 @@ describe('createEncryptionOperatorsV3 - gating errors', () => { ) }) - it('ilike on a column without match throws', async () => { + it('contains on a column without match throws', async () => { const { ops } = setup() - await expect(ops.ilike(users.nickname, 'ada')).rejects.toBeInstanceOf( + await expect(ops.contains(users.nickname, 'ada')).rejects.toBeInstanceOf( EncryptionOperatorError, ) }) + it('null operands throw and point callers to null checks', async () => { + const { ops } = setup() + await expect(ops.eq(users.nickname, null)).rejects.toThrow(/isNull/) + await expect(ops.contains(users.email, undefined)).rejects.toThrow(/isNull/) + }) + it('eq on a storage-only column throws', async () => { const { ops } = setup() await expect(ops.eq(users.flag, true)).rejects.toBeInstanceOf( diff --git a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts index bea97013..fa9efb2f 100644 --- a/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts +++ b/packages/stack/__tests__/drizzle-v3/schema-extraction.test.ts @@ -31,13 +31,13 @@ describe('extractEncryptionSchemaV3', () => { const users = pgTable('users', { id: integer().primaryKey(), email: types.TextSearch('email'), - age: types.Int4Ord('age'), + age: types.IntegerOrd('age'), }) const extracted = extractEncryptionSchemaV3(users) const authored = encryptedTable('users', { email: v3Types.TextSearch('email'), - age: v3Types.Int4Ord('age'), + age: v3Types.IntegerOrd('age'), }) expect(extracted.build()).toEqual(authored.build()) @@ -48,7 +48,7 @@ describe('extractEncryptionSchemaV3', () => { email: types.TextEq('email'), }) const metrics = pgTable('metrics', { - email: types.Int4Ord('email'), + email: types.IntegerOrd('email'), }) const accountsSchema = extractEncryptionSchemaV3(accounts) @@ -61,7 +61,7 @@ describe('extractEncryptionSchemaV3', () => { ) expect(metricsSchema.build()).toEqual( encryptedTable('metrics', { - email: v3Types.Int4Ord('email'), + email: v3Types.IntegerOrd('email'), }).build(), ) }) diff --git a/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts b/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts index 0ef7da07..e3ad75c2 100644 --- a/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts +++ b/packages/stack/__tests__/drizzle-v3/sql-dialect.test.ts @@ -10,26 +10,20 @@ const enc = sql`${'{"v":"t"}'}` describe('v3Dialect', () => { it('equality via HMAC', () => { - expect(render(v3Dialect.equality('eq', col, enc, false))).toBe( - 'eql_v3.eq_term("users"."x") = eql_v3.hmac_256($1::jsonb)', + expect(render(v3Dialect.equality('eq', col, enc))).toBe( + 'eql_v3.eq("users"."x", $1::jsonb)', ) }) it('inequality via HMAC', () => { - expect(render(v3Dialect.equality('ne', col, enc, false))).toBe( - 'eql_v3.eq_term("users"."x") <> eql_v3.hmac_256($1::jsonb)', - ) - }) - - it('equality via ORE (order-only numeric/date domains)', () => { - expect(render(v3Dialect.equality('eq', col, enc, true))).toBe( - 'eql_v3.ord_term("users"."x") = eql_v3.ore_block_256($1::jsonb)', + expect(render(v3Dialect.equality('ne', col, enc))).toBe( + 'eql_v3.neq("users"."x", $1::jsonb)', ) }) it('comparison via ORE', () => { expect(render(v3Dialect.comparison('gte', col, enc))).toBe( - 'eql_v3.ord_term("users"."x") >= eql_v3.ore_block_256($1::jsonb)', + 'eql_v3.gte("users"."x", $1::jsonb)', ) }) @@ -37,13 +31,13 @@ describe('v3Dialect', () => { const lo = sql`${'{"v":"lo"}'}` const hi = sql`${'{"v":"hi"}'}` expect(render(v3Dialect.range(col, lo, hi))).toBe( - 'eql_v3.ord_term("users"."x") >= eql_v3.ore_block_256($1::jsonb) AND eql_v3.ord_term("users"."x") <= eql_v3.ore_block_256($2::jsonb)', + 'eql_v3.gte("users"."x", $1::jsonb) AND eql_v3.lte("users"."x", $2::jsonb)', ) }) - it('match via bloom filter containment', () => { - expect(render(v3Dialect.match(col, enc))).toBe( - 'eql_v3.match_term("users"."x") @> eql_v3.bloom_filter($1::jsonb)', + it('contains via two-arg function', () => { + expect(render(v3Dialect.contains(col, enc))).toBe( + 'eql_v3.contains("users"."x", $1::jsonb)', ) }) diff --git a/packages/stack/__tests__/drizzle-v3/types.test-d.ts b/packages/stack/__tests__/drizzle-v3/types.test-d.ts index ba57f118..503157c9 100644 --- a/packages/stack/__tests__/drizzle-v3/types.test-d.ts +++ b/packages/stack/__tests__/drizzle-v3/types.test-d.ts @@ -8,27 +8,27 @@ describe('v3 drizzle types - type-level', () => { }) it('columns infer their concrete plaintext type via the data type slot', () => { - const age = types.Int4Ord('age') - const ageEq = types.Int4Eq('age_eq') - const int2 = types.Int2Ord('int2') + const age = types.IntegerOrd('age') + const ageEq = types.IntegerEq('age_eq') + const smallint = types.SmallintOrd('smallint') const date = types.DateOrd('created_at') const dateOre = types.DateOrdOre('created_at_ore') - const created = types.Timestamptz('created_at') - const tsOrd = types.TimestamptzOrd('created_at_ord') + const created = types.Timestamp('created_at') + const tsOrd = types.TimestampOrd('created_at_ord') const numOre = types.NumericOrdOre('amount_ore') - const flag = types.Bool('flag') + const flag = types.Boolean('flag') const nick = types.TextEq('nickname') const text = types.Text('text') const match = types.TextMatch('bio') const textOrd = types.TextOrd('text_ord') const textOrdOre = types.TextOrdOre('text_ord_ore') const search = types.TextSearch('search') - const float4Eq = types.Float4Eq('float4_eq') - const float8Ord = types.Float8Ord('float8_ord') + const realEq = types.RealEq('real_eq') + const doubleOrd = types.DoubleOrd('double_ord') expectTypeOf(age._.data).toEqualTypeOf() expectTypeOf(ageEq._.data).toEqualTypeOf() - expectTypeOf(int2._.data).toEqualTypeOf() + expectTypeOf(smallint._.data).toEqualTypeOf() expectTypeOf(date._.data).toEqualTypeOf() expectTypeOf(dateOre._.data).toEqualTypeOf() expectTypeOf(created._.data).toEqualTypeOf() @@ -41,7 +41,22 @@ describe('v3 drizzle types - type-level', () => { expectTypeOf(textOrd._.data).toEqualTypeOf() expectTypeOf(textOrdOre._.data).toEqualTypeOf() expectTypeOf(search._.data).toEqualTypeOf() - expectTypeOf(float4Eq._.data).toEqualTypeOf() - expectTypeOf(float8Ord._.data).toEqualTypeOf() + expectTypeOf(realEq._.data).toEqualTypeOf() + expectTypeOf(doubleOrd._.data).toEqualTypeOf() + }) + + it('does not expose obsolete pre-0.27 concrete domain names', () => { + // @ts-expect-error - use IntegerOrd + types.Int4Ord('age') + // @ts-expect-error - use SmallintOrd + types.Int2Ord('age') + // @ts-expect-error - use Timestamp + types.Timestamptz('created_at') + // @ts-expect-error - use Boolean + types.Bool('active') + // @ts-expect-error - use RealEq + types.Float4Eq('score') + // @ts-expect-error - use DoubleOrd + types.Float8Ord('score') }) }) diff --git a/packages/stack/__tests__/schema-v3-client.test.ts b/packages/stack/__tests__/schema-v3-client.test.ts index 9d18db43..9cb4401c 100644 --- a/packages/stack/__tests__/schema-v3-client.test.ts +++ b/packages/stack/__tests__/schema-v3-client.test.ts @@ -1,11 +1,11 @@ import 'dotenv/config' -import { beforeAll, describe, expect, it } from 'vitest' +import { beforeAll, expect, it } from 'vitest' import type { EncryptionClient } from '@/encryption' import { typedClient } from '@/encryption/v3' import { encryptedTable, types } from '@/eql/v3' import { Encryption } from '@/index' import { unwrapResult } from './fixtures' -import { describeLive, LIVE_CIPHERSTASH_ENABLED } from './helpers/live-gate' +import { describeLive } from './helpers/live-gate' const users = encryptedTable('schema_v3_client_users', { email: types.TextSearch('email'), @@ -131,14 +131,16 @@ describeLive('eql_v3 client integration', () => { 'private note', ) - const encryptedBool = unwrapResult( + const encryptedBoolean = unwrapResult( await protectClient.encrypt(true, { table: users, column: users.active, }), ) - expect(encryptedBool).toHaveProperty('c') - expect(unwrapResult(await protectClient.decrypt(encryptedBool))).toBe(true) + expect(encryptedBoolean).toHaveProperty('c') + expect(unwrapResult(await protectClient.decrypt(encryptedBoolean))).toBe( + true, + ) }, 30000) it('encrypts equality and order query terms for typed v3 columns', async () => { diff --git a/packages/stack/__tests__/v3-matrix/matrix-bulk.test.ts b/packages/stack/__tests__/v3-matrix/matrix-bulk.test.ts index 61915f15..88a13266 100644 --- a/packages/stack/__tests__/v3-matrix/matrix-bulk.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix-bulk.test.ts @@ -6,10 +6,10 @@ * scale. Live soft-skip. */ import 'dotenv/config' -import { beforeAll, describe, expect, it } from 'vitest' +import { beforeAll, expect, it } from 'vitest' import { EncryptionV3, encryptedTable, types } from '@/encryption/v3' import { unwrapResult } from '../fixtures' -import { describeLive, LIVE_CIPHERSTASH_ENABLED } from '../helpers/live-gate' +import { describeLive } from '../helpers/live-gate' const people = encryptedTable('v3_bulk_people', { nickname: types.TextEq('nickname'), diff --git a/packages/stack/__tests__/v3-matrix/matrix-live.test.ts b/packages/stack/__tests__/v3-matrix/matrix-live.test.ts index e8960416..b939b98a 100644 --- a/packages/stack/__tests__/v3-matrix/matrix-live.test.ts +++ b/packages/stack/__tests__/v3-matrix/matrix-live.test.ts @@ -20,10 +20,10 @@ * client-side before any network — and so stay cheap even one at a time. */ import 'dotenv/config' -import { beforeAll, describe, expect, it } from 'vitest' +import { beforeAll, expect, it } from 'vitest' import { EncryptionV3, encryptedTable } from '@/encryption/v3' import { unwrapResult } from '../fixtures' -import { describeLive, LIVE_CIPHERSTASH_ENABLED } from '../helpers/live-gate' +import { describeLive } from '../helpers/live-gate' import { type DomainSpec, type EqlV3TypeName, diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index ac3f6555..7ed4ca78 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -16,9 +16,10 @@ import { } from 'drizzle-orm' import type { PgTable } from 'drizzle-orm/pg-core' import type { EncryptionClient } from '@/encryption' +import type { AuditConfig } from '@/encryption/operations/base-operation' import type { AnyEncryptedV3Column, AnyV3Table } from '@/eql/v3' +import type { LockContext } from '@/identity' import type { ColumnSchema } from '@/schema' -import type { BuildableQueryColumn, Plaintext, QueryTypeName } from '@/types' import { getEqlV3Column } from './column.js' import { extractEncryptionSchemaV3 } from './schema-extraction.js' import { type ComparisonOp, type EqualityOp, v3Dialect } from './sql-dialect.js' @@ -47,6 +48,20 @@ interface ColumnContext { tableName: string } +export type EncryptionOperatorCallOpts = { + lockContext?: LockContext + audit?: AuditConfig +} + +type ChainableOperation = { + withLockContext(lockContext: LockContext): ChainableOperation + audit(config: AuditConfig): ChainableOperation + then: PromiseLike<{ + data?: unknown + failure?: { message: string } + }>['then'] +} + async function mapWithConcurrency( values: T[], limit: number, @@ -70,7 +85,10 @@ async function mapWithConcurrency( return results } -export function createEncryptionOperatorsV3(client: EncryptionClient) { +export function createEncryptionOperatorsV3( + client: EncryptionClient, + defaults: EncryptionOperatorCallOpts = {}, +) { const tableCache = new WeakMap() function drizzleTableOf(column: SQLWrapper): PgTable | undefined { @@ -131,20 +149,45 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { } } - async function encryptTerm( + function applyOperationOptions( + op: ChainableOperation, + opts?: EncryptionOperatorCallOpts, + ): ChainableOperation { + const lockContext = opts?.lockContext ?? defaults.lockContext + const audit = opts?.audit ?? defaults.audit + const withLock = lockContext ? op.withLockContext(lockContext) : op + if (audit) withLock.audit(audit) + return withLock + } + + async function encryptOperand( ctx: ColumnContext, value: unknown, - queryType: QueryTypeName, + operator: string, + opts?: EncryptionOperatorCallOpts, ): Promise { - const result = await client.encryptQuery(value as Plaintext, { - table: ctx.table, - column: ctx.builder as BuildableQueryColumn, - queryType, - }) + if (value == null) { + throw new EncryptionOperatorError( + `Operator "${operator}" cannot encrypt a null operand for column "${ctx.columnName}". Use isNull() or isNotNull() for NULL checks.`, + { + columnName: ctx.columnName, + tableName: ctx.tableName, + operator, + }, + ) + } + + const result = await applyOperationOptions( + client.encrypt(value as never, { + table: ctx.table, + column: ctx.builder as never, + }) as unknown as ChainableOperation, + opts, + ) if (result.failure) { throw new EncryptionOperatorError( - `Failed to encrypt query term for "${ctx.columnName}": ${result.failure.message}`, - { columnName: ctx.columnName, tableName: ctx.tableName }, + `Failed to encrypt query operand for "${ctx.columnName}": ${result.failure.message}`, + { columnName: ctx.columnName, tableName: ctx.tableName, operator }, ) } return sql`${JSON.stringify(result.data)}` @@ -156,6 +199,7 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { op: EqualityOp, left: SQLWrapper, right: unknown, + opts?: EncryptionOperatorCallOpts, ): Promise { const ctx = resolveContext(left, op) if (!ctx.indexes.unique && !ctx.indexes.ore) { @@ -164,23 +208,19 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { { columnName: ctx.columnName, tableName: ctx.tableName, operator: op }, ) } - const viaOre = !ctx.indexes.unique - const enc = await encryptTerm( - ctx, - right, - viaOre ? 'orderAndRange' : 'equality', - ) - return v3Dialect.equality(op, colSql(left), enc, viaOre) + const enc = await encryptOperand(ctx, right, op, opts) + return v3Dialect.equality(op, colSql(left), enc) } async function comparison( op: ComparisonOp, left: SQLWrapper, right: unknown, + opts?: EncryptionOperatorCallOpts, ): Promise { const ctx = resolveContext(left, op) requireIndex(ctx, 'ore', op, 'order/range') - const enc = await encryptTerm(ctx, right, 'orderAndRange') + const enc = await encryptOperand(ctx, right, op, opts) return v3Dialect.comparison(op, colSql(left), enc) } @@ -190,39 +230,46 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { max: unknown, negate: boolean, operator: string, + opts?: EncryptionOperatorCallOpts, ): Promise { const ctx = resolveContext(left, operator) requireIndex(ctx, 'ore', operator, 'order/range') - const encMin = await encryptTerm(ctx, min, 'orderAndRange') - const encMax = await encryptTerm(ctx, max, 'orderAndRange') + const encMin = await encryptOperand(ctx, min, operator, opts) + const encMax = await encryptOperand(ctx, max, operator, opts) const condition = v3Dialect.range(colSql(left), encMin, encMax) return negate ? sql`NOT (${condition})` : condition } - async function match( + async function contains( left: SQLWrapper, right: unknown, - negate: boolean, operator: string, + opts?: EncryptionOperatorCallOpts, ): Promise { const ctx = resolveContext(left, operator) requireIndex(ctx, 'match', operator, 'free-text search') - const enc = await encryptTerm(ctx, right, 'freeTextSearch') - const condition = v3Dialect.match(colSql(left), enc) - return negate ? sql`NOT (${condition})` : condition + const enc = await encryptOperand(ctx, right, operator, opts) + return v3Dialect.contains(colSql(left), enc) } async function inArrayOp( left: SQLWrapper, values: unknown[], negate: boolean, - _operator: string, + operator: string, + opts?: EncryptionOperatorCallOpts, ): Promise { - if (values.length === 0) return negate ? sql`true` : sql`false` + if (values.length === 0) { + const ctx = resolveContext(left, operator) + throw new EncryptionOperatorError( + `Operator "${operator}" requires a non-empty list of values for column "${ctx.columnName}".`, + { columnName: ctx.columnName, tableName: ctx.tableName, operator }, + ) + } const conditions = await mapWithConcurrency( values, MAX_IN_ARRAY_CONCURRENCY, - (v) => equality(negate ? 'ne' : 'eq', left, v), + (v) => equality(negate ? 'ne' : 'eq', left, v, opts), ) const combined = negate ? and(...conditions) : or(...conditions) return combined ?? (negate ? sql`true` : sql`false`) @@ -247,23 +294,42 @@ export function createEncryptionOperatorsV3(client: EncryptionClient) { } return { - eq: (l: SQLWrapper, r: unknown) => equality('eq', l, r), - ne: (l: SQLWrapper, r: unknown) => equality('ne', l, r), - gt: (l: SQLWrapper, r: unknown) => comparison('gt', l, r), - gte: (l: SQLWrapper, r: unknown) => comparison('gte', l, r), - lt: (l: SQLWrapper, r: unknown) => comparison('lt', l, r), - lte: (l: SQLWrapper, r: unknown) => comparison('lte', l, r), - between: (l: SQLWrapper, min: unknown, max: unknown) => - range(l, min, max, false, 'between'), - notBetween: (l: SQLWrapper, min: unknown, max: unknown) => - range(l, min, max, true, 'notBetween'), - like: (l: SQLWrapper, r: unknown) => match(l, r, false, 'like'), - ilike: (l: SQLWrapper, r: unknown) => match(l, r, false, 'ilike'), - notIlike: (l: SQLWrapper, r: unknown) => match(l, r, true, 'notIlike'), - inArray: (l: SQLWrapper, values: unknown[]) => - inArrayOp(l, values, false, 'inArray'), - notInArray: (l: SQLWrapper, values: unknown[]) => - inArrayOp(l, values, true, 'notInArray'), + eq: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + equality('eq', l, r, opts), + ne: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + equality('ne', l, r, opts), + gt: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + comparison('gt', l, r, opts), + gte: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + comparison('gte', l, r, opts), + lt: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + comparison('lt', l, r, opts), + lte: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + comparison('lte', l, r, opts), + between: ( + l: SQLWrapper, + min: unknown, + max: unknown, + opts?: EncryptionOperatorCallOpts, + ) => range(l, min, max, false, 'between', opts), + notBetween: ( + l: SQLWrapper, + min: unknown, + max: unknown, + opts?: EncryptionOperatorCallOpts, + ) => range(l, min, max, true, 'notBetween', opts), + contains: (l: SQLWrapper, r: unknown, opts?: EncryptionOperatorCallOpts) => + contains(l, r, 'contains', opts), + inArray: ( + l: SQLWrapper, + values: unknown[], + opts?: EncryptionOperatorCallOpts, + ) => inArrayOp(l, values, false, 'inArray', opts), + notInArray: ( + l: SQLWrapper, + values: unknown[], + opts?: EncryptionOperatorCallOpts, + ) => inArrayOp(l, values, true, 'notInArray', opts), asc: (c: SQLWrapper) => asc(orderTerm(c, 'asc')), desc: (c: SQLWrapper) => desc(orderTerm(c, 'desc')), and: (...conds: (SQL | SQLWrapper | Promise | undefined)[]) => diff --git a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts index 153ca5b8..8080c231 100644 --- a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts +++ b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts @@ -3,31 +3,22 @@ import { type SQL, sql } from 'drizzle-orm' export type EqualityOp = 'eq' | 'ne' export type ComparisonOp = 'gt' | 'gte' | 'lt' | 'lte' -const ORD_SYMBOL: Record = { - gt: '>', - gte: '>=', - lt: '<', - lte: '<=', -} - export const v3Dialect = { - equality(op: EqualityOp, left: SQL, enc: SQL, viaOre: boolean): SQL { - const cmp = op === 'eq' ? sql.raw('=') : sql.raw('<>') - return viaOre - ? sql`eql_v3.ord_term(${left}) ${cmp} eql_v3.ore_block_256(${enc}::jsonb)` - : sql`eql_v3.eq_term(${left}) ${cmp} eql_v3.hmac_256(${enc}::jsonb)` + equality(op: EqualityOp, left: SQL, enc: SQL): SQL { + const fn = op === 'eq' ? sql.raw('eq') : sql.raw('neq') + return sql`eql_v3.${fn}(${left}, ${enc}::jsonb)` }, comparison(op: ComparisonOp, left: SQL, enc: SQL): SQL { - return sql`eql_v3.ord_term(${left}) ${sql.raw(ORD_SYMBOL[op])} eql_v3.ore_block_256(${enc}::jsonb)` + return sql`eql_v3.${sql.raw(op)}(${left}, ${enc}::jsonb)` }, range(left: SQL, min: SQL, max: SQL): SQL { - return sql`eql_v3.ord_term(${left}) >= eql_v3.ore_block_256(${min}::jsonb) AND eql_v3.ord_term(${left}) <= eql_v3.ore_block_256(${max}::jsonb)` + return sql`eql_v3.gte(${left}, ${min}::jsonb) AND eql_v3.lte(${left}, ${max}::jsonb)` }, - match(left: SQL, enc: SQL): SQL { - return sql`eql_v3.match_term(${left}) @> eql_v3.bloom_filter(${enc}::jsonb)` + contains(left: SQL, enc: SQL): SQL { + return sql`eql_v3.contains(${left}, ${enc}::jsonb)` }, orderBy(left: SQL): SQL { From ed8da5cda40e6e22e8cd283879ba74aff45b0bfc Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 7 Jul 2026 11:07:58 +1000 Subject: [PATCH 21/24] test: drive drizzle v3 operators from matrix --- .../__tests__/drizzle-v3/operators.test.ts | 204 +++++++++++------- 1 file changed, 127 insertions(+), 77 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 61ce6249..fb8a0b9c 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -12,12 +12,14 @@ import { import { integer, PgDialect, pgTable } from 'drizzle-orm/pg-core' import { describe, expect, it, vi } from 'vitest' import type { EncryptionClient } from '@/encryption' +import { makeEqlV3Column } from '@/eql/v3/drizzle/column' import { createEncryptionOperatorsV3, EncryptionOperatorError, } from '@/eql/v3/drizzle/operators' import { extractEncryptionSchemaV3 } from '@/eql/v3/drizzle/schema-extraction' import { types } from '@/eql/v3/drizzle/types' +import { typedEntries, V3_MATRIX } from '../v3-matrix/catalog' const TERM = { c: 'ct', v: 1 } const TERM_JSON = JSON.stringify(TERM) @@ -51,6 +53,32 @@ function setup( return { ops, encrypt, render } } +const slug = (eqlType: string) => eqlType.replace('eql_v3.', '') +const matrixEntries = typedEntries(V3_MATRIX) +const matrixTable = pgTable( + 'matrix_users', + Object.fromEntries( + matrixEntries.map(([eqlType, spec]) => [ + slug(eqlType), + makeEqlV3Column(spec.builder(slug(eqlType))), + ]), + ) as never, +) +const matrixColumn = (eqlType: string) => + (matrixTable as Record)[slug(eqlType)] +const sampleFor = (spec: (typeof V3_MATRIX)[keyof typeof V3_MATRIX]) => + spec.samples[0] + +const equalityDomains = matrixEntries.filter( + ([, spec]) => spec.indexes.unique || spec.indexes.ore, +) +const orderDomains = matrixEntries.filter(([, spec]) => spec.indexes.ore) +const matchDomains = matrixEntries.filter(([, spec]) => spec.indexes.match) +const storageDomains = matrixEntries.filter( + ([, spec]) => + !spec.indexes.unique && !spec.indexes.ore && !spec.indexes.match, +) + const users = pgTable('users', { id: integer().primaryKey(), email: types.TextSearch('email'), @@ -69,44 +97,28 @@ const users = pgTable('users', { }) describe('createEncryptionOperatorsV3 - equality', () => { - it('eq emits the latest two-arg eql_v3.eq with a full-envelope operand', async () => { + it.each( + equalityDomains, + )('%s eq emits the latest two-arg eql_v3.eq with a full-envelope operand', async (eqlType, spec) => { const { ops, encrypt, render } = setup() - const q = render(await ops.eq(users.nickname, 'ada')) - expect(q.sql).toContain('eql_v3.eq("users"."nickname", $1::jsonb)') - expect(q.params[0]).toBe(TERM_JSON) - expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('nickname') - }) + const q = render(await ops.eq(matrixColumn(eqlType), sampleFor(spec))) - it('ne emits eql_v3.neq', async () => { - const { ops, render } = setup() - const q = render(await ops.ne(users.nickname, 'ada')) - expect(q.sql).toContain('eql_v3.neq("users"."nickname", $1::jsonb)') - }) - - it('eq on an order-only numeric column uses the same two-arg equality function', async () => { - const { ops, encrypt, render } = setup() - const q = render(await ops.eq(users.age, 37)) - expect(q.sql).toContain('eql_v3.eq("users"."age", $1::jsonb)') - expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('age') - }) - - it('ne on an order-only numeric column emits eql_v3.neq', async () => { - const { ops, render } = setup() - const q = render(await ops.ne(users.age, 37)) - expect(q.sql).toContain('eql_v3.neq("users"."age", $1::jsonb)') + expect(q.sql).toContain( + `eql_v3.eq("matrix_users"."${slug(eqlType)}", $1::jsonb)`, + ) + expect(q.params).toEqual([TERM_JSON]) + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe(slug(eqlType)) }) - it('eq on text order domains uses the same two-arg equality function', async () => { + it.each(equalityDomains)('%s ne emits eql_v3.neq', async (eqlType, spec) => { const { ops, encrypt, render } = setup() + const q = render(await ops.ne(matrixColumn(eqlType), sampleFor(spec))) - const textOrd = render(await ops.eq(users.textOrd, 'ada')) - expect(textOrd.sql).toContain('eql_v3.eq("users"."text_ord", $1::jsonb)') - expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('text_ord') - - const textOrdOre = render(await ops.eq(users.textOrdOre, 'grace')) - expect(textOrdOre.sql).toContain( - 'eql_v3.eq("users"."text_ord_ore", $1::jsonb)', + expect(q.sql).toContain( + `eql_v3.neq("matrix_users"."${slug(eqlType)}", $1::jsonb)`, ) + expect(q.params).toEqual([TERM_JSON]) + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe(slug(eqlType)) }) it('same-named columns on different tables use their own equality capability', async () => { @@ -155,56 +167,80 @@ describe('createEncryptionOperatorsV3 - comparison & range', () => { ['gte', 'eql_v3.gte'], ['lt', 'eql_v3.lt'], ['lte', 'eql_v3.lte'], - ] as const)('%s emits %s', async (op, fn) => { - const { ops, encrypt, render } = setup() - // biome-ignore lint/suspicious/noExplicitAny: dynamic operator dispatch in test - const q = render(await (ops as any)[op](users.age, 30)) - expect(q.sql).toContain(`${fn}("users"."age", $1::jsonb)`) - expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('age') - }) + ] as const)('%s emits %s for every ORE domain', async (op, fn) => { + for (const [eqlType, spec] of orderDomains) { + const { ops, encrypt, render } = setup() + const q = render(await ops[op](matrixColumn(eqlType), sampleFor(spec))) + + expect(q.sql).toContain( + `${fn}("matrix_users"."${slug(eqlType)}", $1::jsonb)`, + ) + expect(q.params).toEqual([TERM_JSON]) + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe(slug(eqlType)) + } + }) + + it.each( + orderDomains, + )('%s between emits a bounded range with two full-envelope operands', async (eqlType, spec) => { + const { ops, render } = setup() + const value = sampleFor(spec) + const q = render(await ops.between(matrixColumn(eqlType), value, value)) - it.each([ - ['int2_age', users.int2Age, 21], - ['created_on', users.createdOn, new Date('2026-07-01T00:00:00.000Z')], - ['created_at', users.createdAt, new Date('2026-07-01T00:00:00.000Z')], - ['amount', users.amount, 123.45], - ['score4', users.score4, 12.5], - ['score8', users.score8, 12.5], - ] as const)('%s eq on an order-only column uses ORE equality', async (_label, column, value) => { - const { ops, encrypt, render } = setup() - const q = render(await ops.eq(column, value)) - expect(q.sql).toContain('eql_v3.eq(') - expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe(column.name) + expect(q.sql).toContain( + `eql_v3.gte("matrix_users"."${slug(eqlType)}", $1::jsonb)`, + ) + expect(q.sql).toContain( + `eql_v3.lte("matrix_users"."${slug(eqlType)}", $2::jsonb)`, + ) + expect(q.params).toEqual([TERM_JSON, TERM_JSON]) }) - it('between emits a bounded range with two full-envelope operands', async () => { + it.each( + orderDomains, + )('%s notBetween wraps the range in NOT (...)', async (eqlType, spec) => { const { ops, render } = setup() - const q = render(await ops.between(users.age, 20, 40)) - expect(q.sql).toContain('eql_v3.gte("users"."age", $1::jsonb)') - expect(q.sql).toContain('eql_v3.lte("users"."age", $2::jsonb)') + const value = sampleFor(spec) + const q = render(await ops.notBetween(matrixColumn(eqlType), value, value)) + + expect(q.sql).toMatch(/^not \(/i) + expect(q.sql).toContain( + `eql_v3.gte("matrix_users"."${slug(eqlType)}", $1::jsonb)`, + ) + expect(q.sql).toContain( + `eql_v3.lte("matrix_users"."${slug(eqlType)}", $2::jsonb)`, + ) + expect(q.params).toEqual([TERM_JSON, TERM_JSON]) }) - it('notBetween wraps the range in NOT (...)', async () => { + it.each(orderDomains)('%s asc / desc extract the ord term', (eqlType) => { const { ops, render } = setup() - const q = render(await ops.notBetween(users.age, 20, 40)) - expect(q.sql).toMatch(/^not \(/i) - expect(q.sql).toContain('eql_v3.gte(') - expect(q.sql).toContain('eql_v3.lte(') + const ascq = render(ops.asc(matrixColumn(eqlType))) + expect(ascq.sql).toContain( + `eql_v3.ord_term("matrix_users"."${slug(eqlType)}")`, + ) + expect(ascq.sql.toLowerCase()).toContain('asc') + + const descq = render(ops.desc(matrixColumn(eqlType))) + expect(descq.sql).toContain( + `eql_v3.ord_term("matrix_users"."${slug(eqlType)}")`, + ) + expect(descq.sql.toLowerCase()).toContain('desc') }) }) describe('createEncryptionOperatorsV3 - free-text match', () => { - it('contains emits latest eql_v3.contains with a full-envelope operand', async () => { + it.each( + matchDomains, + )('%s contains emits latest eql_v3.contains with a full-envelope operand', async (eqlType, spec) => { const { ops, encrypt, render } = setup() - const q = render(await ops.contains(users.email, 'example.com')) - expect(q.sql).toContain('eql_v3.contains("users"."email", $1::jsonb)') - expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe('email') - }) + const q = render(await ops.contains(matrixColumn(eqlType), sampleFor(spec))) - it('contains on a text_match column uses the same path', async () => { - const { ops, render } = setup() - const q = render(await ops.contains(users.textMatch, 'engineer')) - expect(q.sql).toContain('eql_v3.contains("users"."text_match", $1::jsonb)') + expect(q.sql).toContain( + `eql_v3.contains("matrix_users"."${slug(eqlType)}", $1::jsonb)`, + ) + expect(q.params).toEqual([TERM_JSON]) + expect(encrypt.mock.calls[0]?.[1]?.column.getName()).toBe(slug(eqlType)) }) it('negation is expressed through the passthrough Drizzle not operator', async () => { @@ -222,6 +258,29 @@ describe('createEncryptionOperatorsV3 - free-text match', () => { }) }) +describe('createEncryptionOperatorsV3 - storage-only domains', () => { + it.each(storageDomains)('%s eq throws', async (eqlType, spec) => { + const { ops } = setup() + await expect( + ops.eq(matrixColumn(eqlType), sampleFor(spec)), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) + + it.each(storageDomains)('%s contains throws', async (eqlType, spec) => { + const { ops } = setup() + await expect( + ops.contains(matrixColumn(eqlType), sampleFor(spec)), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) + + it.each(storageDomains)('%s asc throws synchronously', (eqlType) => { + const { ops } = setup() + expect(() => ops.asc(matrixColumn(eqlType))).toThrow( + EncryptionOperatorError, + ) + }) +}) + describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { it('inArray ORs one eq term per value; empty array throws', async () => { const { ops, render } = setup() @@ -254,15 +313,6 @@ describe('createEncryptionOperatorsV3 - array, ordering, combinators', () => { expect((q.sql.match(/eql_v3\.neq/g) ?? []).length).toBe(2) }) - it('asc / desc extract the ord term', () => { - const { ops, render } = setup() - const ascq = render(ops.asc(users.age)) - expect(ascq.sql).toContain('eql_v3.ord_term("users"."age")') - expect(ascq.sql.toLowerCase()).toContain('asc') - const descq = render(ops.desc(users.age)) - expect(descq.sql.toLowerCase()).toContain('desc') - }) - it('and ignores undefined conditions and keeps the encrypted predicates', async () => { const { ops, render } = setup() const q = render( From 83aacae90382794994c56c08c52bf84567005486 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 7 Jul 2026 11:28:47 +1000 Subject: [PATCH 22/24] test: expand live drizzle v3 matrix coverage --- .../drizzle-v3/operators-live-pg.test.ts | 552 ++++++++++++++---- 1 file changed, 426 insertions(+), 126 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts index 9cc65a89..5ae1b271 100644 --- a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -1,5 +1,11 @@ import 'dotenv/config' -import { and, eq as drizzleEq, type SQL } from 'drizzle-orm' +import { + and, + asc as drizzleAsc, + eq as drizzleEq, + type SQL, + type SQLWrapper, +} from 'drizzle-orm' import { integer, pgTable, text } from 'drizzle-orm/pg-core' import { drizzle } from 'drizzle-orm/postgres-js' import postgres from 'postgres' @@ -7,34 +13,164 @@ import { afterAll, beforeAll, expect, it } from 'vitest' import { EncryptionV3 } from '@/encryption/v3' import { createEncryptionOperatorsV3, + EncryptionOperatorError, extractEncryptionSchemaV3, - types, } from '@/eql/v3/drizzle' +import { makeEqlV3Column } from '@/eql/v3/drizzle/column' import { installEqlV3IfNeeded } from '../helpers/eql-v3' import { describeLivePg, LIVE_EQL_V3_PG_ENABLED } from '../helpers/live-gate' +import { + type DomainSpec, + type EqlV3TypeName, + typedEntries, + V3_MATRIX, +} from '../v3-matrix/catalog' const url = process.env.DATABASE_URL const sqlClient = LIVE_EQL_V3_PG_ENABLED ? postgres(url as string, { prepare: false }) : (undefined as unknown as postgres.Sql) -const usersTable = pgTable('protect_ci_v3_drizzle', { +const TABLE_NAME = 'protect_ci_v3_drizzle_matrix' +const ACCOUNT_TABLE_NAME = 'protect_ci_v3_drizzle_matrix_accounts' +const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` +const ROW_A = 'row-a' +const ROW_B = 'row-b' + +const slug = (eqlType: EqlV3TypeName): string => eqlType.replace('eql_v3.', '') +const matrixEntries = typedEntries(V3_MATRIX) +const matrixColumns = Object.fromEntries( + matrixEntries.map(([eqlType, spec]) => [ + slug(eqlType), + makeEqlV3Column(spec.builder(slug(eqlType))), + ]), +) + +const matrixTable = pgTable(TABLE_NAME, { id: integer('id').primaryKey().generatedAlwaysAsIdentity(), - email: types.TextSearch('email'), - nickname: types.TextEq('nickname'), - age: types.IntegerOrd('age'), + rowKey: text('row_key').notNull(), + testRunId: text('test_run_id').notNull(), + nullableTextEq: makeEqlV3Column( + V3_MATRIX['eql_v3.text_eq'].builder('nullable_text_eq'), + ), + ...matrixColumns, +} as never) + +const accountsTable = pgTable(ACCOUNT_TABLE_NAME, { + id: integer('id').primaryKey().generatedAlwaysAsIdentity(), + rowKey: text('row_key').notNull(), + label: text('label').notNull(), testRunId: text('test_run_id').notNull(), }) -const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` -const schema = extractEncryptionSchemaV3(usersTable) +const schema = extractEncryptionSchemaV3(matrixTable) + +type PlainValue = string | number | boolean | Date +type RowKey = typeof ROW_A | typeof ROW_B +type MatrixPlainRow = Record +type MatrixDbRow = Record +type SelectRow = { rowKey: string } +type Db = ReturnType +type Client = Awaited> +type Ops = ReturnType +type ComparisonOperator = 'gt' | 'gte' | 'lt' | 'lte' + +let client: Client +let ops: Ops +let db: Db + +const equalityDomains = matrixEntries.filter( + ([, spec]) => spec.indexes.unique || spec.indexes.ore, +) +const orderDomains = matrixEntries.filter(([, spec]) => spec.indexes.ore) +const matchDomains = matrixEntries.filter(([, spec]) => spec.indexes.match) +const noEqualityDomains = matrixEntries.filter( + ([, spec]) => !spec.indexes.unique && !spec.indexes.ore, +) +const noOrderDomains = matrixEntries.filter(([, spec]) => !spec.indexes.ore) +const noMatchDomains = matrixEntries.filter(([, spec]) => !spec.indexes.match) +const comparisonOperators: Array< + [ComparisonOperator, (cmp: number) => boolean] +> = [ + ['gt', (cmp) => cmp > 0], + ['gte', (cmp) => cmp >= 0], + ['lt', (cmp) => cmp < 0], + ['lte', (cmp) => cmp <= 0], +] +const comparisonDomains = orderDomains.flatMap(([eqlType, spec]) => + comparisonOperators.map( + ([operator, predicate]) => [eqlType, spec, operator, predicate] as const, + ), +) + +const matrixColumn = (eqlType: EqlV3TypeName): SQLWrapper => + (matrixTable as unknown as Record)[slug(eqlType)] + +const scoped = (cond: SQL | undefined): SQL | undefined => + cond ? and(drizzleEq(matrixTable.testRunId, RUN), cond) : cond + +const plainValue = (spec: DomainSpec, rowKey: RowKey): PlainValue => + spec.samples[rowKey === ROW_A ? 0 : 1] + +function comparePlain(left: PlainValue, right: PlainValue): number { + if (left instanceof Date && right instanceof Date) { + return left.getTime() - right.getTime() + } + if (typeof left === 'number' && typeof right === 'number') { + return left - right + } + if (typeof left === 'string' && typeof right === 'string') { + return left.localeCompare(right) + } + throw new Error( + `Unsupported ordered values: ${String(left)}, ${String(right)}`, + ) +} -// biome-ignore lint/suspicious/noExplicitAny: test client typing -let client: any -// biome-ignore lint/suspicious/noExplicitAny: test operator typing -let ops: any -// biome-ignore lint/suspicious/noExplicitAny: drizzle db typing -let db: any +function expectedKeysFor( + spec: DomainSpec, + predicate: (value: PlainValue) => boolean, +): RowKey[] { + return ([ROW_A, ROW_B] as const).filter((rowKey) => + predicate(plainValue(spec, rowKey)), + ) +} + +function sortedKeysFor(spec: DomainSpec, direction: 'asc' | 'desc'): RowKey[] { + return ([ROW_A, ROW_B] as const).sort((left, right) => { + const cmp = comparePlain(plainValue(spec, left), plainValue(spec, right)) + return direction === 'asc' ? cmp : -cmp + }) +} + +async function selectRowKeys(condition: SQL | undefined): Promise { + if (!condition) throw new Error('Expected query condition') + const rows = (await db + .select({ rowKey: matrixTable.rowKey }) + .from(matrixTable) + .where(scoped(condition)) + .orderBy(drizzleAsc(matrixTable.rowKey))) as SelectRow[] + return rows.map((row) => row.rowKey) +} + +function unwrap(result: { data?: T; failure?: { message: string } }): T { + if (result.failure) throw new Error(result.failure.message) + return result.data as T +} + +function encryptedInsertRows(): MatrixPlainRow[] { + return ([ROW_A, ROW_B] as const).map((rowKey) => { + const row: MatrixPlainRow = { + rowKey, + testRunId: RUN, + nullableTextEq: rowKey === ROW_A ? null : 'nullable-present', + } + for (const [eqlType, spec] of matrixEntries) { + row[slug(eqlType)] = plainValue(spec, rowKey) + } + return row + }) +} beforeAll(async () => { if (!LIVE_EQL_V3_PG_ENABLED) return @@ -43,145 +179,309 @@ beforeAll(async () => { ops = createEncryptionOperatorsV3(client) db = drizzle({ client: sqlClient }) - await sqlClient` - CREATE TABLE IF NOT EXISTS protect_ci_v3_drizzle ( + const columnDefs = matrixEntries + .map(([eqlType]) => `"${slug(eqlType)}" ${eqlType} NOT NULL`) + .join(',\n ') + + await sqlClient.unsafe(` + CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + row_key TEXT NOT NULL, + test_run_id TEXT NOT NULL, + nullable_text_eq eql_v3.text_eq, + ${columnDefs} + ) + `) + await sqlClient.unsafe(` + CREATE TABLE IF NOT EXISTS ${ACCOUNT_TABLE_NAME} ( id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, - email eql_v3.text_search NOT NULL, - nickname eql_v3.text_eq NOT NULL, - age eql_v3.integer_ord NOT NULL, + row_key TEXT NOT NULL, + label TEXT NOT NULL, test_run_id TEXT NOT NULL ) - ` - - const seed = [ - { email: 'ada@example.com', nickname: 'ada', age: 37 }, - { email: 'grace@example.com', nickname: 'grace', age: 30 }, - { email: 'alan@example.net', nickname: 'alan', age: 42 }, - ] - for (const row of seed) { - const enc = unwrap(await client.encryptModel({ ...row }, schema)) - await db.insert(usersTable).values({ - email: enc.email, - nickname: enc.nickname, - age: enc.age, - testRunId: RUN, - }) - } -}, 60000) + `) + + const encryptedRows = unwrap( + await client.bulkEncryptModels(encryptedInsertRows(), schema), + ) + await db.insert(matrixTable).values(encryptedRows) + await db.insert(accountsTable).values([ + { rowKey: ROW_A, label: 'primary', testRunId: RUN }, + { rowKey: ROW_B, label: 'secondary', testRunId: RUN }, + ]) +}, 120000) afterAll(async () => { if (!LIVE_EQL_V3_PG_ENABLED) return - await sqlClient`DELETE FROM protect_ci_v3_drizzle WHERE test_run_id = ${RUN}` + await sqlClient`DELETE FROM ${sqlClient(TABLE_NAME)} WHERE test_run_id = ${RUN}` + await sqlClient`DELETE FROM ${sqlClient(ACCOUNT_TABLE_NAME)} WHERE test_run_id = ${RUN}` await sqlClient.end() }, 30000) -// biome-ignore lint/suspicious/noExplicitAny: result unwrap -function unwrap(r: any) { - if (r.failure) throw new Error(r.failure.message) - return r.data -} +describeLivePg('v3 drizzle operators (live pg matrix)', () => { + it.each(equalityDomains)( + '%s eq selects the exact row', + async (eqlType, spec) => { + const rows = await selectRowKeys( + await ops.eq(matrixColumn(eqlType), plainValue(spec, ROW_A)), + ) + expect(rows).toEqual([ROW_A]) + }, + 30000, + ) -describeLivePg('v3 drizzle operators (live pg)', () => { - const scoped = (cond: SQL) => and(drizzleEq(usersTable.testRunId, RUN), cond) + it.each(equalityDomains)( + '%s ne selects the complement row', + async (eqlType, spec) => { + const rows = await selectRowKeys( + await ops.ne(matrixColumn(eqlType), plainValue(spec, ROW_A)), + ) + expect(rows).toEqual([ROW_B]) + }, + 30000, + ) - it('eq selects the exact nickname row', async () => { - const rows = await db - .select({ id: usersTable.id, age: usersTable.age }) - .from(usersTable) - .where(scoped(await ops.eq(usersTable.nickname, 'grace'))) - const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) - expect(decrypted.map((r: { age: number }) => r.age)).toEqual([30]) - }, 30000) + it.each(equalityDomains)( + '%s inArray selects all listed rows', + async (eqlType, spec) => { + const rows = await selectRowKeys( + await ops.inArray(matrixColumn(eqlType), [ + plainValue(spec, ROW_A), + plainValue(spec, ROW_B), + ]), + ) + expect(rows).toEqual([ROW_A, ROW_B]) + }, + 30000, + ) - it('gte selects rows at or above the bound', async () => { - const rows = await db - .select({ id: usersTable.id }) - .from(usersTable) - .where(scoped(await ops.gte(usersTable.age, 37))) - expect(rows.length).toBe(2) - }, 30000) + it.each(equalityDomains)( + '%s notInArray excludes listed rows', + async (eqlType, spec) => { + const rows = await selectRowKeys( + await ops.notInArray(matrixColumn(eqlType), [plainValue(spec, ROW_A)]), + ) + expect(rows).toEqual([ROW_B]) + }, + 30000, + ) - it('between selects the inclusive range', async () => { - const rows = await db - .select({ id: usersTable.id }) - .from(usersTable) - .where(scoped(await ops.between(usersTable.age, 30, 37))) - expect(rows.length).toBe(2) - }, 30000) + it.each(comparisonDomains)( + '%s %s selects rows by encrypted ordering', + async (eqlType, spec, operator, predicate) => { + const bound = plainValue(spec, ROW_A) + const rows = await selectRowKeys( + await ops[operator](matrixColumn(eqlType), bound), + ) + expect(rows).toEqual( + expectedKeysFor(spec, (value) => predicate(comparePlain(value, bound))), + ) + }, + 30000, + ) + + it.each(orderDomains)( + '%s between selects the inclusive range', + async (eqlType, spec) => { + const sortedValues = [ + plainValue(spec, ROW_A), + plainValue(spec, ROW_B), + ].sort(comparePlain) + const rows = await selectRowKeys( + await ops.between( + matrixColumn(eqlType), + sortedValues[0], + sortedValues[1], + ), + ) + expect(rows).toEqual([ROW_A, ROW_B]) + }, + 30000, + ) + + it.each(orderDomains)( + '%s notBetween excludes the inclusive range', + async (eqlType, spec) => { + const sortedValues = [ + plainValue(spec, ROW_A), + plainValue(spec, ROW_B), + ].sort(comparePlain) + const rows = await selectRowKeys( + await ops.notBetween( + matrixColumn(eqlType), + sortedValues[0], + sortedValues[1], + ), + ) + expect(rows).toEqual([]) + }, + 30000, + ) + + it.each(orderDomains)( + '%s asc orders by encrypted order term', + async (eqlType, spec) => { + const rows = (await db + .select({ rowKey: matrixTable.rowKey }) + .from(matrixTable) + .where(drizzleEq(matrixTable.testRunId, RUN)) + .orderBy(ops.asc(matrixColumn(eqlType)))) as SelectRow[] + expect(rows.map((row) => row.rowKey)).toEqual(sortedKeysFor(spec, 'asc')) + }, + 30000, + ) + + it.each(orderDomains)( + '%s desc orders by encrypted order term', + async (eqlType, spec) => { + const rows = (await db + .select({ rowKey: matrixTable.rowKey }) + .from(matrixTable) + .where(drizzleEq(matrixTable.testRunId, RUN)) + .orderBy(ops.desc(matrixColumn(eqlType)))) as SelectRow[] + expect(rows.map((row) => row.rowKey)).toEqual(sortedKeysFor(spec, 'desc')) + }, + 30000, + ) + + it.each(matchDomains)( + '%s contains matches plaintext terms', + async (eqlType, spec) => { + const rows = await selectRowKeys( + await ops.contains(matrixColumn(eqlType), 'ada'), + ) + expect(rows).toEqual( + expectedKeysFor(spec, (value) => + String(value).toLowerCase().includes('ada'), + ), + ) + }, + 30000, + ) + + it.each( + noEqualityDomains, + )('%s eq rejects unsupported equality', async (eqlType, spec) => { + await expect( + ops.eq(matrixColumn(eqlType), plainValue(spec, ROW_A)), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) + + it.each( + noOrderDomains, + )('%s gt rejects unsupported ordering', async (eqlType, spec) => { + await expect( + ops.gt(matrixColumn(eqlType), plainValue(spec, ROW_A)), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) - it('notBetween excludes the inclusive range', async () => { - const rows = await db - .select({ id: usersTable.id, age: usersTable.age }) - .from(usersTable) - .where(scoped(await ops.notBetween(usersTable.age, 30, 37))) - const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) - expect(decrypted.map((r: { age: number }) => r.age).sort()).toEqual([42]) + it.each(noOrderDomains)('%s asc rejects unsupported ordering', (eqlType) => { + expect(() => ops.asc(matrixColumn(eqlType))).toThrow( + EncryptionOperatorError, + ) + }) + + it.each( + noMatchDomains, + )('%s contains rejects unsupported match', async (eqlType, spec) => { + await expect( + ops.contains(matrixColumn(eqlType), plainValue(spec, ROW_A)), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) + + it('and combines encrypted predicates', async () => { + const rows = await selectRowKeys( + await ops.and( + ops.eq(matrixColumn('eql_v3.text_eq'), 'ada@example.com'), + ops.lt(matrixColumn('eql_v3.integer_ord'), 0), + ), + ) + expect(rows).toEqual([ROW_B]) }, 30000) - it('contains matches on the free-text email column', async () => { - const rows = await db - .select({ id: usersTable.id }) - .from(usersTable) - .where(scoped(await ops.contains(usersTable.email, 'example.com'))) - expect(rows.length).toBe(2) + it('or combines encrypted predicates', async () => { + const rows = await selectRowKeys( + await ops.or( + ops.eq(matrixColumn('eql_v3.text_eq'), ''), + ops.eq(matrixColumn('eql_v3.text_eq'), 'ada@example.com'), + ), + ) + expect(rows).toEqual([ROW_A, ROW_B]) }, 30000) - it('inArray selects the exact nickname rows', async () => { - const rows = await db - .select({ id: usersTable.id, nickname: usersTable.nickname }) - .from(usersTable) - .where(scoped(await ops.inArray(usersTable.nickname, ['ada', 'grace']))) - const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) - expect( - decrypted.map((r: { nickname: string }) => r.nickname).sort(), - ).toEqual(['ada', 'grace']) + it('not negates an encrypted predicate', async () => { + const rows = await selectRowKeys( + ops.not(await ops.eq(matrixColumn('eql_v3.text_eq'), '')), + ) + expect(rows).toEqual([ROW_B]) }, 30000) - it('notInArray excludes the listed nickname rows', async () => { - const rows = await db - .select({ id: usersTable.id, nickname: usersTable.nickname }) - .from(usersTable) - .where(scoped(await ops.notInArray(usersTable.nickname, ['grace']))) - const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) + it('isNull and isNotNull work on nullable encrypted columns', async () => { + expect(await selectRowKeys(ops.isNull(matrixTable.nullableTextEq))).toEqual( + [ROW_A], + ) expect( - decrypted.map((r: { nickname: string }) => r.nickname).sort(), - ).toEqual(['ada', 'alan']) + await selectRowKeys(ops.isNotNull(matrixTable.nullableTextEq)), + ).toEqual([ROW_B]) }, 30000) - it('orderBy asc returns ascending ages', async () => { - const rows = await db - .select({ age: usersTable.age }) - .from(usersTable) - .where(scoped(await ops.gte(usersTable.age, 0))) - .orderBy(ops.asc(usersTable.age)) - const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) - const ages = decrypted.map((r: { age: number }) => r.age) - expect(ages).toEqual([...ages].sort((a, b) => a - b)) - }, 30000) + it('exists and notExists work with correlated subqueries', async () => { + const primaryAccount = db + .select({ id: accountsTable.id }) + .from(accountsTable) + .where( + and( + drizzleEq(accountsTable.testRunId, RUN), + drizzleEq(accountsTable.rowKey, matrixTable.rowKey), + drizzleEq(accountsTable.label, 'primary'), + ), + ) + const missingAccount = db + .select({ id: accountsTable.id }) + .from(accountsTable) + .where( + and( + drizzleEq(accountsTable.testRunId, RUN), + drizzleEq(accountsTable.rowKey, matrixTable.rowKey), + drizzleEq(accountsTable.label, 'missing'), + ), + ) - it('orderBy desc returns descending ages', async () => { - const rows = await db - .select({ age: usersTable.age }) - .from(usersTable) - .where(scoped(await ops.gte(usersTable.age, 0))) - .orderBy(ops.desc(usersTable.age)) - const decrypted = unwrap(await client.bulkDecryptModels(rows, schema)) - const ages = decrypted.map((r: { age: number }) => r.age) - expect(ages).toEqual([...ages].sort((a, b) => b - a)) + expect(await selectRowKeys(ops.exists(primaryAccount))).toEqual([ROW_A]) + expect(await selectRowKeys(ops.notExists(missingAccount))).toEqual([ + ROW_A, + ROW_B, + ]) }, 30000) - it('or mixes encrypted and plain predicates', async () => { - const rows = await db - .select({ id: usersTable.id }) - .from(usersTable) - .where( - scoped( - await ops.or( - ops.eq(usersTable.nickname, 'grace'), - drizzleEq(usersTable.testRunId, RUN), - ), + it('joins plain tables while filtering encrypted columns', async () => { + const rows = (await db + .select({ rowKey: matrixTable.rowKey }) + .from(matrixTable) + .innerJoin( + accountsTable, + and( + drizzleEq(accountsTable.testRunId, RUN), + drizzleEq(accountsTable.rowKey, matrixTable.rowKey), ), ) - expect(rows.length).toBe(3) + .where( + scoped(await ops.eq(matrixColumn('eql_v3.text_eq'), 'ada@example.com')), + )) as SelectRow[] + expect(rows.map((row) => row.rowKey)).toEqual([ROW_B]) + }, 30000) + + it('paginates encrypted ordering results with limit and offset', async () => { + const spec = V3_MATRIX['eql_v3.integer_ord'] + const rows = (await db + .select({ rowKey: matrixTable.rowKey }) + .from(matrixTable) + .where(drizzleEq(matrixTable.testRunId, RUN)) + .orderBy(ops.asc(matrixColumn('eql_v3.integer_ord'))) + .limit(1) + .offset(1)) as SelectRow[] + expect(rows.map((row) => row.rowKey)).toEqual( + sortedKeysFor(spec, 'asc').slice(1, 2), + ) }, 30000) }) From fa0c370855aa0caa88c5e841188ffca1a0f1aa83 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 7 Jul 2026 12:21:50 +1000 Subject: [PATCH 23/24] test(stack): strengthen v3 live coverage (range/boundary/lock-context/null) Close assertion-strength gaps in the EQL v3 live suites where a real regression could pass green: - between/notBetween: add narrow single-point range cases that prove EXCLUSION (the spanning cases only ever proved inclusion). - ordering oracle: compare text bytewise, not via localeCompare, to match ORE byte order. - inArray/notInArray: exercise the >4-value concurrency pool. - matrix-boundary-live-pg: drive catalog boundary samples (INT4/smallint bounds, 1e15) through real eql_v3. columns, not just the FFI. - operators-lock-context-live-pg: query lock-context-bound rows through the Drizzle operator path (ops.eq + lockContext/audit); soft-skips on USER_JWT. - operators-null-live-pg: NULL persistence across storage/eq/ord/match tiers. - live-gate-required guard + CI: fail loudly (REQUIRE_LIVE/REQUIRE_LIVE_PG) instead of silently skipping the live matrices when creds are absent. --- .github/workflows/tests.yml | 10 + .../drizzle-v3/operators-live-pg.test.ts | 74 ++++++- .../operators-lock-context-live-pg.test.ts | 181 ++++++++++++++++++ .../drizzle-v3/operators-null-live-pg.test.ts | 177 +++++++++++++++++ .../__tests__/live-gate-required.test.ts | 30 +++ .../v3-matrix/matrix-boundary-live-pg.test.ts | 153 +++++++++++++++ 6 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 packages/stack/__tests__/drizzle-v3/operators-lock-context-live-pg.test.ts create mode 100644 packages/stack/__tests__/drizzle-v3/operators-null-live-pg.test.ts create mode 100644 packages/stack/__tests__/live-gate-required.test.ts create mode 100644 packages/stack/__tests__/v3-matrix/matrix-boundary-live-pg.test.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a0b5a59..81626656 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,6 +93,11 @@ jobs: echo "CS_CLIENT_KEY=${{ secrets.CS_CLIENT_KEY }}" >> ./packages/stack/.env echo "CS_CLIENT_ACCESS_KEY=${{ secrets.CS_CLIENT_ACCESS_KEY }}" >> ./packages/stack/.env echo "DATABASE_URL=postgres://cipherstash:password@localhost:5432/cipherstash" >> ./packages/stack/.env + # Fail loudly if the live suites would silently skip (missing creds): + # the skip-guard in __tests__/live-gate-required.test.ts asserts the + # live gates are actually enabled when these are set. + echo "REQUIRE_LIVE=1" >> ./packages/stack/.env + echo "REQUIRE_LIVE_PG=1" >> ./packages/stack/.env - name: Create .env file in ./packages/protect-dynamodb/ run: | @@ -305,6 +310,11 @@ jobs: echo "CS_CLIENT_KEY=${{ secrets.CS_CLIENT_KEY }}" >> ./packages/stack/.env echo "CS_CLIENT_ACCESS_KEY=${{ secrets.CS_CLIENT_ACCESS_KEY }}" >> ./packages/stack/.env echo "DATABASE_URL=postgres://cipherstash:password@localhost:5432/cipherstash" >> ./packages/stack/.env + # Fail loudly if the live suites would silently skip (missing creds): + # the skip-guard in __tests__/live-gate-required.test.ts asserts the + # live gates are actually enabled when these are set. + echo "REQUIRE_LIVE=1" >> ./packages/stack/.env + echo "REQUIRE_LIVE_PG=1" >> ./packages/stack/.env - name: Create .env file in ./packages/protect-dynamodb/ run: | diff --git a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts index 5ae1b271..a5fea646 100644 --- a/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators-live-pg.test.ts @@ -120,7 +120,12 @@ function comparePlain(left: PlainValue, right: PlainValue): number { return left - right } if (typeof left === 'string' && typeof right === 'string') { - return left.localeCompare(right) + // eql_v3 text ordering (ORE) is BYTEWISE, not locale-collated: the oracle + // must model codepoint order, not `localeCompare` (which folds case, + // reorders punctuation vs letters, and is locale-dependent). Text samples + // must stay ASCII/unambiguous so UTF-16 code-unit order == the byte order + // the DB actually sorts by. + return left < right ? -1 : left > right ? 1 : 0 } throw new Error( `Unsupported ordered values: ${String(left)}, ${String(right)}`, @@ -318,6 +323,39 @@ describeLivePg('v3 drizzle operators (live pg matrix)', () => { 30000, ) + // The spanning cases above only ever prove INCLUSION (both rows are inside + // the range, so `between` -> [A,B] and `notBetween` -> []). These narrow + // cases use a single-point range at ROW_A's value to prove the operators + // also EXCLUDE: `between` must drop ROW_B, `notBetween` must keep it. Without + // these, a `between` that matched everything (or a `notBetween` no-op) passes. + it.each(orderDomains)( + '%s between at a single point excludes the out-of-range row', + async (eqlType, spec) => { + const bound = plainValue(spec, ROW_A) + const rows = await selectRowKeys( + await ops.between(matrixColumn(eqlType), bound, bound), + ) + expect(rows).toEqual( + expectedKeysFor(spec, (value) => comparePlain(value, bound) === 0), + ) + }, + 30000, + ) + + it.each(orderDomains)( + '%s notBetween at a single point keeps the out-of-range row', + async (eqlType, spec) => { + const bound = plainValue(spec, ROW_A) + const rows = await selectRowKeys( + await ops.notBetween(matrixColumn(eqlType), bound, bound), + ) + expect(rows).toEqual( + expectedKeysFor(spec, (value) => comparePlain(value, bound) !== 0), + ) + }, + 30000, + ) + it.each(orderDomains)( '%s asc orders by encrypted order term', async (eqlType, spec) => { @@ -484,4 +522,38 @@ describeLivePg('v3 drizzle operators (live pg matrix)', () => { sortedKeysFor(spec, 'asc').slice(1, 2), ) }, 30000) + + // The matrix inArray/notInArray cases above use 2-element lists, so the + // MAX_IN_ARRAY_CONCURRENCY=4 worker pool (operators.ts) never actually + // concurrently encrypts more terms than the serial path would. These cross + // the pool boundary: 5 values (> 4) forces the pool to reuse workers, and + // must still produce the correct OR/AND of eq/ne terms. + it('inArray encrypts a >4-value list through the concurrency pool', async () => { + const rows = await selectRowKeys( + await ops.inArray(matrixColumn('eql_v3.text_eq'), [ + 'ada@example.com', + '', + 'nobody-1@example.com', + 'nobody-2@example.com', + 'nobody-3@example.com', + ]), + ) + // '' -> ROW_A, 'ada@example.com' -> ROW_B; the three "nobody" terms match + // nothing, exercising the pool without changing the expected set. + expect(rows).toEqual([ROW_A, ROW_B]) + }, 30000) + + it('notInArray encrypts a >4-value list through the concurrency pool', async () => { + const rows = await selectRowKeys( + await ops.notInArray(matrixColumn('eql_v3.text_eq'), [ + '', + 'nobody-1@example.com', + 'nobody-2@example.com', + 'nobody-3@example.com', + 'nobody-4@example.com', + ]), + ) + // Only '' is excluded (ROW_A); ROW_B ('ada@example.com') survives. + expect(rows).toEqual([ROW_B]) + }, 30000) }) diff --git a/packages/stack/__tests__/drizzle-v3/operators-lock-context-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-lock-context-live-pg.test.ts new file mode 100644 index 00000000..49c59b71 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/operators-lock-context-live-pg.test.ts @@ -0,0 +1,181 @@ +/** + * Live, identity-bound querying through the v3 Drizzle operator path. + * + * `matrix-identity-live.test.ts` proves lock context round-trips through the + * typed CLIENT (`encryptModel`/`decryptModel`), and `operators.test.ts` proves + * the Drizzle operators FORWARD `lockContext`/`audit` — but only against a + * MOCKED FFI. Nothing exercises the one end-to-end shape that matters most: + * seed rows bound to an identity, then query them with `ops.eq(col, value, + * { lockContext })` against a real database and assert the encrypted term + * actually matches the stored row and decrypts. + * + * Wiring mirrors `lock-context.test.ts` (the current, non-deprecated + * strategy-based flow): the client authenticates as the end user via + * `OidcFederationStrategy` and binds the key to the `sub` claim with a plain + * `.withLockContext({ identityClaim })`. + * + * Gated twice: `describeLivePg` (needs `DATABASE_URL` + CS creds) and an inner + * `USER_JWT` guard (soft-skip, matching the existing identity/lock-context + * suites). Whether the searchable index terms are themselves identity-bound is + * decided inside `@cipherstash/protect-ffi`, not this repo — so we assert the + * SYMMETRIC behaviour (same lock context on seed + query matches and decrypts), + * not a cross-identity non-match. + */ +import 'dotenv/config' +import { OidcFederationStrategy } from '@cipherstash/auth' +import { and, asc as drizzleAsc, eq as drizzleEq, type SQL } from 'drizzle-orm' +import { integer, pgTable, text } from 'drizzle-orm/pg-core' +import { drizzle } from 'drizzle-orm/postgres-js' +import postgres from 'postgres' +import { afterAll, beforeAll, expect, it } from 'vitest' +import { EncryptionV3 } from '@/encryption/v3' +import { + createEncryptionOperatorsV3, + extractEncryptionSchemaV3, +} from '@/eql/v3/drizzle' +import { makeEqlV3Column } from '@/eql/v3/drizzle/column' +import { installEqlV3IfNeeded } from '../helpers/eql-v3' +import { describeLivePg, LIVE_EQL_V3_PG_ENABLED } from '../helpers/live-gate' +import { V3_MATRIX } from '../v3-matrix/catalog' + +const url = process.env.DATABASE_URL +const sqlClient = LIVE_EQL_V3_PG_ENABLED + ? postgres(url as string, { prepare: false }) + : (undefined as unknown as postgres.Sql) + +const TABLE_NAME = 'protect_ci_v3_drizzle_lock_context' +const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` +const ROW_A = 'row-a' +const ROW_B = 'row-b' +const SECRET_A = 'ada@example.com' +const SECRET_B = 'grace@example.com' + +// A fixed identity claim; the same value must be supplied on encrypt and query +// for the terms/keys to reproduce. +const IDENTITY_CLAIM = { identityClaim: ['sub'] } + +const secretTable = pgTable(TABLE_NAME, { + id: integer('id').primaryKey().generatedAlwaysAsIdentity(), + rowKey: text('row_key').notNull(), + testRunId: text('test_run_id').notNull(), + secret: makeEqlV3Column(V3_MATRIX['eql_v3.text_eq'].builder('secret')), +} as never) + +const schema = extractEncryptionSchemaV3(secretTable) + +type SelectRow = { rowKey: string } + +let client: Awaited> +let ops: ReturnType +let db: ReturnType +let userJwt: string | undefined + +function unwrap(result: { data?: T; failure?: { message: string } }): T { + if (result.failure) throw new Error(result.failure.message) + return result.data as T +} + +/** Run-scoped SELECT of row keys under an already-encrypted SQL condition. */ +async function selectRowKeys(condition: SQL): Promise { + const rows = (await db + .select({ rowKey: secretTable.rowKey }) + .from(secretTable) + .where(and(drizzleEq(secretTable.testRunId, RUN), condition)) + .orderBy(drizzleAsc(secretTable.rowKey))) as SelectRow[] + return rows.map((row) => row.rowKey) +} + +beforeAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + userJwt = process.env.USER_JWT + if (!userJwt) return + + const crn = process.env.CS_WORKSPACE_CRN + if (!crn) + throw new Error('CS_WORKSPACE_CRN must be set for lock-context tests') + + await installEqlV3IfNeeded(sqlClient) + client = await EncryptionV3({ + schemas: [schema], + config: { + strategy: OidcFederationStrategy.create(crn, () => + Promise.resolve(userJwt as string), + ), + }, + }) + ops = createEncryptionOperatorsV3(client) + db = drizzle({ client: sqlClient }) + + await sqlClient.unsafe(` + CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + row_key TEXT NOT NULL, + test_run_id TEXT NOT NULL, + secret eql_v3.text_eq NOT NULL + ) + `) + + // Seed BOTH rows bound to the same lock context. + const encryptedRows = unwrap>>( + await client + .bulkEncryptModels( + [ + { rowKey: ROW_A, testRunId: RUN, secret: SECRET_A }, + { rowKey: ROW_B, testRunId: RUN, secret: SECRET_B }, + ] as never, + schema, + ) + .withLockContext(IDENTITY_CLAIM), + ) + await db.insert(secretTable).values(encryptedRows as never) +}, 120000) + +afterAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + if (userJwt) { + await sqlClient`DELETE FROM ${sqlClient(TABLE_NAME)} WHERE test_run_id = ${RUN}` + } + await sqlClient.end() +}, 30000) + +describeLivePg('v3 drizzle operators with lock context (live pg)', () => { + const skipUnlessJwt = (): boolean => { + if (!userJwt) { + console.log('Skipping lock-context operator test - no USER_JWT provided') + return true + } + return false + } + + it('eq with a matching lock context selects the exact row', async () => { + if (skipUnlessJwt()) return + const condition = await ops.eq(secretTable.secret, SECRET_A, { + // Runtime accepts a plain { identityClaim } (forwarded to + // withLockContext); the operator opts type is narrowed to LockContext. + lockContext: IDENTITY_CLAIM as never, + }) + expect(await selectRowKeys(condition)).toEqual([ROW_A]) + }, 30000) + + it('a lock-context-bound row decrypts with the same lock context', async () => { + if (skipUnlessJwt()) return + const [row] = await sqlClient.unsafe>( + `SELECT secret::jsonb AS value FROM ${TABLE_NAME} + WHERE test_run_id = $1 AND row_key = $2`, + [RUN, ROW_A], + ) + const decrypted = unwrap( + await client.decrypt(row.value as never).withLockContext(IDENTITY_CLAIM), + ) + expect(decrypted).toBe(SECRET_A) + }, 30000) + + it('eq threads an audit config alongside the lock context', async () => { + if (skipUnlessJwt()) return + const condition = await ops.eq(secretTable.secret, SECRET_B, { + lockContext: IDENTITY_CLAIM as never, + audit: { metadata: { sub: 'toby@cipherstash.com', type: 'query' } }, + }) + expect(await selectRowKeys(condition)).toEqual([ROW_B]) + }, 30000) +}) diff --git a/packages/stack/__tests__/drizzle-v3/operators-null-live-pg.test.ts b/packages/stack/__tests__/drizzle-v3/operators-null-live-pg.test.ts new file mode 100644 index 00000000..263f73b0 --- /dev/null +++ b/packages/stack/__tests__/drizzle-v3/operators-null-live-pg.test.ts @@ -0,0 +1,177 @@ +/** + * Live NULL persistence for encrypted columns across every capability tier. + * + * `operators-live-pg.test.ts` proves `isNull`/`isNotNull` — but only for a + * single `text_eq` column (`nullableTextEq`). NULL storage and retrieval for + * the other tiers (storage-only, order/ORE, free-text match) is untested live: + * a bug that mangled a NULL cell, or a domain that rejected NULL, would only + * show up on those column kinds. + * + * This file seeds two rows — row A all-NULL, row B all-present — across one + * representative column per tier, then asserts, per column: `isNull` selects + * the NULL row, `isNotNull` selects the present row, the NULL cell reads back + * as SQL NULL, and the present cell still decrypts to its plaintext. + */ +import 'dotenv/config' +import { and, asc as drizzleAsc, eq as drizzleEq, type SQL } from 'drizzle-orm' +import { integer, pgTable, text } from 'drizzle-orm/pg-core' +import { drizzle } from 'drizzle-orm/postgres-js' +import postgres from 'postgres' +import { afterAll, beforeAll, expect, it } from 'vitest' +import { EncryptionV3 } from '@/encryption/v3' +import { + createEncryptionOperatorsV3, + extractEncryptionSchemaV3, +} from '@/eql/v3/drizzle' +import { makeEqlV3Column } from '@/eql/v3/drizzle/column' +import { installEqlV3IfNeeded } from '../helpers/eql-v3' +import { describeLivePg, LIVE_EQL_V3_PG_ENABLED } from '../helpers/live-gate' +import { V3_MATRIX } from '../v3-matrix/catalog' + +const url = process.env.DATABASE_URL +const sqlClient = LIVE_EQL_V3_PG_ENABLED + ? postgres(url as string, { prepare: false }) + : (undefined as unknown as postgres.Sql) + +const TABLE_NAME = 'protect_ci_v3_drizzle_nullable' +const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` +const ROW_A = 'row-a' // all NULL +const ROW_B = 'row-b' // all present + +// One representative column per capability tier, each NULLABLE. +const nullableTable = pgTable(TABLE_NAME, { + id: integer('id').primaryKey().generatedAlwaysAsIdentity(), + rowKey: text('row_key').notNull(), + testRunId: text('test_run_id').notNull(), + storageText: makeEqlV3Column( + V3_MATRIX['eql_v3.text'].builder('storage_text'), + ), + eqText: makeEqlV3Column(V3_MATRIX['eql_v3.text_eq'].builder('eq_text')), + ordInt: makeEqlV3Column(V3_MATRIX['eql_v3.integer_ord'].builder('ord_int')), + matchText: makeEqlV3Column( + V3_MATRIX['eql_v3.text_match'].builder('match_text'), + ), +} as never) + +// Tier metadata: property (drizzle) + DB column + a present-row plaintext. +const TIERS = [ + { + key: 'storageText', + db: 'storage_text', + domain: 'eql_v3.text', + sample: 'stored-secret', + }, + { + key: 'eqText', + db: 'eq_text', + domain: 'eql_v3.text_eq', + sample: 'ada@example.com', + }, + { key: 'ordInt', db: 'ord_int', domain: 'eql_v3.integer_ord', sample: 42 }, + { + key: 'matchText', + db: 'match_text', + domain: 'eql_v3.text_match', + sample: 'ada lovelace', + }, +] as const + +const schema = extractEncryptionSchemaV3(nullableTable) + +type SelectRow = { rowKey: string } + +let client: Awaited> +let ops: ReturnType +let db: ReturnType + +function unwrap(result: { data?: T; failure?: { message: string } }): T { + if (result.failure) throw new Error(result.failure.message) + return result.data as T +} + +const columnFor = (key: string): SQL => + (nullableTable as unknown as Record)[key] + +async function selectRowKeys(condition: SQL): Promise { + const rows = (await db + .select({ rowKey: nullableTable.rowKey }) + .from(nullableTable) + .where(and(drizzleEq(nullableTable.testRunId, RUN), condition)) + .orderBy(drizzleAsc(nullableTable.rowKey))) as SelectRow[] + return rows.map((row) => row.rowKey) +} + +beforeAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + await installEqlV3IfNeeded(sqlClient) + client = await EncryptionV3({ schemas: [schema] }) + ops = createEncryptionOperatorsV3(client) + db = drizzle({ client: sqlClient }) + + const columnDefs = TIERS.map((t) => `"${t.db}" ${t.domain}`).join(',\n ') + await sqlClient.unsafe(` + CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + row_key TEXT NOT NULL, + test_run_id TEXT NOT NULL, + ${columnDefs} + ) + `) + + // Row A: every encrypted column NULL. Row B: every column present. + const rowA: Record = { rowKey: ROW_A, testRunId: RUN } + const rowB: Record = { rowKey: ROW_B, testRunId: RUN } + for (const t of TIERS) { + rowA[t.key] = null + rowB[t.key] = t.sample + } + + const encryptedRows = unwrap>>( + await client.bulkEncryptModels([rowA, rowB] as never, schema), + ) + await db.insert(nullableTable).values(encryptedRows as never) +}, 120000) + +afterAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + await sqlClient`DELETE FROM ${sqlClient(TABLE_NAME)} WHERE test_run_id = ${RUN}` + await sqlClient.end() +}, 30000) + +describeLivePg('v3 drizzle NULL persistence across tiers (live pg)', () => { + it.each(TIERS)('$domain isNull selects the NULL row', async (tier) => { + expect(await selectRowKeys(ops.isNull(columnFor(tier.key)))).toEqual([ + ROW_A, + ]) + }, 30000) + + it.each(TIERS)('$domain isNotNull selects the present row', async (tier) => { + expect(await selectRowKeys(ops.isNotNull(columnFor(tier.key)))).toEqual([ + ROW_B, + ]) + }, 30000) + + it.each( + TIERS, + )('$domain stores a real NULL for the null row', async (tier) => { + const [row] = await sqlClient.unsafe>( + `SELECT "${tier.db}"::jsonb AS value FROM ${TABLE_NAME} + WHERE test_run_id = $1 AND row_key = $2`, + [RUN, ROW_A], + ) + expect(row.value).toBeNull() + }, 30000) + + it.each( + TIERS, + )('$domain present cell decrypts to its plaintext', async (tier) => { + const [row] = await sqlClient.unsafe>( + `SELECT "${tier.db}"::jsonb AS value FROM ${TABLE_NAME} + WHERE test_run_id = $1 AND row_key = $2`, + [RUN, ROW_B], + ) + expect(row.value).toHaveProperty('c') + const decrypted = unwrap(await client.decrypt(row.value as never)) + expect(decrypted).toBe(tier.sample) + }, 30000) +}) diff --git a/packages/stack/__tests__/live-gate-required.test.ts b/packages/stack/__tests__/live-gate-required.test.ts new file mode 100644 index 00000000..c7211b62 --- /dev/null +++ b/packages/stack/__tests__/live-gate-required.test.ts @@ -0,0 +1,30 @@ +/** + * Guard against silent "live green": the live matrices (`matrix-live*`, + * `operators-*-live-pg`, `matrix-boundary-live-pg`, `matrix-identity-live`) + * all `describe.skip` when credentials are absent — so a CI job that lost its + * `DATABASE_URL` / `CS_*` secrets would still pass, having run NONE of them. + * + * These guards turn that into a hard failure ONLY when the environment opts in: + * - `REQUIRE_LIVE` -> live CipherStash suites must be enabled + * - `REQUIRE_LIVE_PG` -> live Postgres suites must be enabled + * Set both in CI (see `.github/workflows/tests.yml`). Locally, with neither set, + * these are no-ops, so a dev machine without creds still runs green. + */ +import 'dotenv/config' +import { describe, expect, it } from 'vitest' +import { + LIVE_CIPHERSTASH_ENABLED, + LIVE_EQL_V3_PG_ENABLED, +} from './helpers/live-gate' + +describe('live-suite skip guard', () => { + it('live CipherStash suites are enabled when REQUIRE_LIVE is set', () => { + if (!process.env.REQUIRE_LIVE) return + expect(LIVE_CIPHERSTASH_ENABLED).toBe(true) + }) + + it('live Postgres suites are enabled when REQUIRE_LIVE_PG is set', () => { + if (!process.env.REQUIRE_LIVE_PG) return + expect(LIVE_EQL_V3_PG_ENABLED).toBe(true) + }) +}) diff --git a/packages/stack/__tests__/v3-matrix/matrix-boundary-live-pg.test.ts b/packages/stack/__tests__/v3-matrix/matrix-boundary-live-pg.test.ts new file mode 100644 index 00000000..a01b91fa --- /dev/null +++ b/packages/stack/__tests__/v3-matrix/matrix-boundary-live-pg.test.ts @@ -0,0 +1,153 @@ +/** + * Boundary-value coverage through REAL narrow Postgres domain columns. + * + * `matrix-live.test.ts` round-trips every catalog sample — including the type + * EDGE values (INT4 max/min, smallint bounds, `1e15`) — but only through the + * FFI: it never inserts them into an actual `eql_v3.*` column. `matrix-live-pg` + * inserts real columns but only decrypts `samples[0]`. So the boundary samples + * (`samples[2]`/`[3]` in the catalog) have never made the full trip through a + * real, domain-typed Postgres column. + * + * eql_v3 domains store the ENCRYPTED envelope (not a narrow int4/float4), so + * this is not an at-rest overflow test — the value is encrypted client-side. + * What it DOES prove, in the same defence-in-depth spirit as `matrix-live-pg` + * ("an SDK-side bug can hide behind a clean FFI round-trip and only surface + * against real Postgres"): each boundary value passes the domain's CHECK + * constraints and survives the real encrypt -> INSERT (cast to `eql_v3.`) + * -> SELECT -> decrypt path — catching envelope/encoding edge cases (e.g. int32 + * vs int64 handling, `1e15` precision) that only bite at the PG boundary. + */ +import 'dotenv/config' +import postgres from 'postgres' +import { afterAll, beforeAll, expect, it } from 'vitest' +import { EncryptionV3, encryptedTable } from '@/encryption/v3' +import { unwrapResult } from '../fixtures' +import { installEqlV3IfNeeded } from '../helpers/eql-v3' +import { describeLivePg, LIVE_EQL_V3_PG_ENABLED } from '../helpers/live-gate' +import { type EqlV3TypeName, typedEntries, V3_MATRIX } from './catalog' + +const databaseUrl = process.env.DATABASE_URL +const sql = LIVE_EQL_V3_PG_ENABLED + ? postgres(databaseUrl as string, { prepare: false }) + : (undefined as unknown as postgres.Sql) + +const TABLE_NAME = 'v3_matrix_boundary_live_pg' +const TEST_RUN_ID = `matrix-boundary-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + +/** `eql_v3.integer_ord` -> `int4_ord`: a valid, unique Postgres column name. */ +const slug = (t: EqlV3TypeName): string => t.replace('eql_v3.', '') + +// Only domains that actually carry boundary samples (index >= 2). Numeric +// domains have 4 samples (the type bounds live at [2]/[3]); the 3-sample text +// domains contribute [2]. Boolean/date/timestamp have just two and are absent. +const boundaryDomains = typedEntries(V3_MATRIX).filter( + ([, spec]) => spec.samples.length > 2, +) + +// One case per (domain, boundary sample index >= 2), labelled so a failure +// names the exact domain + sample. +const boundaryCases = boundaryDomains.flatMap(([t, spec]) => + spec.samples + .map((sample, i) => [t, sample, i] as const) + .filter(([, , i]) => i >= 2) + .map(([, sample, i]) => [`${t} #${i}`, slug(t), sample, i] as const), +) + +// The distinct boundary indices we need to seed (2 for text, 2 and 3 for +// numeric). Row `i` carries every boundary domain's `samples[i]` where present. +const boundaryIndices = [...new Set(boundaryCases.map(([, , , i]) => i))].sort( + (a, b) => a - b, +) + +const columns = Object.fromEntries( + boundaryDomains.map(([t, spec]) => [slug(t), spec.builder(slug(t))]), +) +const table = encryptedTable(TABLE_NAME, columns as never) + +type Row = { id: number } + +let client: Awaited> +// Postgres row id per boundary sample index. +const rowIdByIndex: Record = {} + +beforeAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + + await installEqlV3IfNeeded(sql) + client = await EncryptionV3({ schemas: [table] as never }) + + // Columns are NULLABLE: an index-3 row has no value for a 3-sample text + // domain, so that column must be allowed to be NULL for that row. + const columnDefs = boundaryDomains + .map(([t]) => `"${slug(t)}" ${t}`) + .join(',\n ') + + await sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + test_run_id TEXT NOT NULL, + ${columnDefs} + ) + `) + + // One model row per boundary index, carrying only the domains that HAVE a + // sample at that index (encryptModel skips absent fields — matches + // matrix-live.test.ts's batching). + const modelRows = boundaryIndices.map((i) => { + const row: Record = {} + for (const [t, spec] of boundaryDomains) { + if (i < spec.samples.length) row[slug(t)] = spec.samples[i] + } + return row + }) + + const encryptedRows = unwrapResult( + await client.bulkEncryptModels(modelRows as never, table as never), + ) as Array> + + // Insert each seeded row, populating only the columns present for that index + // and RETURNING the id so each case can target its exact row. + for (let r = 0; r < boundaryIndices.length; r++) { + const index = boundaryIndices[r] + const enc = encryptedRows[r] + const present = boundaryDomains.filter( + ([, spec]) => index < spec.samples.length, + ) + const colNames = present.map(([t]) => `"${slug(t)}"`) + const casts = present.map(([t], i) => `$${i + 2}::${t}`) + const values = present.map(([t]) => sql.json(enc[slug(t)] as never)) + const [row] = await sql.unsafe( + `INSERT INTO ${TABLE_NAME} (test_run_id, ${colNames.join(', ')}) + VALUES ($1, ${casts.join(', ')}) + RETURNING id`, + [TEST_RUN_ID, ...values], + ) + rowIdByIndex[index] = row.id + } +}, 120000) + +afterAll(async () => { + if (!LIVE_EQL_V3_PG_ENABLED) return + await sql.unsafe(`DELETE FROM ${TABLE_NAME} WHERE test_run_id = $1`, [ + TEST_RUN_ID, + ]) + await sql.end() +}, 30000) + +describeLivePg('v3 boundary-value coverage through real domain columns', () => { + it.each( + boundaryCases, + )('%s survives a real INSERT/SELECT and decrypts', async (_label, col, sample, index) => { + const [row] = await sql.unsafe>( + `SELECT "${col}"::jsonb AS value FROM ${TABLE_NAME} WHERE id = $1`, + [rowIdByIndex[index]], + ) + // Guard against a false pass: the stored cell must be real ciphertext. + expect(row.value).toHaveProperty('c') + + const decrypted = unwrapResult(await client.decrypt(row.value as never)) + // Boundary domains are numeric or text only (no date/timestamp), so the + // single-value decrypt returns the primitive directly. + expect(decrypted).toBe(sample) + }) +}) From dbe1ec07650ac18eb675231ee43f929f4a982296 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 7 Jul 2026 14:11:18 +1000 Subject: [PATCH 24/24] fix(stack): align batch query types with ffi 0.27 --- .../encryption/operations/batch-encrypt-query.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/stack/src/encryption/operations/batch-encrypt-query.ts b/packages/stack/src/encryption/operations/batch-encrypt-query.ts index ca5cfb3b..00f65a0b 100644 --- a/packages/stack/src/encryption/operations/batch-encrypt-query.ts +++ b/packages/stack/src/encryption/operations/batch-encrypt-query.ts @@ -2,7 +2,6 @@ import { type Result, withResult } from '@byteslice/result' import type { Encrypted as CipherStashEncrypted, EncryptedQuery as CipherStashEncryptedQuery, - EncryptedV3Query as CipherStashEncryptedV3Query, } from '@cipherstash/protect-ffi' import { encryptQueryBulk as ffiEncryptQueryBulk, @@ -83,17 +82,17 @@ function buildQueryPayload( */ function assembleResults( totalLength: number, - encryptedValues: ( - | CipherStashEncrypted - | CipherStashEncryptedQuery - | CipherStashEncryptedV3Query - )[], + // Typed as the FFI bulk-query return so it tracks the upstream union. As of + // protect-ffi 0.27 that union also includes `SteVecQuery`, but scalar query + // terms (all this operation builds) never produce a ste_vec result, so each + // element is narrowed back to the scalar shape `formatEncryptedResult` takes. + encryptedValues: Awaited>, nonNullTerms: { term: ScalarQueryTerm; originalIndex: number }[], ): EncryptedQueryResult[] { const results: EncryptedQueryResult[] = new Array(totalLength).fill(null) nonNullTerms.forEach(({ term, originalIndex }, i) => { results[originalIndex] = formatEncryptedResult( - encryptedValues[i], + encryptedValues[i] as CipherStashEncrypted | CipherStashEncryptedQuery, term.returnType, ) })