From 5f3370d01487c75971e7b4f81d97d19c52cf8a48 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 7 Aug 2026 16:48:17 +0200 Subject: [PATCH 01/14] feat: store event definitions in the sync plan --- .changeset/compiled-event-plans.md | 6 + .../implementation/MongoBucketBatch.ts | 6 +- .../src/storage/batch/PostgresBucketBatch.ts | 4 +- .../src/storage/BucketStorageFactory.ts | 12 +- .../src/storage/PersistedSyncConfigContent.ts | 25 +-- .../src/storage/ReplicationEventPayload.ts | 2 +- .../src/PersistedSyncConfigContent.test.ts | 92 +++++++++++ packages/sync-rules/src/HydratedSyncConfig.ts | 8 +- packages/sync-rules/src/SyncConfig.ts | 21 ++- packages/sync-rules/src/compiler/compiler.ts | 150 ++++++++++++++++- .../src/compiler/ir_to_sync_plan.ts | 57 ++++--- packages/sync-rules/src/compiler/rows.ts | 29 +++- .../src/events/CompiledEventSourceQuery.ts | 112 +++++++++++++ .../sync-rules/src/events/EventDescriptor.ts | 31 ++++ .../src/events/SqlEventDescriptor.ts | 67 -------- .../src/events/SqlEventSourceQuery.ts | 152 ------------------ packages/sync-rules/src/from_yaml.ts | 46 +++--- packages/sync-rules/src/index.ts | 4 +- .../sync_plan/evaluator/bucket_data_source.ts | 83 ++-------- .../src/sync_plan/evaluator/index.ts | 14 +- .../src/sync_plan/evaluator/row_projection.ts | 98 +++++++++++ packages/sync-rules/src/sync_plan/plan.ts | 41 ++++- .../src/sync_plan/plan_equality_serialized.ts | 99 ++++++++++++ .../sync-rules/src/sync_plan/serialize.ts | 115 ++++++++++++- .../test/src/compiler/events.test.ts | 139 ++++++++++++++++ .../test/src/legacy/sync_rules.test.ts | 14 +- .../sync_plan/evaluator/table_valued.test.ts | 3 +- .../src/sync_plan/schema_inference.test.ts | 2 +- 28 files changed, 1036 insertions(+), 396 deletions(-) create mode 100644 .changeset/compiled-event-plans.md create mode 100644 packages/service-core/test/src/PersistedSyncConfigContent.test.ts create mode 100644 packages/sync-rules/src/events/CompiledEventSourceQuery.ts create mode 100644 packages/sync-rules/src/events/EventDescriptor.ts delete mode 100644 packages/sync-rules/src/events/SqlEventDescriptor.ts delete mode 100644 packages/sync-rules/src/events/SqlEventSourceQuery.ts create mode 100644 packages/sync-rules/src/sync_plan/evaluator/row_projection.ts create mode 100644 packages/sync-rules/test/src/compiler/events.test.ts diff --git a/.changeset/compiled-event-plans.md b/.changeset/compiled-event-plans.md new file mode 100644 index 000000000..c50d948f7 --- /dev/null +++ b/.changeset/compiled-event-plans.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-sync-rules': minor +'@powersync/service-core': minor +--- + +Compile replication events for every sync-config edition into additive serialized sync-plan data and normalize legacy sidecar events when loading older plans while preserving raw SQL for older services. diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 02dce03e4..d6873a02e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,5 +1,5 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { HydratedSyncConfig, SqlEventDescriptor, SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; +import { HydratedEventDescriptor, HydratedSyncConfig, SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { @@ -938,9 +938,9 @@ export abstract class MongoBucketBatch } /** - * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} + * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable} */ - protected getTableEvents(table: storage.SourceTable): SqlEventDescriptor[] { + protected getTableEvents(table: storage.SourceTable): HydratedEventDescriptor[] { return this.sync_rules.eventDescriptors.filter((evt) => [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) ); diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index d3e883d8f..8e4809610 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -1334,10 +1334,10 @@ export class PostgresBucketBatch } /** - * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} + * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable} * TODO maybe share this with an abstract class */ - protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + protected getTableEvents(table: storage.SourceTable): sync_rules.HydratedEventDescriptor[] { return this.sync_rules.eventDescriptors.filter((evt) => [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) ); diff --git a/packages/service-core/src/storage/BucketStorageFactory.ts b/packages/service-core/src/storage/BucketStorageFactory.ts index 97adff689..398bbd9c8 100644 --- a/packages/service-core/src/storage/BucketStorageFactory.ts +++ b/packages/service-core/src/storage/BucketStorageFactory.ts @@ -180,11 +180,10 @@ export interface SerializedSyncPlan { plan: RawSerializedSyncPlan; compatibility: SerializedCompatibilityContext; /** - * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler - * yet. + * Raw event SQL persisted as a compatibility mirror for compiled {@link plan} events. * - * We might revisit that in the future, but for now we store SQL text of their definitions here to be able to restore - * them. + * Compiled events are an additive plan field. Older services ignore that field and restore these descriptors through + * the legacy evaluator. Keep dual-writing this field until a future plan version explicitly removes that support. */ eventDescriptors: Record; errors?: ReplicationError[]; @@ -212,13 +211,14 @@ export function updateSyncRulesFromConfig( const { config, errors } = parsed; if (config instanceof PrecompiledSyncConfig) { const eventDescriptors: Record = {}; - for (const event of config.eventDescriptors) { - eventDescriptors[event.name] = event.sourceQueries.map((q) => q.sql); + for (const event of config.plan.events) { + eventDescriptors[event.name] = event.sourceQueries.map((query) => query.sql); } plan = { compatibility: config.compatibility.serialize(), plan: serializeSyncPlan(config.plan), + // Dual-write raw SQL so older services can ignore additive compiled plan events without losing event behavior. eventDescriptors, errors: errors.map((e) => syncConfigYamlErrorToReplicationError(e)) }; diff --git a/packages/service-core/src/storage/PersistedSyncConfigContent.ts b/packages/service-core/src/storage/PersistedSyncConfigContent.ts index 6ee3fc5a7..f98c43941 100644 --- a/packages/service-core/src/storage/PersistedSyncConfigContent.ts +++ b/packages/service-core/src/storage/PersistedSyncConfigContent.ts @@ -2,6 +2,7 @@ import { logger as defaultLogger, ErrorCode, ServiceError } from '@powersync/lib import { CompatibilityContext, CompatibilityOption, + compileEventDefinitions, DEFAULT_HYDRATION_STATE, deserializeSyncPlan, ErrorLocation, @@ -9,7 +10,6 @@ import { HydrationState, nodeSqlite, PrecompiledSyncConfig, - SqlEventDescriptor, SqlSyncRules, SyncConfigWithErrors, versionedHydrationState, @@ -40,17 +40,23 @@ export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfi const plan = deserializeSyncPlan(compiledPlan.plan); const compatibility = CompatibilityContext.deserialize(compiledPlan.compatibility); - const eventDefinitions: SqlEventDescriptor[] = []; - for (const [name, queries] of Object.entries(compiledPlan.eventDescriptors)) { - const descriptor = new SqlEventDescriptor(name, compatibility); - for (const query of queries) { - descriptor.addSourceQuery(query, parseOptions); + const errors: YamlError[] = []; + // Compiled events are additive to plan versions 1 and 2. New readers prefer them when present; when an older plan + // does not contain them, normalize the dual-written raw SQL at this loading boundary. This keeps legacy event + // evaluators out of PrecompiledSyncConfig while older binaries can continue reading the same persisted config. + if (compiledPlan.plan.events == null) { + const normalized = compileEventDefinitions(compiledPlan.eventDescriptors, parseOptions); + const fatalErrors = normalized.errors.filter((error) => error.type == 'fatal'); + if (fatalErrors.length != 0) { + throw new Error( + `Failed to compile persisted replication events: ${fatalErrors.map((error) => error.message).join(', ')}` + ); } - - eventDefinitions.push(descriptor); + plan.events = normalized.events; + errors.push(...normalized.errors.map((error) => new YamlError(error))); } - const precompiled = new PrecompiledSyncConfig(plan, compatibility, eventDefinitions, { + const precompiled = new PrecompiledSyncConfig(plan, compatibility, { defaultSchema: parseOptions.defaultSchema, sourceText: content }); @@ -59,7 +65,6 @@ export function parsePersistedSyncConfigContent(options: ParsePersistedSyncConfi // This means asUpdateOptions will not change the storage version, even if the default changes. precompiled.storageVersion = storageVersion; - const errors: YamlError[] = []; if (compiledPlan.errors) { for (const error of compiledPlan.errors) { const location: ErrorLocation | undefined = error.location && { diff --git a/packages/service-core/src/storage/ReplicationEventPayload.ts b/packages/service-core/src/storage/ReplicationEventPayload.ts index 38cddbc24..31229aab1 100644 --- a/packages/service-core/src/storage/ReplicationEventPayload.ts +++ b/packages/service-core/src/storage/ReplicationEventPayload.ts @@ -11,6 +11,6 @@ export type EventData = { export type ReplicationEventPayload = { batch: BucketStorageBatch; data: EventData; - event: sync_rules.SqlEventDescriptor; + event: sync_rules.HydratedEventDescriptor; table: SourceTable; }; diff --git a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts new file mode 100644 index 000000000..4fe8c926a --- /dev/null +++ b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts @@ -0,0 +1,92 @@ +import { + DEFAULT_HYDRATION_STATE, + DEFAULT_TAG, + nodeSqlite, + PrecompiledSyncConfig, + SqlSyncRules +} from '@powersync/service-sync-rules'; +import * as sqlite from 'node:sqlite'; +import { describe, expect, test } from 'vitest'; +import { + parsePersistedSyncConfigContent, + SerializedSyncPlan, + updateSyncRulesFromConfig +} from '../../src/storage/storage-index.js'; + +const EVENT_QUERY = 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true'; +const yamlWithoutEvents = ` +config: + edition: 3 + +streams: + checkpoints: + query: SELECT * FROM checkpoints +`; +const yamlWithEvents = `${yamlWithoutEvents} +event_definitions: + write_checkpoints: + payloads: + - ${EVENT_QUERY} +`; + +describe('persisted compiled replication events', () => { + test('dual-writes additive compiled events and restores the compiled evaluator', () => { + const parsed = SqlSyncRules.fromYaml(yamlWithEvents, { defaultSchema: 'test_schema' }); + const update = updateSyncRulesFromConfig(parsed); + const compiled = update.config.plan!; + + expect(compiled.plan.version).toBeLessThanOrEqual(2); + expect(compiled.plan.events).toHaveLength(1); + expect(compiled.eventDescriptors).toEqual({ write_checkpoints: [EVENT_QUERY] }); + + const restored = restore(compiled); + expect(restored.config).toBeInstanceOf(PrecompiledSyncConfig); + expect(restored.config).not.toHaveProperty('eventDescriptors'); + expect(restored.config.eventDefinitions).toHaveLength(1); + + const hydrated = restored.config.hydrate({ + hydrationState: DEFAULT_HYDRATION_STATE, + sqlite: nodeSqlite(sqlite) + }); + expect( + hydrated.eventDescriptors[0].evaluateRowWithErrors({ + sourceTable: { connectionTag: DEFAULT_TAG, schema: 'test_schema', name: 'checkpoints' }, + record: { user_id: 'user-1', checkpoint: 4n, active: 1 } + }) + ).toEqual({ result: { data: { user_id: 'user-1', checkpoint: 4n } }, errors: [] }); + + // This represents what an older compiler sees after ignoring the additive plan.events field. The raw descriptor + // mirror is normalized into the compiled representation by the new loading boundary. + const { events: _ignored, ...planWithoutCompiledEvents } = compiled.plan; + const legacyView = restore({ ...compiled, plan: planWithoutCompiledEvents }); + expect(legacyView.config).not.toHaveProperty('eventDescriptors'); + expect(legacyView.config.eventDefinitions).toHaveLength(1); + expect((legacyView.config as PrecompiledSyncConfig).plan.events).toHaveLength(1); + }); + + test('restores raw event descriptors attached to version 1 and 2 plans', () => { + const parsed = SqlSyncRules.fromYaml(yamlWithoutEvents, { defaultSchema: 'test_schema' }); + const update = updateSyncRulesFromConfig(parsed); + const legacy: SerializedSyncPlan = { + ...update.config.plan!, + eventDescriptors: { write_checkpoints: [EVENT_QUERY] } + }; + + expect(legacy.plan.version).toBeLessThanOrEqual(2); + const restored = restore(legacy); + expect(restored.config).not.toHaveProperty('eventDescriptors'); + expect(restored.config.eventDefinitions).toHaveLength(1); + expect((restored.config as PrecompiledSyncConfig).plan.events).toMatchObject([ + { name: 'write_checkpoints', sourceQueries: [{ sql: EVENT_QUERY }] } + ]); + }); +}); + +function restore(compiledPlan: SerializedSyncPlan) { + return parsePersistedSyncConfigContent({ + content: yamlWithEvents, + compiledPlan, + storageVersion: 1, + parseOptions: { defaultSchema: 'test_schema' } + }); +} diff --git a/packages/sync-rules/src/HydratedSyncConfig.ts b/packages/sync-rules/src/HydratedSyncConfig.ts index bbcbf0dd9..211bf90cd 100644 --- a/packages/sync-rules/src/HydratedSyncConfig.ts +++ b/packages/sync-rules/src/HydratedSyncConfig.ts @@ -8,6 +8,7 @@ import { EvaluationError, GetBucketParameterQuerierResult, GetQuerierOptions, + HydratedEventDescriptor, HydrateSyncConfigParams, HydrationInput, isEvaluatedParameters, @@ -21,7 +22,6 @@ import { QuerierError, ScopedEvaluateParameterRow, ScopedEvaluateRow, - SqlEventDescriptor, SqliteInputValue, SqliteValue, SyncConfig, @@ -52,7 +52,7 @@ export class HydratedSyncConfig { */ private bucketSources: HydratedBucketSource[] = []; - eventDescriptors: SqlEventDescriptor[] = []; + eventDescriptors: HydratedEventDescriptor[] = []; /** * Only a single compatibility context is supported across all merged SyncConfigs. @@ -118,7 +118,9 @@ export class HydratedSyncConfig { this.bucketParameterLookupSources ).evaluateParameterRow; - this.eventDescriptors = definitions.flatMap((definition) => definition.eventDescriptors); + this.eventDescriptors = definitions.flatMap((definition) => + definition.eventDefinitions.map((event) => event.createEvaluator(this.hydrationInput)) + ); if (definitions.length == 1) { this.#bucketSourceDefinitions = definitions[0].bucketSources; diff --git a/packages/sync-rules/src/SyncConfig.ts b/packages/sync-rules/src/SyncConfig.ts index fc3d50da0..f777c83f3 100644 --- a/packages/sync-rules/src/SyncConfig.ts +++ b/packages/sync-rules/src/SyncConfig.ts @@ -6,7 +6,7 @@ import { } from './BucketSource.js'; import { CompatibilityContext } from './compatibility.js'; import { YamlError } from './errors.js'; -import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; +import { EventDefinition } from './events/EventDescriptor.js'; import { HydratedSyncConfig } from './HydratedSyncConfig.js'; import { SourceTableRef } from './SourceTableRef.js'; import { TablePattern } from './TablePattern.js'; @@ -21,6 +21,8 @@ export abstract class SyncConfig { bucketDataSources: BucketDataSource[] = []; bucketParameterLookupSources: ParameterIndexLookupCreator[] = []; bucketSources: BucketSource[] = []; + /** Prepared event definitions. Executable event descriptors only exist on {@link HydratedSyncConfig}. */ + eventDefinitions: EventDefinition[] = []; compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; /** * If not defined, the storage module picks the latest stable version. @@ -28,7 +30,6 @@ export abstract class SyncConfig { * Only supported storage versions can be set here when parsing from yaml. */ storageVersion: number | undefined; - eventDescriptors: SqlEventDescriptor[] = []; /** * The (YAML-based) source contents from which this sync config has been derived. @@ -71,9 +72,9 @@ export abstract class SyncConfig { sourceTables.set(r.key(), r); } } - for (const event of this.eventDescriptors) { - for (const r of event.getSourceTables()) { - sourceTables.set(r.key(), r); + for (const event of this.eventDefinitions) { + for (const table of event.getSourceTables()) { + sourceTables.set(table.key(), table); } } } @@ -87,11 +88,9 @@ export abstract class SyncConfig { getEventTables(): TablePattern[] { const eventTables = new Map(); - if (this.eventDescriptors) { - for (const event of this.eventDescriptors) { - for (const r of event.getSourceTables()) { - eventTables.set(r.key(), r); - } + for (const event of this.eventDefinitions) { + for (const table of event.getSourceTables()) { + eventTables.set(table.key(), table); } } @@ -99,7 +98,7 @@ export abstract class SyncConfig { } tableTriggersEvent(table: SourceTableRef): boolean { - return this.eventDescriptors.some((bucket) => bucket.tableTriggersEvent(table)); + return this.eventDefinitions.some((event) => event.tableTriggersEvent(table)); } tableSyncsData(table: SourceTableRef): boolean { diff --git a/packages/sync-rules/src/compiler/compiler.ts b/packages/sync-rules/src/compiler/compiler.ts index 13735744c..f7f9b68a2 100644 --- a/packages/sync-rules/src/compiler/compiler.ts +++ b/packages/sync-rules/src/compiler/compiler.ts @@ -1,16 +1,19 @@ import { NodeLocation, parse, PGNode, Statement } from 'pgsql-ast-parser'; -import { StreamOptions, SyncPlan } from '../sync_plan/plan.js'; +import { SqlRuleError } from '../errors.js'; +import { CompiledEventDescriptor, StreamOptions, SyncPlan } from '../sync_plan/plan.js'; import { SourceSchema } from '../types.js'; import { StreamResolver } from './bucket_resolver.js'; import { DangerousParameterDetector } from './detect_dangerous_parameters.js'; import { HashSet } from './equality.js'; import { NodeLocations } from './expression.js'; +import { RowExpression, SingleDependencyExpression } from './filter.js'; import { CompilerModelToSyncPlan } from './ir_to_sync_plan.js'; import { StreamQueryParser } from './parser.js'; import { QuerierGraphBuilder } from './querier_graph.js'; -import { PointLookup, RowEvaluator } from './rows.js'; +import { EventRowEvaluator, PointLookup, RowEvaluator } from './rows.js'; import { SqlScope } from './scope.js'; import { CommonTableExpression, PreparedSubquery } from './sqlite.js'; +import { PhysicalSourceResultSet } from './table.js'; export interface SyncStreamsCompilerOptions { /** @@ -34,6 +37,17 @@ export interface ParseStreamOptions extends StreamOptions { warnOnDangerousParameter: boolean; } +export interface CompiledEvent { + name: string; + sourceQueries: CompiledEventSourceQueryModel[]; +} + +export interface CompiledEventSourceQueryModel { + sql: string; + sourceTable: PhysicalSourceResultSet; + variants: EventRowEvaluator[]; +} + /** * State for compiling sync streams. * @@ -124,11 +138,133 @@ export class SyncStreamsCompiler { } }; } + + /** + * Compiles the payload queries for a named replication event. + * + * Event queries intentionally support a smaller surface than stream queries: They must project and filter a single + * physical source table and cannot depend on request parameters, joins, subqueries or table-valued functions. + */ + event(name: string): IndividualEventCompiler { + const event: CompiledEvent = { name, sourceQueries: [] }; + this.output.events.push(event); + + return { + addSourceQuery: (sql: string, errors: ParsingErrorListener) => { + const stmt = tryParse(sql, errors); + if (stmt == null) { + return; + } + + const parser = new StreamQueryParser({ + compiler: this, + originalText: sql, + locations: this.locations, + parentScope: new SqlScope({}), + errors + }); + const query = parser.parse(stmt); + if (query == null) { + return; + } + + if (query.joined.length != 0) { + errors.report('Event payload queries must SELECT from a single physical source table.', query.span.location); + return; + } + + const defaultSchema = this.options.defaultSchema ?? ''; + const sourceTable = query.sourceTable.tablePattern.toTablePattern(defaultSchema); + if ( + event.sourceQueries.some((source) => + source.sourceTable.tablePattern.toTablePattern(defaultSchema).equals(sourceTable) + ) + ) { + errors.report('Each payload query should query a unique table', query.span.location); + return; + } + + const variants: EventRowEvaluator[] = []; + let valid = true; + for (const variant of query.where.terms) { + const filters: RowExpression[] = []; + for (const term of variant.terms) { + if ( + !(term instanceof SingleDependencyExpression) || + term.dependsOnConnection || + (term.resultSet != null && term.resultSet !== query.sourceTable) + ) { + errors.report( + 'Event payload queries cannot depend on request parameters or other tables.', + term instanceof SingleDependencyExpression ? term.expression.location.location : term.location! + ); + valid = false; + continue; + } + + filters.push(new RowExpression(term)); + } + + variants.push( + new EventRowEvaluator({ + columns: query.resultColumns, + syntacticSource: query.sourceTable, + filters, + partitionBy: [], + addedFunctions: [] + }) + ); + } + + if (valid) { + event.sourceQueries.push({ sql, sourceTable: query.sourceTable, variants }); + } + } + }; + } +} + +/** + * Compiles raw event SQL stored alongside older sync plans into the current plan representation. + * + * This is the compatibility boundary for plans written before compiled events were added. Callers should reject fatal + * errors rather than carrying legacy event evaluators into a {@link SyncPlan}. + */ +export function compileEventDefinitions( + definitions: Readonly>, + options: SyncStreamsCompilerOptions +): { events: CompiledEventDescriptor[]; errors: SqlRuleError[] } { + const compiler = new SyncStreamsCompiler(options); + const errors: SqlRuleError[] = []; + + for (const [name, queries] of Object.entries(definitions)) { + const event = compiler.event(name); + for (const sql of queries) { + event.addSourceQuery(sql, { + report(message, location, reportOptions) { + const error = new SqlRuleError(message, sql, location); + error.type = reportOptions?.isWarning ? 'warning' : 'fatal'; + errors.push(error); + } + }); + } + } + + return { events: compiler.output.toSyncPlan().events, errors }; } function tryParse(sql: string, errors: ParsingErrorListener): Statement | null { try { - const [stmt] = parse(sql, { locationTracking: true }); + const statements = parse(sql, { locationTracking: true }); + if (statements.length != 1) { + errors.report( + 'Only a single SELECT statement is supported', + statements[1]?._location ?? { start: 0, end: sql.length } + ); + return null; + } + + const [stmt] = statements; return stmt; } catch (e: any) { const location: NodeLocation | undefined = e.token?._location; @@ -161,6 +297,13 @@ export interface IndividualSyncStreamCompiler { finish(): void; } +export interface IndividualEventCompiler { + /** + * Validates and adds one payload query to this event. + */ + addSourceQuery(sql: string, errors: ParsingErrorListener): void; +} + /** * Something reporting errors. * @@ -189,6 +332,7 @@ export class CompiledStreamQueries { }); readonly resolvers: StreamResolver[] = []; + readonly events: CompiledEvent[] = []; get evaluators(): RowEvaluator[] { return [...this._evaluators]; diff --git a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts index 5cc3a3992..157e723ac 100644 --- a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts +++ b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts @@ -60,7 +60,15 @@ export class CompilerModelToSyncPlan { queriers: resolvers!.map((e) => this.translateStreamResolver(e)) }; }), - buckets: this.buckets + buckets: this.buckets, + events: source.events.map((event) => ({ + name: event.name, + sourceQueries: event.sourceQueries.map((query) => ({ + sql: query.sql, + sourceTable: query.sourceTable.tablePattern, + variants: query.variants.map((variant) => this.translateRowProjection(variant)) + })) + })) }; } @@ -112,32 +120,39 @@ export class CompilerModelToSyncPlan { private translateRowEvaluator(value: rows.RowEvaluator): plan.StreamDataSource { return this.translateStatefulObject(value, () => { - const hasher = new StableHasher(); - value.buildBehaviorHashCode(hasher); const mapped = { - sourceTable: value.tablePattern, - hashCode: hasher.buildHashCode(), - tableValuedFunctions: this.translateAddedTableValuedFunctions(value.addedFunctions, value), - columns: value.columns.map((e) => { - if (e instanceof rows.StarColumnSource) { - return 'star'; - } else { - return { - expr: this.translateExpression(e.expression.expression, value.syntacticSource, value.addedFunctions), - alias: e.alias ?? null - }; - } - }), - outputTableName: value.outputName, - filters: value.filters.map((e) => - this.translateExpression(e.expression, value.syntacticSource, value.addedFunctions) - ), - parameters: value.partitionBy.map((e) => this.translatePartitionKey(e, value)) + ...this.translateRowProjection(value), + outputTableName: value.outputName } satisfies plan.StreamDataSource; return mapped; }); } + private translateRowProjection(value: rows.RowEvaluator | rows.EventRowEvaluator): plan.RowProjection { + const hasher = new StableHasher(); + value.buildBehaviorHashCode(hasher); + + return { + sourceTable: value.tablePattern, + hashCode: hasher.buildHashCode(), + tableValuedFunctions: this.translateAddedTableValuedFunctions(value.addedFunctions, value), + columns: value.columns.map((e) => { + if (e instanceof rows.StarColumnSource) { + return 'star'; + } else { + return { + expr: this.translateExpression(e.expression.expression, value.syntacticSource, value.addedFunctions), + alias: e.alias ?? null + }; + } + }), + filters: value.filters.map((e) => + this.translateExpression(e.expression, value.syntacticSource, value.addedFunctions) + ), + parameters: value.partitionBy.map((e) => this.translatePartitionKey(e, value)) + }; + } + private translatePointLookup(value: rows.PointLookup, index: number): plan.StreamParameterIndexLookupCreator { return this.translateStatefulObject(value, () => { const hasher = new StableHasher(); diff --git a/packages/sync-rules/src/compiler/rows.ts b/packages/sync-rules/src/compiler/rows.ts index 73f544e6d..e100b2b6f 100644 --- a/packages/sync-rules/src/compiler/rows.ts +++ b/packages/sync-rules/src/compiler/rows.ts @@ -71,7 +71,7 @@ export class TableValuedPartitionKey extends PartitionKey { * This includes {@link RowEvaluator}s, which assigns rows into buckets, and {@link PointLookup}, which creates * parameter lookups used to resolve bucket ids when a user connects. */ -export type SourceRowProcessor = RowEvaluator | PointLookup; +export type SourceRowProcessor = RowEvaluator | EventRowEvaluator | PointLookup; interface SourceProcessorOptions { readonly syntacticSource: PhysicalSourceResultSet; @@ -205,6 +205,33 @@ export class RowEvaluator extends BaseSourceRowProcessor { } } +/** + * A row evaluator producing an event payload. + * + * Unlike {@link RowEvaluator}, the alias of the source table does not affect behavior because event payloads don't + * have a logical output table name. + */ +export class EventRowEvaluator extends BaseSourceRowProcessor { + readonly columns: ColumnSource[]; + + constructor(options: SourceProcessorOptions & { columns: ColumnSource[] }) { + super(options); + this.columns = options.columns; + } + + buildBehaviorHashCode(hasher: StableHasher): void { + this.addBaseHashCode(hasher); + // An event's projected columns, expressions and aliases define the payload delivered to its handler. Changing + // them therefore changes event behavior, so include them here to keep this hash consistent with + // behavesIdenticalTo() and with the compiled definition that will be reprocessed. + equalsIgnoringResultSetList.hash(hasher, this.columns); + } + + behavesIdenticalTo(other: EventRowEvaluator): boolean { + return this.baseMatchesOther(other) && equalsIgnoringResultSetList.equals(other.columns, this.columns); + } +} + /** * A point lookup, creating a materialized index. * diff --git a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts new file mode 100644 index 000000000..e274cdd0f --- /dev/null +++ b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts @@ -0,0 +1,112 @@ +import { HydrationInput } from '../BucketSource.js'; +import { SourceTableRef } from '../SourceTableRef.js'; +import { EvaluatedRowProjection, PendingRowProjection } from '../sync_plan/evaluator/row_projection.js'; +import { + CompiledEventDescriptor as CompiledEventDescriptorPlan, + CompiledEventSourceQuery as CompiledEventSourceQueryPlan +} from '../sync_plan/plan.js'; +import { TablePattern } from '../TablePattern.js'; +import { EvaluateRowOptions, SqliteRow } from '../types.js'; +import { EvaluatedEventRowWithErrors, EventDefinition, HydratedEventDescriptor } from './EventDescriptor.js'; + +/** A named event prepared from a compiled sync plan, before scalar expressions are prepared for evaluation. */ +export class PreparedEventDefinition implements EventDefinition { + readonly name: string; + readonly sourceQueries: PreparedEventSourceQuery[]; + + constructor(source: CompiledEventDescriptorPlan, defaultSchema: string) { + this.name = source.name; + this.sourceQueries = source.sourceQueries.map((query) => new PreparedEventSourceQuery(query, defaultSchema)); + } + + createEvaluator(input: HydrationInput): HydratedEventDescriptor { + return new HydratedCompiledEventDescriptor( + this.name, + this.sourceQueries.map((query) => query.createEvaluator(input)) + ); + } + + getSourceTables(): Set { + return sourceTables(this.sourceQueries); + } + + tableTriggersEvent(table: SourceTableRef): boolean { + return this.sourceQueries.some((query) => query.applies(table)); + } +} + +/** An event source query prepared from a compiled sync plan, before scalar expressions are evaluated. */ +export class PreparedEventSourceQuery { + readonly sourceTable: TablePattern; + private readonly variants: PendingRowProjection[]; + + constructor(source: CompiledEventSourceQueryPlan, defaultSchema: string) { + this.sourceTable = source.sourceTable.toTablePattern(defaultSchema); + this.variants = source.variants.map((variant) => new PendingRowProjection(variant, defaultSchema)); + } + + applies(table: SourceTableRef): boolean { + return this.sourceTable.matches(table); + } + + createEvaluator(input: HydrationInput): HydratedCompiledEventSourceQuery { + return new HydratedCompiledEventSourceQuery( + this.sourceTable, + this.variants.map((variant) => variant.instantiate(input.scalarExpressions)) + ); + } +} + +class HydratedCompiledEventDescriptor implements HydratedEventDescriptor { + constructor( + readonly name: string, + readonly sourceQueries: HydratedCompiledEventSourceQuery[] + ) {} + + evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors { + const matchingQuery = this.sourceQueries.find((query) => query.applies(options.sourceTable)); + if (matchingQuery == null) { + return { errors: [{ error: `No matching source query found for table ${options.sourceTable.name}` }] }; + } + + return matchingQuery.evaluateRowWithErrors(options.sourceTable, options.record); + } + + getSourceTables(): Set { + return sourceTables(this.sourceQueries); + } + + tableTriggersEvent(table: SourceTableRef): boolean { + return this.sourceQueries.some((query) => query.applies(table)); + } +} + +class HydratedCompiledEventSourceQuery { + constructor( + readonly sourceTable: TablePattern, + private readonly variants: ((options: EvaluateRowOptions) => EvaluatedRowProjection[])[] + ) {} + + applies(table: SourceTableRef): boolean { + return this.sourceTable.matches(table); + } + + evaluateRowWithErrors(table: SourceTableRef, row: SqliteRow): EvaluatedEventRowWithErrors { + try { + for (const evaluate of this.variants) { + const [result] = evaluate({ sourceTable: table, record: row }); + if (result != null) { + return { result: { data: result.data }, errors: [] }; + } + } + + return { errors: [] }; + } catch (error) { + return { errors: [{ error: error instanceof Error ? error.message : 'Evaluating event query failed' }] }; + } + } +} + +function sourceTables(queries: readonly { sourceTable: TablePattern }[]): Set { + return new Set(queries.map((query) => query.sourceTable)); +} diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts new file mode 100644 index 000000000..cc9da2474 --- /dev/null +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -0,0 +1,31 @@ +import { HydrationInput } from '../BucketSource.js'; +import { SourceTableRef } from '../SourceTableRef.js'; +import { TablePattern } from '../TablePattern.js'; +import { EvaluateRowOptions, EvaluationError, SqliteJsonRow } from '../types.js'; + +export type EvaluatedEventSourceRow = { + data: SqliteJsonRow; +}; + +export type EvaluatedEventRowWithErrors = { + result?: EvaluatedEventSourceRow; + errors: EvaluationError[]; +}; + +/** A parsed event definition whose compiled expressions have not yet been prepared for evaluation. */ +export interface EventDefinition { + readonly name: string; + + createEvaluator(input: HydrationInput): HydratedEventDescriptor; + getSourceTables(): Set; + tableTriggersEvent(table: SourceTableRef): boolean; +} + +/** An event definition whose payload queries can evaluate replicated rows. */ +export interface HydratedEventDescriptor { + readonly name: string; + + evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors; + getSourceTables(): Set; + tableTriggersEvent(table: SourceTableRef): boolean; +} diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts deleted file mode 100644 index 9de432d6c..000000000 --- a/packages/sync-rules/src/events/SqlEventDescriptor.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { CompatibilityContext } from '../compatibility.js'; -import { SqlRuleError } from '../errors.js'; -import { QueryParseResult } from '../legacy/SqlBucketDescriptor.js'; -import { SourceTableRef } from '../SourceTableRef.js'; -import { SyncRulesOptions } from '../SqlSyncRules.js'; -import { TablePattern } from '../TablePattern.js'; -import { EvaluateRowOptions } from '../types.js'; -import { EvaluatedEventRowWithErrors, SqlEventSourceQuery } from './SqlEventSourceQuery.js'; - -/** - * A sync config event which is triggered from a SQL table change. - */ -export class SqlEventDescriptor { - name: string; - sourceQueries: SqlEventSourceQuery[] = []; - - constructor( - name: string, - private readonly compatibility: CompatibilityContext - ) { - this.name = name; - } - - addSourceQuery(sql: string, options: SyncRulesOptions): QueryParseResult { - const source = SqlEventSourceQuery.fromSql(sql, options, this.compatibility); - - // Each source query should be for a unique table - const existingSourceQuery = this.sourceQueries.find((q) => q.table == source.table); - if (existingSourceQuery) { - return { - parsed: false, - errors: [new SqlRuleError('Each payload query should query a unique table', sql)] - }; - } - - this.sourceQueries.push(source); - - return { - parsed: true, - errors: source.errors - }; - } - - evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors { - // There should only be 1 payload result per source query - const matchingQuery = this.sourceQueries.find((q) => q.applies(options.sourceTable)); - if (!matchingQuery) { - return { - errors: [{ error: `No matching source query found for table ${options.sourceTable.name}` }] - }; - } - - return matchingQuery.evaluateRowWithErrors(options.sourceTable, options.record); - } - - getSourceTables(): Set { - let result = new Set(); - for (let query of this.sourceQueries) { - result.add(query.sourceTable!); - } - return result; - } - - tableTriggersEvent(table: SourceTableRef): boolean { - return this.sourceQueries.some((query) => query.applies(table)); - } -} diff --git a/packages/sync-rules/src/events/SqlEventSourceQuery.ts b/packages/sync-rules/src/events/SqlEventSourceQuery.ts deleted file mode 100644 index f19fc5b25..000000000 --- a/packages/sync-rules/src/events/SqlEventSourceQuery.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { parse } from 'pgsql-ast-parser'; -import { CompatibilityContext } from '../compatibility.js'; -import { SqlRuleError } from '../errors.js'; -import { ExpressionType } from '../ExpressionType.js'; -import { BaseSqlDataQuery, BaseSqlDataQueryOptions, RowValueExtractor } from '../legacy/BaseSqlDataQuery.js'; -import { AvailableTable, SqlTools } from '../legacy/sql_filters.js'; -import { checkUnsupportedFeatures, isClauseError } from '../legacy/sql_support.js'; -import { TableQuerySchema } from '../legacy/TableQuerySchema.js'; -import { SourceTableRef } from '../SourceTableRef.js'; -import { SyncRulesOptions } from '../SqlSyncRules.js'; -import { TablePattern } from '../TablePattern.js'; -import { EvaluationError, QuerySchema, SqliteJsonRow, SqliteRow } from '../types.js'; -import { isSelectStatement } from '../utils.js'; - -export type EvaluatedEventSourceRow = { - data: SqliteJsonRow; -}; - -export type EvaluatedEventRowWithErrors = { - result?: EvaluatedEventSourceRow; - errors: EvaluationError[]; -}; - -/** - * Defines how a Replicated Row is mapped to source parameters for events. - */ -// TODO: Use Sync Streams compiler infrastruture instead of legacy data queries -export class SqlEventSourceQuery extends BaseSqlDataQuery { - static fromSql(sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) { - const parsed = parse(sql, { locationTracking: true }); - const schema = options.schema; - - if (parsed.length > 1) { - throw new SqlRuleError('Only a single SELECT statement is supported', sql, parsed[1]?._location); - } - const q = parsed[0]; - if (!isSelectStatement(q)) { - throw new SqlRuleError('Only SELECT statements are supported', sql, q._location); - } - - let errors: SqlRuleError[] = []; - - errors.push(...checkUnsupportedFeatures(sql, q)); - - if (q.from == null || q.from.length != 1 || q.from[0].type != 'table') { - throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); - } - - const tableRef = q.from?.[0].name; - if (tableRef?.name == null) { - throw new SqlRuleError('Must SELECT from a single table', sql, q.from?.[0]._location); - } - const alias = AvailableTable.fromAst(tableRef); - - const sourceTable = new TablePattern(tableRef.schema ?? options.defaultSchema, tableRef.name); - let querySchema: QuerySchema | undefined = undefined; - if (schema) { - const tables = schema.getTables(sourceTable); - if (tables.length == 0) { - const e = new SqlRuleError( - `Table ${sourceTable.schema}.${sourceTable.tablePattern} not found`, - sql, - q.from?.[0]?._location - ); - e.type = 'warning'; - - errors.push(e); - } else { - querySchema = new TableQuerySchema(tables, alias); - } - } - - const tools = new SqlTools({ - table: alias, - parameterTables: [], - valueTables: [alias], - sql, - schema: querySchema, - compatibilityContext: compatibility - }); - - let extractors: RowValueExtractor[] = []; - for (let column of q.columns ?? []) { - const name = tools.getOutputName(column); - if (name != '*') { - const clause = tools.compileRowValueExtractor(column.expr); - if (isClauseError(clause)) { - // Error logged already - continue; - } - extractors.push({ - extract: (tables, output) => { - output[name] = clause.evaluate(tables); - }, - getTypes(schema, into) { - const def = clause.getColumnDefinition(schema); - into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType }; - } - }); - } else { - extractors.push({ - extract: (tables, output) => { - const row = tables[alias.nameInSchema]; - for (let key in row) { - if (key.startsWith('_')) { - continue; - } - output[key] ??= row[key]; - } - }, - getTypes(schema, into) { - for (let column of schema.getColumns(alias.nameInSchema)) { - into[column.name] ??= column; - } - } - }); - } - } - errors.push(...tools.errors); - - return new SqlEventSourceQuery({ - sourceTable, - table: alias, - sql, - columns: q.columns ?? [], - extractors: extractors, - tools, - bucketParameters: [], - errors: errors - }); - } - - constructor(options: BaseSqlDataQueryOptions) { - super(options); - } - - evaluateRowWithErrors(table: SourceTableRef, row: SqliteRow): EvaluatedEventRowWithErrors { - try { - const tables = { [this.table!.nameInSchema]: this.addSpecialParameters(table, row) }; - - const data = this.transformRow(tables); - return { - result: { - data - }, - errors: [] - }; - } catch (e) { - return { errors: [e.message ?? `Evaluating data query failed`] }; - } - } -} diff --git a/packages/sync-rules/src/from_yaml.ts b/packages/sync-rules/src/from_yaml.ts index e03501667..c923c08eb 100644 --- a/packages/sync-rules/src/from_yaml.ts +++ b/packages/sync-rules/src/from_yaml.ts @@ -9,13 +9,14 @@ import { import { ParsingErrorListener, SyncStreamsCompiler } from './compiler/compiler.js'; import { CommonTableExpression } from './compiler/sqlite.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; -import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; +import { PreparedEventDefinition } from './events/CompiledEventSourceQuery.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { QueryParseResult, SqlBucketDescriptor } from './legacy/SqlBucketDescriptor.js'; import { syncStreamFromSql } from './legacy/streams/from_sql.js'; import { SqlSyncRules } from './SqlSyncRules.js'; import { validateStorageVersion } from './StorageVersion.js'; import { PrecompiledSyncConfig } from './sync_plan/evaluator/index.js'; +import { CompiledEventDescriptor } from './sync_plan/plan.js'; import { SyncConfig, SyncConfigWithErrors } from './SyncConfig.js'; import { TablePattern } from './TablePattern.js'; import { QueryParseOptions, SourceSchema, StreamParseOptions } from './types.js'; @@ -118,23 +119,24 @@ export class SyncConfigFromYaml { const bucketMap = rootState.get('bucket_definitions')?.requireMap(); const streamMap = rootState.get('streams')?.requireMap(); const globalCtes = rootState.get('with')?.requireMap(); + const eventMap = rootState.get('event_definitions')?.requireMap(); let result: SyncConfig; if (compatibility.edition >= CompatibilityEdition.COMPILED_STREAMS) { - result = this.#compileSyncPlan(bucketMap, streamMap, globalCtes, compatibility); + result = this.#compileSyncPlan(bucketMap, streamMap, globalCtes, eventMap, compatibility); this.#warnOnUnusedCtes(); } else { // We don't support CTEs at all in this compiler implementation. globalCtes?.reportError('Common table expressions require edition 3.'); - result = this.#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility); + const eventCompiler = new SyncStreamsCompiler(this.options); + this.#compileEventDefinitions(eventMap, eventCompiler); + const eventPlan = eventCompiler.output.toSyncPlan(); + result = this.#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility, eventPlan.events); } result.storageVersion = storageVersion; - const eventDefinitions = this.#parseEventDefinitions(rootState, compatibility); - result.eventDescriptors.push(...eventDefinitions); - return result; } @@ -190,6 +192,7 @@ export class SyncConfigFromYaml { bucketMap: YamlMapState | undefined, streamMap: YamlMapState | undefined, globalCtes: YamlMapState | undefined, + eventMap: YamlMapState | undefined, compatibility: CompatibilityContext ) { bucketMap?.reportError( @@ -286,8 +289,9 @@ export class SyncConfigFromYaml { streamCompiler.finish(); } - // We pass an empty array for eventDefinitions here because those will get parsed in #parseEventDefinitions. - return new PrecompiledSyncConfig(compiler.output.toSyncPlan(), compatibility, [], { + this.#compileEventDefinitions(eventMap, compiler); + + return new PrecompiledSyncConfig(compiler.output.toSyncPlan(), compatibility, { defaultSchema: this.options.defaultSchema, sourceText: this.yaml }); @@ -304,10 +308,14 @@ export class SyncConfigFromYaml { #legacyParseBucketDefinitionsAndStreams( bucketMap: YamlMapState | undefined, streamMap: YamlMapState | undefined, - compatibility: CompatibilityContext + compatibility: CompatibilityContext, + events: CompiledEventDescriptor[] ) { const rules = new SqlSyncRules(this.yaml); rules.compatibility = compatibility; + rules.eventDefinitions.push( + ...events.map((event) => new PreparedEventDefinition(event, this.options.defaultSchema)) + ); if (bucketMap == null && streamMap == null) { this.#errors.push(new YamlError(new Error(`'bucket_definitions' or 'streams' is required`))); @@ -438,31 +446,23 @@ export class SyncConfigFromYaml { return undefined; } - #parseEventDefinitions(parsed: YamlMapState, compatibility: CompatibilityContext) { - const eventMap = parsed.get('event_definitions')?.requireMap(); - const eventDescriptors: SqlEventDescriptor[] = []; - - for (const { key: name, keyScalar, value: maybeMap } of eventMap?.stringKeyedItems() ?? []) { + #compileEventDefinitions(eventMap: YamlMapState | undefined, compiler: SyncStreamsCompiler): void { + for (const { key: name, value: maybeMap } of eventMap?.stringKeyedItems() ?? []) { using value = maybeMap.requireMap(`Event definitions must be objects.`); if (value == null) continue; const payloads = value.get('payloads')?.requireSequence(`Event definition payloads must be an array.`); if (payloads == null) continue; - const eventDescriptor = new SqlEventDescriptor(name, compatibility); - for (let item of payloads.items) { + const eventCompiler = compiler.event(name); + for (const item of payloads.items) { const itemScalar = item.requireScalar(`Payload queries for events must be scalar.`); if (itemScalar == null) continue; - this.#withScalar(item, (q) => { - return eventDescriptor.addSourceQuery(q, this.options); - }); + const [sql, errorListener] = this.#scalarErrorListener(itemScalar.node); + eventCompiler.addSourceQuery(sql, errorListener); } - - eventDescriptors.push(eventDescriptor); } - - return eventDescriptors; } #checkUniqueName(name: string, literal: YamlState): boolean { diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index bcde2840b..d3073d508 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -4,8 +4,8 @@ export * from './BucketSource.js'; export * from './cast.js'; export * from './compatibility.js'; export * from './errors.js'; -export * from './events/SqlEventDescriptor.js'; -export * from './events/SqlEventSourceQuery.js'; +export * from './events/CompiledEventSourceQuery.js'; +export * from './events/EventDescriptor.js'; export * from './ExpressionType.js'; export * from './HydratedSyncConfig.js'; export * from './HydrationState.js'; diff --git a/packages/sync-rules/src/sync_plan/evaluator/bucket_data_source.ts b/packages/sync-rules/src/sync_plan/evaluator/bucket_data_source.ts index 914cda014..618f63087 100644 --- a/packages/sync-rules/src/sync_plan/evaluator/bucket_data_source.ts +++ b/packages/sync-rules/src/sync_plan/evaluator/bucket_data_source.ts @@ -3,26 +3,14 @@ import { idFromData } from '../../cast.js'; import { ColumnDefinition } from '../../ExpressionType.js'; import { SourceTableRef } from '../../SourceTableRef.js'; import { TablePattern } from '../../TablePattern.js'; -import { - EvaluateRowOptions, - SourceSchema, - SqliteJsonRow, - UnscopedEvaluatedRow, - UnscopedEvaluationResult -} from '../../types.js'; -import { filterJsonRow, isJsonValue, isValidParameterValue, JSONBucketNameSerialize } from '../../utils.js'; -import { - ScalarExpressionEngine, - ScalarStatement, - scalarStatementToSql, - TableValuedFunctionOutput -} from '../engine/scalar_expression_engine.js'; -import { SqlExpression } from '../expression.js'; +import { EvaluateRowOptions, SourceSchema, UnscopedEvaluatedRow, UnscopedEvaluationResult } from '../../types.js'; +import { isValidParameterValue, JSONBucketNameSerialize } from '../../utils.js'; +import { ScalarExpressionEngine } from '../engine/scalar_expression_engine.js'; import { ExpressionToSqlite } from '../expression_to_sql.js'; import * as plan from '../plan.js'; import { SyncPlanSchemaAnalyzer } from '../schema_inference.js'; import { StreamEvaluationContext } from './index.js'; -import { resolveRowMetadata, TableProcessorToSqlHelper } from './table_processor_to_sql.js'; +import { PendingRowProjection } from './row_projection.js'; export class PreparedStreamBucketDataSource implements BucketDataSource { private readonly sourceTables = new Set(); @@ -107,72 +95,27 @@ export class PreparedStreamBucketDataSource implements BucketDataSource { class PendingStreamDataSource { readonly tablePattern: TablePattern; - private readonly outputs: ('star' | { index: number; alias: string })[] = []; - private readonly numberOfOutputExpressions: number; - private readonly numberOfParameters: number; - private readonly evaluatorInputs: (plan.ColumnSqlParameterValue | plan.RowMetadataSqlValue)[]; - private readonly statement: ScalarStatement; + private readonly projection: PendingRowProjection; readonly fixedOutputTableName?: string; constructor(evaluator: plan.StreamDataSource, defaultSchema: string) { - const translationHelper = new TableProcessorToSqlHelper(evaluator); - const outputExpressions: SqlExpression[] = []; - - for (const column of evaluator.columns) { - if (column === 'star') { - this.outputs.push('star'); - } else { - const expressionIndex = outputExpressions.length; - outputExpressions.push(translationHelper.mapper.transform(column.expr)); - this.outputs.push({ index: expressionIndex, alias: column.alias }); - } - } - - this.numberOfOutputExpressions = outputExpressions.length; - for (const parameter of evaluator.parameters) { - outputExpressions.push(translationHelper.mapper.transform(parameter.expr)); - } - this.numberOfParameters = evaluator.parameters.length; - - this.statement = { - outputs: outputExpressions, - filters: translationHelper.filterExpressions, - tableValuedFunctions: translationHelper.tableValuedFunctions - }; + this.projection = new PendingRowProjection(evaluator, defaultSchema); this.fixedOutputTableName = evaluator.outputTableName; - this.tablePattern = evaluator.sourceTable.toTablePattern(defaultSchema); - this.evaluatorInputs = translationHelper.mapper.instantiation; + this.tablePattern = this.projection.tablePattern; } get debugSql(): string { - return scalarStatementToSql(this.statement); + return this.projection.debugSql; } instantiate(engine: ScalarExpressionEngine) { - const evaluator = engine.prepareEvaluator(this.statement); - const pattern = this.tablePattern; + const evaluate = this.projection.instantiate(engine); return (options: EvaluateRowOptions, results: UnscopedEvaluationResult[]) => { try { - const inputInstantiation = this.evaluatorInputs.map((input) => - 'column' in input ? options.record[input.column] : resolveRowMetadata(input, pattern, options.sourceTable) - ); - row: for (const source of evaluator.evaluate(inputInstantiation)) { - const record: SqliteJsonRow = {}; - for (const output of this.outputs) { - if (output === 'star') { - Object.assign(record, filterJsonRow(options.record)); - } else { - const value = source[output.index]; - if (isJsonValue(value)) { - record[output.alias] = value; - } - } - } - const id = idFromData(record); - // source is [...outputs, ...partitionValues] - const partitionValues = source.splice(this.numberOfOutputExpressions, this.numberOfParameters); - + row: for (const projected of evaluate(options)) { + const id = idFromData(projected.data); + const partitionValues = projected.partitionValues; for (const bucketParameter of partitionValues) { if (!isValidParameterValue(bucketParameter)) { continue row; @@ -181,7 +124,7 @@ class PendingStreamDataSource { results.push({ id, - data: record, + data: projected.data, table: this.fixedOutputTableName ?? options.sourceTable.name, serializedBucketParameters: JSONBucketNameSerialize.stringify(partitionValues) } satisfies UnscopedEvaluatedRow); diff --git a/packages/sync-rules/src/sync_plan/evaluator/index.ts b/packages/sync-rules/src/sync_plan/evaluator/index.ts index a779f5a09..2fbbdcb45 100644 --- a/packages/sync-rules/src/sync_plan/evaluator/index.ts +++ b/packages/sync-rules/src/sync_plan/evaluator/index.ts @@ -1,6 +1,6 @@ import { SyncConfig } from '../../SyncConfig.js'; import { CompatibilityContext } from '../../compatibility.js'; -import { SqlEventDescriptor } from '../../index.js'; +import { PreparedEventDefinition } from '../../events/CompiledEventSourceQuery.js'; import * as plan from '../plan.js'; import { PreparedStreamBucketDataSource } from './bucket_data_source.js'; import { StreamBucketSource, StreamInput } from './bucket_source.js'; @@ -17,24 +17,22 @@ export interface StreamEvaluationContext { } export class PrecompiledSyncConfig extends SyncConfig { - /** - * The default schema for this sync config. - * - * This is independent of the loaded {@link plan} (the same sync plan can be loaded with different default schemas). - */ + /** Default schema used to prepare unqualified source-table references from the compiled plan. */ readonly defaultSchema: string; constructor( readonly plan: plan.SyncPlan, compatibility: CompatibilityContext, - eventDefinitions: SqlEventDescriptor[], context: StreamEvaluationContext ) { super(context.sourceText); this.compatibility = compatibility; - this.eventDescriptors = eventDefinitions; this.defaultSchema = context.defaultSchema; + for (const event of plan.events) { + this.eventDefinitions.push(new PreparedEventDefinition(event, context.defaultSchema)); + } + const preparedBuckets = new Map(); const preparedLookups = new Map(); diff --git a/packages/sync-rules/src/sync_plan/evaluator/row_projection.ts b/packages/sync-rules/src/sync_plan/evaluator/row_projection.ts new file mode 100644 index 000000000..d6304252c --- /dev/null +++ b/packages/sync-rules/src/sync_plan/evaluator/row_projection.ts @@ -0,0 +1,98 @@ +import { TablePattern } from '../../TablePattern.js'; +import { EvaluateRowOptions, SqliteJsonRow, SqliteValue } from '../../types.js'; +import { filterJsonRow, isJsonValue } from '../../utils.js'; +import { + ScalarExpressionEngine, + ScalarStatement, + scalarStatementToSql, + TableValuedFunctionOutput +} from '../engine/scalar_expression_engine.js'; +import { SqlExpression } from '../expression.js'; +import * as plan from '../plan.js'; +import { resolveRowMetadata, TableProcessorToSqlHelper } from './table_processor_to_sql.js'; + +export interface EvaluatedRowProjection { + data: SqliteJsonRow; + partitionValues: SqliteValue[]; +} + +/** + * Prepares the shared projection/filter portion of bucket and event row evaluators. + */ +export class PendingRowProjection { + readonly tablePattern: TablePattern; + private readonly outputs: ('star' | { index: number; alias: string })[] = []; + private readonly numberOfOutputExpressions: number; + private readonly numberOfParameters: number; + private readonly evaluatorInputs: (plan.ColumnSqlParameterValue | plan.RowMetadataSqlValue)[]; + private readonly statement: ScalarStatement; + + constructor(evaluator: plan.RowProjection, defaultSchema: string) { + const translationHelper = new TableProcessorToSqlHelper(evaluator); + const outputExpressions: SqlExpression[] = []; + + for (const column of evaluator.columns) { + if (column === 'star') { + this.outputs.push('star'); + } else { + const expressionIndex = outputExpressions.length; + outputExpressions.push(translationHelper.mapper.transform(column.expr)); + this.outputs.push({ index: expressionIndex, alias: column.alias }); + } + } + + this.numberOfOutputExpressions = outputExpressions.length; + for (const parameter of evaluator.parameters) { + outputExpressions.push(translationHelper.mapper.transform(parameter.expr)); + } + this.numberOfParameters = evaluator.parameters.length; + + this.statement = { + outputs: outputExpressions, + filters: translationHelper.filterExpressions, + tableValuedFunctions: translationHelper.tableValuedFunctions + }; + this.tablePattern = evaluator.sourceTable.toTablePattern(defaultSchema); + this.evaluatorInputs = translationHelper.mapper.instantiation; + } + + get debugSql(): string { + return scalarStatementToSql(this.statement); + } + + instantiate(engine: ScalarExpressionEngine): (options: EvaluateRowOptions) => EvaluatedRowProjection[] { + const evaluator = engine.prepareEvaluator(this.statement); + const pattern = this.tablePattern; + + return (options) => { + const inputInstantiation = this.evaluatorInputs.map((input) => + 'column' in input ? options.record[input.column] : resolveRowMetadata(input, pattern, options.sourceTable) + ); + const results: EvaluatedRowProjection[] = []; + + for (const source of evaluator.evaluate(inputInstantiation)) { + const record: SqliteJsonRow = {}; + for (const output of this.outputs) { + if (output === 'star') { + Object.assign(record, filterJsonRow(options.record)); + } else { + const value = source[output.index]; + if (isJsonValue(value)) { + record[output.alias] = value; + } + } + } + + results.push({ + data: record, + partitionValues: source.slice( + this.numberOfOutputExpressions, + this.numberOfOutputExpressions + this.numberOfParameters + ) + }); + } + + return results; + }; + } +} diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index 23d39cd45..2a1be0d09 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -26,6 +26,7 @@ export interface SyncPlan { buckets: StreamBucketDataSource[]; parameterIndexes: StreamParameterIndexLookupCreator[]; streams: CompiledSyncStream[]; + events: CompiledEventDescriptor[]; } /** @@ -118,21 +119,53 @@ export interface PartitionKey { * duplicate bucket data but can still share the actual processing logic between multiple streams (to avoid evaluating * the same filters and expressions multiple times). */ -export interface StreamDataSource extends TableProcessor { +export interface RowProjection extends TableProcessor { + /** + * Output columns produced when a matching source row is evaluated. + */ + columns: ColumnSource[]; +} + +export interface StreamDataSource extends RowProjection { /** * The name of the output table for evaluated rows. * * If null, the name of the table being evaluated should be used instead. */ outputTableName?: string; +} +export type ColumnSource = 'star' | { expr: SqlExpression; alias: string }; + +/** + * A named replication event compiled from `event_definitions`. + */ +export interface CompiledEventDescriptor { + name: string; + sourceQueries: CompiledEventSourceQuery[]; +} + +/** + * A single payload query for an event. + * + * Event payload queries are restricted to one physical source table. A query can have multiple variants after its + * filter has been normalized to disjunctive normal form; evaluation stops after the first matching variant so one + * source row produces at most one event payload for this query. + */ +export interface CompiledEventSourceQuery { /** - * Output columns describing the row to store in buckets. + * Original SQL retained as a compatibility mirror for services using the legacy event evaluator. + * Semantic event identity is derived from the compiled variants, not this string. */ - columns: ColumnSource[]; + sql: string; + sourceTable: ImplicitSchemaTablePattern; + variants: EventRowEvaluator[]; } -export type ColumnSource = 'star' | { expr: SqlExpression; alias: string }; +/** + * A row projection used by one normalized filter variant of an event payload query. + */ +export interface EventRowEvaluator extends RowProjection {} /** * A mapping describing how {@link StreamDataSource}s are combined into buckets. diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index 4b321b5ce..d9d381e16 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -1,7 +1,10 @@ import { Equality } from '../compiler/equality.js'; +import { DEFAULT_TAG } from '../TablePattern.js'; import { SerializedBucketDataSource, SerializedDataSource, + SerializedEventRowEvaluator, + SerializedEventSourceQuery, SerializedParameterIndexLookupCreator } from './serialize.js'; @@ -10,6 +13,102 @@ export interface SerializedBucketDataSourceWithDataSources { dataSources: readonly SerializedDataSource[]; } +export interface SerializedEventSourceDefinition { + eventName: string; + defaultSchema: string; + source: SerializedEventSourceQuery; +} + +/** + * Semantic equality for an individual event source query. + * + * Event source identity is independent of the containing sync config. The identity deliberately excludes raw SQL and + * compiler hash codes, and normalizes unordered filter variants so callers can use it as stable input to a persisted + * fingerprint. Callers must still verify equality after a fingerprint lookup. + */ +export const serializedEventSourceDefinitionEquality: Equality = { + hash(hasher, value) { + hasher.addString(serializedEventSourceDefinitionIdentity(value)); + }, + equals(a, b) { + return a === b || serializedEventSourceDefinitionIdentity(a) == serializedEventSourceDefinitionIdentity(b); + } +}; + +/** + * Returns the canonical, versioned identity input for one event source query. + * + * This is intentionally not a durable identifier by itself. Storage implementations can hash the returned value for + * lookup and then use {@link serializedEventSourceDefinitionEquality} to guard against collisions. + */ +export function serializedEventSourceDefinitionIdentity(value: SerializedEventSourceDefinition): string { + const variants = value.source.variants + .map((variant) => eventVariantIdentity(variant, value.defaultSchema)) + .map((variant) => JSON.stringify(variant)) + .sort(); + + return JSON.stringify({ + version: 1, + eventName: value.eventName, + sourceTable: resolvedTableIdentity(value.source.table, value.defaultSchema), + variants + }); +} + +function eventVariantIdentity(value: SerializedEventRowEvaluator, defaultSchema: string) { + return { + table: resolvedTableIdentity(value.table, defaultSchema), + columns: value.columns.map((column) => { + return column == 'star' ? column : { ...column, expr: canonicalExpression(column.expr) }; + }), + // A conjunction's filter order does not affect behavior. + filters: value.filters.map((filter) => JSON.stringify(canonicalExpression(filter))).sort(), + tableValuedFunctions: value.tableValuedFunctions.map((fn) => ({ + ...fn, + functionInputs: fn.functionInputs.map(canonicalExpression) + })), + partitionBy: value.partitionBy.map((key) => ({ expr: canonicalExpression(key.expr) })) + }; +} + +function resolvedTableIdentity(table: SerializedEventRowEvaluator['table'], defaultSchema: string) { + return { ...table, connection: table.connection ?? DEFAULT_TAG, schema: table.schema ?? defaultSchema }; +} + +function canonicalExpression(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalExpression); + } + if (value == null || typeof value != 'object') { + return value; + } + + const expression = value as Record; + if (expression.type == 'binary' && (expression.operator == 'and' || expression.operator == 'or')) { + const operator = expression.operator; + const operands: unknown[] = []; + const addOperand = (operand: unknown) => { + if ( + operand != null && + typeof operand == 'object' && + (operand as Record).type == 'binary' && + (operand as Record).operator == operator + ) { + addOperand((operand as Record).left); + addOperand((operand as Record).right); + } else { + operands.push(canonicalExpression(operand)); + } + }; + + addOperand(expression.left); + addOperand(expression.right); + return { type: 'commutative', operator, operands: operands.map((operand) => JSON.stringify(operand)).sort() }; + } + + return Object.fromEntries(Object.entries(expression).map(([key, nested]) => [key, canonicalExpression(nested)])); +} + /** * Equality for SerializedParameterIndexLookupCreator. * diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 7db39fd98..6d969a26b 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -5,8 +5,10 @@ import { MapSourceVisitor, visitExpr } from './expression_visitor.js'; import { ColumnSource, ColumnSqlParameterValue, + CompiledEventDescriptor, CompiledSyncStream, EvaluateTableValuedFunction, + EventRowEvaluator, ExpandingLookup, ParameterLookup, ParameterValue, @@ -110,6 +112,34 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { }); } + function serializeEventRowEvaluator(source: EventRowEvaluator): SerializedEventRowEvaluator { + return { + hash: source.hashCode, + table: serializeTablePattern(source.sourceTable), + tableValuedFunctions: serializeTableValued(source), + filters: source.filters.map(serializeTableProcessorDataExpr), + partitionBy: translateParameters(source), + columns: source.columns.map((column): SerializedColumnSource => { + if (column == 'star') { + return 'star'; + } + + return { expr: serializeTableProcessorDataExpr(column.expr), alias: column.alias }; + }) + }; + } + + function serializeEvents(): SerializedEventDescriptor[] { + return plan.events.map((event) => ({ + name: event.name, + sourceQueries: event.sourceQueries.map((query) => ({ + sql: query.sql, + table: serializeTablePattern(query.sourceTable), + variants: query.variants.map(serializeEventRowEvaluator) + })) + })); + } + function serializeParameterIndexes(): SerializedParameterIndexLookupCreator[] { return plan.parameterIndexes.map((source, i) => { parameterIndex.set(source, i); @@ -178,7 +208,8 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { }; } - return { + const events = serializeEvents(); + const serialized: SerializedSyncPlan = { dataSources: serializeDataSources(), buckets: plan.buckets.map((bkt, index) => { bucketIndex.set(bkt, index); @@ -195,6 +226,14 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { })), version: usesRowMetadataSqlValue ? 2 : 1 }; + + // Compiled events are intentionally additive to plan versions 1 and 2. The service also persists their raw SQL in + // the legacy eventDescriptors field, so older readers can ignore this field and retain equivalent event behavior. + if (events.length != 0) { + serialized.events = events; + } + + return serialized; } export function deserializeSyncPlan(serialized: unknown): SyncPlan { @@ -282,6 +321,40 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { }; }); + function deserializeEventRowEvaluator(source: SerializedEventRowEvaluator): EventRowEvaluator { + const functions = (tableValuedFunctionsInScope = source.tableValuedFunctions); + + return { + hashCode: source.hash, + sourceTable: deserializeTablePattern(source.table), + tableValuedFunctions: functions, + filters: source.filters.map(deserializeTableProcessorDataExpr), + parameters: deserializeParameters(source.partitionBy), + columns: source.columns.map((column): ColumnSource => { + if (column == 'star') { + return 'star'; + } + + return { expr: deserializeTableProcessorDataExpr(column.expr), alias: column.alias }; + }) + }; + } + + if (plan.events != null && !Array.isArray(plan.events)) { + throw new Error('Compiled sync plan events must be an array.'); + } + const serializedEvents = plan.events ?? []; + const events = serializedEvents.map((event): CompiledEventDescriptor => { + return { + name: event.name, + sourceQueries: event.sourceQueries.map((query) => ({ + sql: query.sql, + sourceTable: deserializeTablePattern(query.table), + variants: query.variants.map(deserializeEventRowEvaluator) + })) + }; + }); + function deserializeParameterValue(stages: ExpandingLookup[][], value: SerializedParameterValue): ParameterValue { switch (value.type) { case 'request': @@ -346,15 +419,19 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { dataSources, buckets, parameterIndexes, - streams + streams, + events }; } /** - * Every change to the format of {@link SerializedSyncPlan} needs a version bump and a changelog entry in this - * documentation comment. Even for seemingly backward-compatible changes, like adding new fields, older services would - * be unaware of them and thus interpret the sync plan incorrectly. Increasing this version ensures that older services - * wouldn't even try to deserialize sync plans. + * Changes to {@link SerializedSyncPlan} require a version bump when older services would interpret the plan + * incorrectly. Optional additive fields are only safe without a bump when older readers can ignore them while another + * persisted representation preserves equivalent behavior. + * + * Compiled `events` are an explicit additive exception: service-core continues to persist raw event SQL alongside the + * plan for the legacy evaluator. Older readers ignore `events` and use that legacy mirror. Removing the mirror or + * relying on compiled-only event semantics will require a version bump. * * ### Version 2 * @@ -376,6 +453,11 @@ export interface SerializedSyncPlan { buckets: SerializedBucketDataSource[]; parameterIndexes: SerializedParameterIndexLookupCreator[]; streams: SerializedStream[]; + /** + * Optional additive compiled event definitions. Older readers safely ignore this because service-core dual-writes + * equivalent raw SQL in its legacy `eventDescriptors` field. + */ + events?: SerializedEventDescriptor[]; } export interface SerializedBucketDataSource { @@ -416,6 +498,27 @@ export interface SerializedDataSource { partitionBy: SerializedPartitionKey[]; } +export interface SerializedEventDescriptor { + name: string; + sourceQueries: SerializedEventSourceQuery[]; +} + +export interface SerializedEventSourceQuery { + /** Raw SQL retained only for the legacy compatibility mirror; compiled variants define semantic identity. */ + sql: string; + table: SerializedTablePattern; + variants: SerializedEventRowEvaluator[]; +} + +export interface SerializedEventRowEvaluator { + table: SerializedTablePattern; + hash: number; + columns: SerializedColumnSource[]; + filters: SqlExpression[]; + tableValuedFunctions: TableProcessorTableValuedFunction[]; + partitionBy: SerializedPartitionKey[]; +} + export interface SerializedParameterIndexLookupCreator { table: SerializedTablePattern; hash: number; diff --git a/packages/sync-rules/test/src/compiler/events.test.ts b/packages/sync-rules/test/src/compiler/events.test.ts new file mode 100644 index 000000000..e35af35c2 --- /dev/null +++ b/packages/sync-rules/test/src/compiler/events.test.ts @@ -0,0 +1,139 @@ +import * as sqlite from 'node:sqlite'; +import { describe, expect, test } from 'vitest'; +import { + DEFAULT_HYDRATION_STATE, + deserializeSyncPlan, + nodeSqlite, + PrecompiledSyncConfig, + serializedEventSourceDefinitionEquality, + serializedEventSourceDefinitionIdentity, + serializeSyncPlan, + SqlSyncRules +} from '../../../src/index.js'; +import { TestSourceTable } from '../util.js'; +import { yamlToSyncPlan } from './utils.js'; + +const CHECKPOINT_EVENT_YAML = ` +config: + edition: 3 + +streams: + checkpoints: + query: SELECT * FROM checkpoints + +event_definitions: + write_checkpoints: + payloads: + - SELECT user_id, checkpoint, client_id FROM checkpoints WHERE active = true +`; + +describe('compiled replication events', () => { + test('compiles, serializes and evaluates event payload queries', () => { + const { config, errors } = SqlSyncRules.fromYaml(CHECKPOINT_EVENT_YAML, { + defaultSchema: 'test_schema', + throwOnError: false + }); + expect(errors).toStrictEqual([]); + expect(config).toBeInstanceOf(PrecompiledSyncConfig); + + const compiled = config as PrecompiledSyncConfig; + const serialized = serializeSyncPlan(compiled.plan); + // Compiled events are additive and do not require a new plan version. + expect(serialized.version).toBe(1); + expect(serialized.events).toHaveLength(1); + expect(deserializeSyncPlan(JSON.parse(JSON.stringify(serialized))).events).toEqual(compiled.plan.events); + + const hydrated = compiled.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) }); + const event = hydrated.eventDescriptors[0]; + const checkpoints = new TestSourceTable('checkpoints'); + + expect( + event.evaluateRowWithErrors({ + sourceTable: checkpoints, + record: { user_id: 'user-1', checkpoint: 42n, client_id: 'client-1', active: 1, ignored: 'value' } + }) + ).toEqual({ + result: { data: { user_id: 'user-1', checkpoint: 42n, client_id: 'client-1' } }, + errors: [] + }); + expect( + event.evaluateRowWithErrors({ + sourceTable: checkpoints, + record: { user_id: 'user-1', checkpoint: 42n, client_id: 'client-1', active: 0 } + }) + ).toEqual({ errors: [] }); + }); + + test('uses canonical semantic identity independent of formatting and filter order', () => { + const first = eventSourceFromQuery( + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0' + ); + const equivalent = eventSourceFromQuery( + ' select user_id, checkpoint from test_schema.checkpoints AS c where c.checkpoint > 0 and c.active = true ' + ); + const changed = eventSourceFromQuery( + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1' + ); + + expect(serializedEventSourceDefinitionEquality.equals(first, equivalent)).toBe(true); + expect(serializedEventSourceDefinitionIdentity(first)).toBe(serializedEventSourceDefinitionIdentity(equivalent)); + expect(serializedEventSourceDefinitionEquality.equals(first, changed)).toBe(false); + }); + + test.each([ + ['joins', 'SELECT c.user_id FROM checkpoints c JOIN users u ON u.id = c.user_id', 'single physical source table'], + [ + 'request parameters', + 'SELECT user_id FROM checkpoints WHERE user_id = auth.user_id()', + 'cannot depend on request parameters' + ] + ])('rejects %s', (_label, query, message) => { + const [errors] = yamlToSyncPlan(yamlWithEventQueries(query), { + defaultSchema: 'test_schema', + throwOnError: false + }); + + expect(errors.map((error) => error.message)).toContainEqual(expect.stringContaining(message)); + }); + + test('requires payload queries within an event to use unique source tables', () => { + const [errors] = yamlToSyncPlan( + yamlWithEventQueries( + 'SELECT user_id FROM checkpoints', + 'SELECT checkpoint FROM test_schema.checkpoints WHERE checkpoint > 0' + ), + { defaultSchema: 'test_schema', throwOnError: false } + ); + + expect(errors.map((error) => error.message)).toContain('Each payload query should query a unique table'); + }); +}); + +function eventSourceFromQuery(query: string) { + const plan = serializeSyncPlan( + ( + SqlSyncRules.fromYaml(yamlWithEventQueries(query), { + defaultSchema: 'test_schema' + }).config as PrecompiledSyncConfig + ).plan + ); + const source = plan.events![0].sourceQueries[0]; + + return { eventName: 'write_checkpoints', defaultSchema: 'test_schema', source }; +} + +function yamlWithEventQueries(...queries: string[]): string { + return ` +config: + edition: 3 + +streams: + checkpoints: + query: SELECT * FROM checkpoints + +event_definitions: + write_checkpoints: + payloads: +${queries.map((query) => ` - ${query}`).join('\n')} +`; +} diff --git a/packages/sync-rules/test/src/legacy/sync_rules.test.ts b/packages/sync-rules/test/src/legacy/sync_rules.test.ts index dc6be9efe..b0cf44b76 100644 --- a/packages/sync-rules/test/src/legacy/sync_rules.test.ts +++ b/packages/sync-rules/test/src/legacy/sync_rules.test.ts @@ -1108,7 +1108,19 @@ event_definitions: PARSE_OPTIONS ); expect(errors).toStrictEqual([]); - expect(rules.eventDescriptors).toHaveLength(1); + expect(rules).not.toHaveProperty('eventDescriptors'); + expect(rules.eventDefinitions).toHaveLength(1); + + const [event] = rules.hydrate(hydrationParams).eventDescriptors; + expect( + event.evaluateRowWithErrors({ + sourceTable: new TestSourceTable('checkpoints'), + record: { user_id: 'user-1', checkpoint: 2n, client_id: 'client-1' } + }) + ).toEqual({ + result: { data: { user_id: 'user-1', checkpoint: 2n, client_id: 'client-1' } }, + errors: [] + }); }); test('suggests upgrading for streams on edition 2', () => { diff --git a/packages/sync-rules/test/src/sync_plan/evaluator/table_valued.test.ts b/packages/sync-rules/test/src/sync_plan/evaluator/table_valued.test.ts index 9d1bc3519..d64da9a40 100644 --- a/packages/sync-rules/test/src/sync_plan/evaluator/table_valued.test.ts +++ b/packages/sync-rules/test/src/sync_plan/evaluator/table_valued.test.ts @@ -130,7 +130,8 @@ streams: dataSources: [source], buckets: [{ hashCode: 0, uniqueName: 'a', sources: [source] }], parameterIndexes: [], - streams: [] + streams: [], + events: [] }) ); diff --git a/packages/sync-rules/test/src/sync_plan/schema_inference.test.ts b/packages/sync-rules/test/src/sync_plan/schema_inference.test.ts index cf39c98da..8bb292e50 100644 --- a/packages/sync-rules/test/src/sync_plan/schema_inference.test.ts +++ b/packages/sync-rules/test/src/sync_plan/schema_inference.test.ts @@ -39,7 +39,7 @@ describe('schema inference', () => { function generateSchema(...queries: string[]) { const serializedPlan = compileSingleStreamAndSerialize(...queries); const plan = deserializeSyncPlan(serializedPlan); - const rules = new PrecompiledSyncConfig(plan, new CompatibilityContext({ edition: 3 }), [], { + const rules = new PrecompiledSyncConfig(plan, new CompatibilityContext({ edition: 3 }), { sourceText: '', defaultSchema: 'test_schema' }); From 48661802fefd549bcdb21ecfea56c83303117dca Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 11 Aug 2026 09:37:18 +0200 Subject: [PATCH 02/14] add id for event plans --- .../src/PersistedSyncConfigContent.test.ts | 4 + .../src/compiler/ir_to_sync_plan.ts | 21 +-- .../src/events/CompiledEventSourceQuery.ts | 11 +- .../sync-rules/src/events/EventDescriptor.ts | 7 + packages/sync-rules/src/sync_plan/plan.ts | 5 +- .../src/sync_plan/plan_equality_serialized.ts | 63 +++++---- .../sync-rules/src/sync_plan/serialize.ts | 124 ++++++++++++------ .../test/src/compiler/events.test.ts | 58 +++++++- 8 files changed, 217 insertions(+), 76 deletions(-) diff --git a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts index 4fe8c926a..ce448ed15 100644 --- a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts +++ b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts @@ -34,15 +34,18 @@ describe('persisted compiled replication events', () => { const parsed = SqlSyncRules.fromYaml(yamlWithEvents, { defaultSchema: 'test_schema' }); const update = updateSyncRulesFromConfig(parsed); const compiled = update.config.plan!; + const eventDefinitionId = parsed.config.eventDefinitions[0].id; expect(compiled.plan.version).toBeLessThanOrEqual(2); expect(compiled.plan.events).toHaveLength(1); + expect(compiled.plan.events![0].id).toBe(eventDefinitionId); expect(compiled.eventDescriptors).toEqual({ write_checkpoints: [EVENT_QUERY] }); const restored = restore(compiled); expect(restored.config).toBeInstanceOf(PrecompiledSyncConfig); expect(restored.config).not.toHaveProperty('eventDescriptors'); expect(restored.config.eventDefinitions).toHaveLength(1); + expect(restored.config.eventDefinitions[0].id).toBe(eventDefinitionId); const hydrated = restored.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, @@ -61,6 +64,7 @@ describe('persisted compiled replication events', () => { const legacyView = restore({ ...compiled, plan: planWithoutCompiledEvents }); expect(legacyView.config).not.toHaveProperty('eventDescriptors'); expect(legacyView.config.eventDefinitions).toHaveLength(1); + expect(legacyView.config.eventDefinitions[0].id).toBe(eventDefinitionId); expect((legacyView.config as PrecompiledSyncConfig).plan.events).toHaveLength(1); }); diff --git a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts index 157e723ac..cb2bf51d8 100644 --- a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts +++ b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts @@ -1,6 +1,7 @@ import { SqlExpression } from '../sync_plan/expression.js'; import { MapSourceVisitor, visitExpr } from '../sync_plan/expression_visitor.js'; import * as plan from '../sync_plan/plan.js'; +import { compiledEventDefinitionId } from '../sync_plan/serialize.js'; import * as resolver from './bucket_resolver.js'; import { CompiledStreamQueries } from './compiler.js'; import { Equality, HashMap, StableHasher, unorderedEquality } from './equality.js'; @@ -61,14 +62,18 @@ export class CompilerModelToSyncPlan { }; }), buckets: this.buckets, - events: source.events.map((event) => ({ - name: event.name, - sourceQueries: event.sourceQueries.map((query) => ({ - sql: query.sql, - sourceTable: query.sourceTable.tablePattern, - variants: query.variants.map((variant) => this.translateRowProjection(variant)) - })) - })) + events: source.events.map((event) => { + const definition: Omit = { + name: event.name, + sourceQueries: event.sourceQueries.map((query) => ({ + sql: query.sql, + sourceTable: query.sourceTable.tablePattern, + variants: query.variants.map((variant) => this.translateRowProjection(variant)) + })) + }; + + return { id: compiledEventDefinitionId(definition), ...definition }; + }) }; } diff --git a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts index e274cdd0f..0996841e1 100644 --- a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts +++ b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts @@ -7,20 +7,28 @@ import { } from '../sync_plan/plan.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, SqliteRow } from '../types.js'; -import { EvaluatedEventRowWithErrors, EventDefinition, HydratedEventDescriptor } from './EventDescriptor.js'; +import { + EvaluatedEventRowWithErrors, + EventDefinition, + EventDefinitionId, + HydratedEventDescriptor +} from './EventDescriptor.js'; /** A named event prepared from a compiled sync plan, before scalar expressions are prepared for evaluation. */ export class PreparedEventDefinition implements EventDefinition { + readonly id: EventDefinitionId; readonly name: string; readonly sourceQueries: PreparedEventSourceQuery[]; constructor(source: CompiledEventDescriptorPlan, defaultSchema: string) { + this.id = source.id; this.name = source.name; this.sourceQueries = source.sourceQueries.map((query) => new PreparedEventSourceQuery(query, defaultSchema)); } createEvaluator(input: HydrationInput): HydratedEventDescriptor { return new HydratedCompiledEventDescriptor( + this.id, this.name, this.sourceQueries.map((query) => query.createEvaluator(input)) ); @@ -59,6 +67,7 @@ export class PreparedEventSourceQuery { class HydratedCompiledEventDescriptor implements HydratedEventDescriptor { constructor( + readonly id: EventDefinitionId, readonly name: string, readonly sourceQueries: HydratedCompiledEventSourceQuery[] ) {} diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index cc9da2474..5245f7fcb 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -12,8 +12,13 @@ export type EvaluatedEventRowWithErrors = { errors: EvaluationError[]; }; +/** Content-addressed identity of a complete named event definition. */ +export type EventDefinitionId = string; + /** A parsed event definition whose compiled expressions have not yet been prepared for evaluation. */ export interface EventDefinition { + /** Deterministic identity generated from the serialized compiled definition. */ + readonly id: EventDefinitionId; readonly name: string; createEvaluator(input: HydrationInput): HydratedEventDescriptor; @@ -23,6 +28,8 @@ export interface EventDefinition { /** An event definition whose payload queries can evaluate replicated rows. */ export interface HydratedEventDescriptor { + /** Deterministic identity generated from the serialized compiled definition. */ + readonly id: EventDefinitionId; readonly name: string; evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors; diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index 2a1be0d09..9c92fd91e 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -1,4 +1,5 @@ import { BucketPriority } from '../BucketDescription.js'; +import type { EventDefinitionId } from '../events/EventDescriptor.js'; import { ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern } from '../TablePattern.js'; import { UnscopedEvaluatedParameters } from '../types.js'; @@ -141,6 +142,8 @@ export type ColumnSource = 'star' | { expr: SqlExpression; a * A named replication event compiled from `event_definitions`. */ export interface CompiledEventDescriptor { + /** Content-addressed identity assigned when compiler output is finalized or restored from a serialized plan. */ + id: EventDefinitionId; name: string; sourceQueries: CompiledEventSourceQuery[]; } @@ -155,7 +158,7 @@ export interface CompiledEventDescriptor { export interface CompiledEventSourceQuery { /** * Original SQL retained as a compatibility mirror for services using the legacy event evaluator. - * Semantic event identity is derived from the compiled variants, not this string. + * It remains part of the exact serialized event definition used to derive the event ID. */ sql: string; sourceTable: ImplicitSchemaTablePattern; diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index d9d381e16..18fd7ca78 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -1,13 +1,17 @@ -import { Equality } from '../compiler/equality.js'; -import { DEFAULT_TAG } from '../TablePattern.js'; -import { +import * as uuid from 'uuid'; +import type { Equality } from '../compiler/equality.js'; +import type { EventDefinitionId } from '../events/EventDescriptor.js'; +import type { SerializedBucketDataSource, SerializedDataSource, + SerializedEventDescriptor, SerializedEventRowEvaluator, SerializedEventSourceQuery, SerializedParameterIndexLookupCreator } from './serialize.js'; +const EVENT_DEFINITION_ID_NAMESPACE = uuid.v5('powersync-replication-event-definition-v1', uuid.v5.URL); + export interface SerializedBucketDataSourceWithDataSources { bucket: SerializedBucketDataSource; dataSources: readonly SerializedDataSource[]; @@ -15,16 +19,30 @@ export interface SerializedBucketDataSourceWithDataSources { export interface SerializedEventSourceDefinition { eventName: string; - defaultSchema: string; source: SerializedEventSourceQuery; } +/** Returns the serialized event definition without its derived ID. */ +export function serializedEventDefinitionIdentity( + event: Pick +): string { + return JSON.stringify({ name: event.name, sourceQueries: event.sourceQueries }); +} + +/** Generate the content-addressed ID persisted with and exposed by a compiled event definition. */ +export function serializedEventDefinitionId( + event: Pick +): EventDefinitionId { + return uuid.v5(serializedEventDefinitionIdentity(event), EVENT_DEFINITION_ID_NAMESPACE); +} + /** - * Semantic equality for an individual event source query. + * Compiled-plan equality for an individual event source query. * * Event source identity is independent of the containing sync config. The identity deliberately excludes raw SQL and - * compiler hash codes, and normalizes unordered filter variants so callers can use it as stable input to a persisted - * fingerprint. Callers must still verify equality after a fingerprint lookup. + * compiler hash codes, preserves table references as represented in the plan, and normalizes unordered filter variants + * so callers can use it as stable input to a persisted fingerprint. Callers must still verify equality after a + * fingerprint lookup. */ export const serializedEventSourceDefinitionEquality: Equality = { hash(hasher, value) { @@ -38,26 +56,31 @@ export const serializedEventSourceDefinitionEquality: Equality eventVariantIdentity(variant, value.defaultSchema)) - .map((variant) => JSON.stringify(variant)) - .sort(); - return JSON.stringify({ version: 1, eventName: value.eventName, - sourceTable: resolvedTableIdentity(value.source.table, value.defaultSchema), - variants + ...eventSourceQueryIdentity(value.source) }); } -function eventVariantIdentity(value: SerializedEventRowEvaluator, defaultSchema: string) { +/** Canonical identity fields for a source query without the containing event name. */ +function eventSourceQueryIdentity(source: SerializedEventSourceQuery) { return { - table: resolvedTableIdentity(value.table, defaultSchema), + sourceTable: source.table, + variants: source.variants + .map(eventVariantIdentity) + .map((variant) => JSON.stringify(variant)) + .sort() + }; +} + +function eventVariantIdentity(value: SerializedEventRowEvaluator) { + return { + table: value.table, columns: value.columns.map((column) => { return column == 'star' ? column : { ...column, expr: canonicalExpression(column.expr) }; }), @@ -71,10 +94,6 @@ function eventVariantIdentity(value: SerializedEventRowEvaluator, defaultSchema: }; } -function resolvedTableIdentity(table: SerializedEventRowEvaluator['table'], defaultSchema: string) { - return { ...table, connection: table.connection ?? DEFAULT_TAG, schema: table.schema ?? defaultSchema }; -} - function canonicalExpression(value: unknown): unknown { if (Array.isArray(value)) { return value.map(canonicalExpression); diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 6d969a26b..9ccfa1fb6 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -1,3 +1,4 @@ +import type { EventDefinitionId } from '../events/EventDescriptor.js'; import { ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern, TablePattern } from '../TablePattern.js'; import { SqlExpression } from './expression.js'; @@ -26,19 +27,9 @@ import { TableProcessorTableValuedFunction, TableProcessorTableValuedFunctionOutput } from './plan.js'; +import { serializedEventDefinitionId } from './plan_equality_serialized.js'; -/** - * Serializes a sync plan into a simple JSON object. - * - * While {@link SyncPlan}s are already serializable for the most part, it contains a graph of references from e.g. - * queriers to bucket creators. To represent this efficiently, we assign numbers to referenced elements while - * serializing instead of duplicating definitions. - */ -export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { - const dataSourceIndex = new Map(); - const bucketIndex = new Map(); - const parameterIndex = new Map(); - const expandingLookups = new Map(); +function createTableProcessorSerializer() { const addedTableValuedFunctions = new Map(); let usesRowMetadataSqlValue = false; @@ -90,28 +81,6 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { }); } - function serializeDataSources(): SerializedDataSource[] { - return plan.dataSources.map((source, i) => { - dataSourceIndex.set(source, i); - - return { - hash: source.hashCode, - table: serializeTablePattern(source.sourceTable), - outputTableName: source.outputTableName, - tableValuedFunctions: serializeTableValued(source), - filters: source.filters.map(serializeTableProcessorDataExpr), - partitionBy: translateParameters(source), - columns: source.columns.map((c): SerializedColumnSource => { - if (c == 'star') { - return 'star'; - } else { - return { expr: serializeTableProcessorDataExpr(c.expr), alias: c.alias }; - } - }) - } satisfies SerializedDataSource; - }); - } - function serializeEventRowEvaluator(source: EventRowEvaluator): SerializedEventRowEvaluator { return { hash: source.hashCode, @@ -129,15 +98,72 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { }; } - function serializeEvents(): SerializedEventDescriptor[] { - return plan.events.map((event) => ({ + function serializeEventDefinition( + event: Pick + ): Omit { + return { name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ sql: query.sql, table: serializeTablePattern(query.sourceTable), variants: query.variants.map(serializeEventRowEvaluator) })) - })); + }; + } + + function serializeEvent(event: CompiledEventDescriptor): SerializedEventDescriptor { + return { id: event.id, ...serializeEventDefinition(event) }; + } + + return { + get usesRowMetadataSqlValue() { + return usesRowMetadataSqlValue; + }, + serializeTableProcessorDataExpr, + serializeTablePattern, + serializeTableValued, + translateParameters, + serializeEventDefinition, + serializeEvent + }; +} + +/** + * Serializes a sync plan into a simple JSON object. + * + * While {@link SyncPlan}s are already serializable for the most part, it contains a graph of references from e.g. + * queriers to bucket creators. To represent this efficiently, we assign numbers to referenced elements while + * serializing instead of duplicating definitions. + */ +export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { + const dataSourceIndex = new Map(); + const bucketIndex = new Map(); + const parameterIndex = new Map(); + const expandingLookups = new Map(); + const tableProcessorSerializer = createTableProcessorSerializer(); + const { serializeTableProcessorDataExpr, serializeTablePattern, serializeTableValued, translateParameters } = + tableProcessorSerializer; + + function serializeDataSources(): SerializedDataSource[] { + return plan.dataSources.map((source, i) => { + dataSourceIndex.set(source, i); + + return { + hash: source.hashCode, + table: serializeTablePattern(source.sourceTable), + outputTableName: source.outputTableName, + tableValuedFunctions: serializeTableValued(source), + filters: source.filters.map(serializeTableProcessorDataExpr), + partitionBy: translateParameters(source), + columns: source.columns.map((c): SerializedColumnSource => { + if (c == 'star') { + return 'star'; + } else { + return { expr: serializeTableProcessorDataExpr(c.expr), alias: c.alias }; + } + }) + } satisfies SerializedDataSource; + }); } function serializeParameterIndexes(): SerializedParameterIndexLookupCreator[] { @@ -150,7 +176,7 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { tableValuedFunctions: serializeTableValued(source), filters: source.filters.map(serializeTableProcessorDataExpr), partitionBy: translateParameters(source), - output: source.outputs.map((out) => visitExpr(replaceFunctionReferenceWithIndex, out, null)), + output: source.outputs.map(serializeTableProcessorDataExpr), lookupScope: source.defaultLookupScope } satisfies SerializedParameterIndexLookupCreator; }); @@ -208,7 +234,7 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { }; } - const events = serializeEvents(); + const events = plan.events.map(tableProcessorSerializer.serializeEvent); const serialized: SerializedSyncPlan = { dataSources: serializeDataSources(), buckets: plan.buckets.map((bkt, index) => { @@ -224,7 +250,7 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { stream: s.stream, queriers: s.queriers.map(serializeStreamQuerier) })), - version: usesRowMetadataSqlValue ? 2 : 1 + version: tableProcessorSerializer.usesRowMetadataSqlValue ? 2 : 1 }; // Compiled events are intentionally additive to plan versions 1 and 2. The service also persists their raw SQL in @@ -346,6 +372,7 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { const serializedEvents = plan.events ?? []; const events = serializedEvents.map((event): CompiledEventDescriptor => { return { + id: event.id, name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ sql: query.sql, @@ -424,6 +451,19 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { }; } +/** Derive the ID assigned while finalizing a compiled event definition. */ +export function compiledEventDefinitionId( + event: Pick +): EventDefinitionId { + const definition = createTableProcessorSerializer().serializeEventDefinition(event); + return serializedEventDefinitionId(definition); +} + +/** Serialize a single compiled event using the exact representation persisted in a sync plan. */ +export function serializeEventDescriptor(event: CompiledEventDescriptor): SerializedEventDescriptor { + return createTableProcessorSerializer().serializeEvent(event); +} + /** * Changes to {@link SerializedSyncPlan} require a version bump when older services would interpret the plan * incorrectly. Optional additive fields are only safe without a bump when older readers can ignore them while another @@ -499,12 +539,14 @@ export interface SerializedDataSource { } export interface SerializedEventDescriptor { + /** Content-addressed identity derived from the rest of this event definition. */ + id: EventDefinitionId; name: string; sourceQueries: SerializedEventSourceQuery[]; } export interface SerializedEventSourceQuery { - /** Raw SQL retained only for the legacy compatibility mirror; compiled variants define semantic identity. */ + /** Raw SQL retained for the legacy compatibility mirror and as part of the exact serialized event definition. */ sql: string; table: SerializedTablePattern; variants: SerializedEventRowEvaluator[]; diff --git a/packages/sync-rules/test/src/compiler/events.test.ts b/packages/sync-rules/test/src/compiler/events.test.ts index e35af35c2..51342eea6 100644 --- a/packages/sync-rules/test/src/compiler/events.test.ts +++ b/packages/sync-rules/test/src/compiler/events.test.ts @@ -5,6 +5,8 @@ import { deserializeSyncPlan, nodeSqlite, PrecompiledSyncConfig, + serializedEventDefinitionId, + serializedEventDefinitionIdentity, serializedEventSourceDefinitionEquality, serializedEventSourceDefinitionIdentity, serializeSyncPlan, @@ -41,10 +43,17 @@ describe('compiled replication events', () => { // Compiled events are additive and do not require a new plan version. expect(serialized.version).toBe(1); expect(serialized.events).toHaveLength(1); - expect(deserializeSyncPlan(JSON.parse(JSON.stringify(serialized))).events).toEqual(compiled.plan.events); + expect(compiled.eventDefinitions[0].id).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + ); + expect(serialized.events![0].id).toBe(compiled.eventDefinitions[0].id); + const deserialized = deserializeSyncPlan(JSON.parse(JSON.stringify(serialized))); + expect(deserialized.events).toMatchObject(compiled.plan.events); + expect(deserialized.events[0].id).toBe(compiled.eventDefinitions[0].id); const hydrated = compiled.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) }); const event = hydrated.eventDescriptors[0]; + expect(event.id).toBe(compiled.eventDefinitions[0].id); const checkpoints = new TestSourceTable('checkpoints'); expect( @@ -69,7 +78,7 @@ describe('compiled replication events', () => { 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0' ); const equivalent = eventSourceFromQuery( - ' select user_id, checkpoint from test_schema.checkpoints AS c where c.checkpoint > 0 and c.active = true ' + ' select user_id, checkpoint from checkpoints AS c where c.checkpoint > 0 and c.active = true ' ); const changed = eventSourceFromQuery( 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1' @@ -80,6 +89,36 @@ describe('compiled replication events', () => { expect(serializedEventSourceDefinitionEquality.equals(first, changed)).toBe(false); }); + test('derives a content-addressed id from the exact serialized event definition', () => { + const first = eventDefinitionFromQueries( + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true', + 'SELECT user_id, checkpoint FROM archived_checkpoints' + ); + const reordered = eventDefinitionFromQueries( + ' select user_id, checkpoint from archived_checkpoints ', + 'select user_id, checkpoint from checkpoints c where c.active = true' + ); + const changed = eventDefinitionFromQueries( + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = false', + 'SELECT user_id, checkpoint FROM archived_checkpoints' + ); + + expect(first.id).toBe(serializedEventDefinitionId(first.event)); + expect(reordered.id).not.toBe(first.id); + expect(changed.id).not.toBe(first.id); + + const { id: _id, ...definition } = first.event; + expect(serializedEventDefinitionIdentity(first.event)).toBe(JSON.stringify(definition)); + }); + + test('derives the id from the plan without loading context', () => { + const query = 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true'; + + expect(eventDefinitionForSchema('first_schema', query).id).toBe( + eventDefinitionForSchema('second_schema', query).id + ); + }); + test.each([ ['joins', 'SELECT c.user_id FROM checkpoints c JOIN users u ON u.id = c.user_id', 'single physical source table'], [ @@ -119,7 +158,20 @@ function eventSourceFromQuery(query: string) { ); const source = plan.events![0].sourceQueries[0]; - return { eventName: 'write_checkpoints', defaultSchema: 'test_schema', source }; + return { eventName: 'write_checkpoints', source }; +} + +function eventDefinitionFromQueries(...queries: string[]) { + return eventDefinitionForSchema('test_schema', ...queries); +} + +function eventDefinitionForSchema(defaultSchema: string, ...queries: string[]) { + const config = SqlSyncRules.fromYaml(yamlWithEventQueries(...queries), { + defaultSchema + }).config as PrecompiledSyncConfig; + const plan = serializeSyncPlan(config.plan); + + return { id: config.eventDefinitions[0].id, event: plan.events![0] }; } function yamlWithEventQueries(...queries: string[]): string { From 719d19278a8d925253abc91a7b1fc597e734d50c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 19 Aug 2026 15:31:26 +0200 Subject: [PATCH 03/14] cleanup --- packages/sync-rules/src/compiler/compiler.ts | 45 ++++++++++++------- .../src/compiler/ir_to_sync_plan.ts | 12 ++--- .../sync-rules/src/compiler/querier_graph.ts | 6 +-- .../src/events/CompiledEventSourceQuery.ts | 11 ++--- .../sync-rules/src/events/EventDescriptor.ts | 7 +-- packages/sync-rules/src/from_yaml.ts | 4 +- packages/sync-rules/src/sync_plan/plan.ts | 13 +++--- .../src/sync_plan/plan_equality_serialized.ts | 11 ++--- .../sync-rules/src/sync_plan/serialize.ts | 19 ++++---- .../sync-rules/test/src/compiler/utils.ts | 2 +- 10 files changed, 66 insertions(+), 64 deletions(-) diff --git a/packages/sync-rules/src/compiler/compiler.ts b/packages/sync-rules/src/compiler/compiler.ts index f7f9b68a2..3ca56281e 100644 --- a/packages/sync-rules/src/compiler/compiler.ts +++ b/packages/sync-rules/src/compiler/compiler.ts @@ -49,20 +49,22 @@ export interface CompiledEventSourceQueryModel { } /** - * State for compiling sync streams. + * State for compiling sync streams and replication events into a sync plan. * - * The output of compiling all sync streams is a {@link SyncPlan}, a declarative description of the sync process that - * can be serialized to bucket storage. The compiler stores a mutable intermediate representation that is essentially a - * copy of the sync plan, except that we're using JavaScript classes with methods to compute hash codes and equality - * relations. This allows the compiler to efficiently de-duplicate parameters and buckets. + * The compiler stores a mutable intermediate representation that is essentially a copy of the resulting + * {@link SyncPlan}, except that we're using JavaScript classes with methods to compute hash codes and equality + * relations. Stream queries and event definitions remain separate within that model. * - * Overall, the compilation process is as follows: Each data query for a stream is first parsed by + * The stream compilation process is as follows: Each data query for a stream is first parsed by * {@link StreamQueryParser} into a canonicalized intermediate representation (see that class for details). * Then, {@link QuerierGraphBuilder} analyzes a chain of `AND` expressions to identify parameters (as partition keys) * and their instantiation, as well as static filters that need to be added to reach row. */ export class SyncStreamsCompiler { - readonly output = new CompiledStreamQueries(); + readonly output: SyncPlanCompilerModel = { + streams: new CompiledStreamQueries(), + events: [] + }; private readonly locations = new NodeLocations(); constructor(readonly options: SyncStreamsCompilerOptions) {} @@ -222,6 +224,14 @@ export class SyncStreamsCompiler { } }; } + + /** + * @returns A sync plan representing an immutable snapshot of the compiler output. + */ + toSyncPlan(): SyncPlan { + const translator = new CompilerModelToSyncPlan(); + return translator.translate(this.output); + } } /** @@ -250,7 +260,7 @@ export function compileEventDefinitions( } } - return { events: compiler.output.toSyncPlan().events, errors }; + return { events: compiler.toSyncPlan().events, errors }; } function tryParse(sql: string, errors: ParsingErrorListener): Statement | null { @@ -332,7 +342,6 @@ export class CompiledStreamQueries { }); readonly resolvers: StreamResolver[] = []; - readonly events: CompiledEvent[] = []; get evaluators(): RowEvaluator[] { return [...this._evaluators]; @@ -349,12 +358,16 @@ export class CompiledStreamQueries { canonicalizePointLookup(lookup: PointLookup): PointLookup { return this._pointLookups.getOrInsert(lookup)[0]; } +} - /** - * @returns A sync plan representing an immutable snapshot of this intermediate representation. - */ - toSyncPlan(): SyncPlan { - const translator = new CompilerModelToSyncPlan(); - return translator.translate(this); - } +/** + * Top-level compiler output used to assemble a complete sync plan. + * + * Streams and events are sibling sync-config concerns. Keeping their intermediate state separate prevents stream + * compilation abstractions from acquiring event-specific responsibilities just because both are persisted in one + * {@link SyncPlan}. + */ +export interface SyncPlanCompilerModel { + readonly streams: CompiledStreamQueries; + readonly events: CompiledEvent[]; } diff --git a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts index cb2bf51d8..8f8ce265d 100644 --- a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts +++ b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts @@ -3,7 +3,7 @@ import { MapSourceVisitor, visitExpr } from '../sync_plan/expression_visitor.js' import * as plan from '../sync_plan/plan.js'; import { compiledEventDefinitionId } from '../sync_plan/serialize.js'; import * as resolver from './bucket_resolver.js'; -import { CompiledStreamQueries } from './compiler.js'; +import type { SyncPlanCompilerModel } from './compiler.js'; import { Equality, HashMap, StableHasher, unorderedEquality } from './equality.js'; import { ColumnInRow, ExpressionInput, RowMetadata, SyncExpression } from './expression.js'; import * as rows from './rows.js'; @@ -47,12 +47,12 @@ export class CompilerModelToSyncPlan { return mapped; } - translate(source: CompiledStreamQueries): plan.SyncPlan { - const queriersByStream = Object.groupBy(source.resolvers, (r) => r.options.name); + translate(source: SyncPlanCompilerModel): plan.SyncPlan { + const queriersByStream = Object.groupBy(source.streams.resolvers, (r) => r.options.name); return { - dataSources: source.evaluators.map((e) => this.translateRowEvaluator(e)), - parameterIndexes: source.pointLookups.map((p, i) => this.translatePointLookup(p, i)), + dataSources: source.streams.evaluators.map((e) => this.translateRowEvaluator(e)), + parameterIndexes: source.streams.pointLookups.map((p, i) => this.translatePointLookup(p, i)), // Note: data sources and parameter indexes must be translated first because we reference them in stream // resolvers. streams: Object.values(queriersByStream).map((resolvers) => { @@ -63,7 +63,7 @@ export class CompilerModelToSyncPlan { }), buckets: this.buckets, events: source.events.map((event) => { - const definition: Omit = { + const definition: plan.CompiledEventDescriptorContent = { name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ sql: query.sql, diff --git a/packages/sync-rules/src/compiler/querier_graph.ts b/packages/sync-rules/src/compiler/querier_graph.ts index 05bd84d57..60595ee4d 100644 --- a/packages/sync-rules/src/compiler/querier_graph.ts +++ b/packages/sync-rules/src/compiler/querier_graph.ts @@ -69,7 +69,7 @@ export class QuerierGraphBuilder { return []; } - this.compiler.output.resolvers.push(...buckets); + this.compiler.output.streams.resolvers.push(...buckets); return buckets; } @@ -164,7 +164,7 @@ class PendingQuerierPath { const state = this.resolveResultSet(this.query.sourceTable); const [partitions, partitionValues] = state.resolvePartitions(); - const evaluator = this.builder.compiler.output.canonicalizeEvaluator( + const evaluator = this.builder.compiler.output.streams.canonicalizeEvaluator( new RowEvaluator({ columns: this.query.resultColumns, syntacticSource: this.query.sourceTable, @@ -435,7 +435,7 @@ class PendingQuerierPath { if (data.type == 'point') { const resultSet = data.resultSet; const [partitionKeys, partitionInputs] = resultSet.resolvePartitions(); - const canonicalized = this.builder.compiler.output.canonicalizePointLookup( + const canonicalized = this.builder.compiler.output.streams.canonicalizePointLookup( new PointLookup({ syntacticSource: data.source, filters: resultSet.filters, diff --git a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts index 0996841e1..8fd2b3013 100644 --- a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts +++ b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts @@ -7,16 +7,11 @@ import { } from '../sync_plan/plan.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, SqliteRow } from '../types.js'; -import { - EvaluatedEventRowWithErrors, - EventDefinition, - EventDefinitionId, - HydratedEventDescriptor -} from './EventDescriptor.js'; +import { EvaluatedEventRowWithErrors, EventDefinition, HydratedEventDescriptor } from './EventDescriptor.js'; /** A named event prepared from a compiled sync plan, before scalar expressions are prepared for evaluation. */ export class PreparedEventDefinition implements EventDefinition { - readonly id: EventDefinitionId; + readonly id: string; readonly name: string; readonly sourceQueries: PreparedEventSourceQuery[]; @@ -67,7 +62,7 @@ export class PreparedEventSourceQuery { class HydratedCompiledEventDescriptor implements HydratedEventDescriptor { constructor( - readonly id: EventDefinitionId, + readonly id: string, readonly name: string, readonly sourceQueries: HydratedCompiledEventSourceQuery[] ) {} diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index 5245f7fcb..3d66c6594 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -12,13 +12,10 @@ export type EvaluatedEventRowWithErrors = { errors: EvaluationError[]; }; -/** Content-addressed identity of a complete named event definition. */ -export type EventDefinitionId = string; - /** A parsed event definition whose compiled expressions have not yet been prepared for evaluation. */ export interface EventDefinition { /** Deterministic identity generated from the serialized compiled definition. */ - readonly id: EventDefinitionId; + readonly id: string; readonly name: string; createEvaluator(input: HydrationInput): HydratedEventDescriptor; @@ -29,7 +26,7 @@ export interface EventDefinition { /** An event definition whose payload queries can evaluate replicated rows. */ export interface HydratedEventDescriptor { /** Deterministic identity generated from the serialized compiled definition. */ - readonly id: EventDefinitionId; + readonly id: string; readonly name: string; evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors; diff --git a/packages/sync-rules/src/from_yaml.ts b/packages/sync-rules/src/from_yaml.ts index c923c08eb..4080a3b67 100644 --- a/packages/sync-rules/src/from_yaml.ts +++ b/packages/sync-rules/src/from_yaml.ts @@ -131,7 +131,7 @@ export class SyncConfigFromYaml { const eventCompiler = new SyncStreamsCompiler(this.options); this.#compileEventDefinitions(eventMap, eventCompiler); - const eventPlan = eventCompiler.output.toSyncPlan(); + const eventPlan = eventCompiler.toSyncPlan(); result = this.#legacyParseBucketDefinitionsAndStreams(bucketMap, streamMap, compatibility, eventPlan.events); } @@ -291,7 +291,7 @@ export class SyncConfigFromYaml { this.#compileEventDefinitions(eventMap, compiler); - return new PrecompiledSyncConfig(compiler.output.toSyncPlan(), compatibility, { + return new PrecompiledSyncConfig(compiler.toSyncPlan(), compatibility, { defaultSchema: this.options.defaultSchema, sourceText: this.yaml }); diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index 9c92fd91e..ea3f2a5dd 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -1,5 +1,4 @@ import { BucketPriority } from '../BucketDescription.js'; -import type { EventDefinitionId } from '../events/EventDescriptor.js'; import { ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern } from '../TablePattern.js'; import { UnscopedEvaluatedParameters } from '../types.js'; @@ -139,15 +138,19 @@ export interface StreamDataSource extends RowProjection { export type ColumnSource = 'star' | { expr: SqlExpression; alias: string }; /** - * A named replication event compiled from `event_definitions`. + * The content of a named replication event compiled from `event_definitions`. */ -export interface CompiledEventDescriptor { - /** Content-addressed identity assigned when compiler output is finalized or restored from a serialized plan. */ - id: EventDefinitionId; +export interface CompiledEventDescriptorContent { name: string; sourceQueries: CompiledEventSourceQuery[]; } +/** A compiled replication event together with its content-addressed identity. */ +export interface CompiledEventDescriptor extends CompiledEventDescriptorContent { + /** Content-addressed identity assigned when compiler output is finalized or restored from a serialized plan. */ + id: string; +} + /** * A single payload query for an event. * diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index 18fd7ca78..d49b58b2d 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -1,10 +1,9 @@ import * as uuid from 'uuid'; import type { Equality } from '../compiler/equality.js'; -import type { EventDefinitionId } from '../events/EventDescriptor.js'; import type { SerializedBucketDataSource, SerializedDataSource, - SerializedEventDescriptor, + SerializedEventDescriptorContent, SerializedEventRowEvaluator, SerializedEventSourceQuery, SerializedParameterIndexLookupCreator @@ -23,16 +22,12 @@ export interface SerializedEventSourceDefinition { } /** Returns the serialized event definition without its derived ID. */ -export function serializedEventDefinitionIdentity( - event: Pick -): string { +export function serializedEventDefinitionIdentity(event: SerializedEventDescriptorContent): string { return JSON.stringify({ name: event.name, sourceQueries: event.sourceQueries }); } /** Generate the content-addressed ID persisted with and exposed by a compiled event definition. */ -export function serializedEventDefinitionId( - event: Pick -): EventDefinitionId { +export function serializedEventDefinitionId(event: SerializedEventDescriptorContent): string { return uuid.v5(serializedEventDefinitionIdentity(event), EVENT_DEFINITION_ID_NAMESPACE); } diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 9ccfa1fb6..48f07e756 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -1,4 +1,3 @@ -import type { EventDefinitionId } from '../events/EventDescriptor.js'; import { ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern, TablePattern } from '../TablePattern.js'; import { SqlExpression } from './expression.js'; @@ -7,6 +6,7 @@ import { ColumnSource, ColumnSqlParameterValue, CompiledEventDescriptor, + CompiledEventDescriptorContent, CompiledSyncStream, EvaluateTableValuedFunction, EventRowEvaluator, @@ -98,9 +98,7 @@ function createTableProcessorSerializer() { }; } - function serializeEventDefinition( - event: Pick - ): Omit { + function serializeEventDefinition(event: CompiledEventDescriptorContent): SerializedEventDescriptorContent { return { name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ @@ -452,9 +450,7 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { } /** Derive the ID assigned while finalizing a compiled event definition. */ -export function compiledEventDefinitionId( - event: Pick -): EventDefinitionId { +export function compiledEventDefinitionId(event: CompiledEventDescriptorContent): string { const definition = createTableProcessorSerializer().serializeEventDefinition(event); return serializedEventDefinitionId(definition); } @@ -538,13 +534,16 @@ export interface SerializedDataSource { partitionBy: SerializedPartitionKey[]; } -export interface SerializedEventDescriptor { - /** Content-addressed identity derived from the rest of this event definition. */ - id: EventDefinitionId; +export interface SerializedEventDescriptorContent { name: string; sourceQueries: SerializedEventSourceQuery[]; } +export interface SerializedEventDescriptor extends SerializedEventDescriptorContent { + /** Content-addressed identity derived from the rest of this event definition. */ + id: string; +} + export interface SerializedEventSourceQuery { /** Raw SQL retained for the legacy compatibility mirror and as part of the exact serialized event definition. */ sql: string; diff --git a/packages/sync-rules/test/src/compiler/utils.ts b/packages/sync-rules/test/src/compiler/utils.ts index 21cfd6ccb..16a9ab7f4 100644 --- a/packages/sync-rules/test/src/compiler/utils.ts +++ b/packages/sync-rules/test/src/compiler/utils.ts @@ -69,7 +69,7 @@ function compileSingleStream(...sql: string[]): [TranslationError[], SyncPlan] { } builder.finish(); - const originalPlan = compiler.output.toSyncPlan(); + const originalPlan = compiler.toSyncPlan(); // Add a serialization roundtrip to ensure sync plans are correctly evaluated even after being deserialized. const afterSerializationRoundtrip = deserializeSyncPlan(JSON.parse(JSON.stringify(serializeSyncPlan(originalPlan)))); From 3f94b21d8748f342e1ff09198c56a189d30080a9 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 20 Aug 2026 15:27:55 +0200 Subject: [PATCH 04/14] cleanup --- .../sync-rules/src/events/EventDescriptor.ts | 12 +++- packages/sync-rules/src/sync_plan/plan.ts | 6 +- .../src/sync_plan/plan_equality_serialized.ts | 62 +++++++------------ .../sync-rules/src/sync_plan/serialize.ts | 14 ++--- .../test/src/compiler/events.test.ts | 62 +++++++------------ 5 files changed, 67 insertions(+), 89 deletions(-) diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index 3d66c6594..a6b4aa0ff 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -14,7 +14,11 @@ export type EvaluatedEventRowWithErrors = { /** A parsed event definition whose compiled expressions have not yet been prepared for evaluation. */ export interface EventDefinition { - /** Deterministic identity generated from the serialized compiled definition. */ + /** + * Deterministic identity generated from canonical compiled behavior. Stable across SQL formatting and ordering, so + * unchanged definitions are recognized and not reprocessed; behaviorally-equal but differently-written definitions + * may still differ, which only over-reprocesses and never misses a change. + */ readonly id: string; readonly name: string; @@ -25,7 +29,11 @@ export interface EventDefinition { /** An event definition whose payload queries can evaluate replicated rows. */ export interface HydratedEventDescriptor { - /** Deterministic identity generated from the serialized compiled definition. */ + /** + * Deterministic identity generated from canonical compiled behavior. Stable across SQL formatting and ordering, so + * unchanged definitions are recognized and not reprocessed; behaviorally-equal but differently-written definitions + * may still differ, which only over-reprocesses and never misses a change. + */ readonly id: string; readonly name: string; diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index ea3f2a5dd..dd7906525 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -145,9 +145,9 @@ export interface CompiledEventDescriptorContent { sourceQueries: CompiledEventSourceQuery[]; } -/** A compiled replication event together with its content-addressed identity. */ +/** A compiled replication event together with its canonical behavioral identity. */ export interface CompiledEventDescriptor extends CompiledEventDescriptorContent { - /** Content-addressed identity assigned when compiler output is finalized or restored from a serialized plan. */ + /** Identity excluding non-functional SQL formatting, ordering, and compiler hash changes. */ id: string; } @@ -161,7 +161,7 @@ export interface CompiledEventDescriptor extends CompiledEventDescriptorContent export interface CompiledEventSourceQuery { /** * Original SQL retained as a compatibility mirror for services using the legacy event evaluator. - * It remains part of the exact serialized event definition used to derive the event ID. + * It is deliberately excluded from the event ID. */ sql: string; sourceTable: ImplicitSchemaTablePattern; diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index d49b58b2d..6730c0943 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -16,53 +16,39 @@ export interface SerializedBucketDataSourceWithDataSources { dataSources: readonly SerializedDataSource[]; } -export interface SerializedEventSourceDefinition { - eventName: string; - source: SerializedEventSourceQuery; -} - -/** Returns the serialized event definition without its derived ID. */ -export function serializedEventDefinitionIdentity(event: SerializedEventDescriptorContent): string { - return JSON.stringify({ name: event.name, sourceQueries: event.sourceQueries }); -} - -/** Generate the content-addressed ID persisted with and exposed by a compiled event definition. */ -export function serializedEventDefinitionId(event: SerializedEventDescriptorContent): string { - return uuid.v5(serializedEventDefinitionIdentity(event), EVENT_DEFINITION_ID_NAMESPACE); -} - /** - * Compiled-plan equality for an individual event source query. + * Returns the canonical, versioned identity input for a complete named event. * - * Event source identity is independent of the containing sync config. The identity deliberately excludes raw SQL and - * compiler hash codes, preserves table references as represented in the plan, and normalizes unordered filter variants - * so callers can use it as stable input to a persisted fingerprint. Callers must still verify equality after a - * fingerprint lookup. - */ -export const serializedEventSourceDefinitionEquality: Equality = { - hash(hasher, value) { - hasher.addString(serializedEventSourceDefinitionIdentity(value)); - }, - equals(a, b) { - return a === b || serializedEventSourceDefinitionIdentity(a) == serializedEventSourceDefinitionIdentity(b); - } -}; - -/** - * Returns the canonical, versioned identity input for one event source query. + * Raw SQL and compiler hash codes are excluded because they do not define event behavior. Filter variants and source + * queries are sorted because their order is not significant. * - * This is intentionally not a durable identifier by itself. Complete named events use - * {@link serializedEventDefinitionId}; source-level callers can use this value for semantic comparisons. + * This normalizes formatting and ordering, but not deeper SQL equivalences (e.g. commutative operands like + * `a = b` vs `b = a`). Two behaviorally-identical definitions may therefore still produce different identities. That + * only ever causes redundant reprocessing, never a missed change — which is the safe direction for incremental + * reprocessing, and it still avoids reprocessing on the common formatting/ordering edits. */ -export function serializedEventSourceDefinitionIdentity(value: SerializedEventSourceDefinition): string { +export function serializedEventDefinitionIdentity(event: SerializedEventDescriptorContent): string { return JSON.stringify({ version: 1, - eventName: value.eventName, - ...eventSourceQueryIdentity(value.source) + name: event.name, + sourceQueries: event.sourceQueries.map((source) => JSON.stringify(eventSourceQueryIdentity(source))).sort() }); } -/** Canonical identity fields for a source query without the containing event name. */ +/** + * Generate the content-addressed ID persisted with and exposed by a compiled event definition. + * + * We hash the canonical identity into a wide UUIDv5 rather than reusing the compiler's structural hash code, because + * event ids are compared directly across sync configs (equality of persisted id strings) with no `equals()` fallback + * to resolve collisions. The identifier must therefore be collision-resistant: a collision would make two different + * events look identical and silently skip an event's reprocessing. The 32-bit structural hash is only safe where it is + * paired with a full-equality check (e.g. bucket data sources). + */ +export function serializedEventDefinitionId(event: SerializedEventDescriptorContent): string { + return uuid.v5(serializedEventDefinitionIdentity(event), EVENT_DEFINITION_ID_NAMESPACE); +} + +/** Canonical identity fields for a source query without raw SQL or compiler hashes. */ function eventSourceQueryIdentity(source: SerializedEventSourceQuery) { return { sourceTable: source.table, diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 48f07e756..3c2e4bdda 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -455,11 +455,6 @@ export function compiledEventDefinitionId(event: CompiledEventDescriptorContent) return serializedEventDefinitionId(definition); } -/** Serialize a single compiled event using the exact representation persisted in a sync plan. */ -export function serializeEventDescriptor(event: CompiledEventDescriptor): SerializedEventDescriptor { - return createTableProcessorSerializer().serializeEvent(event); -} - /** * Changes to {@link SerializedSyncPlan} require a version bump when older services would interpret the plan * incorrectly. Optional additive fields are only safe without a bump when older readers can ignore them while another @@ -540,12 +535,12 @@ export interface SerializedEventDescriptorContent { } export interface SerializedEventDescriptor extends SerializedEventDescriptorContent { - /** Content-addressed identity derived from the rest of this event definition. */ + /** Canonical behavioral identity derived without raw SQL or compiler hashes. */ id: string; } export interface SerializedEventSourceQuery { - /** Raw SQL retained for the legacy compatibility mirror and as part of the exact serialized event definition. */ + /** Raw SQL retained for the legacy compatibility mirror, but excluded from the event ID. */ sql: string; table: SerializedTablePattern; variants: SerializedEventRowEvaluator[]; @@ -553,6 +548,11 @@ export interface SerializedEventSourceQuery { export interface SerializedEventRowEvaluator { table: SerializedTablePattern; + /** + * The compiler's structural hash, retained only for round-trip symmetry with data sources (which reuse the same + * projection shape). It is NOT part of event identity: the event {@link SerializedEventDescriptor.id} is derived from + * the canonical definition and deliberately excludes this hash, and events are never deduplicated by it at runtime. + */ hash: number; columns: SerializedColumnSource[]; filters: SqlExpression[]; diff --git a/packages/sync-rules/test/src/compiler/events.test.ts b/packages/sync-rules/test/src/compiler/events.test.ts index 51342eea6..5220e37d1 100644 --- a/packages/sync-rules/test/src/compiler/events.test.ts +++ b/packages/sync-rules/test/src/compiler/events.test.ts @@ -7,8 +7,6 @@ import { PrecompiledSyncConfig, serializedEventDefinitionId, serializedEventDefinitionIdentity, - serializedEventSourceDefinitionEquality, - serializedEventSourceDefinitionIdentity, serializeSyncPlan, SqlSyncRules } from '../../../src/index.js'; @@ -73,42 +71,41 @@ describe('compiled replication events', () => { ).toEqual({ errors: [] }); }); - test('uses canonical semantic identity independent of formatting and filter order', () => { - const first = eventSourceFromQuery( - 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0' - ); - const equivalent = eventSourceFromQuery( - ' select user_id, checkpoint from checkpoints AS c where c.checkpoint > 0 and c.active = true ' - ); - const changed = eventSourceFromQuery( - 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1' - ); - - expect(serializedEventSourceDefinitionEquality.equals(first, equivalent)).toBe(true); - expect(serializedEventSourceDefinitionIdentity(first)).toBe(serializedEventSourceDefinitionIdentity(equivalent)); - expect(serializedEventSourceDefinitionEquality.equals(first, changed)).toBe(false); - }); - - test('derives a content-addressed id from the exact serialized event definition', () => { + test('derives a canonical id independent of formatting, filter order and payload query order', () => { const first = eventDefinitionFromQueries( - 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true', + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0', 'SELECT user_id, checkpoint FROM archived_checkpoints' ); const reordered = eventDefinitionFromQueries( - ' select user_id, checkpoint from archived_checkpoints ', - 'select user_id, checkpoint from checkpoints c where c.active = true' + ' select "user_id", "checkpoint" from "archived_checkpoints" ', + 'select "user_id", "checkpoint" from "checkpoints" c where c."checkpoint" > 0 and c."active" = true' ); const changed = eventDefinitionFromQueries( - 'SELECT user_id, checkpoint FROM checkpoints WHERE active = false', + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1', 'SELECT user_id, checkpoint FROM archived_checkpoints' ); expect(first.id).toBe(serializedEventDefinitionId(first.event)); - expect(reordered.id).not.toBe(first.id); + expect(reordered.id).toBe(first.id); expect(changed.id).not.toBe(first.id); + expect(serializedEventDefinitionIdentity(reordered.event)).toBe(serializedEventDefinitionIdentity(first.event)); + }); - const { id: _id, ...definition } = first.event; - expect(serializedEventDefinitionIdentity(first.event)).toBe(JSON.stringify(definition)); + test('excludes raw SQL, compiler hashes and variant order from the id', () => { + const first = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); + const second = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE checkpoint > 0'); + const definition = structuredClone(first.event); + definition.sourceQueries[0].variants.push(structuredClone(second.event.sourceQueries[0].variants[0])); + const modified = structuredClone(definition); + modified.sourceQueries[0].sql = 'raw sql is compatibility metadata'; + modified.sourceQueries[0].variants.reverse(); + for (const variant of modified.sourceQueries[0].variants) { + variant.hash += 1; + } + + expect(modified.sourceQueries[0].variants).toHaveLength(2); + expect(serializedEventDefinitionIdentity(modified)).toBe(serializedEventDefinitionIdentity(definition)); + expect(serializedEventDefinitionId(modified)).toBe(serializedEventDefinitionId(definition)); }); test('derives the id from the plan without loading context', () => { @@ -148,19 +145,6 @@ describe('compiled replication events', () => { }); }); -function eventSourceFromQuery(query: string) { - const plan = serializeSyncPlan( - ( - SqlSyncRules.fromYaml(yamlWithEventQueries(query), { - defaultSchema: 'test_schema' - }).config as PrecompiledSyncConfig - ).plan - ); - const source = plan.events![0].sourceQueries[0]; - - return { eventName: 'write_checkpoints', source }; -} - function eventDefinitionFromQueries(...queries: string[]) { return eventDefinitionForSchema('test_schema', ...queries); } From 04d9376dc4cb9311377441bbc95dabd0ca6216bf Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 20 Aug 2026 16:20:17 +0200 Subject: [PATCH 05/14] cleanup types --- packages/sync-rules/src/HydrationState.ts | 1 + packages/sync-rules/src/events/CompiledEventSourceQuery.ts | 5 +++-- packages/sync-rules/src/events/EventDescriptor.ts | 5 +++-- packages/sync-rules/src/sync_plan/plan.ts | 4 ++-- .../sync-rules/src/sync_plan/plan_equality_serialized.ts | 3 ++- packages/sync-rules/src/sync_plan/serialize.ts | 6 +++--- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 5911f68ad..567a7f765 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -2,6 +2,7 @@ import { BucketDataSource, ParameterIndexLookupCreator } from './BucketSource.js export type BucketDefinitionId = string; export type ParameterIndexId = string; +export type EventDefinitionId = string; export interface BucketDataScope { /** diff --git a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts index 8fd2b3013..8acebbd50 100644 --- a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts +++ b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts @@ -1,4 +1,5 @@ import { HydrationInput } from '../BucketSource.js'; +import { EventDefinitionId } from '../HydrationState.js'; import { SourceTableRef } from '../SourceTableRef.js'; import { EvaluatedRowProjection, PendingRowProjection } from '../sync_plan/evaluator/row_projection.js'; import { @@ -11,7 +12,7 @@ import { EvaluatedEventRowWithErrors, EventDefinition, HydratedEventDescriptor } /** A named event prepared from a compiled sync plan, before scalar expressions are prepared for evaluation. */ export class PreparedEventDefinition implements EventDefinition { - readonly id: string; + readonly id: EventDefinitionId; readonly name: string; readonly sourceQueries: PreparedEventSourceQuery[]; @@ -62,7 +63,7 @@ export class PreparedEventSourceQuery { class HydratedCompiledEventDescriptor implements HydratedEventDescriptor { constructor( - readonly id: string, + readonly id: EventDefinitionId, readonly name: string, readonly sourceQueries: HydratedCompiledEventSourceQuery[] ) {} diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index a6b4aa0ff..318e728b8 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -1,4 +1,5 @@ import { HydrationInput } from '../BucketSource.js'; +import { EventDefinitionId } from '../HydrationState.js'; import { SourceTableRef } from '../SourceTableRef.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, EvaluationError, SqliteJsonRow } from '../types.js'; @@ -19,7 +20,7 @@ export interface EventDefinition { * unchanged definitions are recognized and not reprocessed; behaviorally-equal but differently-written definitions * may still differ, which only over-reprocesses and never misses a change. */ - readonly id: string; + readonly id: EventDefinitionId; readonly name: string; createEvaluator(input: HydrationInput): HydratedEventDescriptor; @@ -34,7 +35,7 @@ export interface HydratedEventDescriptor { * unchanged definitions are recognized and not reprocessed; behaviorally-equal but differently-written definitions * may still differ, which only over-reprocesses and never misses a change. */ - readonly id: string; + readonly id: EventDefinitionId; readonly name: string; evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors; diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index dd7906525..f307b09bf 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -1,5 +1,5 @@ import { BucketPriority } from '../BucketDescription.js'; -import { ParameterLookupDefinitionId } from '../HydrationState.js'; +import { EventDefinitionId, ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern } from '../TablePattern.js'; import { UnscopedEvaluatedParameters } from '../types.js'; import { SqlExpression } from './expression.js'; @@ -148,7 +148,7 @@ export interface CompiledEventDescriptorContent { /** A compiled replication event together with its canonical behavioral identity. */ export interface CompiledEventDescriptor extends CompiledEventDescriptorContent { /** Identity excluding non-functional SQL formatting, ordering, and compiler hash changes. */ - id: string; + id: EventDefinitionId; } /** diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index 6730c0943..279a2b972 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -1,5 +1,6 @@ import * as uuid from 'uuid'; import type { Equality } from '../compiler/equality.js'; +import type { EventDefinitionId } from '../HydrationState.js'; import type { SerializedBucketDataSource, SerializedDataSource, @@ -44,7 +45,7 @@ export function serializedEventDefinitionIdentity(event: SerializedEventDescript * events look identical and silently skip an event's reprocessing. The 32-bit structural hash is only safe where it is * paired with a full-equality check (e.g. bucket data sources). */ -export function serializedEventDefinitionId(event: SerializedEventDescriptorContent): string { +export function serializedEventDefinitionId(event: SerializedEventDescriptorContent): EventDefinitionId { return uuid.v5(serializedEventDefinitionIdentity(event), EVENT_DEFINITION_ID_NAMESPACE); } diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 3c2e4bdda..5e5cee189 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -1,4 +1,4 @@ -import { ParameterLookupDefinitionId } from '../HydrationState.js'; +import { EventDefinitionId, ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern, TablePattern } from '../TablePattern.js'; import { SqlExpression } from './expression.js'; import { MapSourceVisitor, visitExpr } from './expression_visitor.js'; @@ -450,7 +450,7 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { } /** Derive the ID assigned while finalizing a compiled event definition. */ -export function compiledEventDefinitionId(event: CompiledEventDescriptorContent): string { +export function compiledEventDefinitionId(event: CompiledEventDescriptorContent): EventDefinitionId { const definition = createTableProcessorSerializer().serializeEventDefinition(event); return serializedEventDefinitionId(definition); } @@ -536,7 +536,7 @@ export interface SerializedEventDescriptorContent { export interface SerializedEventDescriptor extends SerializedEventDescriptorContent { /** Canonical behavioral identity derived without raw SQL or compiler hashes. */ - id: string; + id: EventDefinitionId; } export interface SerializedEventSourceQuery { From 5ef205f363c55f88cbc8aabd0832b206ac09b6e0 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 21 Aug 2026 09:20:55 +0200 Subject: [PATCH 06/14] use equality rather than a uuid for comparing events --- .../src/PersistedSyncConfigContent.test.ts | 14 ++- .../src/compiler/ir_to_sync_plan.ts | 7 +- .../src/events/CompiledEventSourceQuery.ts | 5 - .../sync-rules/src/events/EventDescriptor.ts | 13 -- packages/sync-rules/src/sync_plan/plan.ts | 15 +-- .../src/sync_plan/plan_equality_serialized.ts | 112 ++++-------------- .../sync-rules/src/sync_plan/serialize.ts | 35 ++---- .../test/src/compiler/events.test.ts | 57 ++++----- 8 files changed, 67 insertions(+), 191 deletions(-) diff --git a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts index ce448ed15..f883f3ea9 100644 --- a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts +++ b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts @@ -3,6 +3,8 @@ import { DEFAULT_TAG, nodeSqlite, PrecompiledSyncConfig, + serializedEventDefinitionEquality, + serializeSyncPlan, SqlSyncRules } from '@powersync/service-sync-rules'; import * as sqlite from 'node:sqlite'; @@ -34,18 +36,18 @@ describe('persisted compiled replication events', () => { const parsed = SqlSyncRules.fromYaml(yamlWithEvents, { defaultSchema: 'test_schema' }); const update = updateSyncRulesFromConfig(parsed); const compiled = update.config.plan!; - const eventDefinitionId = parsed.config.eventDefinitions[0].id; + const originalEvent = compiled.plan.events![0]; expect(compiled.plan.version).toBeLessThanOrEqual(2); expect(compiled.plan.events).toHaveLength(1); - expect(compiled.plan.events![0].id).toBe(eventDefinitionId); + expect(originalEvent.name).toBe('write_checkpoints'); expect(compiled.eventDescriptors).toEqual({ write_checkpoints: [EVENT_QUERY] }); const restored = restore(compiled); expect(restored.config).toBeInstanceOf(PrecompiledSyncConfig); expect(restored.config).not.toHaveProperty('eventDescriptors'); expect(restored.config.eventDefinitions).toHaveLength(1); - expect(restored.config.eventDefinitions[0].id).toBe(eventDefinitionId); + expect(restored.config.eventDefinitions[0].name).toBe('write_checkpoints'); const hydrated = restored.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, @@ -64,8 +66,10 @@ describe('persisted compiled replication events', () => { const legacyView = restore({ ...compiled, plan: planWithoutCompiledEvents }); expect(legacyView.config).not.toHaveProperty('eventDescriptors'); expect(legacyView.config.eventDefinitions).toHaveLength(1); - expect(legacyView.config.eventDefinitions[0].id).toBe(eventDefinitionId); - expect((legacyView.config as PrecompiledSyncConfig).plan.events).toHaveLength(1); + expect(legacyView.config.eventDefinitions[0].name).toBe('write_checkpoints'); + // The event recompiled from the raw SQL mirror is structurally identical to the original. + const legacyEvent = serializeSyncPlan((legacyView.config as PrecompiledSyncConfig).plan).events![0]; + expect(serializedEventDefinitionEquality.equals(legacyEvent, originalEvent)).toBe(true); }); test('restores raw event descriptors attached to version 1 and 2 plans', () => { diff --git a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts index 8f8ce265d..3d5bcf56a 100644 --- a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts +++ b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts @@ -1,7 +1,6 @@ import { SqlExpression } from '../sync_plan/expression.js'; import { MapSourceVisitor, visitExpr } from '../sync_plan/expression_visitor.js'; import * as plan from '../sync_plan/plan.js'; -import { compiledEventDefinitionId } from '../sync_plan/serialize.js'; import * as resolver from './bucket_resolver.js'; import type { SyncPlanCompilerModel } from './compiler.js'; import { Equality, HashMap, StableHasher, unorderedEquality } from './equality.js'; @@ -62,8 +61,8 @@ export class CompilerModelToSyncPlan { }; }), buckets: this.buckets, - events: source.events.map((event) => { - const definition: plan.CompiledEventDescriptorContent = { + events: source.events.map((event): plan.CompiledEventDescriptor => { + return { name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ sql: query.sql, @@ -71,8 +70,6 @@ export class CompilerModelToSyncPlan { variants: query.variants.map((variant) => this.translateRowProjection(variant)) })) }; - - return { id: compiledEventDefinitionId(definition), ...definition }; }) }; } diff --git a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts index 8acebbd50..e274cdd0f 100644 --- a/packages/sync-rules/src/events/CompiledEventSourceQuery.ts +++ b/packages/sync-rules/src/events/CompiledEventSourceQuery.ts @@ -1,5 +1,4 @@ import { HydrationInput } from '../BucketSource.js'; -import { EventDefinitionId } from '../HydrationState.js'; import { SourceTableRef } from '../SourceTableRef.js'; import { EvaluatedRowProjection, PendingRowProjection } from '../sync_plan/evaluator/row_projection.js'; import { @@ -12,19 +11,16 @@ import { EvaluatedEventRowWithErrors, EventDefinition, HydratedEventDescriptor } /** A named event prepared from a compiled sync plan, before scalar expressions are prepared for evaluation. */ export class PreparedEventDefinition implements EventDefinition { - readonly id: EventDefinitionId; readonly name: string; readonly sourceQueries: PreparedEventSourceQuery[]; constructor(source: CompiledEventDescriptorPlan, defaultSchema: string) { - this.id = source.id; this.name = source.name; this.sourceQueries = source.sourceQueries.map((query) => new PreparedEventSourceQuery(query, defaultSchema)); } createEvaluator(input: HydrationInput): HydratedEventDescriptor { return new HydratedCompiledEventDescriptor( - this.id, this.name, this.sourceQueries.map((query) => query.createEvaluator(input)) ); @@ -63,7 +59,6 @@ export class PreparedEventSourceQuery { class HydratedCompiledEventDescriptor implements HydratedEventDescriptor { constructor( - readonly id: EventDefinitionId, readonly name: string, readonly sourceQueries: HydratedCompiledEventSourceQuery[] ) {} diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index 318e728b8..cc9da2474 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -1,5 +1,4 @@ import { HydrationInput } from '../BucketSource.js'; -import { EventDefinitionId } from '../HydrationState.js'; import { SourceTableRef } from '../SourceTableRef.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, EvaluationError, SqliteJsonRow } from '../types.js'; @@ -15,12 +14,6 @@ export type EvaluatedEventRowWithErrors = { /** A parsed event definition whose compiled expressions have not yet been prepared for evaluation. */ export interface EventDefinition { - /** - * Deterministic identity generated from canonical compiled behavior. Stable across SQL formatting and ordering, so - * unchanged definitions are recognized and not reprocessed; behaviorally-equal but differently-written definitions - * may still differ, which only over-reprocesses and never misses a change. - */ - readonly id: EventDefinitionId; readonly name: string; createEvaluator(input: HydrationInput): HydratedEventDescriptor; @@ -30,12 +23,6 @@ export interface EventDefinition { /** An event definition whose payload queries can evaluate replicated rows. */ export interface HydratedEventDescriptor { - /** - * Deterministic identity generated from canonical compiled behavior. Stable across SQL formatting and ordering, so - * unchanged definitions are recognized and not reprocessed; behaviorally-equal but differently-written definitions - * may still differ, which only over-reprocesses and never misses a change. - */ - readonly id: EventDefinitionId; readonly name: string; evaluateRowWithErrors(options: EvaluateRowOptions): EvaluatedEventRowWithErrors; diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index f307b09bf..108067539 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -1,5 +1,5 @@ import { BucketPriority } from '../BucketDescription.js'; -import { EventDefinitionId, ParameterLookupDefinitionId } from '../HydrationState.js'; +import { ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern } from '../TablePattern.js'; import { UnscopedEvaluatedParameters } from '../types.js'; import { SqlExpression } from './expression.js'; @@ -138,19 +138,16 @@ export interface StreamDataSource extends RowProjection { export type ColumnSource = 'star' | { expr: SqlExpression; alias: string }; /** - * The content of a named replication event compiled from `event_definitions`. + * A named replication event compiled from `event_definitions`. + * + * Events have no content id of their own: storage assigns and persists a stable id for each one, matching definitions + * across sync configs with {@link serializedEventDefinitionEquality}. */ -export interface CompiledEventDescriptorContent { +export interface CompiledEventDescriptor { name: string; sourceQueries: CompiledEventSourceQuery[]; } -/** A compiled replication event together with its canonical behavioral identity. */ -export interface CompiledEventDescriptor extends CompiledEventDescriptorContent { - /** Identity excluding non-functional SQL formatting, ordering, and compiler hash changes. */ - id: EventDefinitionId; -} - /** * A single payload query for an event. * diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index 279a2b972..7447568f6 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -1,113 +1,43 @@ -import * as uuid from 'uuid'; import type { Equality } from '../compiler/equality.js'; -import type { EventDefinitionId } from '../HydrationState.js'; import type { SerializedBucketDataSource, SerializedDataSource, - SerializedEventDescriptorContent, - SerializedEventRowEvaluator, - SerializedEventSourceQuery, + SerializedEventDescriptor, SerializedParameterIndexLookupCreator } from './serialize.js'; -const EVENT_DEFINITION_ID_NAMESPACE = uuid.v5('powersync-replication-event-definition-v1', uuid.v5.URL); - export interface SerializedBucketDataSourceWithDataSources { bucket: SerializedBucketDataSource; dataSources: readonly SerializedDataSource[]; } /** - * Returns the canonical, versioned identity input for a complete named event. + * Structural equality for a compiled event definition. * - * Raw SQL and compiler hash codes are excluded because they do not define event behavior. Filter variants and source - * queries are sorted because their order is not significant. + * This decides whether an event in a new sync config matches one in an active config, so it can keep that config's + * assigned storage id during incremental reprocessing instead of being treated as new. * - * This normalizes formatting and ordering, but not deeper SQL equivalences (e.g. commutative operands like - * `a = b` vs `b = a`). Two behaviorally-identical definitions may therefore still produce different identities. That - * only ever causes redundant reprocessing, never a missed change — which is the safe direction for incremental - * reprocessing, and it still avoids reprocessing on the common formatting/ordering edits. + * The raw `sql` mirror is excluded so that formatting, quoting and aliasing changes don't count as a change - the + * compiled structure already normalizes those. Payload-query order is not significant, so those are sorted. Everything + * else is compared verbatim, mirroring {@link serializedStreamBucketDataSourceEquality}. A behaviorally-neutral change + * we don't normalize (e.g. reordering conjunction terms) simply reprocesses, which is the safe direction. */ -export function serializedEventDefinitionIdentity(event: SerializedEventDescriptorContent): string { +export const serializedEventDefinitionEquality: Equality = { + hash(hasher, value) { + hasher.addString(eventDefinitionIdentity(value)); + }, + equals(a, b) { + return a === b || eventDefinitionIdentity(a) == eventDefinitionIdentity(b); + } +}; + +function eventDefinitionIdentity(event: SerializedEventDescriptor): string { return JSON.stringify({ - version: 1, name: event.name, - sourceQueries: event.sourceQueries.map((source) => JSON.stringify(eventSourceQueryIdentity(source))).sort() - }); -} - -/** - * Generate the content-addressed ID persisted with and exposed by a compiled event definition. - * - * We hash the canonical identity into a wide UUIDv5 rather than reusing the compiler's structural hash code, because - * event ids are compared directly across sync configs (equality of persisted id strings) with no `equals()` fallback - * to resolve collisions. The identifier must therefore be collision-resistant: a collision would make two different - * events look identical and silently skip an event's reprocessing. The 32-bit structural hash is only safe where it is - * paired with a full-equality check (e.g. bucket data sources). - */ -export function serializedEventDefinitionId(event: SerializedEventDescriptorContent): EventDefinitionId { - return uuid.v5(serializedEventDefinitionIdentity(event), EVENT_DEFINITION_ID_NAMESPACE); -} - -/** Canonical identity fields for a source query without raw SQL or compiler hashes. */ -function eventSourceQueryIdentity(source: SerializedEventSourceQuery) { - return { - sourceTable: source.table, - variants: source.variants - .map(eventVariantIdentity) - .map((variant) => JSON.stringify(variant)) + sourceQueries: event.sourceQueries + .map((query) => JSON.stringify({ table: query.table, variants: query.variants })) .sort() - }; -} - -function eventVariantIdentity(value: SerializedEventRowEvaluator) { - return { - table: value.table, - columns: value.columns.map((column) => { - return column == 'star' ? column : { ...column, expr: canonicalExpression(column.expr) }; - }), - // A conjunction's filter order does not affect behavior. - filters: value.filters.map((filter) => JSON.stringify(canonicalExpression(filter))).sort(), - tableValuedFunctions: value.tableValuedFunctions.map((fn) => ({ - ...fn, - functionInputs: fn.functionInputs.map(canonicalExpression) - })), - partitionBy: value.partitionBy.map((key) => ({ expr: canonicalExpression(key.expr) })) - }; -} - -function canonicalExpression(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(canonicalExpression); - } - if (value == null || typeof value != 'object') { - return value; - } - - const expression = value as Record; - if (expression.type == 'binary' && (expression.operator == 'and' || expression.operator == 'or')) { - const operator = expression.operator; - const operands: unknown[] = []; - const addOperand = (operand: unknown) => { - if ( - operand != null && - typeof operand == 'object' && - (operand as Record).type == 'binary' && - (operand as Record).operator == operator - ) { - addOperand((operand as Record).left); - addOperand((operand as Record).right); - } else { - operands.push(canonicalExpression(operand)); - } - }; - - addOperand(expression.left); - addOperand(expression.right); - return { type: 'commutative', operator, operands: operands.map((operand) => JSON.stringify(operand)).sort() }; - } - - return Object.fromEntries(Object.entries(expression).map(([key, nested]) => [key, canonicalExpression(nested)])); + }); } /** diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 5e5cee189..24cca809f 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -1,4 +1,4 @@ -import { EventDefinitionId, ParameterLookupDefinitionId } from '../HydrationState.js'; +import { ParameterLookupDefinitionId } from '../HydrationState.js'; import { ImplicitSchemaTablePattern, TablePattern } from '../TablePattern.js'; import { SqlExpression } from './expression.js'; import { MapSourceVisitor, visitExpr } from './expression_visitor.js'; @@ -6,7 +6,6 @@ import { ColumnSource, ColumnSqlParameterValue, CompiledEventDescriptor, - CompiledEventDescriptorContent, CompiledSyncStream, EvaluateTableValuedFunction, EventRowEvaluator, @@ -27,7 +26,6 @@ import { TableProcessorTableValuedFunction, TableProcessorTableValuedFunctionOutput } from './plan.js'; -import { serializedEventDefinitionId } from './plan_equality_serialized.js'; function createTableProcessorSerializer() { const addedTableValuedFunctions = new Map(); @@ -98,7 +96,7 @@ function createTableProcessorSerializer() { }; } - function serializeEventDefinition(event: CompiledEventDescriptorContent): SerializedEventDescriptorContent { + function serializeEventDefinition(event: CompiledEventDescriptor): SerializedEventDescriptor { return { name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ @@ -109,10 +107,6 @@ function createTableProcessorSerializer() { }; } - function serializeEvent(event: CompiledEventDescriptor): SerializedEventDescriptor { - return { id: event.id, ...serializeEventDefinition(event) }; - } - return { get usesRowMetadataSqlValue() { return usesRowMetadataSqlValue; @@ -121,8 +115,7 @@ function createTableProcessorSerializer() { serializeTablePattern, serializeTableValued, translateParameters, - serializeEventDefinition, - serializeEvent + serializeEventDefinition }; } @@ -232,7 +225,7 @@ export function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlan { }; } - const events = plan.events.map(tableProcessorSerializer.serializeEvent); + const events = plan.events.map(tableProcessorSerializer.serializeEventDefinition); const serialized: SerializedSyncPlan = { dataSources: serializeDataSources(), buckets: plan.buckets.map((bkt, index) => { @@ -370,7 +363,6 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { const serializedEvents = plan.events ?? []; const events = serializedEvents.map((event): CompiledEventDescriptor => { return { - id: event.id, name: event.name, sourceQueries: event.sourceQueries.map((query) => ({ sql: query.sql, @@ -449,12 +441,6 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { }; } -/** Derive the ID assigned while finalizing a compiled event definition. */ -export function compiledEventDefinitionId(event: CompiledEventDescriptorContent): EventDefinitionId { - const definition = createTableProcessorSerializer().serializeEventDefinition(event); - return serializedEventDefinitionId(definition); -} - /** * Changes to {@link SerializedSyncPlan} require a version bump when older services would interpret the plan * incorrectly. Optional additive fields are only safe without a bump when older readers can ignore them while another @@ -529,18 +515,13 @@ export interface SerializedDataSource { partitionBy: SerializedPartitionKey[]; } -export interface SerializedEventDescriptorContent { +export interface SerializedEventDescriptor { name: string; sourceQueries: SerializedEventSourceQuery[]; } -export interface SerializedEventDescriptor extends SerializedEventDescriptorContent { - /** Canonical behavioral identity derived without raw SQL or compiler hashes. */ - id: EventDefinitionId; -} - export interface SerializedEventSourceQuery { - /** Raw SQL retained for the legacy compatibility mirror, but excluded from the event ID. */ + /** Raw SQL retained for the legacy compatibility mirror, but excluded from event identity. */ sql: string; table: SerializedTablePattern; variants: SerializedEventRowEvaluator[]; @@ -550,8 +531,8 @@ export interface SerializedEventRowEvaluator { table: SerializedTablePattern; /** * The compiler's structural hash, retained only for round-trip symmetry with data sources (which reuse the same - * projection shape). It is NOT part of event identity: the event {@link SerializedEventDescriptor.id} is derived from - * the canonical definition and deliberately excludes this hash, and events are never deduplicated by it at runtime. + * projection shape). It is not part of event identity ({@link serializedEventDefinitionEquality} compares the + * structure directly) and events are never deduplicated by it at runtime. */ hash: number; columns: SerializedColumnSource[]; diff --git a/packages/sync-rules/test/src/compiler/events.test.ts b/packages/sync-rules/test/src/compiler/events.test.ts index 5220e37d1..4ea3c56f8 100644 --- a/packages/sync-rules/test/src/compiler/events.test.ts +++ b/packages/sync-rules/test/src/compiler/events.test.ts @@ -5,8 +5,7 @@ import { deserializeSyncPlan, nodeSqlite, PrecompiledSyncConfig, - serializedEventDefinitionId, - serializedEventDefinitionIdentity, + serializedEventDefinitionEquality, serializeSyncPlan, SqlSyncRules } from '../../../src/index.js'; @@ -41,17 +40,12 @@ describe('compiled replication events', () => { // Compiled events are additive and do not require a new plan version. expect(serialized.version).toBe(1); expect(serialized.events).toHaveLength(1); - expect(compiled.eventDefinitions[0].id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ - ); - expect(serialized.events![0].id).toBe(compiled.eventDefinitions[0].id); const deserialized = deserializeSyncPlan(JSON.parse(JSON.stringify(serialized))); expect(deserialized.events).toMatchObject(compiled.plan.events); - expect(deserialized.events[0].id).toBe(compiled.eventDefinitions[0].id); const hydrated = compiled.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) }); const event = hydrated.eventDescriptors[0]; - expect(event.id).toBe(compiled.eventDefinitions[0].id); + expect(event.name).toBe('write_checkpoints'); const checkpoints = new TestSourceTable('checkpoints'); expect( @@ -71,49 +65,41 @@ describe('compiled replication events', () => { ).toEqual({ errors: [] }); }); - test('derives a canonical id independent of formatting, filter order and payload query order', () => { + test('matches definitions independent of formatting, quoting and payload-query order', () => { const first = eventDefinitionFromQueries( 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0', 'SELECT user_id, checkpoint FROM archived_checkpoints' ); - const reordered = eventDefinitionFromQueries( + const equivalent = eventDefinitionFromQueries( ' select "user_id", "checkpoint" from "archived_checkpoints" ', - 'select "user_id", "checkpoint" from "checkpoints" c where c."checkpoint" > 0 and c."active" = true' + 'select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0' ); const changed = eventDefinitionFromQueries( 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1', 'SELECT user_id, checkpoint FROM archived_checkpoints' ); - expect(first.id).toBe(serializedEventDefinitionId(first.event)); - expect(reordered.id).toBe(first.id); - expect(changed.id).not.toBe(first.id); - expect(serializedEventDefinitionIdentity(reordered.event)).toBe(serializedEventDefinitionIdentity(first.event)); + expect(serializedEventDefinitionEquality.equals(first, equivalent)).toBe(true); + expect(serializedEventDefinitionEquality.equals(first, changed)).toBe(false); }); - test('excludes raw SQL, compiler hashes and variant order from the id', () => { - const first = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); - const second = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE checkpoint > 0'); - const definition = structuredClone(first.event); - definition.sourceQueries[0].variants.push(structuredClone(second.event.sourceQueries[0].variants[0])); - const modified = structuredClone(definition); - modified.sourceQueries[0].sql = 'raw sql is compatibility metadata'; - modified.sourceQueries[0].variants.reverse(); - for (const variant of modified.sourceQueries[0].variants) { - variant.hash += 1; - } - - expect(modified.sourceQueries[0].variants).toHaveLength(2); - expect(serializedEventDefinitionIdentity(modified)).toBe(serializedEventDefinitionIdentity(definition)); - expect(serializedEventDefinitionId(modified)).toBe(serializedEventDefinitionId(definition)); + test('excludes the raw SQL mirror from the identity', () => { + const original = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); + const differentSql = structuredClone(original); + differentSql.sourceQueries[0].sql = 'raw sql is compatibility metadata only'; + + expect(serializedEventDefinitionEquality.equals(original, differentSql)).toBe(true); }); - test('derives the id from the plan without loading context', () => { + test('matches independent of the default schema used to compile', () => { const query = 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true'; - expect(eventDefinitionForSchema('first_schema', query).id).toBe( - eventDefinitionForSchema('second_schema', query).id - ); + expect( + serializedEventDefinitionEquality.equals( + eventDefinitionForSchema('first_schema', query), + eventDefinitionForSchema('second_schema', query) + ) + ).toBe(true); }); test.each([ @@ -153,9 +139,8 @@ function eventDefinitionForSchema(defaultSchema: string, ...queries: string[]) { const config = SqlSyncRules.fromYaml(yamlWithEventQueries(...queries), { defaultSchema }).config as PrecompiledSyncConfig; - const plan = serializeSyncPlan(config.plan); - return { id: config.eventDefinitions[0].id, event: plan.events![0] }; + return serializeSyncPlan(config.plan).events![0]; } function yamlWithEventQueries(...queries: string[]): string { From 9a22ff68aa6655f23dc4b16614b0a36aea7d5a24 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 21 Aug 2026 17:27:47 +0200 Subject: [PATCH 07/14] moving code around between PRs --- .../src/PersistedSyncConfigContent.test.ts | 5 +- packages/sync-rules/src/HydrationState.ts | 1 - packages/sync-rules/src/compiler/compiler.ts | 90 +++++++++++++++++- .../sync-rules/src/compiler/expression.ts | 94 +++++++++++++++++++ packages/sync-rules/src/sync_plan/plan.ts | 7 +- .../src/sync_plan/plan_equality_serialized.ts | 30 ------ .../sync-rules/src/sync_plan/serialize.ts | 5 +- .../test/src/compiler/events.test.ts | 26 ++--- 8 files changed, 199 insertions(+), 59 deletions(-) diff --git a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts index f883f3ea9..693b623e0 100644 --- a/packages/service-core/test/src/PersistedSyncConfigContent.test.ts +++ b/packages/service-core/test/src/PersistedSyncConfigContent.test.ts @@ -3,7 +3,6 @@ import { DEFAULT_TAG, nodeSqlite, PrecompiledSyncConfig, - serializedEventDefinitionEquality, serializeSyncPlan, SqlSyncRules } from '@powersync/service-sync-rules'; @@ -67,9 +66,9 @@ describe('persisted compiled replication events', () => { expect(legacyView.config).not.toHaveProperty('eventDescriptors'); expect(legacyView.config.eventDefinitions).toHaveLength(1); expect(legacyView.config.eventDefinitions[0].name).toBe('write_checkpoints'); - // The event recompiled from the raw SQL mirror is structurally identical to the original. + // Recompiling the raw SQL mirror produces the same serialized event plan. const legacyEvent = serializeSyncPlan((legacyView.config as PrecompiledSyncConfig).plan).events![0]; - expect(serializedEventDefinitionEquality.equals(legacyEvent, originalEvent)).toBe(true); + expect(legacyEvent).toEqual(originalEvent); }); test('restores raw event descriptors attached to version 1 and 2 plans', () => { diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 567a7f765..5911f68ad 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -2,7 +2,6 @@ import { BucketDataSource, ParameterIndexLookupCreator } from './BucketSource.js export type BucketDefinitionId = string; export type ParameterIndexId = string; -export type EventDefinitionId = string; export interface BucketDataScope { /** diff --git a/packages/sync-rules/src/compiler/compiler.ts b/packages/sync-rules/src/compiler/compiler.ts index 3ca56281e..c11b6a6ba 100644 --- a/packages/sync-rules/src/compiler/compiler.ts +++ b/packages/sync-rules/src/compiler/compiler.ts @@ -4,13 +4,13 @@ import { CompiledEventDescriptor, StreamOptions, SyncPlan } from '../sync_plan/p import { SourceSchema } from '../types.js'; import { StreamResolver } from './bucket_resolver.js'; import { DangerousParameterDetector } from './detect_dangerous_parameters.js'; -import { HashSet } from './equality.js'; -import { NodeLocations } from './expression.js'; +import { Equality, HashSet } from './equality.js'; +import { expressionBehaviorIdentity, NodeLocations } from './expression.js'; import { RowExpression, SingleDependencyExpression } from './filter.js'; import { CompilerModelToSyncPlan } from './ir_to_sync_plan.js'; import { StreamQueryParser } from './parser.js'; import { QuerierGraphBuilder } from './querier_graph.js'; -import { EventRowEvaluator, PointLookup, RowEvaluator } from './rows.js'; +import { EventRowEvaluator, ExpressionColumnSource, PointLookup, RowEvaluator, StarColumnSource } from './rows.js'; import { SqlScope } from './scope.js'; import { CommonTableExpression, PreparedSubquery } from './sqlite.js'; import { PhysicalSourceResultSet } from './table.js'; @@ -48,6 +48,56 @@ export interface CompiledEventSourceQueryModel { variants: EventRowEvaluator[]; } +/** + * Behavioral equality for replication events in the compiler's JavaScript plan model. + * + * Raw SQL and payload-query order are deliberately ignored. The row evaluators compare the tables, filters and + * projected payload through the same compatibility model used while compiling stream plans. + */ +export const compiledEventDefinitionEquality: Equality = { + hash(hasher, value) { + hasher.addString(compiledEventDefinitionIdentity(value)); + }, + equals(a, b) { + return a === b || compiledEventDefinitionIdentity(a) == compiledEventDefinitionIdentity(b); + } +}; + +function compiledEventDefinitionIdentity(event: CompiledEvent): string { + return JSON.stringify([ + event.name, + event.sourceQueries + .map((query) => + JSON.stringify([ + tablePatternIdentity(query.sourceTable), + query.variants.map(compiledEventVariantIdentity).sort() + ]) + ) + .sort() + ]); +} + +function compiledEventVariantIdentity(variant: EventRowEvaluator): string { + return JSON.stringify([ + tablePatternIdentity(variant.syntacticSource), + variant.filters.map((filter) => expressionBehaviorIdentity(filter.expression.node)).sort(), + variant.columns.map((column) => { + if (column instanceof StarColumnSource) { + return 'star'; + } + const expressionColumn = column as ExpressionColumnSource; + return [expressionColumn.alias, expressionBehaviorIdentity(expressionColumn.expression.expression.node)]; + }), + variant.partitionBy.length, + variant.addedFunctions.length + ]); +} + +function tablePatternIdentity(source: PhysicalSourceResultSet): readonly (string | null)[] { + const pattern = source.tablePattern; + return [pattern.connectionTag, pattern.schema, pattern.tablePattern]; +} + /** * State for compiling sync streams and replication events into a sync plan. * @@ -244,14 +294,46 @@ export function compileEventDefinitions( definitions: Readonly>, options: SyncStreamsCompilerOptions ): { events: CompiledEventDescriptor[]; errors: SqlRuleError[] } { + const { compiler, errors } = compileEventDefinitionsIntoCompiler(definitions, options); + return { events: compiler.toSyncPlan().events, errors }; +} + +/** + * Compiles raw event SQL and retains the compiler's JavaScript model for behavioral compatibility checks. + */ +export function compileEventDefinitionsToCompilerModel( + definitions: Readonly>, + options: SyncStreamsCompilerOptions +): { events: CompiledEvent[]; errors: SqlRuleError[] } { + const { compiler, errors } = compileEventDefinitionsIntoCompiler(definitions, options); + return { events: compiler.output.events, errors }; +} + +/** + * Shared compilation step for the two event representations above. + * + * Returning the compiler, rather than one of its outputs, is intentional: persisted-plan normalization translates the + * result into serializable sync-plan data, while compatibility matching needs the JavaScript compiler model and its + * behavioral structure. Keeping both paths on this helper ensures they compile the same SQL with the same options. + */ +function compileEventDefinitionsIntoCompiler( + definitions: Readonly>, + options: SyncStreamsCompilerOptions +): { compiler: SyncStreamsCompiler; errors: SqlRuleError[] } { + // Use one compiler for the complete event collection, matching normal sync-config compilation. In particular, this + // keeps all compiler-owned expression locations and intermediate event models in one coherent output snapshot. const compiler = new SyncStreamsCompiler(options); const errors: SqlRuleError[] = []; for (const [name, queries] of Object.entries(definitions)) { + // event() registers the named model immediately; each payload query then contributes one validated physical-table + // source to that model. Invalid queries report errors and are not added to the compiled event. const event = compiler.event(name); for (const sql of queries) { event.addSourceQuery(sql, { report(message, location, reportOptions) { + // Locations are offsets into this specific payload SQL string, so bind every reported compiler diagnostic to + // that source before accumulating it with diagnostics from the other event queries. const error = new SqlRuleError(message, sql, location); error.type = reportOptions?.isWarning ? 'warning' : 'fatal'; errors.push(error); @@ -260,7 +342,7 @@ export function compileEventDefinitions( } } - return { events: compiler.toSyncPlan().events, errors }; + return { compiler, errors }; } function tryParse(sql: string, errors: ParsingErrorListener): Statement | null { diff --git a/packages/sync-rules/src/compiler/expression.ts b/packages/sync-rules/src/compiler/expression.ts index fee1eb130..78cf926e7 100644 --- a/packages/sync-rules/src/compiler/expression.ts +++ b/packages/sync-rules/src/compiler/expression.ts @@ -77,6 +77,100 @@ export class SyncExpression implements EqualsIgnoringResultSet { } } +/** + * A stable identity for expression behavior across syntactic operand ordering changes. + * + * Boolean conjunction/disjunction and equality are commutative. Comparisons are normalized to one direction, so + * `a > b` and `b < a` also match. Operators and constructs where order affects behavior retain their original order. + */ +export function expressionBehaviorIdentity(expression: SqlExpression): string { + switch (expression.type) { + case 'data': + return JSON.stringify(['data', expressionInputIdentity(expression.source)]); + case 'unary': + return JSON.stringify(['unary', expression.operator, expressionBehaviorIdentity(expression.operand)]); + case 'binary': { + if (expression.operator == 'and' || expression.operator == 'or') { + const operands: SqlExpression[] = []; + collectAssociativeOperands(expression, expression.operator, operands); + return JSON.stringify(['binary', expression.operator, operands.map(expressionBehaviorIdentity).sort()]); + } + + let operator = expression.operator; + let left = expression.left; + let right = expression.right; + if (operator == '>' || operator == '>=') { + operator = operator == '>' ? '<' : '<='; + [left, right] = [right, left]; + } + + const operands = [expressionBehaviorIdentity(left), expressionBehaviorIdentity(right)]; + if (operator == '=' || operator == 'is') { + operands.sort(); + } + return JSON.stringify(['binary', operator, operands]); + } + case 'between': + return JSON.stringify([ + 'between', + expressionBehaviorIdentity(expression.value), + expressionBehaviorIdentity(expression.low), + expressionBehaviorIdentity(expression.high) + ]); + case 'scalar_in': + return JSON.stringify([ + 'scalar_in', + expressionBehaviorIdentity(expression.target), + expression.in.map(expressionBehaviorIdentity) + ]); + case 'case_when': + return JSON.stringify([ + 'case_when', + expression.operand == null ? null : expressionBehaviorIdentity(expression.operand), + expression.whens.map((branch) => [ + expressionBehaviorIdentity(branch.when), + expressionBehaviorIdentity(branch.then) + ]), + expression.else == null ? null : expressionBehaviorIdentity(expression.else) + ]); + case 'cast': + return JSON.stringify(['cast', expression.cast_as, expressionBehaviorIdentity(expression.operand)]); + case 'function': + return JSON.stringify(['function', expression.function, expression.parameters.map(expressionBehaviorIdentity)]); + case 'lit_null': + return JSON.stringify(['lit_null']); + case 'lit_double': + return JSON.stringify(['lit_double', expression.value]); + case 'lit_int': + return JSON.stringify(['lit_int', expression.base10]); + case 'lit_string': + return JSON.stringify(['lit_string', expression.value]); + } +} + +function collectAssociativeOperands( + expression: SqlExpression, + operator: 'and' | 'or', + output: SqlExpression[] +): void { + if (expression.type == 'binary' && expression.operator == operator) { + collectAssociativeOperands(expression.left, operator, output); + collectAssociativeOperands(expression.right, operator, output); + } else { + output.push(expression); + } +} + +function expressionInputIdentity(input: ExpressionInput): readonly string[] { + if (input instanceof ColumnInRow) { + return ['column', input.column]; + } else if (input instanceof RowMetadata) { + return ['row_metadata', input.kind]; + } else { + return ['connection_parameter', input.source]; + } +} + class FindExternalData extends RecursiveExpressionVisitor { defaultExpression(expr: SqlExpression, arg: ExpressionInput[]): void { this.visitChildren(expr, arg); diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index 108067539..06cba0529 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -139,9 +139,6 @@ export type ColumnSource = 'star' | { expr: SqlExpression; a /** * A named replication event compiled from `event_definitions`. - * - * Events have no content id of their own: storage assigns and persists a stable id for each one, matching definitions - * across sync configs with {@link serializedEventDefinitionEquality}. */ export interface CompiledEventDescriptor { name: string; @@ -157,8 +154,8 @@ export interface CompiledEventDescriptor { */ export interface CompiledEventSourceQuery { /** - * Original SQL retained as a compatibility mirror for services using the legacy event evaluator. - * It is deliberately excluded from the event ID. + * Original SQL retained as a compatibility mirror for services using the legacy event evaluator. Compiled event + * evaluation uses the remaining fields. */ sql: string; sourceTable: ImplicitSchemaTablePattern; diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index 7447568f6..63f721bbb 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -2,7 +2,6 @@ import type { Equality } from '../compiler/equality.js'; import type { SerializedBucketDataSource, SerializedDataSource, - SerializedEventDescriptor, SerializedParameterIndexLookupCreator } from './serialize.js'; @@ -11,35 +10,6 @@ export interface SerializedBucketDataSourceWithDataSources { dataSources: readonly SerializedDataSource[]; } -/** - * Structural equality for a compiled event definition. - * - * This decides whether an event in a new sync config matches one in an active config, so it can keep that config's - * assigned storage id during incremental reprocessing instead of being treated as new. - * - * The raw `sql` mirror is excluded so that formatting, quoting and aliasing changes don't count as a change - the - * compiled structure already normalizes those. Payload-query order is not significant, so those are sorted. Everything - * else is compared verbatim, mirroring {@link serializedStreamBucketDataSourceEquality}. A behaviorally-neutral change - * we don't normalize (e.g. reordering conjunction terms) simply reprocesses, which is the safe direction. - */ -export const serializedEventDefinitionEquality: Equality = { - hash(hasher, value) { - hasher.addString(eventDefinitionIdentity(value)); - }, - equals(a, b) { - return a === b || eventDefinitionIdentity(a) == eventDefinitionIdentity(b); - } -}; - -function eventDefinitionIdentity(event: SerializedEventDescriptor): string { - return JSON.stringify({ - name: event.name, - sourceQueries: event.sourceQueries - .map((query) => JSON.stringify({ table: query.table, variants: query.variants })) - .sort() - }); -} - /** * Equality for SerializedParameterIndexLookupCreator. * diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 24cca809f..2cf94bb7c 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -521,7 +521,7 @@ export interface SerializedEventDescriptor { } export interface SerializedEventSourceQuery { - /** Raw SQL retained for the legacy compatibility mirror, but excluded from event identity. */ + /** Raw SQL retained for the legacy compatibility mirror. */ sql: string; table: SerializedTablePattern; variants: SerializedEventRowEvaluator[]; @@ -531,8 +531,7 @@ export interface SerializedEventRowEvaluator { table: SerializedTablePattern; /** * The compiler's structural hash, retained only for round-trip symmetry with data sources (which reuse the same - * projection shape). It is not part of event identity ({@link serializedEventDefinitionEquality} compares the - * structure directly) and events are never deduplicated by it at runtime. + * projection shape). It has no behavioral meaning for events. */ hash: number; columns: SerializedColumnSource[]; diff --git a/packages/sync-rules/test/src/compiler/events.test.ts b/packages/sync-rules/test/src/compiler/events.test.ts index 4ea3c56f8..abdcf593e 100644 --- a/packages/sync-rules/test/src/compiler/events.test.ts +++ b/packages/sync-rules/test/src/compiler/events.test.ts @@ -1,11 +1,12 @@ import * as sqlite from 'node:sqlite'; import { describe, expect, test } from 'vitest'; import { + compiledEventDefinitionEquality, + compileEventDefinitionsToCompilerModel, DEFAULT_HYDRATION_STATE, deserializeSyncPlan, nodeSqlite, PrecompiledSyncConfig, - serializedEventDefinitionEquality, serializeSyncPlan, SqlSyncRules } from '../../../src/index.js'; @@ -65,37 +66,37 @@ describe('compiled replication events', () => { ).toEqual({ errors: [] }); }); - test('matches definitions independent of formatting, quoting and payload-query order', () => { + test('matches definitions independent of formatting, equivalent operand order and payload-query order', () => { const first = eventDefinitionFromQueries( 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0', 'SELECT user_id, checkpoint FROM archived_checkpoints' ); const equivalent = eventDefinitionFromQueries( ' select "user_id", "checkpoint" from "archived_checkpoints" ', - 'select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0' + 'select "user_id", "checkpoint" from "checkpoints" where 0 < "checkpoint" and true = "active"' ); const changed = eventDefinitionFromQueries( 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1', 'SELECT user_id, checkpoint FROM archived_checkpoints' ); - expect(serializedEventDefinitionEquality.equals(first, equivalent)).toBe(true); - expect(serializedEventDefinitionEquality.equals(first, changed)).toBe(false); + expect(compiledEventDefinitionEquality.equals(first, equivalent)).toBe(true); + expect(compiledEventDefinitionEquality.equals(first, changed)).toBe(false); }); - test('excludes the raw SQL mirror from the identity', () => { + test('excludes the raw SQL compatibility mirror from equality', () => { const original = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); - const differentSql = structuredClone(original); + const differentSql = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); differentSql.sourceQueries[0].sql = 'raw sql is compatibility metadata only'; - expect(serializedEventDefinitionEquality.equals(original, differentSql)).toBe(true); + expect(compiledEventDefinitionEquality.equals(original, differentSql)).toBe(true); }); test('matches independent of the default schema used to compile', () => { const query = 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true'; expect( - serializedEventDefinitionEquality.equals( + compiledEventDefinitionEquality.equals( eventDefinitionForSchema('first_schema', query), eventDefinitionForSchema('second_schema', query) ) @@ -136,11 +137,10 @@ function eventDefinitionFromQueries(...queries: string[]) { } function eventDefinitionForSchema(defaultSchema: string, ...queries: string[]) { - const config = SqlSyncRules.fromYaml(yamlWithEventQueries(...queries), { - defaultSchema - }).config as PrecompiledSyncConfig; + const compiled = compileEventDefinitionsToCompilerModel({ write_checkpoints: queries }, { defaultSchema }); + expect(compiled.errors.filter((error) => error.type == 'fatal')).toEqual([]); - return serializeSyncPlan(config.plan).events![0]; + return compiled.events[0]; } function yamlWithEventQueries(...queries: string[]): string { From 4494db38004f185ebf065af1dc1910bdc31dbb83 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 21 Aug 2026 17:28:15 +0200 Subject: [PATCH 08/14] update changeset --- .changeset/compiled-event-plans.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/compiled-event-plans.md b/.changeset/compiled-event-plans.md index c50d948f7..44abe1149 100644 --- a/.changeset/compiled-event-plans.md +++ b/.changeset/compiled-event-plans.md @@ -3,4 +3,4 @@ '@powersync/service-core': minor --- -Compile replication events for every sync-config edition into additive serialized sync-plan data and normalize legacy sidecar events when loading older plans while preserving raw SQL for older services. +Compile replication events for every sync-config edition into additive serialized sync-plan data, expose behavioral equality for compiled event models, and normalize legacy sidecar events when loading older plans while preserving raw SQL for older services. From 21b817f20019cb62850c46616eff5991bf08fc37 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 25 Aug 2026 13:15:18 +0200 Subject: [PATCH 09/14] clarify comparison of serialized events --- .changeset/compiled-event-plans.md | 2 +- packages/sync-rules/src/compiler/compiler.ts | 157 ++++++------------ .../sync-rules/src/compiler/expression.ts | 94 ----------- packages/sync-rules/src/compiler/rows.ts | 26 +-- .../src/sync_plan/plan_equality_serialized.ts | 56 +++++++ .../test/src/compiler/events.test.ts | 141 +++++++++++----- 6 files changed, 215 insertions(+), 261 deletions(-) diff --git a/.changeset/compiled-event-plans.md b/.changeset/compiled-event-plans.md index 44abe1149..819eca040 100644 --- a/.changeset/compiled-event-plans.md +++ b/.changeset/compiled-event-plans.md @@ -3,4 +3,4 @@ '@powersync/service-core': minor --- -Compile replication events for every sync-config edition into additive serialized sync-plan data, expose behavioral equality for compiled event models, and normalize legacy sidecar events when loading older plans while preserving raw SQL for older services. +Compile replication events for every sync-config edition into additive serialized sync-plan data, expose behavioral equality for serialized event plans, and normalize legacy sidecar events when loading older plans while preserving raw SQL for older services. diff --git a/packages/sync-rules/src/compiler/compiler.ts b/packages/sync-rules/src/compiler/compiler.ts index c11b6a6ba..ae2d1d9a8 100644 --- a/packages/sync-rules/src/compiler/compiler.ts +++ b/packages/sync-rules/src/compiler/compiler.ts @@ -4,16 +4,16 @@ import { CompiledEventDescriptor, StreamOptions, SyncPlan } from '../sync_plan/p import { SourceSchema } from '../types.js'; import { StreamResolver } from './bucket_resolver.js'; import { DangerousParameterDetector } from './detect_dangerous_parameters.js'; -import { Equality, HashSet } from './equality.js'; -import { expressionBehaviorIdentity, NodeLocations } from './expression.js'; +import { HashSet } from './equality.js'; +import { NodeLocations } from './expression.js'; import { RowExpression, SingleDependencyExpression } from './filter.js'; import { CompilerModelToSyncPlan } from './ir_to_sync_plan.js'; import { StreamQueryParser } from './parser.js'; import { QuerierGraphBuilder } from './querier_graph.js'; -import { EventRowEvaluator, ExpressionColumnSource, PointLookup, RowEvaluator, StarColumnSource } from './rows.js'; +import { EventRowEvaluator, PointLookup, RowEvaluator } from './rows.js'; import { SqlScope } from './scope.js'; import { CommonTableExpression, PreparedSubquery } from './sqlite.js'; -import { PhysicalSourceResultSet } from './table.js'; +import type { PhysicalSourceResultSet } from './table.js'; export interface SyncStreamsCompilerOptions { /** @@ -37,73 +37,12 @@ export interface ParseStreamOptions extends StreamOptions { warnOnDangerousParameter: boolean; } -export interface CompiledEvent { - name: string; - sourceQueries: CompiledEventSourceQueryModel[]; -} - -export interface CompiledEventSourceQueryModel { - sql: string; - sourceTable: PhysicalSourceResultSet; - variants: EventRowEvaluator[]; -} - -/** - * Behavioral equality for replication events in the compiler's JavaScript plan model. - * - * Raw SQL and payload-query order are deliberately ignored. The row evaluators compare the tables, filters and - * projected payload through the same compatibility model used while compiling stream plans. - */ -export const compiledEventDefinitionEquality: Equality = { - hash(hasher, value) { - hasher.addString(compiledEventDefinitionIdentity(value)); - }, - equals(a, b) { - return a === b || compiledEventDefinitionIdentity(a) == compiledEventDefinitionIdentity(b); - } -}; - -function compiledEventDefinitionIdentity(event: CompiledEvent): string { - return JSON.stringify([ - event.name, - event.sourceQueries - .map((query) => - JSON.stringify([ - tablePatternIdentity(query.sourceTable), - query.variants.map(compiledEventVariantIdentity).sort() - ]) - ) - .sort() - ]); -} - -function compiledEventVariantIdentity(variant: EventRowEvaluator): string { - return JSON.stringify([ - tablePatternIdentity(variant.syntacticSource), - variant.filters.map((filter) => expressionBehaviorIdentity(filter.expression.node)).sort(), - variant.columns.map((column) => { - if (column instanceof StarColumnSource) { - return 'star'; - } - const expressionColumn = column as ExpressionColumnSource; - return [expressionColumn.alias, expressionBehaviorIdentity(expressionColumn.expression.expression.node)]; - }), - variant.partitionBy.length, - variant.addedFunctions.length - ]); -} - -function tablePatternIdentity(source: PhysicalSourceResultSet): readonly (string | null)[] { - const pattern = source.tablePattern; - return [pattern.connectionTag, pattern.schema, pattern.tablePattern]; -} - /** * State for compiling sync streams and replication events into a sync plan. * * The compiler stores a mutable intermediate representation that is essentially a copy of the resulting - * {@link SyncPlan}, except that we're using JavaScript classes with methods to compute hash codes and equality - * relations. Stream queries and event definitions remain separate within that model. + * {@link SyncPlan}. Stream compilation uses JavaScript classes with behavioral equality, while events use plain + * compiler records. Stream queries and event definitions remain separate within that model. * * The stream compilation process is as follows: Each data query for a stream is first parsed by * {@link StreamQueryParser} into a canonicalized intermediate representation (see that class for details). @@ -225,12 +164,19 @@ export class SyncStreamsCompiler { return; } - const defaultSchema = this.options.defaultSchema ?? ''; - const sourceTable = query.sourceTable.tablePattern.toTablePattern(defaultSchema); + const defaultSchema = this.options.defaultSchema; + const sourceTable = query.sourceTable.tablePattern; + // Runtime event evaluation selects one payload query for each source table, so accepting duplicate sources + // would make later queries unreachable. Resolve implicit schemas when a real default is available to detect + // `table` and `default_schema.table` as the same source without inventing an empty schema otherwise. + const normalizedSourceTable = defaultSchema == null ? sourceTable : sourceTable.toTablePattern(defaultSchema); if ( - event.sourceQueries.some((source) => - source.sourceTable.tablePattern.toTablePattern(defaultSchema).equals(sourceTable) - ) + event.sourceQueries.some((source) => { + const existingSourceTable = source.sourceTable.tablePattern; + const normalizedExistingSourceTable = + defaultSchema == null ? existingSourceTable : existingSourceTable.toTablePattern(defaultSchema); + return normalizedExistingSourceTable.equals(normalizedSourceTable); + }) ) { errors.report('Each payload query should query a unique table', query.span.location); return; @@ -285,55 +231,23 @@ export class SyncStreamsCompiler { } /** - * Compiles raw event SQL stored alongside older sync plans into the current plan representation. + * Compiles raw event definitions from legacy sync plans into the current plan representation. * - * This is the compatibility boundary for plans written before compiled events were added. Callers should reject fatal - * errors rather than carrying legacy event evaluators into a {@link SyncPlan}. + * All definitions share one compiler to match normal sync-config compilation. Callers should reject fatal errors + * rather than carrying invalid legacy event evaluators into a {@link SyncPlan}. */ export function compileEventDefinitions( definitions: Readonly>, options: SyncStreamsCompilerOptions ): { events: CompiledEventDescriptor[]; errors: SqlRuleError[] } { - const { compiler, errors } = compileEventDefinitionsIntoCompiler(definitions, options); - return { events: compiler.toSyncPlan().events, errors }; -} - -/** - * Compiles raw event SQL and retains the compiler's JavaScript model for behavioral compatibility checks. - */ -export function compileEventDefinitionsToCompilerModel( - definitions: Readonly>, - options: SyncStreamsCompilerOptions -): { events: CompiledEvent[]; errors: SqlRuleError[] } { - const { compiler, errors } = compileEventDefinitionsIntoCompiler(definitions, options); - return { events: compiler.output.events, errors }; -} - -/** - * Shared compilation step for the two event representations above. - * - * Returning the compiler, rather than one of its outputs, is intentional: persisted-plan normalization translates the - * result into serializable sync-plan data, while compatibility matching needs the JavaScript compiler model and its - * behavioral structure. Keeping both paths on this helper ensures they compile the same SQL with the same options. - */ -function compileEventDefinitionsIntoCompiler( - definitions: Readonly>, - options: SyncStreamsCompilerOptions -): { compiler: SyncStreamsCompiler; errors: SqlRuleError[] } { - // Use one compiler for the complete event collection, matching normal sync-config compilation. In particular, this - // keeps all compiler-owned expression locations and intermediate event models in one coherent output snapshot. const compiler = new SyncStreamsCompiler(options); const errors: SqlRuleError[] = []; for (const [name, queries] of Object.entries(definitions)) { - // event() registers the named model immediately; each payload query then contributes one validated physical-table - // source to that model. Invalid queries report errors and are not added to the compiled event. const event = compiler.event(name); for (const sql of queries) { event.addSourceQuery(sql, { report(message, location, reportOptions) { - // Locations are offsets into this specific payload SQL string, so bind every reported compiler diagnostic to - // that source before accumulating it with diagnostics from the other event queries. const error = new SqlRuleError(message, sql, location); error.type = reportOptions?.isWarning ? 'warning' : 'fatal'; errors.push(error); @@ -342,7 +256,7 @@ function compileEventDefinitionsIntoCompiler( } } - return { compiler, errors }; + return { events: compiler.toSyncPlan().events, errors }; } function tryParse(sql: string, errors: ParsingErrorListener): Statement | null { @@ -442,6 +356,33 @@ export class CompiledStreamQueries { } } +/** + * Compiler model for one SQL query in a named event's `payloads` list. + * + * It binds that query to its single physical source table and contains one row evaluator for each normalized `OR` + * branch. Within each evaluator, filters determine whether a source row triggers the event, while projected columns + * determine the payload produced for a matching row. + * + * {@link sql} is retained so service-core can dual-write the legacy `eventDescriptors` field for older services during + * rolling upgrades. The executable plan uses the parsed source table and row evaluators instead. + */ +export interface CompiledEventSourceQueryModel { + sql: string; + sourceTable: PhysicalSourceResultSet; + variants: EventRowEvaluator[]; +} + +/** + * Compiler model for one named entry under `event_definitions`. + * + * {@link name} identifies the event exposed to handlers. {@link sourceQueries} represents its complete `payloads` + * list, which may define how rows from different source tables trigger that event and produce its payload. + */ +export interface CompiledEvent { + name: string; + sourceQueries: CompiledEventSourceQueryModel[]; +} + /** * Top-level compiler output used to assemble a complete sync plan. * diff --git a/packages/sync-rules/src/compiler/expression.ts b/packages/sync-rules/src/compiler/expression.ts index 78cf926e7..fee1eb130 100644 --- a/packages/sync-rules/src/compiler/expression.ts +++ b/packages/sync-rules/src/compiler/expression.ts @@ -77,100 +77,6 @@ export class SyncExpression implements EqualsIgnoringResultSet { } } -/** - * A stable identity for expression behavior across syntactic operand ordering changes. - * - * Boolean conjunction/disjunction and equality are commutative. Comparisons are normalized to one direction, so - * `a > b` and `b < a` also match. Operators and constructs where order affects behavior retain their original order. - */ -export function expressionBehaviorIdentity(expression: SqlExpression): string { - switch (expression.type) { - case 'data': - return JSON.stringify(['data', expressionInputIdentity(expression.source)]); - case 'unary': - return JSON.stringify(['unary', expression.operator, expressionBehaviorIdentity(expression.operand)]); - case 'binary': { - if (expression.operator == 'and' || expression.operator == 'or') { - const operands: SqlExpression[] = []; - collectAssociativeOperands(expression, expression.operator, operands); - return JSON.stringify(['binary', expression.operator, operands.map(expressionBehaviorIdentity).sort()]); - } - - let operator = expression.operator; - let left = expression.left; - let right = expression.right; - if (operator == '>' || operator == '>=') { - operator = operator == '>' ? '<' : '<='; - [left, right] = [right, left]; - } - - const operands = [expressionBehaviorIdentity(left), expressionBehaviorIdentity(right)]; - if (operator == '=' || operator == 'is') { - operands.sort(); - } - return JSON.stringify(['binary', operator, operands]); - } - case 'between': - return JSON.stringify([ - 'between', - expressionBehaviorIdentity(expression.value), - expressionBehaviorIdentity(expression.low), - expressionBehaviorIdentity(expression.high) - ]); - case 'scalar_in': - return JSON.stringify([ - 'scalar_in', - expressionBehaviorIdentity(expression.target), - expression.in.map(expressionBehaviorIdentity) - ]); - case 'case_when': - return JSON.stringify([ - 'case_when', - expression.operand == null ? null : expressionBehaviorIdentity(expression.operand), - expression.whens.map((branch) => [ - expressionBehaviorIdentity(branch.when), - expressionBehaviorIdentity(branch.then) - ]), - expression.else == null ? null : expressionBehaviorIdentity(expression.else) - ]); - case 'cast': - return JSON.stringify(['cast', expression.cast_as, expressionBehaviorIdentity(expression.operand)]); - case 'function': - return JSON.stringify(['function', expression.function, expression.parameters.map(expressionBehaviorIdentity)]); - case 'lit_null': - return JSON.stringify(['lit_null']); - case 'lit_double': - return JSON.stringify(['lit_double', expression.value]); - case 'lit_int': - return JSON.stringify(['lit_int', expression.base10]); - case 'lit_string': - return JSON.stringify(['lit_string', expression.value]); - } -} - -function collectAssociativeOperands( - expression: SqlExpression, - operator: 'and' | 'or', - output: SqlExpression[] -): void { - if (expression.type == 'binary' && expression.operator == operator) { - collectAssociativeOperands(expression.left, operator, output); - collectAssociativeOperands(expression.right, operator, output); - } else { - output.push(expression); - } -} - -function expressionInputIdentity(input: ExpressionInput): readonly string[] { - if (input instanceof ColumnInRow) { - return ['column', input.column]; - } else if (input instanceof RowMetadata) { - return ['row_metadata', input.kind]; - } else { - return ['connection_parameter', input.source]; - } -} - class FindExternalData extends RecursiveExpressionVisitor { defaultExpression(expr: SqlExpression, arg: ExpressionInput[]): void { this.visitChildren(expr, arg); diff --git a/packages/sync-rules/src/compiler/rows.ts b/packages/sync-rules/src/compiler/rows.ts index e100b2b6f..ee6ad051e 100644 --- a/packages/sync-rules/src/compiler/rows.ts +++ b/packages/sync-rules/src/compiler/rows.ts @@ -207,28 +207,12 @@ export class RowEvaluator extends BaseSourceRowProcessor { /** * A row evaluator producing an event payload. - * - * Unlike {@link RowEvaluator}, the alias of the source table does not affect behavior because event payloads don't - * have a logical output table name. */ -export class EventRowEvaluator extends BaseSourceRowProcessor { - readonly columns: ColumnSource[]; - - constructor(options: SourceProcessorOptions & { columns: ColumnSource[] }) { - super(options); - this.columns = options.columns; - } - - buildBehaviorHashCode(hasher: StableHasher): void { - this.addBaseHashCode(hasher); - // An event's projected columns, expressions and aliases define the payload delivered to its handler. Changing - // them therefore changes event behavior, so include them here to keep this hash consistent with - // behavesIdenticalTo() and with the compiled definition that will be reprocessed. - equalsIgnoringResultSetList.hash(hasher, this.columns); - } - - behavesIdenticalTo(other: EventRowEvaluator): boolean { - return this.baseMatchesOther(other) && equalsIgnoringResultSetList.equals(other.columns, this.columns); +export class EventRowEvaluator extends RowEvaluator { + override get outputName(): undefined { + // Stream output names determine the logical destination table. Event handlers instead receive the named event and + // physical source table, so changing a payload query's source alias does not change event behavior. + return undefined; } } diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts index 63f721bbb..54ceb03f5 100644 --- a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -2,6 +2,8 @@ import type { Equality } from '../compiler/equality.js'; import type { SerializedBucketDataSource, SerializedDataSource, + SerializedEventDescriptor, + SerializedEventRowEvaluator, SerializedParameterIndexLookupCreator } from './serialize.js'; @@ -21,6 +23,24 @@ export interface SerializedBucketDataSourceWithDataSources { export const serializedStreamParameterIndexLookupCreatorEquality = jsonEquality(); +/** + * Equality for persisted replication events. + * + * Raw SQL is a rolling-upgrade compatibility mirror and cached evaluator hashes are not equality checks, so neither is + * part of event behavior. Payload queries are unordered because runtime selects them by source table, and normalized + * variants are unordered because they share a projection and only determine whether a row matches. The remaining + * serialized plan is a stable, self-contained behavior representation: Expression ASTs include both their shape and + * external-data bindings. + */ +export const serializedEventDefinitionEquality: Equality = { + hash(hasher, value) { + hasher.addString(JSON.stringify(eventIdentity(value))); + }, + equals(a, b) { + return a === b || JSON.stringify(eventIdentity(a)) == JSON.stringify(eventIdentity(b)); + } +}; + /** * SerializedBucketDataSource is not safe to compare _directly_, since it contains index references to SerializedDataSource * in the serialized sync plan. However, each SerializedDataSource is self-contained and safe to compare directly. @@ -50,6 +70,42 @@ function bucketIdentity(value: SerializedBucketDataSourceWithDataSources) { }; } +/** + * Builds a normalized identity for comparing serialized event descriptors from persisted sync plans. + * + * Raw SQL is excluded, while source queries and variants are sorted because their order does not affect event + * behavior. + */ +function eventIdentity(event: SerializedEventDescriptor) { + return { + name: event.name, + sourceQueries: event.sourceQueries + .map((query) => + JSON.stringify({ + table: query.table, + variants: query.variants.map((variant) => JSON.stringify(eventRowEvaluatorIdentity(variant))).sort() + }) + ) + .sort() + }; +} + +/** + * Selects the behavioral fields used when comparing row evaluators within serialized event descriptors. + * + * The cached evaluator hash is deliberately omitted. Ordering inside an evaluator is retained conservatively so + * changes to projected column order or the compiler's filter-expression order invalidate persisted compatibility. + */ +function eventRowEvaluatorIdentity(evaluator: SerializedEventRowEvaluator) { + return { + table: evaluator.table, + tableValuedFunctions: evaluator.tableValuedFunctions, + filters: evaluator.filters, + partitionBy: evaluator.partitionBy, + columns: evaluator.columns + }; +} + function jsonEquality(): Equality { return { hash(hasher, value) { diff --git a/packages/sync-rules/test/src/compiler/events.test.ts b/packages/sync-rules/test/src/compiler/events.test.ts index abdcf593e..395db0e3d 100644 --- a/packages/sync-rules/test/src/compiler/events.test.ts +++ b/packages/sync-rules/test/src/compiler/events.test.ts @@ -1,12 +1,12 @@ import * as sqlite from 'node:sqlite'; import { describe, expect, test } from 'vitest'; +import { StableHasher } from '../../../src/compiler/equality.js'; import { - compiledEventDefinitionEquality, - compileEventDefinitionsToCompilerModel, DEFAULT_HYDRATION_STATE, deserializeSyncPlan, nodeSqlite, PrecompiledSyncConfig, + serializedEventDefinitionEquality, serializeSyncPlan, SqlSyncRules } from '../../../src/index.js'; @@ -66,41 +66,109 @@ describe('compiled replication events', () => { ).toEqual({ errors: [] }); }); - test('matches definitions independent of formatting, equivalent operand order and payload-query order', () => { - const first = eventDefinitionFromQueries( - 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0', - 'SELECT user_id, checkpoint FROM archived_checkpoints' - ); - const equivalent = eventDefinitionFromQueries( - ' select "user_id", "checkpoint" from "archived_checkpoints" ', - 'select "user_id", "checkpoint" from "checkpoints" where 0 < "checkpoint" and true = "active"' - ); - const changed = eventDefinitionFromQueries( - 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 1', - 'SELECT user_id, checkpoint FROM archived_checkpoints' + // Persisted equality ignores representation-only differences while retaining every field that affects event + // evaluation and payloads. + test.each([ + { + description: 'formatting and payload-query order', + first: [ + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0', + 'SELECT user_id, checkpoint FROM archived_checkpoints' + ], + second: [ + ' select "user_id", "checkpoint" from "archived_checkpoints" ', + 'select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0' + ], + equal: true + }, + { + description: 'source aliases', + first: ['SELECT checkpoint.user_id FROM checkpoints AS checkpoint WHERE checkpoint.active = true'], + second: ['SELECT source_row.user_id FROM checkpoints AS source_row WHERE source_row.active = true'], + equal: true + }, + { + description: 'projected payload aliases', + first: ['SELECT user_id AS payload_user FROM checkpoints'], + second: ['SELECT user_id AS account FROM checkpoints'], + equal: false + }, + { + description: 'external-data bindings', + first: ['SELECT user_id FROM checkpoints WHERE active = true'], + second: ['SELECT user_id FROM checkpoints WHERE archived = true'], + equal: false + }, + { + description: 'filter literals', + first: ['SELECT user_id FROM checkpoints WHERE checkpoint > 0'], + second: ['SELECT user_id FROM checkpoints WHERE checkpoint > 1'], + equal: false + }, + { + description: 'reordered expression operands', + first: ['SELECT user_id FROM checkpoints WHERE active = true'], + second: ['SELECT user_id FROM checkpoints WHERE true = active'], + equal: false + }, + { + description: 'reordered filter clauses', + first: ['SELECT user_id FROM checkpoints WHERE active = true AND checkpoint > 0'], + second: ['SELECT user_id FROM checkpoints WHERE checkpoint > 0 AND active = true'], + equal: false + }, + { + description: 'reordered projected columns', + first: ['SELECT user_id, checkpoint FROM checkpoints'], + second: ['SELECT checkpoint, user_id FROM checkpoints'], + equal: false + } + ])('compares serialized event behavior for $description', ({ first, second, equal }) => { + const firstSerialized = serializedEventDefinitionFromQueries(...first); + const secondSerialized = serializedEventDefinitionFromQueries(...second); + + expect(serializedEventDefinitionEquality.equals(firstSerialized, secondSerialized)).toBe(equal); + if (equal) { + expect(StableHasher.hashWith(serializedEventDefinitionEquality, firstSerialized)).toEqual( + StableHasher.hashWith(serializedEventDefinitionEquality, secondSerialized) + ); + } + }); + + // Raw SQL and cached evaluator hashes are persisted metadata, not event behavior. + test('excludes serialized compatibility metadata from equality', () => { + const original = serializedEventDefinitionFromQueries( + 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true' ); + const metadataChanged = structuredClone(original); + metadataChanged.sourceQueries[0].sql = 'raw sql is compatibility metadata only'; + metadataChanged.sourceQueries[0].variants[0].hash++; - expect(compiledEventDefinitionEquality.equals(first, equivalent)).toBe(true); - expect(compiledEventDefinitionEquality.equals(first, changed)).toBe(false); + expect(serializedEventDefinitionEquality.equals(original, metadataChanged)).toBe(true); + expect(StableHasher.hashWith(serializedEventDefinitionEquality, original)).toEqual( + StableHasher.hashWith(serializedEventDefinitionEquality, metadataChanged) + ); }); - test('excludes the raw SQL compatibility mirror from equality', () => { - const original = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); - const differentSql = eventDefinitionFromQueries('SELECT user_id, checkpoint FROM checkpoints WHERE active = true'); - differentSql.sourceQueries[0].sql = 'raw sql is compatibility metadata only'; + // Persisted equality must compare the evaluator that produced existing data rather than recompiling its retained SQL. + test('detects changed serialized evaluator behavior when raw SQL is unchanged', () => { + const original = serializedEventDefinitionFromQueries( + 'SELECT user_id FROM checkpoints WHERE active = true AND checkpoint > 0' + ); + const changedEvaluator = structuredClone(original); + changedEvaluator.sourceQueries[0].variants[0].filters = []; - expect(compiledEventDefinitionEquality.equals(original, differentSql)).toBe(true); + expect(changedEvaluator.sourceQueries[0].sql).toEqual(original.sourceQueries[0].sql); + expect(serializedEventDefinitionEquality.equals(original, changedEvaluator)).toBe(false); }); - test('matches independent of the default schema used to compile', () => { - const query = 'SELECT user_id, checkpoint FROM checkpoints WHERE active = true'; + // Event names are observable by handlers and remain part of persisted compatibility matching. + test('compares serialized event names', () => { + const original = serializedEventDefinitionFromQueries('SELECT user_id FROM checkpoints'); + const renamed = structuredClone(original); + renamed.name = 'other_event'; - expect( - compiledEventDefinitionEquality.equals( - eventDefinitionForSchema('first_schema', query), - eventDefinitionForSchema('second_schema', query) - ) - ).toBe(true); + expect(serializedEventDefinitionEquality.equals(original, renamed)).toBe(false); }); test.each([ @@ -132,15 +200,14 @@ describe('compiled replication events', () => { }); }); -function eventDefinitionFromQueries(...queries: string[]) { - return eventDefinitionForSchema('test_schema', ...queries); -} - -function eventDefinitionForSchema(defaultSchema: string, ...queries: string[]) { - const compiled = compileEventDefinitionsToCompilerModel({ write_checkpoints: queries }, { defaultSchema }); - expect(compiled.errors.filter((error) => error.type == 'fatal')).toEqual([]); +function serializedEventDefinitionFromQueries(...queries: string[]) { + const [errors, plan] = yamlToSyncPlan(yamlWithEventQueries(...queries), { + defaultSchema: 'test_schema', + throwOnError: false + }); + expect(errors).toEqual([]); - return compiled.events[0]; + return serializeSyncPlan(plan).events![0]; } function yamlWithEventQueries(...queries: string[]): string { From d1019ba7a6ee480764b5f57d9c52a339244fea8e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 19 Aug 2026 16:41:33 +0200 Subject: [PATCH 10/14] incrementally reprocess events based off stable event ids --- .changeset/incremental-event-reprocessing.md | 7 + docs/replication/01-core-concepts.md | 2 +- .../replication/04-storage-writer-overview.md | 2 +- docs/replication/09-resolve-tables-flow.md | 4 +- docs/storage/storage-v3.md | 9 +- .../src/storage/MongoBucketStorage.ts | 3 +- .../implementation/MongoBucketBatch.ts | 12 +- .../implementation/v3/MongoBucketBatchV3.ts | 17 +- .../v3/MongoStoppedSyncConfigCleanup.ts | 158 ++++++------ .../src/storage/implementation/v3/models.ts | 5 + .../implementation/v3/source-table-utils.ts | 71 +++--- .../src/cleanup-stopped-sync-configs.test.ts | 24 +- .../test/src/storage_sync.test.ts | 227 +++++++++++++++--- .../service-core/src/storage/SourceTable.ts | 18 +- .../implementation/BucketDefinitionMapping.ts | 49 +++- .../IncrementalReprocessingSyncConfigLog.ts | 52 +++- .../test/src/storage/SourceTable.test.ts | 5 +- packages/sync-rules/src/HydratedSyncConfig.ts | 7 +- .../sync-rules/src/events/EventDescriptor.ts | 3 + 19 files changed, 452 insertions(+), 223 deletions(-) create mode 100644 .changeset/incremental-event-reprocessing.md diff --git a/.changeset/incremental-event-reprocessing.md b/.changeset/incremental-event-reprocessing.md new file mode 100644 index 000000000..d57f8b48a --- /dev/null +++ b/.changeset/incremental-event-reprocessing.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-sync-rules': minor +'@powersync/service-core': minor +'@powersync/service-module-mongodb-storage': minor +--- + +Track replication events as MongoDB storage v3 source-table memberships so unchanged event definitions retain their assigned ids while new or changed definitions are resnapshotted during incremental reprocessing. diff --git a/docs/replication/01-core-concepts.md b/docs/replication/01-core-concepts.md index 9c823a41c..a77c21631 100644 --- a/docs/replication/01-core-concepts.md +++ b/docs/replication/01-core-concepts.md @@ -98,7 +98,7 @@ A `SourceTable` tracks: - Per-table snapshot completion and progress. - The bucket data sources and parameter lookup sources that use it, plus their persisted definition ids where storage tracks them. -When multiple `SourceTable` records exist for one physical table, storage designates only one record as the event carrier for row-change events. This lets the source connector save a row change to each relevant table record without firing duplicate events. +With v3 incremental storage, each compiled event definition id is assigned to exactly one `SourceTable` record for a physical table. This lets the source connector save a row change to each relevant table record without firing the same event twice, while a changed event definition can receive its own snapshot work. For a fuller walkthrough, see [09-resolve-tables-flow.md](./09-resolve-tables-flow.md). diff --git a/docs/replication/04-storage-writer-overview.md b/docs/replication/04-storage-writer-overview.md index fd014f72c..c0158d69c 100644 --- a/docs/replication/04-storage-writer-overview.md +++ b/docs/replication/04-storage-writer-overview.md @@ -22,7 +22,7 @@ The source connector is responsible for closing each writer cleanly so pending w Before source rows can be evaluated, discovered table or collection metadata has to be matched to the stream's parsed sync config set and the persisted source table state. That resolution decides which `SourceTable` records should receive data, which persisted bucket or parameter definition ids they cover, and which outdated mappings should be removed. -In incremental storage, one physical source entity can map to multiple `SourceTable` records. Each record owns a disjoint set of bucket data definition ids and parameter index ids, or exists only to carry row-change events. New definitions can therefore snapshot without forcing already-compatible definitions to be reprocessed. +In incremental storage, one physical source entity can map to multiple `SourceTable` records. Each record owns a disjoint set of bucket data definition ids, parameter index ids, and compiled event definition ids. New definitions can therefore snapshot without forcing already-compatible definitions to be reprocessed. An event-only record owns event ids but no bucket or parameter memberships. See [09-resolve-tables-flow.md](./09-resolve-tables-flow.md) for the detailed source table lifecycle. diff --git a/docs/replication/09-resolve-tables-flow.md b/docs/replication/09-resolve-tables-flow.md index 00f42a99e..88ae354e2 100644 --- a/docs/replication/09-resolve-tables-flow.md +++ b/docs/replication/09-resolve-tables-flow.md @@ -49,11 +49,11 @@ A `SourceTable` is a replicated table with state: 2. It stores the specific metadata from the `SourceEntityDescriptor` - any changes would result in a new `SourceTable`. 3. It tracks snapshot lifecycle state (complete/in-progress, progress markers). 4. It carries resolved sync participation flags (used for data, parameters, events). -5. It tracks which persisted bucket data definitions and parameter indexes are used with it. +5. It tracks which persisted bucket data definitions, parameter indexes, and compiled event definitions are used with it. There may be multiple `SourceTable`s per `SourceTableRef`. Historically it was generally 1:1, but incremental reprocessing now uses multiple records when a new bucket data source or parameter index is added. Instead of re-snapshotting an existing `SourceTable`, storage creates a new `SourceTable` with the same `SourceTableRef`. The new snapshot then only affects the new definitions, not existing compatible ones. -When multiple records exist for one physical table, their bucket and parameter memberships must be disjoint so each definition receives each source row once. Storage also designates a single event carrier so row-change events are not duplicated. +When multiple records exist for one physical table, their bucket, parameter, and event memberships must be disjoint so each definition receives each source row once. An unchanged event id can reuse a snapshotted record, while a new or changed event id creates new snapshot work. `SourceTable` is also used to track changes that may require a re-snapshot: diff --git a/docs/storage/storage-v3.md b/docs/storage/storage-v3.md index de7f60b15..b3f2a8e11 100644 --- a/docs/storage/storage-v3.md +++ b/docs/storage/storage-v3.md @@ -73,8 +73,9 @@ Each source table document stores: 2. Snapshot state. 3. `bucket_data_source_ids`: bucket definitions covered by this source table. 4. `parameter_lookup_source_ids`: parameter indexes covered by this source table. +5. `event_definition_ids`: content-addressed compiled event definitions covered by this source table. -Memberships are narrowed when stopped configs are cleaned up. New memberships are covered by creating a new source table document rather than expanding an existing one. Empty memberships represent an event-only source table. +Memberships are narrowed when stopped configs are cleaned up. New memberships are covered by creating a new source table document rather than expanding an existing one. An event-only source table has empty bucket and parameter memberships and one or more event definition ids. ## source_records (previously current_data) @@ -128,7 +129,7 @@ Incremental streams can contain stopped sync config state while the stream conti 1. Bucket data collections, parameter index collections, and bucket state are removed only for ids no live config still uses. 2. Source table memberships for unused ids are removed from retained source tables. -3. Source tables whose data and parameter memberships become empty are deleted with their source records collections unless a live config still triggers events for that table. -4. Source tables kept only for live events become event-only; their source records collections are dropped. -5. Event-only source tables are deleted with their source records collections when no live config still triggers events for them. +3. Unused event definition ids are removed from source table memberships using the same stopped-versus-live comparison. +4. Source tables whose data, parameter, and event memberships become empty are deleted with their source records collections. +5. Source tables kept only by event memberships become event-only; their source records collections are dropped. 6. The stopped sync config entries are pruned from `sync_rules.sync_configs`. diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 14c28a4f0..4a438d7d0 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -469,7 +469,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ), newMapping: mapping, newSyncConfig: updateOptions.config.parsed, - mappingChanges: mappingResult.changes + mappingChanges: mappingResult.changes, + activeEventDefinitions: existingConfigDocs.flatMap((doc) => doc.serialized_plan?.plan.events ?? []) }) ); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index d6873a02e..b524cc086 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -766,9 +766,9 @@ export abstract class MongoBucketBatch async save(record: storage.SaveOptions): Promise { const { after, before, sourceTable, tag } = record; const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData; - // syncEvent is the per-table designation from resolveTables. With v3 storage, multiple - // SourceTables can exist for the same ref, with a row change saved once per table - - // only the designated event carrier may fire events, so each event fires once per row. + // V3 source tables own disjoint event-definition ids, so a definition is fired by + // exactly one SourceTable even when bucket and parameter memberships are split. + // Legacy storage leaves eventDefinitionIds undefined and selects by table ref. if (sourceTable.syncEvent) { for (const event of this.getTableEvents(sourceTable)) { this.iterateListeners((cb) => @@ -941,8 +941,10 @@ export abstract class MongoBucketBatch * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable} */ protected getTableEvents(table: storage.SourceTable): HydratedEventDescriptor[] { - return this.sync_rules.eventDescriptors.filter((evt) => - [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) + return this.sync_rules.eventDescriptors.filter( + (event) => + (table.eventDefinitionIds == null || table.eventDefinitionIds.has(event.id)) && + event.tableTriggersEvent(table.ref) ); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index ff3ff0d25..b469d4a7c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -15,7 +15,6 @@ import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; import { ReplicationStreamDocumentV3, SourceTableDocumentV3 } from './models.js'; import { createNewSourceTable, - designateEventCarrier, overlappingSourceTableFilter, planSourceTableReconciliation, sourceTableDesiredResolution, @@ -91,7 +90,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, - parameterLookupSourceIds + parameterLookupSourceIds, + [...table.eventDefinitionIds!] ) ); } @@ -210,7 +210,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { { $set: { bucket_data_source_ids: update.memberships.bucketDataSourceIds, - parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds + parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds, + event_definition_ids: update.memberships.eventDefinitionIds } }, { session } @@ -228,9 +229,6 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { plan.tables.push(table); } - // If memberships are split across multiple source tables, only one may fire events. - designateEventCarrier(plan.tables, context.desired.triggersEvent); - result = { tables: plan.tables, dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping)) @@ -248,12 +246,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { return null; } - const refreshed = sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping); - // The event-carrier designation is decided per resolveTables result and not persisted - - // preserve the caller's designation instead of recomputing it from the ref, so that - // refreshing a non-carrier table does not make it fire events. - refreshed.syncEvent = table.syncEvent; - return refreshed; + return sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping); } async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts index a879d6673..a70ff2a82 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts @@ -1,7 +1,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { Logger, ReplicationAbortedError } from '@powersync/lib-services-framework'; import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core'; -import { BucketDefinitionId, ParameterIndexId, SyncConfig } from '@powersync/service-sync-rules'; +import { BucketDefinitionId, EventDefinitionId, ParameterIndexId, SyncConfig } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { clearCollectionInIdRanges, idPrefixFilter } from '../../../utils/util.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; @@ -47,7 +47,6 @@ export class MongoStoppedSyncConfigCleanup { private readonly logger: Logger; private readonly objectStorage: ObjectStorage | undefined; private readonly defaultSchema: string; - private readonly sourceConnectionTag: string; private readonly clearBatchThrottleRate: number; constructor(options: MongoStoppedSyncConfigCleanupOptions) { @@ -57,7 +56,6 @@ export class MongoStoppedSyncConfigCleanup { this.logger = options.logger; this.objectStorage = options.objectStorage; this.defaultSchema = options.defaultSchema; - this.sourceConnectionTag = options.sourceConnectionTag; this.clearBatchThrottleRate = options.clearBatchThrottleRate; } @@ -91,16 +89,24 @@ export class MongoStoppedSyncConfigCleanup { liveStorageIds.bucketDefinitionIds ); const unusedParameterIndexIds = difference(stoppedStorageIds.parameterIndexIds, liveStorageIds.parameterIndexIds); + const unusedEventDefinitionIds = difference( + stoppedStorageIds.eventDefinitionIds, + liveStorageIds.eventDefinitionIds + ); const result: storage.CleanupStoppedSyncConfigsResult = { ...EMPTY_RESULT }; - if (unusedBucketDefinitionIds.length > 0 || unusedParameterIndexIds.length > 0) { + if ( + unusedBucketDefinitionIds.length > 0 || + unusedParameterIndexIds.length > 0 || + unusedEventDefinitionIds.length > 0 + ) { await this.cleanupSourceTableMemberships( unusedBucketDefinitionIds, unusedParameterIndexIds, - liveConfigDocs, + unusedEventDefinitionIds, result ); result.bucketDataCollectionsDropped = await this.dropBucketDataCollections(unusedBucketDefinitionIds); @@ -108,11 +114,6 @@ export class MongoStoppedSyncConfigCleanup { result.parameterIndexCollectionsDropped = await this.dropParameterIndexCollections(unusedParameterIndexIds); } - // Event-only source tables carry empty membership arrays, so they are never selected by the - // membership filter above. Clean them up separately, regardless of whether any bucket or - // parameter ids became unused (a stopped config may have contributed only an event trigger). - await this.cleanupEventOnlySourceTables(liveConfigDocs, result); - result.stoppedSyncConfigsRemoved = await this.pruneStoppedSyncConfigStates(stoppedStates); if (result.stoppedSyncConfigsRemoved > 0) { @@ -132,20 +133,23 @@ export class MongoStoppedSyncConfigCleanup { return this.db.syncConfigDefinitions.find({ _id: { $in: ids } }).toArray(); } - private storageIdsFor(configs: Pick[]) { + private storageIdsFor(configs: SyncConfigDefinition[]) { const mappings = configs.map((config) => SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping) ); return { bucketDefinitionIds: [...new Set(mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))], - parameterIndexIds: [...new Set(mappings.flatMap((mapping) => mapping.allParameterIndexIds()))] + parameterIndexIds: [...new Set(mappings.flatMap((mapping) => mapping.allParameterIndexIds()))], + eventDefinitionIds: [ + ...new Set(this.parseSyncConfigs(configs).flatMap((config) => config.eventDefinitions.map((event) => event.id))) + ] }; } private async cleanupSourceTableMemberships( unusedBucketDefinitionIds: BucketDefinitionId[], unusedParameterIndexIds: ParameterIndexId[], - liveConfigDocs: SyncConfigDefinition[], + unusedEventDefinitionIds: EventDefinitionId[], result: storage.CleanupStoppedSyncConfigsResult ) { const update: Record = {}; @@ -155,11 +159,18 @@ export class MongoStoppedSyncConfigCleanup { if (unusedParameterIndexIds.length > 0) { update.parameter_lookup_source_ids = { $in: unusedParameterIndexIds }; } + if (unusedEventDefinitionIds.length > 0) { + update.event_definition_ids = { $in: unusedEventDefinitionIds }; + } // Keep obsolete membership ids as the durable cleanup marker until each source table is // either deleted or retained. If interrupted after dropping a source_records collection, // the next run can still rediscover the source table from these obsolete ids and retry. - const filter = this.sourceTableMembershipFilter(unusedBucketDefinitionIds, unusedParameterIndexIds); + const filter = this.sourceTableMembershipFilter( + unusedBucketDefinitionIds, + unusedParameterIndexIds, + unusedEventDefinitionIds + ); const candidateSourceTables = await this.db .sourceTables(this.replicationStreamId) .find(filter, { @@ -167,6 +178,7 @@ export class MongoStoppedSyncConfigCleanup { _id: 1, bucket_data_source_ids: 1, parameter_lookup_source_ids: 1, + event_definition_ids: 1, schema_name: 1, table_name: 1 } @@ -175,12 +187,13 @@ export class MongoStoppedSyncConfigCleanup { if (candidateSourceTables.length == 0) { return; } - const liveSyncConfigs = this.parseSyncConfigs(liveConfigDocs); - - const deletableSourceTables = candidateSourceTables.filter( - (sourceTable) => - this.membershipsBecomeEmpty(sourceTable, unusedBucketDefinitionIds, unusedParameterIndexIds) && - !this.triggersLiveEvent(sourceTable, liveSyncConfigs) + const deletableSourceTables = candidateSourceTables.filter((sourceTable) => + this.membershipsBecomeEmpty( + sourceTable, + unusedBucketDefinitionIds, + unusedParameterIndexIds, + unusedEventDefinitionIds + ) ); const retainedSourceTables = candidateSourceTables.filter( (sourceTable) => !deletableSourceTables.some((deletable) => deletable._id.equals(sourceTable._id)) @@ -189,19 +202,25 @@ export class MongoStoppedSyncConfigCleanup { await this.deleteSourceTables( deletableSourceTables.map((table) => table._id), - (ids) => this.deletableSourceTableFilter(ids, unusedBucketDefinitionIds, unusedParameterIndexIds), + (ids) => + this.deletableSourceTableFilter( + ids, + unusedBucketDefinitionIds, + unusedParameterIndexIds, + unusedEventDefinitionIds + ), result ); - // A retained source table whose memberships become empty is kept alive only by a live event - // (otherwise it would be deletable). It becomes event-only, and event-only save() never reads + // A retained source table whose data and parameter memberships become empty is event-only. + // Event-only save() never reads // or writes current_data, so its source_records collection is now dead weight. Drop it before // the $pull below, so the obsolete membership ids remain as a recovery marker if interrupted. // Existing source-table docs only ever shrink their memberships, so this table cannot resume // data sync on the same doc and need current_data again. const becomingEventOnlySourceTableIds = retainedSourceTables .filter((sourceTable) => - this.membershipsBecomeEmpty(sourceTable, unusedBucketDefinitionIds, unusedParameterIndexIds) + this.dataMembershipsBecomeEmpty(sourceTable, unusedBucketDefinitionIds, unusedParameterIndexIds) ) .map((sourceTable) => sourceTable._id); for (const sourceTableId of becomingEventOnlySourceTableIds) { @@ -223,42 +242,6 @@ export class MongoStoppedSyncConfigCleanup { } } - /** - * Clean up event-only source tables that no live sync config still triggers events for. - * - * Event-only source tables carry empty membership arrays (see source-table-utils: - * "Empty memberships indicate an event-only table"), so the membership filter never selects - * them. Without this, the source_tables row and its source_records collection leak when the - * config that contributed the event trigger is stopped. - */ - private async cleanupEventOnlySourceTables( - liveConfigDocs: SyncConfigDefinition[], - result: storage.CleanupStoppedSyncConfigsResult - ) { - const candidateSourceTables = await this.db - .sourceTables(this.replicationStreamId) - .find(this.eventOnlySourceTableFilter(), { - projection: { - _id: 1, - bucket_data_source_ids: 1, - parameter_lookup_source_ids: 1, - schema_name: 1, - table_name: 1 - } - }) - .toArray(); - if (candidateSourceTables.length == 0) { - return; - } - - const liveSyncConfigs = this.parseSyncConfigs(liveConfigDocs); - const deletableSourceTableIds = candidateSourceTables - .filter((sourceTable) => !this.triggersLiveEvent(sourceTable, liveSyncConfigs)) - .map((sourceTable) => sourceTable._id); - - await this.deleteSourceTables(deletableSourceTableIds, (ids) => this.eventOnlyDeletableFilter(ids), result); - } - /** * Drop the given source tables and their source_records collections. * @@ -298,6 +281,25 @@ export class MongoStoppedSyncConfigCleanup { } private membershipsBecomeEmpty( + sourceTable: Pick< + SourceTableDocumentV3, + 'bucket_data_source_ids' | 'parameter_lookup_source_ids' | 'event_definition_ids' + >, + unusedBucketDefinitionIds: BucketDefinitionId[], + unusedParameterIndexIds: ParameterIndexId[], + unusedEventDefinitionIds: EventDefinitionId[] + ): boolean { + const unusedBucketDefinitionIdSet = new Set(unusedBucketDefinitionIds); + const unusedParameterIndexIdSet = new Set(unusedParameterIndexIds); + const unusedEventDefinitionIdSet = new Set(unusedEventDefinitionIds); + return ( + sourceTable.bucket_data_source_ids.every((id) => unusedBucketDefinitionIdSet.has(id)) && + sourceTable.parameter_lookup_source_ids.every((id) => unusedParameterIndexIdSet.has(id)) && + sourceTable.event_definition_ids.every((id) => unusedEventDefinitionIdSet.has(id)) + ); + } + + private dataMembershipsBecomeEmpty( sourceTable: Pick, unusedBucketDefinitionIds: BucketDefinitionId[], unusedParameterIndexIds: ParameterIndexId[] @@ -313,26 +315,14 @@ export class MongoStoppedSyncConfigCleanup { private deletableSourceTableFilter( ids: bson.ObjectId[], unusedBucketDefinitionIds: BucketDefinitionId[], - unusedParameterIndexIds: ParameterIndexId[] + unusedParameterIndexIds: ParameterIndexId[], + unusedEventDefinitionIds: EventDefinitionId[] ): Record { return { _id: { $in: ids }, bucket_data_source_ids: { $not: { $elemMatch: { $nin: unusedBucketDefinitionIds } } }, - parameter_lookup_source_ids: { $not: { $elemMatch: { $nin: unusedParameterIndexIds } } } - }; - } - - private eventOnlySourceTableFilter(): Record { - return { - bucket_data_source_ids: { $size: 0 }, - parameter_lookup_source_ids: { $size: 0 } - }; - } - - private eventOnlyDeletableFilter(ids: bson.ObjectId[]): Record { - return { - _id: { $in: ids }, - ...this.eventOnlySourceTableFilter() + parameter_lookup_source_ids: { $not: { $elemMatch: { $nin: unusedParameterIndexIds } } }, + event_definition_ids: { $not: { $elemMatch: { $nin: unusedEventDefinitionIds } } } }; } @@ -351,19 +341,10 @@ export class MongoStoppedSyncConfigCleanup { }); } - private triggersLiveEvent(sourceTable: SourceTableDocumentV3, liveSyncConfigs: SyncConfig[]): boolean { - return liveSyncConfigs.some((syncConfig) => - syncConfig.tableTriggersEvent({ - connectionTag: this.sourceConnectionTag, - schema: sourceTable.schema_name, - name: sourceTable.table_name - }) - ); - } - private sourceTableMembershipFilter( bucketDefinitionIds: BucketDefinitionId[], - parameterIndexIds: ParameterIndexId[] + parameterIndexIds: ParameterIndexId[], + eventDefinitionIds: EventDefinitionId[] ): Partial | Record { const clauses: Record[] = []; if (bucketDefinitionIds.length > 0) { @@ -372,6 +353,9 @@ export class MongoStoppedSyncConfigCleanup { if (parameterIndexIds.length > 0) { clauses.push({ parameter_lookup_source_ids: { $in: parameterIndexIds } }); } + if (eventDefinitionIds.length > 0) { + clauses.push({ event_definition_ids: { $in: eventDefinitionIds } }); + } if (clauses.length == 0) { return { _id: { $exists: false } }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 73a69bb31..038045d18 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -8,6 +8,7 @@ import { } from '@powersync/service-core'; import { BucketDefinitionId, + EventDefinitionId, ParameterIndexId, ScopedParameterLookup, SqliteJsonValue @@ -156,6 +157,10 @@ export interface SourceTableDocumentV3 { snapshot_status: SourceTableDocumentSnapshotStatus | undefined; bucket_data_source_ids: BucketDefinitionId[]; parameter_lookup_source_ids: ParameterIndexId[]; + /** + * Content-addressed compiled event definitions evaluated by this source table. + */ + event_definition_ids: EventDefinitionId[]; latest_pending_delete?: InternalOpId | undefined; /** * Source-specific metadata. Absent for legacy records. diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts index 55cfc9477..62e7a8343 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts @@ -3,6 +3,8 @@ import { BucketDefinitionMapping, ColumnDescriptor, JsonValue, storage } from '@ import { BucketDataSource, BucketDefinitionId, + EventDefinitionId, + HydratedEventDescriptor, HydratedSyncConfig, MatchingSources, ParameterIndexId, @@ -22,12 +24,13 @@ export interface SourceTableIdentity { export interface SourceTableMembershipIds { bucketDataSourceIds: BucketDefinitionId[]; parameterLookupSourceIds: ParameterIndexId[]; + eventDefinitionIds: EventDefinitionId[]; } export interface SourceTableDesiredResolution { bucketSourceById: Map; parameterLookupSourceById: Map; - triggersEvent: boolean; + eventDefinitionById: Map; } export interface SourceTableReconciliationContext { @@ -55,7 +58,7 @@ export interface SourceTableReconciliationPlan { narrowingUpdates: SourceTableMembershipUpdate[]; /** * Memberships for a new source-table doc covering desired ids no existing doc covers, - * or null if no new doc is needed. Empty memberships indicate an event-only table. + * or null if no new doc is needed. */ newTableMemberships: SourceTableMembershipIds | null; /** Identity-overlapping docs that conflict with the current identity and must be dropped. */ @@ -85,7 +88,11 @@ export function sourceTableDesiredResolution( parameterLookupSourceById: new Map( matchingSources.parameterLookupSources.map((source) => [mapping.parameterLookupId(source), source] as const) ), - triggersEvent: syncConfig.tableTriggersEvent(ref) + eventDefinitionById: new Map( + syncConfig.eventDescriptors + .filter((event) => event.tableTriggersEvent(ref)) + .map((event) => [event.id, event] as const) + ) }; } @@ -116,28 +123,33 @@ function intersectMembershipIds( ): SourceTableMembershipIds { return { bucketDataSourceIds: doc.bucket_data_source_ids.filter((id) => desired.bucketSourceById.has(id)), - parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id)) + parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id)), + eventDefinitionIds: doc.event_definition_ids.filter((id) => desired.eventDefinitionById.has(id)) }; } function hasMembershipIds(memberships: SourceTableMembershipIds) { - return memberships.bucketDataSourceIds.length > 0 || memberships.parameterLookupSourceIds.length > 0; + return ( + memberships.bucketDataSourceIds.length > 0 || + memberships.parameterLookupSourceIds.length > 0 || + memberships.eventDefinitionIds.length > 0 + ); } function sameMembershipIds(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds) { return ( sameStringArray(doc.bucket_data_source_ids, memberships.bucketDataSourceIds) && - sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds) + sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds) && + sameStringArray(doc.event_definition_ids, memberships.eventDefinitionIds) ); } class SourceTableReconciliationPlanner { private readonly coveredBucketDataSourceIds = new Set(); private readonly coveredParameterLookupSourceIds = new Set(); - private readonly retainedDocIds = new Set(); + private readonly coveredEventDefinitionIds = new Set(); private readonly tables: storage.SourceTable[] = []; private readonly narrowingUpdates: SourceTableMembershipUpdate[] = []; - private retainedEventOnlyTable = false; constructor(private readonly context: SourceTableReconciliationContext) {} @@ -168,24 +180,15 @@ class SourceTableReconciliationPlanner { private retainDoc(doc: SourceTableDocumentV3) { const memberships = intersectMembershipIds(doc, this.context.desired); const coversDesiredMembership = hasMembershipIds(memberships); - const coversEventOnlyTable = - !this.desiredHasMembership() && this.context.desired.triggersEvent && !this.retainedEventOnlyTable; this.recordCoverage(doc, memberships); - this.planNarrowingUpdate(doc, memberships, coversDesiredMembership, coversEventOnlyTable); + this.planNarrowingUpdate(doc, memberships, coversDesiredMembership); - if (coversDesiredMembership || coversEventOnlyTable) { - this.retainedEventOnlyTable ||= coversEventOnlyTable; - this.retainedDocIds.add(doc._id.toHexString()); + if (coversDesiredMembership) { this.tables.push(this.sourceTableFor(doc, memberships)); } } - private desiredHasMembership() { - const { desired } = this.context; - return desired.bucketSourceById.size > 0 || desired.parameterLookupSourceById.size > 0; - } - private recordCoverage(doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds) { this.addCoverage(doc, 'bucket data source', this.coveredBucketDataSourceIds, memberships.bucketDataSourceIds); this.addCoverage( @@ -194,6 +197,7 @@ class SourceTableReconciliationPlanner { this.coveredParameterLookupSourceIds, memberships.parameterLookupSourceIds ); + this.addCoverage(doc, 'event definition', this.coveredEventDefinitionIds, memberships.eventDefinitionIds); } // Membership sets must be pairwise disjoint across the docs of one physical table: @@ -215,11 +219,9 @@ class SourceTableReconciliationPlanner { private planNarrowingUpdate( doc: SourceTableDocumentV3, memberships: SourceTableMembershipIds, - coversDesiredMembership: boolean, - coversEventOnlyTable: boolean + coversDesiredMembership: boolean ) { - const shouldNarrow = - (coversDesiredMembership || coversEventOnlyTable) && !doc.snapshot_done && !sameMembershipIds(doc, memberships); + const shouldNarrow = coversDesiredMembership && !doc.snapshot_done && !sameMembershipIds(doc, memberships); if (!shouldNarrow) { return; @@ -239,9 +241,12 @@ class SourceTableReconciliationPlanner { ), parameterLookupSourceIds: [...desired.parameterLookupSourceById.keys()].filter( (id) => !this.coveredParameterLookupSourceIds.has(id) + ), + eventDefinitionIds: [...desired.eventDefinitionById.keys()].filter( + (id) => !this.coveredEventDefinitionIds.has(id) ) }; - if (hasMembershipIds(uncovered) || (desired.triggersEvent && this.tables.length == 0)) { + if (hasMembershipIds(uncovered)) { return uncovered; } return null; @@ -306,6 +311,7 @@ export function createNewSourceTable( snapshot_status: undefined, bucket_data_source_ids: memberships.bucketDataSourceIds, parameter_lookup_source_ids: memberships.parameterLookupSourceIds, + event_definition_ids: memberships.eventDefinitionIds, // Records created together share the same source metadata. source_metadata: newTableSourceMetadata }; @@ -322,17 +328,6 @@ export function createNewSourceTable( return { doc, table }; } -export function designateEventCarrier(tables: storage.SourceTable[], triggersEvent: boolean) { - if (!triggersEvent) { - return; - } - - const eventCarrier = tables.find((table) => table.snapshotComplete) ?? tables[0]; - for (const table of tables) { - table.syncEvent = table === eventCarrier; - } -} - function sourceTableMembershipsFromDocument( doc: SourceTableDocumentV3, syncConfig: HydratedSyncConfig, @@ -364,6 +359,9 @@ export function sourceTableFromDocument( bucketDataSourceIds: resolvedMemberships.bucketDataSources.map((source) => mapping.bucketSourceId(source)), parameterLookupSourceIds: resolvedMemberships.parameterLookupSources.map((source) => mapping.parameterLookupId(source) + ), + eventDefinitionIds: doc.event_definition_ids.filter((id) => + syncConfig.eventDescriptors.some((event) => event.id == id) ) }; const table = new storage.SourceTable({ @@ -383,11 +381,12 @@ export function sourceTableFromDocument( parameterLookupSources: resolvedMemberships.parameterLookupSources, bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds), parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds), + eventDefinitionIds: new Set(resolvedMembershipIds.eventDefinitionIds), sourceMetadata: doc.source_metadata ?? null }); table.syncData = table.bucketDataSources.length > 0; table.syncParameters = table.parameterLookupSources.length > 0; - table.syncEvent = syncConfig.tableTriggersEvent(table.ref); + table.syncEvent = table.eventDefinitionIds!.size > 0; table.snapshotStatus = doc.snapshot_status == null ? undefined diff --git a/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts b/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts index dd828c11d..78d872a0f 100644 --- a/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts +++ b/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts @@ -366,9 +366,10 @@ event_definitions: const sourceTableId = resolved.tables[0].id as bson.ObjectId; const sourceRecordsCollection = db.sourceRecords(first.replicationStreamId, sourceTableId).collectionName; const ownerDefinitionId = first.syncConfigContent[0].mapping.allBucketDefinitionIds()[0]; - expect( - (await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId }))?.bucket_data_source_ids - ).toEqual([ownerDefinitionId]); + const initialSourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId }); + expect(initialSourceTable?.bucket_data_source_ids).toEqual([ownerDefinitionId]); + expect(initialSourceTable?.event_definition_ids).toHaveLength(1); + const eventDefinitionIds = initialSourceTable!.event_definition_ids; // The prior snapshot is persisted in current_data. expect(await collectionExists(db, sourceRecordsCollection)).toBe(true); @@ -414,12 +415,13 @@ event_definitions: sourceTablesDeleted: 0 }); - // The source table row survives (the live event still needs it), narrowed to empty - // memberships, but its now-unused current_data collection is dropped. + // The source table row survives (the live event still needs it), narrowed to empty data and + // parameter memberships, but its event membership remains and current_data is dropped. const sourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId }); expect(sourceTable).not.toBeNull(); expect(sourceTable?.bucket_data_source_ids).toEqual([]); expect(sourceTable?.parameter_lookup_source_ids).toEqual([]); + expect(sourceTable?.event_definition_ids).toEqual(eventDefinitionIds); expect(await collectionExists(db, sourceRecordsCollection)).toBe(false); const streamDoc = (await db.sync_rules.findOne({ _id: first.replicationStreamId })) as ReplicationStreamDocumentV3; @@ -430,8 +432,8 @@ event_definitions: await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); // The first config syncs `todos` and additionally fires events for `audit_log`. The - // `audit_log` table is referenced only by the event trigger, so its source table carries - // empty membership arrays (an event-only table). + // `audit_log` table is referenced only by the event trigger, so its source table has only an + // event membership (an event-only table). const first = await factory.updateSyncRules( updateSyncRulesFromYaml( ` @@ -490,16 +492,16 @@ event_definitions: const todosTableId = todosTable.id as bson.ObjectId; const auditTableId = auditTable.id as bson.ObjectId; - // The event-only table is persisted with empty membership arrays, so the membership filter - // never selects it. + // The event-only table has empty bucket and parameter memberships, plus its event id. const auditSourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: auditTableId }); expect(auditSourceTable?.bucket_data_source_ids).toEqual([]); expect(auditSourceTable?.parameter_lookup_source_ids).toEqual([]); + expect(auditSourceTable?.event_definition_ids).toHaveLength(1); const auditRecordsCollection = db.sourceRecords(first.replicationStreamId, auditTableId).collectionName; // The second (active) config keeps the same `by_owner` stream but drops the audit event. - // The shared bucket definition stays in use, so no bucket/parameter ids become unused and - // the membership-cleanup block is skipped entirely. + // The shared bucket definition stays in use, while the unused event id selects this table for + // membership cleanup. const second = await factory.updateSyncRules( updateSyncRulesFromYaml( ` diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index fc1612427..2d1373c52 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -633,13 +633,11 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(eventOnly.tables[0].syncEvent).toBe(true); }); - test.runIf(storageVersion >= 3)( - 'resolveTables designates a single event carrier across split source tables', - async () => { - // When memberships are split over multiple SourceTables for the same ref, a row change - // is saved once per table. Only one table may fire events, otherwise the same event - // would fire once per table for every row change. - const dataOnlyEventYaml = ` + test.runIf(storageVersion >= 3)('resolveTables assigns an event definition to one split source table', async () => { + // When memberships are split over multiple SourceTables for the same ref, a row change + // is saved once per table. Only one table may fire events, otherwise the same event + // would fire once per table for every row change. + const dataOnlyEventYaml = ` bucket_definitions: by_owner: parameters: @@ -652,7 +650,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor payloads: - SELECT id, owner_id FROM memberships `; - const fullEventYaml = ` + const fullEventYaml = ` bucket_definitions: by_owner: parameters: @@ -666,42 +664,201 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor - SELECT id, owner_id FROM memberships `; + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion })); + const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion); + const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion); + const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' }); + + const initial = await writer.resolveTables({ + connection_id: 1, + source, + idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'), + parsedSyncConfig: dataOnlyRules + }); + expect(initial.tables).toHaveLength(1); + expect(initial.tables[0].syncEvent).toBe(true); + + // Adding the table-based parameter lookup creates a second SourceTable for the same ref. + const split = await writer.resolveTables({ + connection_id: 1, + source, + idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'), + parsedSyncConfig: fullRules + }); + expect(split.tables).toHaveLength(2); + // Both tables match the event by ref, but the event id belongs to only one of them. + const carriers = split.tables.filter((table) => table.syncEvent); + expect(carriers).toHaveLength(1); + + // getSourceTableStatus rehydrates the persisted event memberships rather than recomputing + // them from the ref, so refreshing the other table does not make it fire the event. + const nonCarrier = split.tables.find((table) => !table.syncEvent)!; + const refreshed = await writer.getSourceTableStatus(nonCarrier); + expect(refreshed!.syncEvent).toBe(false); + }); + + test.runIf(storageVersion >= 3)( + 'reuses an unchanged event definition without resnapshotting or firing it twice', + async () => { + const firstYaml = ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') + +event_definitions: + write_checkpoints: + payloads: + - SELECT user_id, checkpoint FROM checkpoints WHERE active = true +`; + const secondYaml = ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') + by_project: + query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id') + +event_definitions: + write_checkpoints: + payloads: + - SELECT user_id, checkpoint FROM checkpoints WHERE active = true +`; + await using factory = await storageConfig.factory(); - const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion); - const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion); - const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' }); + const emittedEventIds: string[] = []; + const disposeListener = factory.registerListener({ + replicationEvent: ({ event }) => emittedEventIds.push(event.id) + }); + try { + const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion })); + const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); + const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' }); + const firstResolved = await firstWriter.resolveTables({ + connection_id: 1, + source, + idGenerator: objectIdGenerator('6544e3899293153fa7b38351') + }); + const eventId = [...firstResolved.tables[0].eventDefinitionIds!][0]; + expect(eventId).toBeDefined(); + await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1'); + await firstWriter.markAllSnapshotDone('1/1'); + await firstWriter.commit('1/1'); + + const second = await factory.updateSyncRules(updateSyncRulesFromYaml(secondYaml, { storageVersion })); + expect(second.replicationStreamId).toBe(first.replicationStreamId); + const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage; + await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); + const resolved = await secondWriter.resolveTables({ + connection_id: 1, + source, + idGenerator: () => { + throw new Error('unchanged event definition should reuse the snapshotted source table'); + } + }); + + expect(resolved.tables).toHaveLength(1); + expect(resolved.tables[0].id).toEqual(firstResolved.tables[0].id); + expect(resolved.tables[0].snapshotComplete).toBe(true); + expect([...resolved.tables[0].eventDefinitionIds!]).toEqual([eventId]); + + emittedEventIds.length = 0; + await secondWriter.save({ + sourceTable: resolved.tables[0], + tag: storage.SaveOperationTag.INSERT, + after: { id: 'checkpoint-1', user_id: 'user-1', checkpoint: 1n, active: 1 }, + afterReplicaId: test_utils.rid('checkpoint-1') + }); + expect(emittedEventIds).toEqual([eventId]); + } finally { + disposeListener(); + } + } + ); + + test.runIf(storageVersion >= 3)('creates snapshot work for a changed event definition', async () => { + const yaml = (active: boolean) => ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') + +event_definitions: + write_checkpoints: + payloads: + - SELECT user_id, checkpoint FROM checkpoints WHERE active = ${active} +`; - const initial = await writer.resolveTables({ + await using factory = await storageConfig.factory(); + const emittedEventIds: string[] = []; + const disposeListener = factory.registerListener({ + replicationEvent: ({ event }) => emittedEventIds.push(event.id) + }); + try { + const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion })); + const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); + const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' }); + const firstResolved = await firstWriter.resolveTables({ connection_id: 1, source, - idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'), - parsedSyncConfig: dataOnlyRules + idGenerator: objectIdGenerator('6544e3899293153fa7b38352') }); - expect(initial.tables).toHaveLength(1); - expect(initial.tables[0].syncEvent).toBe(true); + const oldEventId = [...firstResolved.tables[0].eventDefinitionIds!][0]; + await firstWriter.markTableSnapshotDone(firstResolved.tables, '1/1'); + await firstWriter.markAllSnapshotDone('1/1'); + await firstWriter.commit('1/1'); - // Adding the table-based parameter lookup creates a second SourceTable for the same ref. - const split = await writer.resolveTables({ + const second = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(false), { storageVersion })); + expect(second.replicationStreamId).toBe(first.replicationStreamId); + const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage; + await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); + const resolved = await secondWriter.resolveTables({ connection_id: 1, source, - idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'), - parsedSyncConfig: fullRules + idGenerator: objectIdGenerator('6544e3899293153fa7b38353') }); - expect(split.tables).toHaveLength(2); - // Both tables match the event by ref, but only one may carry it. - const carriers = split.tables.filter((table) => table.syncEvent); - expect(carriers).toHaveLength(1); - - // getSourceTableStatus preserves the carrier designation rather than recomputing it - // from the ref, so refreshing a non-carrier table does not make it fire events. - const nonCarrier = split.tables.find((table) => !table.syncEvent)!; - const refreshed = await writer.getSourceTableStatus(nonCarrier); - expect(refreshed!.syncEvent).toBe(false); + + expect(resolved.tables).toHaveLength(2); + const oldTable = resolved.tables.find((table) => table.eventDefinitionIds?.has(oldEventId))!; + const newTable = resolved.tables.find((table) => !table.eventDefinitionIds?.has(oldEventId))!; + const newEventId = [...newTable.eventDefinitionIds!][0]; + expect(newEventId).toBeDefined(); + expect(newEventId).not.toBe(oldEventId); + expect(oldTable.snapshotComplete).toBe(true); + expect(newTable.snapshotComplete).toBe(false); + expect(newTable.bucketDataSources).toEqual([]); + expect(newTable.parameterLookupSources).toEqual([]); + + await expect(secondWriter.markSnapshotDone('2/1')).rejects.toThrow( + 'Cannot mark snapshot done while source tables still require snapshotting' + ); + + emittedEventIds.length = 0; + await secondWriter.save({ + sourceTable: newTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'checkpoint-2', user_id: 'user-2', checkpoint: 2n, active: 0 }, + afterReplicaId: test_utils.rid('checkpoint-2') + }); + expect(emittedEventIds).toEqual([newEventId]); + + await secondWriter.markTableSnapshotDone([newTable], '2/1'); + await secondWriter.markSnapshotDone('2/1'); + } finally { + disposeListener(); } - ); + }); test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => { await using factory = await storageConfig.factory(); @@ -1829,6 +1986,7 @@ streams: snapshot_status: undefined, bucket_data_source_ids: [], parameter_lookup_source_ids: [], + event_definition_ids: [], latest_pending_delete: 9n }, { @@ -1842,6 +2000,7 @@ streams: snapshot_status: undefined, bucket_data_source_ids: [], parameter_lookup_source_ids: [], + event_definition_ids: [], latest_pending_delete: 12n } ]); diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index 5caa685f0..a3963b329 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -2,6 +2,7 @@ import { BucketDataSource, BucketDefinitionId, DEFAULT_TAG, + EventDefinitionId, ParameterIndexId, ParameterIndexLookupCreator, SourceTableRef @@ -35,6 +36,13 @@ export interface SourceTableOptions { parameterLookupSources: ParameterIndexLookupCreator[]; bucketDataSourceIds?: Set; parameterLookupSourceIds?: Set; + /** + * Compiled event definitions assigned to this persisted source-table record. + * + * Undefined is the legacy/non-incremental representation where event selection is + * based on the table ref. V3 incremental storage always supplies this set. + */ + eventDefinitionIds?: Set; /** * Source-specific metadata. Null when no metadata has been recorded. */ @@ -77,9 +85,8 @@ export class SourceTable { /** * True if this table should fire events for row changes. * - * This value is resolved externally, and cached here. When multiple SourceTables exist - * for the same SourceTableRef (v3 storage), resolveTables designates exactly one of them - * as the event carrier, so that a row change saved once per table fires each event once. + * This value is resolved externally, and cached here. V3 storage assigns disjoint + * event-definition ids to SourceTables for the same ref, so each event is fired once. * * Defaults to true for tests. */ @@ -150,6 +157,10 @@ export class SourceTable { return this.options.parameterLookupSourceIds; } + get eventDefinitionIds() { + return this.options.eventDefinitionIds; + } + get sourceMetadata() { return this.options.sourceMetadata ?? null; } @@ -192,6 +203,7 @@ export class SourceTable { bucketDataSourceIds: this.bucketDataSourceIds == null ? undefined : new Set(this.bucketDataSourceIds), parameterLookupSourceIds: this.parameterLookupSourceIds == null ? undefined : new Set(this.parameterLookupSourceIds), + eventDefinitionIds: this.eventDefinitionIds == null ? undefined : new Set(this.eventDefinitionIds), sourceMetadata: structuredClone(sourceMetadata) }); copy.syncData = this.syncData; diff --git a/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts b/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts index 671039669..2861aa5ce 100644 --- a/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts +++ b/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts @@ -2,6 +2,7 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { BucketDataSource, BucketDefinitionId, + EventDefinitionId, HashMap, ParameterIndexId, ParameterIndexLookupCreator, @@ -20,12 +21,12 @@ export interface SerializedSyncConfigWithMapping { mapping: SingleSyncConfigBucketDefinitionMapping; } -export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup'; +export type IncrementalMappingDefinitionType = 'bucket_data' | 'parameter_lookup' | 'event'; export interface IncrementalMappingDefinitionChange { type: IncrementalMappingDefinitionType; name: string; - id: BucketDefinitionId | ParameterIndexId; + id: BucketDefinitionId | ParameterIndexId | EventDefinitionId; } export interface IncrementalMappingChanges { @@ -84,7 +85,8 @@ export interface BucketDefinitionMapping { selectedSyncConfigIds: string[], table: SourceTableRef, bucketDataSourceIds: BucketDefinitionId[], - parameterLookupSourceIds: ParameterIndexId[] + parameterLookupSourceIds: ParameterIndexId[], + eventDefinitionIds: EventDefinitionId[] ): string[]; snapshotBlockingSourceTablesFilter(syncConfigId: string): Record; @@ -297,9 +299,12 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition selectedSyncConfigIds: string[], _table: SourceTableRef, bucketDataSourceIds: BucketDefinitionId[], - parameterLookupSourceIds: ParameterIndexId[] + parameterLookupSourceIds: ParameterIndexId[], + eventDefinitionIds: EventDefinitionId[] ): string[] { - return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0 ? selectedSyncConfigIds : []; + return bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0 || eventDefinitionIds.length > 0 + ? selectedSyncConfigIds + : []; } snapshotBlockingSourceTablesFilter(_syncConfigId: string): Record { @@ -337,6 +342,7 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM private bucketDataSourceSyncConfigIdsById = new Map>(); private parameterLookupMappings = new WeakMap(); private parameterLookupSyncConfigIdsById = new Map>(); + private eventDefinitionSyncConfigIdsById = new Map>(); private syncConfigsById = new Map(); private mappings: SingleSyncConfigBucketDefinitionMapping[]; @@ -357,6 +363,9 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM config.syncConfigId ); } + for (const event of config.syncConfig.config.eventDefinitions) { + addSetEntry(this.eventDefinitionSyncConfigIdsById, event.id, config.syncConfigId); + } } } @@ -390,9 +399,10 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM syncConfigIdsForSourceTable( _selectedSyncConfigIds: string[], - table: SourceTableRef, + _table: SourceTableRef, bucketDataSourceIds: BucketDefinitionId[], - parameterLookupSourceIds: ParameterIndexId[] + parameterLookupSourceIds: ParameterIndexId[], + eventDefinitionIds: EventDefinitionId[] ): string[] { const ids = new Set(); for (const sourceId of bucketDataSourceIds) { @@ -401,10 +411,8 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM for (const sourceId of parameterLookupSourceIds) { addAll(ids, this.parameterLookupSyncConfigIdsById.get(sourceId)); } - for (const [syncConfigId, config] of this.syncConfigsById) { - if (config.syncConfig.config.tableTriggersEvent(table)) { - ids.add(syncConfigId); - } + for (const eventDefinitionId of eventDefinitionIds) { + addAll(ids, this.eventDefinitionSyncConfigIdsById.get(eventDefinitionId)); } return [...ids]; } @@ -415,7 +423,24 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM throw new ServiceAssertionError(`No mapping found for sync config ${syncConfigId}`); } - return config.mapping.snapshotBlockingSourceTablesFilter(syncConfigId); + const mappingFilter = config.mapping.snapshotBlockingSourceTablesFilter(syncConfigId) as { + $or?: Record[]; + }; + const clauses = [...(mappingFilter.$or ?? [])]; + const eventDefinitionIds = config.syncConfig.config.eventDefinitions.map((event) => event.id); + if (eventDefinitionIds.length > 0) { + clauses.push({ event_definition_ids: { $in: eventDefinitionIds } }); + } + if (clauses.length == 0) { + return { + snapshot_done: false, + _id: { $exists: false } + }; + } + return { + snapshot_done: false, + $or: clauses + }; } } diff --git a/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts b/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts index b1b776e69..e079caf53 100644 --- a/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +++ b/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts @@ -1,4 +1,10 @@ -import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules'; +import { + BucketDataSource, + EventDefinition, + ParameterIndexLookupCreator, + SerializedEventDescriptor, + SyncConfigWithErrors +} from '@powersync/service-sync-rules'; import { IncrementalMappingChanges, IncrementalMappingDefinitionChange, @@ -28,14 +34,30 @@ export function describeIncrementalSyncConfigUpdate(options: { newMapping: SingleSyncConfigBucketDefinitionMapping; newSyncConfig: SyncConfigWithErrors; mappingChanges: IncrementalMappingChanges; + activeEventDefinitions?: Pick[]; }): IncrementalSyncConfigUpdateLog { - const { activeMappings, newMapping, newSyncConfig, mappingChanges } = options; - const newDefinitionKeys = new Set(newMapping.allDefinitionEntries().map(definitionKey)); - const activeDefinitions = uniqueDefinitions(activeMappings.flatMap((mapping) => mapping.allDefinitionEntries())); + const { activeMappings, newMapping, newSyncConfig, mappingChanges, activeEventDefinitions = [] } = options; + const activeEventIds = new Set(activeEventDefinitions.map((event) => event.id)); + const eventChanges = newSyncConfig.config.eventDefinitions.map( + (event): IncrementalMappingDefinitionChange => ({ + type: 'event', + name: event.name, + id: event.id + }) + ); + const reusedEvents = eventChanges.filter((event) => activeEventIds.has(event.id)); + const addedEvents = eventChanges.filter((event) => !activeEventIds.has(event.id)); + const newDefinitionKeys = new Set([...newMapping.allDefinitionEntries(), ...eventChanges].map(definitionKey)); + const activeDefinitions = uniqueDefinitions([ + ...activeMappings.flatMap((mapping) => mapping.allDefinitionEntries()), + ...activeEventDefinitions.map( + (event): IncrementalMappingDefinitionChange => ({ type: 'event', name: event.name, id: event.id }) + ) + ]); return { - reusedDefinitions: mappingChanges.reusedDefinitions, - addedDefinitions: mappingChanges.addedDefinitions.map((definition) => { + reusedDefinitions: [...mappingChanges.reusedDefinitions, ...reusedEvents], + addedDefinitions: [...mappingChanges.addedDefinitions, ...addedEvents].map((definition) => { const sourceTables = sourceTablesForDefinition(newSyncConfig, newMapping, definition); return { ...definition, @@ -57,17 +79,25 @@ function sourceTablesForDefinition( ); } - return sourceTablesForSources( - syncConfig.config.bucketParameterLookupSources.filter( - (source) => mapping.parameterLookupId(source) == definition.id - ) - ); + if (definition.type == 'parameter_lookup') { + return sourceTablesForSources( + syncConfig.config.bucketParameterLookupSources.filter( + (source) => mapping.parameterLookupId(source) == definition.id + ) + ); + } + + return sourceTablesForEvents(syncConfig.config.eventDefinitions.filter((event) => event.id == definition.id)); } function sourceTablesForSources(sources: Array) { return uniqueSorted(sources.flatMap((source) => [...source.getSourceTables()].map((table) => table.tablePattern))); } +function sourceTablesForEvents(events: EventDefinition[]) { + return uniqueSorted(events.flatMap((event) => [...event.getSourceTables()].map((table) => table.tablePattern))); +} + function definitionKey(definition: IncrementalMappingDefinitionChange) { return `${definition.type}:${definition.id}`; } diff --git a/packages/service-core/test/src/storage/SourceTable.test.ts b/packages/service-core/test/src/storage/SourceTable.test.ts index 5aab131c0..898b3786a 100644 --- a/packages/service-core/test/src/storage/SourceTable.test.ts +++ b/packages/service-core/test/src/storage/SourceTable.test.ts @@ -43,7 +43,8 @@ describe('SourceTable', () => { test('clone preserves all properties including storeCurrentData', () => { const table = makeTable({ replicaIdColumns: [{ name: 'id', type: 'int4' }], - snapshotComplete: false + snapshotComplete: false, + eventDefinitionIds: new Set(['event-1']) }); table.syncData = false; @@ -63,6 +64,8 @@ describe('SourceTable', () => { expect(cloned.syncEvent).toBe(false); expect(cloned.storeCurrentData).toBe(false); expect(cloned.snapshotStatus).toEqual(table.snapshotStatus); + expect(cloned.eventDefinitionIds).toEqual(new Set(['event-1'])); + expect(cloned.eventDefinitionIds).not.toBe(table.eventDefinitionIds); }); }); diff --git a/packages/sync-rules/src/HydratedSyncConfig.ts b/packages/sync-rules/src/HydratedSyncConfig.ts index 211bf90cd..dce0cac07 100644 --- a/packages/sync-rules/src/HydratedSyncConfig.ts +++ b/packages/sync-rules/src/HydratedSyncConfig.ts @@ -118,8 +118,11 @@ export class HydratedSyncConfig { this.bucketParameterLookupSources ).evaluateParameterRow; - this.eventDescriptors = definitions.flatMap((definition) => - definition.eventDefinitions.map((event) => event.createEvaluator(this.hydrationInput)) + this.eventDescriptors = uniqueBy( + definitions.flatMap((definition) => + definition.eventDefinitions.map((event) => event.createEvaluator(this.hydrationInput)) + ), + (event) => event.id ); if (definitions.length == 1) { diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index cc9da2474..62bef1104 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -3,6 +3,9 @@ import { SourceTableRef } from '../SourceTableRef.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, EvaluationError, SqliteJsonRow } from '../types.js'; +/** Deterministic, content-addressed identity of a compiled event definition. */ +export type EventDefinitionId = string; + export type EvaluatedEventSourceRow = { data: SqliteJsonRow; }; From 70e8697000689327d745c1375af559455cf3c160 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 20 Aug 2026 16:13:05 +0200 Subject: [PATCH 11/14] cleanup --- .../src/storage/implementation/MongoBucketBatch.ts | 13 ++++++++----- .../v3/MongoStoppedSyncConfigCleanup.ts | 9 ++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index b524cc086..d7d85ce83 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -941,10 +941,13 @@ export abstract class MongoBucketBatch * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable} */ protected getTableEvents(table: storage.SourceTable): HydratedEventDescriptor[] { - return this.sync_rules.eventDescriptors.filter( - (event) => - (table.eventDefinitionIds == null || table.eventDefinitionIds.has(event.id)) && - event.tableTriggersEvent(table.ref) - ); + // V3 storage assigns event-definition ids to each source table, so membership is authoritative + // (ids are only ever assigned for events that match the ref). Legacy storage leaves this + // undefined and selects by table ref. + if (table.eventDefinitionIds != null) { + return this.sync_rules.eventDescriptors.filter((event) => table.eventDefinitionIds!.has(event.id)); + } + + return this.sync_rules.eventDescriptors.filter((event) => event.tableTriggersEvent(table.ref)); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts index a70ff2a82..13f74617e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts @@ -213,11 +213,10 @@ export class MongoStoppedSyncConfigCleanup { ); // A retained source table whose data and parameter memberships become empty is event-only. - // Event-only save() never reads - // or writes current_data, so its source_records collection is now dead weight. Drop it before - // the $pull below, so the obsolete membership ids remain as a recovery marker if interrupted. - // Existing source-table docs only ever shrink their memberships, so this table cannot resume - // data sync on the same doc and need current_data again. + // Event-only save() never reads or writes current_data, so its source_records collection is now + // dead weight. Drop it before the $pull below, so the obsolete membership ids remain as a recovery + // marker if interrupted. Existing source-table docs only ever shrink their memberships, so this + // table cannot resume data sync on the same doc and need current_data again. const becomingEventOnlySourceTableIds = retainedSourceTables .filter((sourceTable) => this.dataMembershipsBecomeEmpty(sourceTable, unusedBucketDefinitionIds, unusedParameterIndexIds) From 4bb3a1c7e38545e6fe77dc3edefce93d20eaa47b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 20 Aug 2026 16:33:11 +0200 Subject: [PATCH 12/14] cleanup --- packages/sync-rules/src/events/EventDescriptor.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/sync-rules/src/events/EventDescriptor.ts b/packages/sync-rules/src/events/EventDescriptor.ts index 62bef1104..cc9da2474 100644 --- a/packages/sync-rules/src/events/EventDescriptor.ts +++ b/packages/sync-rules/src/events/EventDescriptor.ts @@ -3,9 +3,6 @@ import { SourceTableRef } from '../SourceTableRef.js'; import { TablePattern } from '../TablePattern.js'; import { EvaluateRowOptions, EvaluationError, SqliteJsonRow } from '../types.js'; -/** Deterministic, content-addressed identity of a compiled event definition. */ -export type EventDefinitionId = string; - export type EvaluatedEventSourceRow = { data: SqliteJsonRow; }; From 563fbde067238579f7d05e502d3e7ec958e3986c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 21 Aug 2026 16:15:32 +0200 Subject: [PATCH 13/14] refactor to compute equality and comparisons instead --- docs/replication/01-core-concepts.md | 4 +- docs/storage/storage-v3.md | 16 ++- .../src/storage/MongoBucketStorage.ts | 3 +- .../implementation/MongoBucketBatch.ts | 17 ++- .../MongoParsedSyncConfigSet.ts | 34 +++++ .../implementation/v3/MongoBucketBatchV3.ts | 17 ++- .../v3/MongoStoppedSyncConfigCleanup.ts | 23 +-- .../implementation/v3/source-table-utils.ts | 16 +-- .../test/src/storage_sync.test.ts | 29 ++-- .../implementation/BucketDefinitionMapping.ts | 133 ++++++++++++++++-- .../IncrementalReprocessingSyncConfigLog.ts | 33 ++--- packages/sync-rules/src/HydratedSyncConfig.ts | 19 ++- packages/sync-rules/src/HydrationState.ts | 2 + packages/sync-rules/src/sync_plan/plan.ts | 3 + .../sync-rules/src/sync_plan/serialize.ts | 3 +- .../test/src/hydrated_sync_rules.test.ts | 25 ++++ 16 files changed, 275 insertions(+), 102 deletions(-) diff --git a/docs/replication/01-core-concepts.md b/docs/replication/01-core-concepts.md index a77c21631..50c684eb0 100644 --- a/docs/replication/01-core-concepts.md +++ b/docs/replication/01-core-concepts.md @@ -96,9 +96,9 @@ A `SourceTable` tracks: - Whether it participates in data, parameters, or events. - Whether current row data must be stored for partial update handling. - Per-table snapshot completion and progress. -- The bucket data sources and parameter lookup sources that use it, plus their persisted definition ids where storage tracks them. +- The bucket data sources, parameter lookup sources, and events that use it, plus their persisted definition ids where storage tracks them. -With v3 incremental storage, each compiled event definition id is assigned to exactly one `SourceTable` record for a physical table. This lets the source connector save a row change to each relevant table record without firing the same event twice, while a changed event definition can receive its own snapshot work. +With v3 incremental storage, each event definition's assigned storage id belongs to exactly one `SourceTable` record for a physical table. This lets the source connector save a row change to each relevant table record without firing the same event twice, while a changed event definition can receive its own snapshot work. For a fuller walkthrough, see [09-resolve-tables-flow.md](./09-resolve-tables-flow.md). diff --git a/docs/storage/storage-v3.md b/docs/storage/storage-v3.md index b3f2a8e11..460048bed 100644 --- a/docs/storage/storage-v3.md +++ b/docs/storage/storage-v3.md @@ -48,9 +48,17 @@ A specific sync config definition never moves between replication streams. When ## rule_mapping -Each sync config definition has a `rule_mapping` that maps bucket data sources and parameter lookup sources to stable ids within the replication stream. +Each sync config definition has a `rule_mapping` that maps bucket data sources, parameter lookup sources, and events to stable ids within the replication stream. -Compatible incremental updates reuse ids for equivalent serialized bucket and parameter definitions. Added definitions receive new ids. Historical mappings are included when allocating new ids so dropped ids are not accidentally reused inside the same stream. +Compatible incremental updates reuse ids for equivalent serialized bucket and parameter definitions. Event definitions are recompiled from their retained SQL and compared with the compiler's JavaScript plan model, which ignores payload-query order and normalizes safe operand reordering such as reordered conjunctions. Added definitions receive new ids. Historical mappings are included when allocating new ids so dropped ids are not accidentally reused inside the same stream. + +### Event mapping and matching + +Event ids are opaque, hexadecimal counters scoped to one replication stream. Each sync config persists an event-name-to-id mapping. A new event receives one more than the largest event id in any current or historical mapping. The service does not calculate an id by hashing the event definition: a compatible event reuses the id recorded in an earlier mapping, while an incompatible event receives the next counter value. Historical mappings continue to reserve an id after its event is removed, preventing that value from later being assigned to a different event definition in the same stream. + +When deploying a sync config, each event is matched independently against events in compatible active configs using its name and compiled behavior. Matching includes source tables, filters, and projected payloads while ignoring raw SQL formatting, event-definition order, payload-query order, and safe expression operand reordering. A match reuses the existing id; otherwise, a new id is allocated. Consequently, reordering unchanged `event_definitions` does not create ids or snapshot work. Ordering only determines which opaque counter values are assigned when multiple genuinely new events are introduced together. + +Source-table documents store event ids as memberships. A reused id can reuse existing snapshot-complete source-table coverage. A new or changed event has a new, uncovered id, so reconciliation creates separate source-table snapshot work for it. The in-memory reverse mapping from event id to sync config ids ensures that this work affects only configs using that id. Compatibility matching must remain conservative: failing to match only causes another snapshot, while incorrectly matching could reuse incompatible event state. These ids are used by: @@ -65,7 +73,7 @@ Scoped to a replication stream. Collection: `source_table_${stream_id}` -There may be multiple copies per physical table per stream. This is how incremental reprocessing snapshots new bucket or parameter definitions without reprocessing already-compatible definitions. +There may be multiple copies per physical table per stream. This is how incremental reprocessing snapshots new bucket, parameter, or event definitions without reprocessing already-compatible definitions. Each source table document stores: @@ -73,7 +81,7 @@ Each source table document stores: 2. Snapshot state. 3. `bucket_data_source_ids`: bucket definitions covered by this source table. 4. `parameter_lookup_source_ids`: parameter indexes covered by this source table. -5. `event_definition_ids`: content-addressed compiled event definitions covered by this source table. +5. `event_definition_ids`: assigned event definition ids covered by this source table. Memberships are narrowed when stopped configs are cleaned up. New memberships are covered by creating a new source table document rather than expanding an existing one. An event-only source table has empty bucket and parameter memberships and one or more event definition ids. diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 4a438d7d0..14c28a4f0 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -469,8 +469,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ), newMapping: mapping, newSyncConfig: updateOptions.config.parsed, - mappingChanges: mappingResult.changes, - activeEventDefinitions: existingConfigDocs.flatMap((doc) => doc.serialized_plan?.plan.events ?? []) + mappingChanges: mappingResult.changes }) ); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index d7d85ce83..df65067b3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -941,11 +941,20 @@ export abstract class MongoBucketBatch * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable} */ protected getTableEvents(table: storage.SourceTable): HydratedEventDescriptor[] { - // V3 storage assigns event-definition ids to each source table, so membership is authoritative - // (ids are only ever assigned for events that match the ref). Legacy storage leaves this - // undefined and selects by table ref. + // V3 storage assigns event-definition ids to each source table, so membership is authoritative. + // Iterate the table's distinct ids and resolve each through the stream's deduped event map, so a + // definition reused across configs fires exactly once. Legacy storage leaves this undefined and + // selects by table ref. if (table.eventDefinitionIds != null) { - return this.sync_rules.eventDescriptors.filter((event) => table.eventDefinitionIds!.has(event.id)); + const eventById = this.options.parsedSyncConfig.eventById; + const events: HydratedEventDescriptor[] = []; + for (const id of table.eventDefinitionIds) { + const event = eventById.get(id); + if (event != null && event.tableTriggersEvent(table.ref)) { + events.push(event); + } + } + return events; } return this.sync_rules.eventDescriptors.filter((event) => event.tableTriggersEvent(table.ref)); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParsedSyncConfigSet.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParsedSyncConfigSet.ts index 819fedb41..1faae3bcc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParsedSyncConfigSet.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParsedSyncConfigSet.ts @@ -12,6 +12,8 @@ import { import { CompatibilityOption, DEFAULT_HYDRATION_STATE, + EventDefinitionId, + HydratedEventDescriptor, HydratedSyncConfig, HydrationState, nodeSqlite, @@ -27,6 +29,8 @@ export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet { public readonly replicationStreamName: string; public readonly mapping: BucketDefinitionMapping; + readonly #configsWithMapping: readonly SyncConfigWithMapping[]; + constructor( public readonly replicationStreamId: number, storageConfig: StorageConfig, @@ -34,6 +38,7 @@ export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet { syncConfigs: SyncConfigWithMapping[] ) { this.replicationStreamName = slotName; + this.#configsWithMapping = [...syncConfigs]; this.syncConfigs = syncConfigs.map((config) => config.syncConfig); if (this.syncConfigs.length == 0) { throw new ServiceAssertionError(`At least one sync config is required`); @@ -85,4 +90,33 @@ export class MongoParsedSyncConfigSet implements storage.ParsedSyncConfigSet { }); return this.#hydratedSyncConfig; } + + #eventById: ReadonlyMap | undefined; + + /** + * Hydrated events for the replication stream, keyed by their assigned storage id. + * + * Each config's events are resolved against that config's own (single-config) mapping, so the resolution is + * unambiguous, and the result is deduplicated by assigned id: unchanged events shared across configs collapse to + * one entry, while a changed event keeps a separate entry under its new id. + */ + get eventById(): ReadonlyMap { + if (this.#eventById == null) { + const map = new Map(); + const byDefinition = this.hydratedSyncConfig.eventDescriptorsByDefinition; + for (const config of this.#configsWithMapping) { + if (config.mapping == null) { + continue; + } + for (const event of byDefinition.get(config.syncConfig.config) ?? []) { + const id = config.mapping.eventId(event); + if (!map.has(id)) { + map.set(id, event); + } + } + } + this.#eventById = map; + } + return this.#eventById; + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index b469d4a7c..b52b3e1c7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -147,6 +147,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { const parsedOverride = options.parsedSyncConfig as MongoParsedSyncConfigSet | undefined; const syncConfig = parsedOverride?.hydratedSyncConfig ?? this.sync_rules; const mapping = parsedOverride?.mapping ?? this.mapping; + const eventById = (parsedOverride ?? this.options.parsedSyncConfig).eventById; const { connection_id, source } = options; const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler; @@ -174,7 +175,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { .toArray(); const candidateTables = candidateDocs.map((doc) => - sourceTableFromDocument(doc, source.connectionTag, syncConfig, mapping) + sourceTableFromDocument(doc, source.connectionTag, syncConfig, mapping, eventById) ); const candidates = candidateTables.map((table) => table.clone()); const resolution = await reconcile({ source, candidates }); @@ -192,7 +193,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { storeCurrentData: source.sendsCompleteRows !== true, syncConfig, mapping, - desired: sourceTableDesiredResolution(syncConfig, source, mapping), + desired: sourceTableDesiredResolution(syncConfig, source, mapping, eventById), sourceCompatibleTables: resolution.compatibleTables, newTableSourceMetadata: resolution.newTableValues.sourceMetadata }; @@ -231,7 +232,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { result = { tables: plan.tables, - dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping)) + dropTables: plan.dropDocs.map((doc) => + sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping, eventById) + ) }; }); @@ -246,7 +249,13 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { return null; } - return sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping); + return sourceTableFromDocument( + doc, + table.ref.connectionTag, + this.sync_rules, + this.mapping, + this.options.parsedSyncConfig.eventById + ); } async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts index 13f74617e..4012e7344 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts @@ -1,7 +1,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { Logger, ReplicationAbortedError } from '@powersync/lib-services-framework'; import { SingleSyncConfigBucketDefinitionMapping, storage } from '@powersync/service-core'; -import { BucketDefinitionId, EventDefinitionId, ParameterIndexId, SyncConfig } from '@powersync/service-sync-rules'; +import { BucketDefinitionId, EventDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { clearCollectionInIdRanges, idPrefixFilter } from '../../../utils/util.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; @@ -46,7 +46,6 @@ export class MongoStoppedSyncConfigCleanup { private readonly signal: AbortSignal | undefined; private readonly logger: Logger; private readonly objectStorage: ObjectStorage | undefined; - private readonly defaultSchema: string; private readonly clearBatchThrottleRate: number; constructor(options: MongoStoppedSyncConfigCleanupOptions) { @@ -55,7 +54,6 @@ export class MongoStoppedSyncConfigCleanup { this.signal = options.signal; this.logger = options.logger; this.objectStorage = options.objectStorage; - this.defaultSchema = options.defaultSchema; this.clearBatchThrottleRate = options.clearBatchThrottleRate; } @@ -140,9 +138,7 @@ export class MongoStoppedSyncConfigCleanup { return { bucketDefinitionIds: [...new Set(mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))], parameterIndexIds: [...new Set(mappings.flatMap((mapping) => mapping.allParameterIndexIds()))], - eventDefinitionIds: [ - ...new Set(this.parseSyncConfigs(configs).flatMap((config) => config.eventDefinitions.map((event) => event.id))) - ] + eventDefinitionIds: [...new Set(mappings.flatMap((mapping) => mapping.allEventDefinitionIds()))] }; } @@ -325,21 +321,6 @@ export class MongoStoppedSyncConfigCleanup { }; } - private parseSyncConfigs(configDocs: SyncConfigDefinition[]): SyncConfig[] { - // This is ugly - we should not need to re-parse to achieve this. - // Revisit persistence for this later. - return configDocs.map((config) => { - return storage.parsePersistedSyncConfigContent({ - content: config.content, - compiledPlan: config.serialized_plan ?? null, - storageVersion: config.storage_version, - parseOptions: { - defaultSchema: this.defaultSchema - } - }).config; - }); - } - private sourceTableMembershipFilter( bucketDefinitionIds: BucketDefinitionId[], parameterIndexIds: ParameterIndexId[], diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts index 62e7a8343..2fcf0e354 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/source-table-utils.ts @@ -78,7 +78,8 @@ export interface NewSourceTable { export function sourceTableDesiredResolution( syncConfig: HydratedSyncConfig, ref: SourceTableRef, - mapping: BucketDefinitionMapping + mapping: BucketDefinitionMapping, + eventById: ReadonlyMap ): SourceTableDesiredResolution { const matchingSources = syncConfig.getMatchingSources(ref); return { @@ -88,11 +89,7 @@ export function sourceTableDesiredResolution( parameterLookupSourceById: new Map( matchingSources.parameterLookupSources.map((source) => [mapping.parameterLookupId(source), source] as const) ), - eventDefinitionById: new Map( - syncConfig.eventDescriptors - .filter((event) => event.tableTriggersEvent(ref)) - .map((event) => [event.id, event] as const) - ) + eventDefinitionById: new Map([...eventById].filter(([, event]) => event.tableTriggersEvent(ref))) }; } @@ -259,6 +256,7 @@ class SourceTableReconciliationPlanner { connectionTag, syncConfig, mapping, + desired.eventDefinitionById, matchingSourcesFor(desired, memberships), memberships ); @@ -320,6 +318,7 @@ export function createNewSourceTable( connectionTag, syncConfig, mapping, + desired.eventDefinitionById, matchingSourcesFor(desired, memberships), memberships ); @@ -351,6 +350,7 @@ export function sourceTableFromDocument( connectionTag: string, syncConfig: HydratedSyncConfig, mapping: BucketDefinitionMapping, + eventById: ReadonlyMap, memberships?: MatchingSources, membershipIds?: SourceTableMembershipIds ): storage.SourceTable { @@ -360,9 +360,7 @@ export function sourceTableFromDocument( parameterLookupSourceIds: resolvedMemberships.parameterLookupSources.map((source) => mapping.parameterLookupId(source) ), - eventDefinitionIds: doc.event_definition_ids.filter((id) => - syncConfig.eventDescriptors.some((event) => event.id == id) - ) + eventDefinitionIds: doc.event_definition_ids.filter((id) => eventById.has(id)) }; const table = new storage.SourceTable({ id: doc._id, diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 2d1373c52..48d34f718 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -559,6 +559,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' }); const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348'); const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349'); + const addedEventTableId = new bson.ObjectId('6544e3899293153fa7b3834a'); const dataOnly = await writer.resolveTables({ connection_id: 1, @@ -616,15 +617,13 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const eventOnly = await writer.resolveTables({ connection_id: 1, source, - idGenerator: () => { - throw new Error('resolve should reuse existing v3 source table'); - }, + idGenerator: () => addedEventTableId, parsedSyncConfig: eventOnlyRules }); - // Event-only table can re-use any existing table. + // A newly-added event gets its own source table so that its existing rows are snapshotted. expect(eventOnly.tables).toHaveLength(1); - expect([dataOnlyTableId.toString(), addedParameterTableId.toString()]).toContain(eventOnly.tables[0].id.toString()); + expect(eventOnly.tables[0].id).toEqual(addedEventTableId); expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]); expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0); expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0); @@ -714,7 +713,7 @@ streams: event_definitions: write_checkpoints: payloads: - - SELECT user_id, checkpoint FROM checkpoints WHERE active = true + - SELECT user_id, checkpoint FROM checkpoints WHERE active = true AND checkpoint > 0 `; const secondYaml = ` config: @@ -729,13 +728,13 @@ streams: event_definitions: write_checkpoints: payloads: - - SELECT user_id, checkpoint FROM checkpoints WHERE active = true + - SELECT user_id, checkpoint FROM checkpoints WHERE checkpoint > 0 AND true = active `; await using factory = await storageConfig.factory(); - const emittedEventIds: string[] = []; + const emittedEventNames: string[] = []; const disposeListener = factory.registerListener({ - replicationEvent: ({ event }) => emittedEventIds.push(event.id) + replicationEvent: ({ event }) => emittedEventNames.push(event.name) }); try { const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion })); @@ -770,14 +769,14 @@ event_definitions: expect(resolved.tables[0].snapshotComplete).toBe(true); expect([...resolved.tables[0].eventDefinitionIds!]).toEqual([eventId]); - emittedEventIds.length = 0; + emittedEventNames.length = 0; await secondWriter.save({ sourceTable: resolved.tables[0], tag: storage.SaveOperationTag.INSERT, after: { id: 'checkpoint-1', user_id: 'user-1', checkpoint: 1n, active: 1 }, afterReplicaId: test_utils.rid('checkpoint-1') }); - expect(emittedEventIds).toEqual([eventId]); + expect(emittedEventNames).toEqual(['write_checkpoints']); } finally { disposeListener(); } @@ -800,9 +799,9 @@ event_definitions: `; await using factory = await storageConfig.factory(); - const emittedEventIds: string[] = []; + const emittedEventNames: string[] = []; const disposeListener = factory.registerListener({ - replicationEvent: ({ event }) => emittedEventIds.push(event.id) + replicationEvent: ({ event }) => emittedEventNames.push(event.name) }); try { const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion })); @@ -844,14 +843,14 @@ event_definitions: 'Cannot mark snapshot done while source tables still require snapshotting' ); - emittedEventIds.length = 0; + emittedEventNames.length = 0; await secondWriter.save({ sourceTable: newTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'checkpoint-2', user_id: 'user-2', checkpoint: 2n, active: 0 }, afterReplicaId: test_utils.rid('checkpoint-2') }); - expect(emittedEventIds).toEqual([newEventId]); + expect(emittedEventNames).toEqual(['write_checkpoints']); await secondWriter.markTableSnapshotDone([newTable], '2/1'); await secondWriter.markSnapshotDone('2/1'); diff --git a/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts b/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts index 2861aa5ce..363b98293 100644 --- a/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts +++ b/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts @@ -2,12 +2,16 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { BucketDataSource, BucketDefinitionId, + CompiledEvent, + compiledEventDefinitionEquality, + compileEventDefinitionsToCompilerModel, EventDefinitionId, HashMap, ParameterIndexId, ParameterIndexLookupCreator, ParameterLookupDefinitionId, SerializedBucketDataSourceWithDataSources, + SerializedEventDescriptor, SerializedParameterIndexLookupCreator, serializedStreamBucketDataSourceEquality, serializedStreamParameterIndexLookupCreatorEquality, @@ -16,6 +20,11 @@ import { SyncConfigWithErrors } from '@powersync/service-sync-rules'; +/** Minimal shape required to resolve an event's assigned id: its name is the per-config key. */ +export interface NamedEventDefinition { + readonly name: string; +} + export interface SerializedSyncConfigWithMapping { plan: SerializedSyncPlan; mapping: SingleSyncConfigBucketDefinitionMapping; @@ -59,6 +68,10 @@ export interface PersistedDefinitionMapping { * Map of (lookupName, queryId) -> id, unique per replication stream. */ parameter_indexes: Record; + /** + * Map of event name -> id, unique per replication stream. Absent for mappings persisted before events were tracked. + */ + events?: Record; } /** @@ -81,6 +94,8 @@ export interface BucketDefinitionMapping { allParameterIndexIds(): ParameterIndexId[]; + allEventDefinitionIds(): EventDefinitionId[]; + syncConfigIdsForSourceTable( selectedSyncConfigIds: string[], table: SourceTableRef, @@ -102,7 +117,11 @@ export interface BucketDefinitionMapping { */ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinitionMapping { static fromPersistedMapping(mapping: PersistedDefinitionMapping): SingleSyncConfigBucketDefinitionMapping { - return new SingleSyncConfigBucketDefinitionMapping(mapping.definitions ?? {}, mapping.parameter_indexes ?? {}); + return new SingleSyncConfigBucketDefinitionMapping( + mapping.definitions ?? {}, + mapping.parameter_indexes ?? {}, + mapping.events ?? {} + ); } static fromParsedSyncConfig(syncConfig: SyncConfigWithErrors): SingleSyncConfigBucketDefinitionMapping { @@ -110,9 +129,11 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition const parameterKeys = syncConfig.config.bucketParameterLookupSources .map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`) .sort(); + const eventNames = syncConfig.config.eventDefinitions.map((event) => event.name).sort(); const definitions: Record = {}; const parameterLookups: Record = {}; + const events: Record = {}; for (const [index, uniqueName] of definitionNames.entries()) { definitions[uniqueName] = (index + 1).toString(16); @@ -120,8 +141,11 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition for (const [index, key] of parameterKeys.entries()) { parameterLookups[key] = (index + 1).toString(16); } + for (const [index, name] of eventNames.entries()) { + events[name] = (index + 1).toString(16); + } - return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups); + return new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups, events); } /** @@ -164,9 +188,25 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition nextParameterIndexId++; return id; } + // Event ids are deliberately allocated per replication stream instead of being hashes or UUIDs derived from a + // compiled plan. Compatibility decides whether to reuse an existing id, which keeps persisted storage identity + // independent of compiler serialization and lets safe behavioral equivalences be added later. That comparison + // must remain conservative: a false negative only causes a new id and resnapshot, while a false positive could + // incorrectly reuse existing event state. + let nextEventDefinitionId = + reservedMappings + .map((mapping) => mapping.allEventDefinitionIds()) + .flat() + .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1; + function generateNewEventDefinitionId(): EventDefinitionId { + const id = nextEventDefinitionId.toString(16); + nextEventDefinitionId++; + return id; + } const definitions: Record = {}; const parameterLookups: Record = {}; + const events: Record = {}; const changes: IncrementalMappingChanges = { reusedDefinitions: [], addedDefinitions: [] @@ -177,6 +217,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition const compatibleParameterLookups = new HashMap( serializedStreamParameterIndexLookupCreatorEquality ); + const compatibleEvents = new HashMap(compiledEventDefinitionEquality); for (const config of compatibleConfigs) { for (const bucket of config.plan.buckets) { @@ -190,6 +231,10 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition config.mapping.parameterLookupIdByKey(parameterLookupKey(parameterLookup.lookupScope)) ); } + + for (const event of compileSerializedEventsForCompatibility(config.plan.events ?? [])) { + compatibleEvents.putIfAbsent(event, () => config.mapping.eventDefinitionIdByName(event.name)); + } } for (const bucket of newPlan.buckets) { @@ -225,15 +270,32 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition } } + for (const event of compileSerializedEventsForCompatibility(newPlan.events ?? [])) { + const compatibleId = compatibleEvents.get(event); + const id = compatibleId ?? generateNewEventDefinitionId(); + events[event.name] = id; + const change: IncrementalMappingDefinitionChange = { + type: 'event', + name: event.name, + id + }; + if (compatibleId == null) { + changes.addedDefinitions.push(change); + } else { + changes.reusedDefinitions.push(change); + } + } + return { - mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups), + mapping: new SingleSyncConfigBucketDefinitionMapping(definitions, parameterLookups, events), changes }; } constructor( private definitions: Record = {}, - private parameterLookupMapping: Record = {} + private parameterLookupMapping: Record = {}, + private events: Record = {} ) {} bucketSourceId(source: BucketDataSource): BucketDefinitionId { @@ -272,8 +334,20 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition })); } + allEventDefinitionIds(): EventDefinitionId[] { + return Object.values(this.events); + } + + eventDefinitionEntries(): IncrementalMappingDefinitionChange[] { + return Object.entries(this.events).map(([name, id]) => ({ + type: 'event', + name, + id + })); + } + allDefinitionEntries(): IncrementalMappingDefinitionChange[] { - return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries()]; + return [...this.bucketDefinitionEntries(), ...this.parameterIndexEntries(), ...this.eventDefinitionEntries()]; } parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { @@ -288,10 +362,23 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition return defId; } + eventId(event: NamedEventDefinition): EventDefinitionId { + return this.eventDefinitionIdByName(event.name); + } + + eventDefinitionIdByName(name: string): EventDefinitionId { + const id = this.events[name]; + if (id == null) { + throw new ServiceAssertionError(`No mapping found for event definition ${name}`); + } + return id; + } + serialize(): PersistedDefinitionMapping { return { definitions: { ...this.definitions }, - parameter_indexes: { ...this.parameterLookupMapping } + parameter_indexes: { ...this.parameterLookupMapping }, + events: { ...this.events } }; } @@ -317,6 +404,10 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition if (parameterLookupSourceIds.length > 0) { clauses.push({ parameter_lookup_source_ids: { $in: parameterLookupSourceIds } }); } + const eventDefinitionIds = this.allEventDefinitionIds(); + if (eventDefinitionIds.length > 0) { + clauses.push({ event_definition_ids: { $in: eventDefinitionIds } }); + } if (clauses.length == 0) { return { snapshot_done: false, @@ -330,6 +421,25 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition } } +function compileSerializedEventsForCompatibility(events: readonly SerializedEventDescriptor[]): CompiledEvent[] { + const definitions: Record = Object.create(null); + for (const event of events) { + if (definitions[event.name] != null) { + throw new ServiceAssertionError(`Duplicate compiled replication event ${event.name}`); + } + definitions[event.name] = event.sourceQueries.map((query) => query.sql); + } + + const compiled = compileEventDefinitionsToCompilerModel(definitions, {}); + const fatalErrors = compiled.errors.filter((error) => error.type == 'fatal'); + if (fatalErrors.length != 0) { + throw new ServiceAssertionError( + `Failed to compile replication events for compatibility: ${fatalErrors.map((error) => error.message).join(', ')}` + ); + } + return compiled.events; +} + /** * A BucketDefinitionMapping across all SyncConfigs of one parse of a replication stream. * @@ -364,7 +474,7 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM ); } for (const event of config.syncConfig.config.eventDefinitions) { - addSetEntry(this.eventDefinitionSyncConfigIdsById, event.id, config.syncConfigId); + addSetEntry(this.eventDefinitionSyncConfigIdsById, config.mapping.eventId(event), config.syncConfigId); } } } @@ -397,6 +507,10 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))]; } + allEventDefinitionIds(): EventDefinitionId[] { + return [...new Set(this.mappings.flatMap((mapping) => mapping.allEventDefinitionIds()))]; + } + syncConfigIdsForSourceTable( _selectedSyncConfigIds: string[], _table: SourceTableRef, @@ -426,11 +540,8 @@ export class MultiSyncConfigBucketDefinitionMapping implements BucketDefinitionM const mappingFilter = config.mapping.snapshotBlockingSourceTablesFilter(syncConfigId) as { $or?: Record[]; }; + // The single-config filter already includes this config's event ids; just reuse it directly. const clauses = [...(mappingFilter.$or ?? [])]; - const eventDefinitionIds = config.syncConfig.config.eventDefinitions.map((event) => event.id); - if (eventDefinitionIds.length > 0) { - clauses.push({ event_definition_ids: { $in: eventDefinitionIds } }); - } if (clauses.length == 0) { return { snapshot_done: false, diff --git a/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts b/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts index e079caf53..bc0a2b5d3 100644 --- a/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +++ b/packages/service-core/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts @@ -2,7 +2,6 @@ import { BucketDataSource, EventDefinition, ParameterIndexLookupCreator, - SerializedEventDescriptor, SyncConfigWithErrors } from '@powersync/service-sync-rules'; import { @@ -34,30 +33,16 @@ export function describeIncrementalSyncConfigUpdate(options: { newMapping: SingleSyncConfigBucketDefinitionMapping; newSyncConfig: SyncConfigWithErrors; mappingChanges: IncrementalMappingChanges; - activeEventDefinitions?: Pick[]; }): IncrementalSyncConfigUpdateLog { - const { activeMappings, newMapping, newSyncConfig, mappingChanges, activeEventDefinitions = [] } = options; - const activeEventIds = new Set(activeEventDefinitions.map((event) => event.id)); - const eventChanges = newSyncConfig.config.eventDefinitions.map( - (event): IncrementalMappingDefinitionChange => ({ - type: 'event', - name: event.name, - id: event.id - }) - ); - const reusedEvents = eventChanges.filter((event) => activeEventIds.has(event.id)); - const addedEvents = eventChanges.filter((event) => !activeEventIds.has(event.id)); - const newDefinitionKeys = new Set([...newMapping.allDefinitionEntries(), ...eventChanges].map(definitionKey)); - const activeDefinitions = uniqueDefinitions([ - ...activeMappings.flatMap((mapping) => mapping.allDefinitionEntries()), - ...activeEventDefinitions.map( - (event): IncrementalMappingDefinitionChange => ({ type: 'event', name: event.name, id: event.id }) - ) - ]); + const { activeMappings, newMapping, newSyncConfig, mappingChanges } = options; + // Buckets, parameter lookups and events all flow through the mapping's assigned ids, so their changes are + // already present in mappingChanges and allDefinitionEntries(). + const newDefinitionKeys = new Set(newMapping.allDefinitionEntries().map(definitionKey)); + const activeDefinitions = uniqueDefinitions(activeMappings.flatMap((mapping) => mapping.allDefinitionEntries())); return { - reusedDefinitions: [...mappingChanges.reusedDefinitions, ...reusedEvents], - addedDefinitions: [...mappingChanges.addedDefinitions, ...addedEvents].map((definition) => { + reusedDefinitions: mappingChanges.reusedDefinitions, + addedDefinitions: mappingChanges.addedDefinitions.map((definition) => { const sourceTables = sourceTablesForDefinition(newSyncConfig, newMapping, definition); return { ...definition, @@ -87,7 +72,9 @@ function sourceTablesForDefinition( ); } - return sourceTablesForEvents(syncConfig.config.eventDefinitions.filter((event) => event.id == definition.id)); + return sourceTablesForEvents( + syncConfig.config.eventDefinitions.filter((event) => mapping.eventId(event) == definition.id) + ); } function sourceTablesForSources(sources: Array) { diff --git a/packages/sync-rules/src/HydratedSyncConfig.ts b/packages/sync-rules/src/HydratedSyncConfig.ts index dce0cac07..705fb0db4 100644 --- a/packages/sync-rules/src/HydratedSyncConfig.ts +++ b/packages/sync-rules/src/HydratedSyncConfig.ts @@ -54,6 +54,14 @@ export class HydratedSyncConfig { eventDescriptors: HydratedEventDescriptor[] = []; + /** + * Hydrated events grouped by their source definition. + * + * Deduplication of events across definitions is left to the caller (storage resolves each event to its + * per-config assigned id and dedupes on that), so this keeps every definition's events intact for that. + */ + readonly eventDescriptorsByDefinition = new Map(); + /** * Only a single compatibility context is supported across all merged SyncConfigs. */ @@ -118,12 +126,11 @@ export class HydratedSyncConfig { this.bucketParameterLookupSources ).evaluateParameterRow; - this.eventDescriptors = uniqueBy( - definitions.flatMap((definition) => - definition.eventDefinitions.map((event) => event.createEvaluator(this.hydrationInput)) - ), - (event) => event.id - ); + this.eventDescriptors = definitions.flatMap((definition) => { + const events = definition.eventDefinitions.map((event) => event.createEvaluator(this.hydrationInput)); + this.eventDescriptorsByDefinition.set(definition, events); + return events; + }); if (definitions.length == 1) { this.#bucketSourceDefinitions = definitions[0].bucketSources; diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 5911f68ad..e6373531e 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -2,6 +2,8 @@ import { BucketDataSource, ParameterIndexLookupCreator } from './BucketSource.js export type BucketDefinitionId = string; export type ParameterIndexId = string; +/** Stream-local storage identity assigned to a replication event definition. */ +export type EventDefinitionId = string; export interface BucketDataScope { /** diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index 06cba0529..2e7801c00 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -139,6 +139,9 @@ export type ColumnSource = 'star' | { expr: SqlExpression; a /** * A named replication event compiled from `event_definitions`. + * + * Events have no content id of their own: storage assigns and persists a stable id for each one, matching definitions + * across sync configs with the compiler model's behavioral equality. */ export interface CompiledEventDescriptor { name: string; diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index 2cf94bb7c..5c22afbb1 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -531,7 +531,8 @@ export interface SerializedEventRowEvaluator { table: SerializedTablePattern; /** * The compiler's structural hash, retained only for round-trip symmetry with data sources (which reuse the same - * projection shape). It has no behavioral meaning for events. + * projection shape). It is not part of event identity (the compiler model compares behavior directly), and events + * are never deduplicated by it at runtime. */ hash: number; columns: SerializedColumnSource[]; diff --git a/packages/sync-rules/test/src/hydrated_sync_rules.test.ts b/packages/sync-rules/test/src/hydrated_sync_rules.test.ts index 3c7dc1b1d..b77d8925e 100644 --- a/packages/sync-rules/test/src/hydrated_sync_rules.test.ts +++ b/packages/sync-rules/test/src/hydrated_sync_rules.test.ts @@ -188,6 +188,31 @@ bucket_definitions: ]); }); + test('keeps event descriptors grouped by their source sync config', () => { + const yaml = ` +config: + edition: 3 + +streams: {} + +event_definitions: + write_checkpoints: + payloads: + - SELECT user_id, checkpoint FROM checkpoints +`; + const { config: first } = SqlSyncRules.fromYaml(yaml, PARSE_OPTIONS); + const { config: second } = SqlSyncRules.fromYaml(yaml, PARSE_OPTIONS); + + const hydrated = new HydratedSyncConfig({ + definitions: [first, second], + createParams: hydrationParams + }); + + expect(hydrated.eventDescriptors).toHaveLength(2); + expect(hydrated.eventDescriptorsByDefinition.get(first)).toEqual([hydrated.eventDescriptors[0]]); + expect(hydrated.eventDescriptorsByDefinition.get(second)).toEqual([hydrated.eventDescriptors[1]]); + }); + test('requires matching compatibility contexts for multiple sync configs', () => { const { config: legacy } = SqlSyncRules.fromYaml( ` From f6263910435ab8eb408cd62556863630d204afb4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 25 Aug 2026 14:49:27 +0200 Subject: [PATCH 14/14] update to use serialized event plan equality --- docs/storage/storage-v3.md | 4 +- .../src/storage/implementation/v3/models.ts | 2 +- .../test/src/storage_sync.test.ts | 2 +- .../implementation/BucketDefinitionMapping.ts | 39 +++++-------------- 4 files changed, 14 insertions(+), 33 deletions(-) diff --git a/docs/storage/storage-v3.md b/docs/storage/storage-v3.md index 460048bed..8bef6a282 100644 --- a/docs/storage/storage-v3.md +++ b/docs/storage/storage-v3.md @@ -50,13 +50,13 @@ A specific sync config definition never moves between replication streams. When Each sync config definition has a `rule_mapping` that maps bucket data sources, parameter lookup sources, and events to stable ids within the replication stream. -Compatible incremental updates reuse ids for equivalent serialized bucket and parameter definitions. Event definitions are recompiled from their retained SQL and compared with the compiler's JavaScript plan model, which ignores payload-query order and normalizes safe operand reordering such as reordered conjunctions. Added definitions receive new ids. Historical mappings are included when allocating new ids so dropped ids are not accidentally reused inside the same stream. +Compatible incremental updates reuse ids for equivalent serialized bucket and parameter definitions. Event definitions use a normalized serialized-plan identity that excludes raw SQL and stored hash values while retaining expression ASTs and their external-data bindings. This ignores payload-query order, but otherwise uses the compiler's existing conservative expression equality. Added definitions receive new ids. Historical mappings are included when allocating new ids so dropped ids are not accidentally reused inside the same stream. ### Event mapping and matching Event ids are opaque, hexadecimal counters scoped to one replication stream. Each sync config persists an event-name-to-id mapping. A new event receives one more than the largest event id in any current or historical mapping. The service does not calculate an id by hashing the event definition: a compatible event reuses the id recorded in an earlier mapping, while an incompatible event receives the next counter value. Historical mappings continue to reserve an id after its event is removed, preventing that value from later being assigned to a different event definition in the same stream. -When deploying a sync config, each event is matched independently against events in compatible active configs using its name and compiled behavior. Matching includes source tables, filters, and projected payloads while ignoring raw SQL formatting, event-definition order, payload-query order, and safe expression operand reordering. A match reuses the existing id; otherwise, a new id is allocated. Consequently, reordering unchanged `event_definitions` does not create ids or snapshot work. Ordering only determines which opaque counter values are assigned when multiple genuinely new events are introduced together. +When deploying a sync config, each event is matched independently against events in compatible active configs using its name and serialized compiled behavior. Matching includes source tables, filters, and projected payloads while ignoring raw SQL formatting, event-definition order, and payload-query order. Expression operands retain the compiler's existing ordered comparison, so reordering operands or clauses is conservatively treated as a change. A match reuses the existing id; otherwise, a new id is allocated. Consequently, reordering unchanged `event_definitions` does not create ids or snapshot work. Ordering only determines which opaque counter values are assigned when multiple genuinely new events are introduced together. Source-table documents store event ids as memberships. A reused id can reuse existing snapshot-complete source-table coverage. A new or changed event has a new, uncovered id, so reconciliation creates separate source-table snapshot work for it. The in-memory reverse mapping from event id to sync config ids ensures that this work affects only configs using that id. Compatibility matching must remain conservative: failing to match only causes another snapshot, while incorrectly matching could reuse incompatible event state. diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 038045d18..95306c81c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -158,7 +158,7 @@ export interface SourceTableDocumentV3 { bucket_data_source_ids: BucketDefinitionId[]; parameter_lookup_source_ids: ParameterIndexId[]; /** - * Content-addressed compiled event definitions evaluated by this source table. + * Stream-local ids of the compiled event definitions evaluated by this source table. */ event_definition_ids: EventDefinitionId[]; latest_pending_delete?: InternalOpId | undefined; diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 48d34f718..098fef983 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -728,7 +728,7 @@ streams: event_definitions: write_checkpoints: payloads: - - SELECT user_id, checkpoint FROM checkpoints WHERE checkpoint > 0 AND true = active + - select "user_id", "checkpoint" from "checkpoints" where "active" = true and "checkpoint" > 0 `; await using factory = await storageConfig.factory(); diff --git a/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts b/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts index 363b98293..765e47626 100644 --- a/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts +++ b/packages/service-core/src/storage/implementation/BucketDefinitionMapping.ts @@ -2,15 +2,13 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { BucketDataSource, BucketDefinitionId, - CompiledEvent, - compiledEventDefinitionEquality, - compileEventDefinitionsToCompilerModel, EventDefinitionId, HashMap, ParameterIndexId, ParameterIndexLookupCreator, ParameterLookupDefinitionId, SerializedBucketDataSourceWithDataSources, + serializedEventDefinitionEquality, SerializedEventDescriptor, SerializedParameterIndexLookupCreator, serializedStreamBucketDataSourceEquality, @@ -189,10 +187,10 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition return id; } // Event ids are deliberately allocated per replication stream instead of being hashes or UUIDs derived from a - // compiled plan. Compatibility decides whether to reuse an existing id, which keeps persisted storage identity - // independent of compiler serialization and lets safe behavioral equivalences be added later. That comparison - // must remain conservative: a false negative only causes a new id and resnapshot, while a false positive could - // incorrectly reuse existing event state. + // compiled plan. Compatibility decides whether to reuse an existing id, keeping the storage identifier separate + // from event content and allowing the comparison to evolve independently. That comparison must remain + // conservative: a false negative only causes a new id and resnapshot, while a false positive could incorrectly + // reuse existing event state. let nextEventDefinitionId = reservedMappings .map((mapping) => mapping.allEventDefinitionIds()) @@ -217,7 +215,9 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition const compatibleParameterLookups = new HashMap( serializedStreamParameterIndexLookupCreatorEquality ); - const compatibleEvents = new HashMap(compiledEventDefinitionEquality); + const compatibleEvents = new HashMap( + serializedEventDefinitionEquality + ); for (const config of compatibleConfigs) { for (const bucket of config.plan.buckets) { @@ -232,7 +232,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition ); } - for (const event of compileSerializedEventsForCompatibility(config.plan.events ?? [])) { + for (const event of config.plan.events ?? []) { compatibleEvents.putIfAbsent(event, () => config.mapping.eventDefinitionIdByName(event.name)); } } @@ -270,7 +270,7 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition } } - for (const event of compileSerializedEventsForCompatibility(newPlan.events ?? [])) { + for (const event of newPlan.events ?? []) { const compatibleId = compatibleEvents.get(event); const id = compatibleId ?? generateNewEventDefinitionId(); events[event.name] = id; @@ -421,25 +421,6 @@ export class SingleSyncConfigBucketDefinitionMapping implements BucketDefinition } } -function compileSerializedEventsForCompatibility(events: readonly SerializedEventDescriptor[]): CompiledEvent[] { - const definitions: Record = Object.create(null); - for (const event of events) { - if (definitions[event.name] != null) { - throw new ServiceAssertionError(`Duplicate compiled replication event ${event.name}`); - } - definitions[event.name] = event.sourceQueries.map((query) => query.sql); - } - - const compiled = compileEventDefinitionsToCompilerModel(definitions, {}); - const fatalErrors = compiled.errors.filter((error) => error.type == 'fatal'); - if (fatalErrors.length != 0) { - throw new ServiceAssertionError( - `Failed to compile replication events for compatibility: ${fatalErrors.map((error) => error.message).join(', ')}` - ); - } - return compiled.events; -} - /** * A BucketDefinitionMapping across all SyncConfigs of one parse of a replication stream. *