Add SyncConfig superclass for sync configurations

.changeset/neat-ears-sort.md

@@ -0,0 +1,11 @@
+---
+'@powersync/service-sync-rules': minor
+'@powersync/service-module-postgres-storage': patch
+'@powersync/service-module-mongodb-storage': patch
+'@powersync/service-core-tests': patch
+'@powersync/service-module-postgres': patch
+'@powersync/service-module-mongodb': patch
+'@powersync/service-core': patch
+---
+
+Introduce `BaseSyncConfig` to represent SQL-based sync rules and precompiled sync plans.

modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts

@@ -1,4 +1,4 @@
-import { SqlSyncRules, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
+import { SyncConfigWithErrors, HydratedSyncRules, versionedHydrationState } from '@powersync/service-sync-rules';
 
 import { storage } from '@powersync/service-core';
 
@@ -7,14 +7,14 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules {
 
   constructor(
     public readonly id: number,
-    public readonly sync_rules: SqlSyncRules,
+    public readonly sync_rules: SyncConfigWithErrors,
     public readonly checkpoint_lsn: string | null,
     slot_name: string | null
   ) {
     this.slot_name = slot_name ?? `powersync_${id}`;
   }
 
   hydratedSyncRules(): HydratedSyncRules {
-    return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(this.id) });
+    return this.sync_rules.config.hydrate({ hydrationState: versionedHydrationState(this.id) });
   }
 }

modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts

@@ -81,7 +81,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
 
   async getDebugTablesInfo(
     tablePatterns: sync_rules.TablePattern[],
-    sqlSyncRules: sync_rules.SqlSyncRules
+    sqlSyncRules: sync_rules.SyncConfig
   ): Promise<api.PatternResult[]> {
     let result: api.PatternResult[] = [];
 

modules/module-mongodb/test/src/mongo_test.test.ts

@@ -410,7 +410,7 @@ describe('mongo data types', () => {
     await setupTable(db);
     await insert(collection);
 
-    const rules = SqlSyncRules.fromYaml(
+    const { config: rules } = SqlSyncRules.fromYaml(
       `
 bucket_definitions:
   global:
@@ -457,7 +457,7 @@ bucket_definitions:
     await setupTable(db);
     await insert(collection);
 
-    const rules = SqlSyncRules.fromYaml(
+    const { config: rules } = SqlSyncRules.fromYaml(
       `
 bucket_definitions:
   global:
@@ -505,7 +505,7 @@ bucket_definitions:
     await setupTable(db);
     await insert(collection);
 
-    const rules = SqlSyncRules.fromYaml(
+    const { config: rules } = SqlSyncRules.fromYaml(
       `
 bucket_definitions:
   global:

modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts

@@ -37,7 +37,7 @@ export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncR
       slot_name: this.slot_name,
       sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options),
       hydratedSyncRules() {
-        return this.sync_rules.hydrate({
+        return this.sync_rules.config.hydrate({
           hydrationState: versionedHydrationState(this.id)
         });
       }

modules/module-postgres/src/replication/replication-utils.ts

@@ -196,7 +196,7 @@ export interface GetDebugTablesInfoOptions {
   publicationName: string;
   connectionTag: string;
   tablePatterns: sync_rules.TablePattern[];
-  syncRules: sync_rules.SqlSyncRules;
+  syncRules: sync_rules.SyncConfig;
 }
 
 export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Promise<PatternResult[]> {
@@ -296,7 +296,7 @@ export interface GetDebugTableInfoOptions {
   connectionTag: string;
   tablePattern: sync_rules.TablePattern;
   relationId: number | null;
-  syncRules: sync_rules.SqlSyncRules;
+  syncRules: sync_rules.SyncConfig;
 }
 
 export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Promise<service_types.TableInfo> {

modules/module-postgres/test/src/validation.test.ts

@@ -21,13 +21,13 @@ bucket_definitions:
 
   const syncRules = await context.factory.updateSyncRules({ content: syncRuleContent });
 
-  const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.getSourceTables();
+  const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.config.getSourceTables();
   const tableInfo = await getDebugTablesInfo({
     db: pool,
     publicationName: context.publicationName,
     connectionTag: context.connectionTag,
     tablePatterns: tablePatterns,
-    syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules
+    syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules.config
   });
   expect(tableInfo).toEqual([
     {

packages/service-core-tests/src/test-utils/general-utils.ts

@@ -28,7 +28,7 @@ export function testRules(content: string): storage.PersistedSyncRulesContent {
         sync_rules: SqlSyncRules.fromYaml(content, options),
         slot_name: 'test',
         hydratedSyncRules() {
-          return this.sync_rules.hydrate({ hydrationState: versionedHydrationState(1) });
+          return this.sync_rules.config.hydrate({ hydrationState: versionedHydrationState(1) });
         }
       };
     },

packages/service-core/src/api/RouteAPI.ts

@@ -1,4 +1,4 @@
-import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
 import * as types from '@powersync/service-types';
 import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
 
@@ -41,7 +41,7 @@ export interface RouteAPI {
    *           tables to ensure syncing should function according to the input
    *           pattern. Debug errors and warnings are reported per table.
    */
-  getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SqlSyncRules): Promise<PatternResult[]>;
+  getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SyncConfig): Promise<PatternResult[]>;
 
   /**
    * @returns The replication lag: that is the amount of data which has not been

packages/service-core/src/api/diagnostics.ts

@@ -1,5 +1,5 @@
 import { logger } from '@powersync/lib-services-framework';
-import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules } from '@powersync/service-sync-rules';
+import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules, SyncConfigWithErrors } from '@powersync/service-sync-rules';
 import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types';
 
 import * as storage from '../storage/storage-index.js';
@@ -41,11 +41,11 @@ export async function getSyncRulesStatus(
   const check_connection = options.check_connection ?? false;
   const now = new Date().toISOString();
 
-  let rules: SqlSyncRules;
+  let parsed: SyncConfigWithErrors;
   let persisted: storage.PersistedSyncRules;
   try {
     persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
-    rules = persisted.sync_rules;
+    parsed = persisted.sync_rules;
   } catch (e) {
     return {
       content: include_content ? sync_rules.sync_rules_content : undefined,
@@ -54,6 +54,7 @@ export async function getSyncRulesStatus(
     };
   }
 
+  const { config: rules, errors: syncRuleErrors } = parsed;
   const sourceConfig = await apiHandler.getSourceConfig();
   // This method can run under some situations if no connection is configured yet.
   // It will return a default tag in such a case. This default tag is not module specific.
@@ -131,7 +132,7 @@ export async function getSyncRulesStatus(
     });
   }
   errors.push(
-    ...rules.errors.map((e) => {
+    ...syncRuleErrors.map((e) => {
       return {
         level: e.type,
         message: e.message,

packages/service-core/src/routes/endpoints/admin.ts

@@ -128,7 +128,7 @@ export const reprocess = routeDefinition({
     }
 
     const new_rules = await activeBucketStorage.updateSyncRules({
-      content: active.sync_rules.content,
+      content: active.sync_rules.config.content,
       // These sync rules already passed validation. But if the rules are not valid anymore due
       // to a service change, we do want to report the error here.
       validate: true
@@ -179,7 +179,7 @@ export const validate = routeDefinition({
             schema
           }),
           hydratedSyncRules() {
-            return this.sync_rules.hydrate();
+            return this.sync_rules.config.hydrate();
           }
         };
       },

packages/service-core/src/routes/endpoints/sync-rules.ts

@@ -169,7 +169,7 @@ export const reprocessSyncRules = routeDefinition({
     }
 
     const new_rules = await activeBucketStorage.updateSyncRules({
-      content: sync_rules.sync_rules.content,
+      content: sync_rules.sync_rules.config.content,
       // These sync rules already passed validation. But if the rules are not valid anymore due
       // to a service change, we do want to report the error here.
       validate: true
@@ -197,14 +197,14 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) {
       // No schema-based validation at this point
       schema: undefined
     });
-    const source_table_patterns = rules.getSourceTables();
-    const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
+    const source_table_patterns = rules.config.getSourceTables();
+    const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules.config);
 
     return {
       valid: true,
-      bucket_definitions: rules.debugRepresentation(),
+      bucket_definitions: rules.config.debugRepresentation(),
       source_tables: resolved_tables,
-      data_tables: rules.debugGetOutputTables()
+      data_tables: rules.config.debugGetOutputTables()
     };
   } catch (e) {
     if (e instanceof SyncRulesErrors) {

packages/service-core/src/storage/PersistedSyncRulesContent.ts

@@ -1,4 +1,4 @@
-import { SqlSyncRules, HydratedSyncRules } from '@powersync/service-sync-rules';
+import { HydratedSyncRules, SyncConfig, SyncConfigWithErrors } from '@powersync/service-sync-rules';
 import { ReplicationLock } from './ReplicationLock.js';
 
 export interface ParseSyncRulesOptions {
@@ -28,7 +28,7 @@ export interface PersistedSyncRulesContent {
 
 export interface PersistedSyncRules {
   readonly id: number;
-  readonly sync_rules: SqlSyncRules;
+  readonly sync_rules: SyncConfigWithErrors;
   readonly slot_name: string;
 
   hydratedSyncRules(): HydratedSyncRules;

packages/service-core/test/src/sync/BucketChecksumState.test.ts

@@ -26,7 +26,7 @@ bucket_definitions:
     data: []
     `,
     { defaultSchema: 'public' }
-  ).hydrate({ hydrationState: versionedHydrationState(1) });
+  ).config.hydrate({ hydrationState: versionedHydrationState(1) });
 
   // global[1] and global[2]
   const SYNC_RULES_GLOBAL_TWO = SqlSyncRules.fromYaml(
@@ -39,7 +39,7 @@ bucket_definitions:
     data: []
     `,
     { defaultSchema: 'public' }
-  ).hydrate({ hydrationState: versionedHydrationState(2) });
+  ).config.hydrate({ hydrationState: versionedHydrationState(2) });
 
   // by_project[n]
   const SYNC_RULES_DYNAMIC = SqlSyncRules.fromYaml(
@@ -50,7 +50,7 @@ bucket_definitions:
     data: []
     `,
     { defaultSchema: 'public' }
-  ).hydrate({ hydrationState: versionedHydrationState(3) });
+  ).config.hydrate({ hydrationState: versionedHydrationState(3) });
 
   const syncContext = new SyncContext({
     maxBuckets: 100,
@@ -615,7 +615,7 @@ config:
 
       const rules = SqlSyncRules.fromYaml(source, {
         defaultSchema: 'public'
-      }).hydrate({ hydrationState: versionedHydrationState(1) });
+      }).config.hydrate({ hydrationState: versionedHydrationState(1) });
 
       return new BucketChecksumState({
         syncContext,

packages/sync-rules/src/BaseSyncConfig.ts

@@ -0,0 +1,126 @@
+import { BucketDataSource, BucketSource, CreateSourceParams, ParameterIndexLookupCreator } from './BucketSource.js';
+import { CompatibilityContext, CompatibilityOption } from './compatibility.js';
+import { YamlError } from './errors.js';
+import { SqlEventDescriptor } from './events/SqlEventDescriptor.js';
+import { HydratedSyncRules } from './HydratedSyncRules.js';
+import { DEFAULT_HYDRATION_STATE } from './HydrationState.js';
+import { SourceTableInterface } from './SourceTableInterface.js';
+import { SyncPlan } from './sync_plan/plan.js';
+import { TablePattern } from './TablePattern.js';
+import { SqliteInputValue, SqliteRow, SqliteValue, SyncConfig } from './types.js';
+import { applyRowContext } from './utils.js';
+
+/**
+ * @internal Sealed class, can only be extended by `SqlSyncRules` and `PrecompiledSyncConfig`.
+ */
+export abstract class BaseSyncConfig {
+  bucketDataSources: BucketDataSource[] = [];
+  bucketParameterLookupSources: ParameterIndexLookupCreator[] = [];
+  bucketSources: BucketSource[] = [];
+  compatibility: CompatibilityContext = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY;
+  eventDescriptors: SqlEventDescriptor[] = [];
+
+  /**
+   * The (YAML-based) source contents from which these sync rules have been derived.
+   */
+  content: string;
+
+  constructor(content: string) {
+    this.content = content;
+  }
+
+  // Ensure asSyncConfig is not implemented externally
+  protected abstract asSyncConfig(): SyncConfig & this;
+
+  /**
+   * If this sync config is fully described by a serializable sync plan, returns that plan.
+   */
+  abstract extractSyncPlan(): SyncPlan | null;
+
+  /**
+   * Hydrate the sync rule definitions with persisted state into runnable sync rules.
+   *
+   * @param params.hydrationState Transforms bucket ids based on persisted state. May omit for tests.
+   */
+  hydrate(params?: CreateSourceParams): HydratedSyncRules {
+    let hydrationState = params?.hydrationState;
+    if (hydrationState == null || !this.compatibility.isEnabled(CompatibilityOption.versionedBucketIds)) {
+      hydrationState = DEFAULT_HYDRATION_STATE;
+    }
+    const resolvedParams = { hydrationState };
+    return new HydratedSyncRules({
+      definition: this.asSyncConfig(),
+      createParams: resolvedParams,
+      bucketDataSources: this.bucketDataSources,
+      bucketParameterIndexLookupCreators: this.bucketParameterLookupSources,
+      eventDescriptors: this.eventDescriptors,
+      compatibility: this.compatibility
+    });
+  }
+
+  applyRowContext<MaybeToast extends undefined = never>(
+    source: SqliteRow<SqliteInputValue | MaybeToast>
+  ): SqliteRow<SqliteValue | MaybeToast> {
+    return applyRowContext(source, this.compatibility);
+  }
+
+  protected writeSourceTables(sourceTables: Map<String, TablePattern>): void {
+    for (const bucket of this.bucketDataSources) {
+      for (const r of bucket.getSourceTables()) {
+        const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`;
+        sourceTables.set(key, r);
+      }
+    }
+    for (const bucket of this.bucketParameterLookupSources) {
+      for (const r of bucket.getSourceTables()) {
+        const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`;
+        sourceTables.set(key, r);
+      }
+    }
+  }
+
+  getSourceTables(): TablePattern[] {
+    const sourceTables = new Map<String, TablePattern>();
+    this.writeSourceTables(sourceTables);
+    return [...sourceTables.values()];
+  }
+
+  getEventTables(): TablePattern[] {
+    const eventTables = new Map<String, TablePattern>();
+
+    if (this.eventDescriptors) {
+      for (const event of this.eventDescriptors) {
+        for (const r of event.getSourceTables()) {
+          const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`;
+          eventTables.set(key, r);
+        }
+      }
+    }
+
+    return [...eventTables.values()];
+  }
+
+  tableTriggersEvent(table: SourceTableInterface): boolean {
+    return this.eventDescriptors.some((bucket) => bucket.tableTriggersEvent(table));
+  }
+
+  tableSyncsData(table: SourceTableInterface): boolean {
+    return this.bucketDataSources.some((b) => b.tableSyncsData(table));
+  }
+
+  tableSyncsParameters(table: SourceTableInterface): boolean {
+    return this.bucketParameterLookupSources.some((b) => b.tableSyncsParameters(table));
+  }
+
+  debugGetOutputTables() {
+    let result: Record<string, any[]> = {};
+    for (let bucket of this.bucketDataSources) {
+      bucket.debugWriteOutputTables(result);
+    }
+    return result;
+  }
+
+  debugRepresentation() {
+    return this.bucketSources.map((rules) => rules.debugRepresentation());
+  }
+}