From d2cde703fdf89f125e24da26eb4841364f10d44d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 5 Mar 2026 17:46:22 +0200 Subject: [PATCH 1/8] Add createWriter API. --- .../implementation/MongoBucketBatch.ts | 4 +++ .../implementation/MongoSyncBucketStorage.ts | 31 ++++++++++--------- .../src/storage/PostgresSyncRulesStorage.ts | 31 ++++++++++--------- .../src/storage/batch/PostgresBucketBatch.ts | 4 +++ .../src/storage/BucketStorageBatch.ts | 10 ++++++ .../src/storage/SyncRulesBucketStorage.ts | 18 +++++++++-- 6 files changed, 67 insertions(+), 31 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index c523db533..fc6c2547c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -668,6 +668,10 @@ export class MongoBucketBatch super.clearListeners(); } + async dispose() { + await this[Symbol.asyncDispose](); + } + private lastWaitingLogThottled = 0; async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 1a395ee37..ce3e3f549 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -167,10 +167,7 @@ export class MongoSyncBucketStorage }); } - async startBatch( - options: storage.StartBatchOptions, - callback: (batch: storage.BucketStorageBatch) => Promise - ): Promise { + async createWriter(options: storage.CreateWriterOptions): Promise { const doc = await this.db.sync_rules.findOne( { _id: this.group_id @@ -179,7 +176,7 @@ export class MongoSyncBucketStorage ); const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; - await using batch = new MongoBucketBatch({ + const writer = new MongoBucketBatch({ logger: options.logger, db: this.db, syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), @@ -192,15 +189,21 @@ export class MongoSyncBucketStorage skipExistingRows: options.skipExistingRows ?? false, markRecordUnavailable: options.markRecordUnavailable }); - this.iterateListeners((cb) => cb.batchStarted?.(batch)); - - await callback(batch); - await batch.flush(); - if (batch.last_flushed_op != null) { - return { flushed_op: batch.last_flushed_op }; - } else { - return null; - } + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + /** + * @deprecated Use `createWriter()` with `await using` instead. + */ + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } async resolveTable(options: storage.ResolveTableOptions): Promise { diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 19d1b25bc..ca99ebf1c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -343,10 +343,7 @@ export class PostgresSyncRulesStorage }); } - async startBatch( - options: storage.StartBatchOptions, - callback: (batch: storage.BucketStorageBatch) => Promise - ): Promise { + async createWriter(options: storage.CreateWriterOptions): Promise { const syncRules = await this.db.sql` SELECT last_checkpoint_lsn, @@ -363,7 +360,7 @@ export class PostgresSyncRulesStorage const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null; - const batch = new PostgresBucketBatch({ + const writer = new PostgresBucketBatch({ logger: options.logger ?? framework.logger, db: this.db, sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(), @@ -378,15 +375,21 @@ export class PostgresSyncRulesStorage markRecordUnavailable: options.markRecordUnavailable, storageConfig: this.storageConfig }); - this.iterateListeners((cb) => cb.batchStarted?.(batch)); - - await callback(batch); - await batch.flush(); - if (batch.last_flushed_op != null) { - return { flushed_op: batch.last_flushed_op }; - } else { - return null; - } + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + /** + * @deprecated Use `createWriter()` with `await using` instead. + */ + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } async getParameterSets( diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index a16178daa..b39e622b0 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -129,6 +129,10 @@ export class PostgresBucketBatch super.clearListeners(); } + async dispose() { + await this[Symbol.asyncDispose](); + } + async save(record: storage.SaveOptions): Promise { // TODO maybe share with abstract class const { after, before, sourceTable, tag } = record; diff --git a/packages/service-core/src/storage/BucketStorageBatch.ts b/packages/service-core/src/storage/BucketStorageBatch.ts index 7023fab05..2c5a195e4 100644 --- a/packages/service-core/src/storage/BucketStorageBatch.ts +++ b/packages/service-core/src/storage/BucketStorageBatch.ts @@ -12,6 +12,16 @@ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptio }; export interface BucketStorageBatch extends ObserverClient, AsyncDisposable { + /** + * Alias for [Symbol.asyncDispose] + */ + dispose(): Promise; + + /** + * Last written op, if any. This may not reflect a consistent checkpoint. + */ + last_flushed_op: InternalOpId | null; + /** * Save an op, and potentially flush. * diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 9be697f21..8a623747a 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -30,10 +30,17 @@ export interface SyncRulesBucketStorage resolveTable(options: ResolveTableOptions): Promise; /** - * Use this to get access to update storage data. + * Create a new writer. + * + * The writer must be flushed and disposed when done. + */ + createWriter(options: CreateWriterOptions): Promise; + + /** + * @deprecated Use `createWriter()` with `await using` instead. */ startBatch( - options: StartBatchOptions, + options: CreateWriterOptions, callback: (batch: BucketStorageBatch) => Promise ): Promise; @@ -162,7 +169,7 @@ export interface ResolveTableResult { dropTables: SourceTable[]; } -export interface StartBatchOptions extends ParseSyncRulesOptions { +export interface CreateWriterOptions extends ParseSyncRulesOptions { zeroLSN: string; /** * Whether or not to store a copy of the current data. @@ -192,6 +199,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions { logger?: Logger; } +/** + * @deprecated Use `CreateWriterOptions`. + */ +export interface StartBatchOptions extends CreateWriterOptions {} + export interface CompactOptions { /** * Heap memory limit for the compact process. From 28468be2ba0b7f000df1985f271f074c44e6e7a3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 5 Mar 2026 18:30:28 +0200 Subject: [PATCH 2/8] Initial test refactor. --- .../test/src/storage_compacting.test.ts | 12 +- .../test/src/storage_sync.test.ts | 15 +- .../test/src/change_stream_utils.ts | 6 +- .../test/src/storage.test.ts | 15 +- .../test/src/storage_compacting.test.ts | 34 ++-- .../test/src/storage_sync.test.ts | 15 +- .../src/tests/register-compacting-tests.ts | 158 +++++++++++------- .../register-data-storage-checkpoint-tests.ts | 96 +++++------ .../register-data-storage-parameter-tests.ts | 132 +++++++-------- .../register-parameter-compacting-tests.ts | 50 +++--- 10 files changed, 289 insertions(+), 244 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 9f797a5d3..065f75d05 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -10,10 +10,10 @@ describe('Mongo Sync Bucket Storage Compact', () => { describe('with blank bucket_state', () => { // This can happen when migrating from older service versions, that did not populate bucket_state yet. const populate = async (bucketStorage: SyncRulesBucketStorage) => { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -23,7 +23,7 @@ describe('Mongo Sync Bucket Storage Compact', () => { afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -33,8 +33,8 @@ describe('Mongo Sync Bucket Storage Compact', () => { afterReplicaId: test_utils.rid('t2') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); return bucketStorage.getCheckpoint(); }; diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index b2767fd83..3e2c73667 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -30,12 +30,13 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const bucketStorage = factory.getInstance(syncRules); const globalBucket = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { const sourceTable = TEST_TABLE; const largeDescription = '0123456789'.repeat(2_000_00); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -45,7 +46,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor afterReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -56,7 +57,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor }); // Large enough to split the returned batch - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -66,7 +67,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor afterReplicaId: test_utils.rid('large2') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -75,7 +76,9 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor }, afterReplicaId: test_utils.rid('test3') }); - }); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 8fc3931fc..854a3c438 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -164,9 +164,9 @@ export class ChangeStreamTestContext { async markSnapshotConsistent() { const checkpoint = await createCheckpoint(this.client, this.db, STANDALONE_CHECKPOINT_ID); - await this.storage!.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive(checkpoint); - }); + await using writer = await this.storage!.createWriter(test_utils.BATCH_OPTIONS); + await writer.keepalive(checkpoint); + await writer.flush(); } startStreaming() { diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index f918841c2..fd14eb01c 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -40,12 +40,13 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY); const largeDescription = '0123456789'.repeat(2_000_00); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -55,7 +56,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { afterReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -66,7 +67,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { }); // Large enough to split the returned batch - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -76,7 +77,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { afterReplicaId: test_utils.rid('large2') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -85,7 +86,9 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { }, afterReplicaId: test_utils.rid('test3') }); - }); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts index 78641e758..5d0be7413 100644 --- a/modules/module-postgres-storage/test/src/storage_compacting.test.ts +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -19,22 +19,25 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.markAllSnapshotDone('1/1'); - await batch.commit('1/1'); - }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; @@ -69,34 +72,37 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); const request = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { id: 't1' }, beforeReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { id: 't2' }, afterReplicaId: test_utils.rid('t2') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { id: 't2' }, beforeReplicaId: test_utils.rid('t2') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); diff --git a/modules/module-postgres-storage/test/src/storage_sync.test.ts b/modules/module-postgres-storage/test/src/storage_sync.test.ts index 2596cdabe..32b526f06 100644 --- a/modules/module-postgres-storage/test/src/storage_sync.test.ts +++ b/modules/module-postgres-storage/test/src/storage_sync.test.ts @@ -37,12 +37,13 @@ function registerStorageVersionTests(storageVersion: number) { const bucketStorage = factory.getInstance(syncRules); const globalBucket = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { const sourceTable = TEST_TABLE; const largeDescription = '0123456789'.repeat(2_000_00); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -52,7 +53,7 @@ function registerStorageVersionTests(storageVersion: number) { afterReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -63,7 +64,7 @@ function registerStorageVersionTests(storageVersion: number) { }); // Large enough to split the returned batch - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -73,7 +74,7 @@ function registerStorageVersionTests(storageVersion: number) { afterReplicaId: test_utils.rid('large2') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -82,7 +83,9 @@ function registerStorageVersionTests(storageVersion: number) { }, afterReplicaId: test_utils.rid('test3') }); - }); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index 5b775426b..a9c79420a 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -19,9 +19,10 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -30,7 +31,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -39,7 +40,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -48,8 +49,10 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; @@ -125,9 +128,10 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -136,7 +140,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -145,7 +149,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -154,7 +158,7 @@ bucket_definitions: beforeReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -163,8 +167,10 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; const request = bucketRequest(syncRules, 'global[]'); @@ -240,9 +246,10 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -251,7 +258,7 @@ bucket_definitions: afterReplicaId: 't1' }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -260,7 +267,7 @@ bucket_definitions: afterReplicaId: 't2' }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -269,15 +276,18 @@ bucket_definitions: beforeReplicaId: 't1' }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint1 = result!.flushed_op; const request = bucketRequest(syncRules, 'global[]'); const checksumBefore = await bucketStorage.getChecksums(checkpoint1, [request]); - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result2 = await (async () => { + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -285,8 +295,10 @@ bucket_definitions: }, beforeReplicaId: 't2' }); - await batch.commit('2/1'); - }); + await writer2.commit('2/1'); + await writer2.flush(); + return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; + })(); const checkpoint2 = result2!.flushed_op; await bucketStorage.compact({ @@ -328,8 +340,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.markAllSnapshotDone('1/1'); /** * Repeatedly create operations which fall into different buckets. * The bucket operations are purposely interleaved as the op_id increases. @@ -338,7 +351,7 @@ bucket_definitions: * contain operations from multiple buckets. */ for (let count = 0; count < 100; count++) { - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -349,7 +362,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -360,7 +373,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -371,7 +384,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -382,7 +395,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -393,9 +406,11 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.commit('1/1'); + await writer.commit('1/1'); } - }); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint = result!.flushed_op; @@ -456,9 +471,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -467,7 +482,7 @@ bucket_definitions: afterReplicaId: 't1' }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -476,7 +491,7 @@ bucket_definitions: afterReplicaId: 't2' }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -485,8 +500,8 @@ bucket_definitions: beforeReplicaId: 't1' }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); await bucketStorage.compact({ clearBatchLimit: 2, @@ -496,8 +511,9 @@ bucket_definitions: minChangeRatio: 0 }); - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result2 = await (async () => { + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -505,8 +521,10 @@ bucket_definitions: }, beforeReplicaId: 't2' }); - await batch.commit('2/1'); - }); + await writer2.commit('2/1'); + await writer2.flush(); + return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; + })(); const checkpoint2 = result2!.flushed_op; const request = bucketRequest(syncRules, 'global[]'); await bucketStorage.clearChecksumCache(); @@ -532,9 +550,10 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result = await (async () => { + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -543,7 +562,7 @@ bucket_definitions: afterReplicaId: 't1' }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -552,13 +571,16 @@ bucket_definitions: afterReplicaId: 't1' }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); // Get checksums here just to populate the cache await bucketStorage.getChecksums(result!.flushed_op, bucketRequests(syncRules, ['global[]'])); - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result2 = await (async () => { + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -566,8 +588,10 @@ bucket_definitions: }, beforeReplicaId: 't1' }); - await batch.commit('2/1'); - }); + await writer2.commit('2/1'); + await writer2.flush(); + return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; + })(); await bucketStorage.compact({ clearBatchLimit: 20, @@ -601,28 +625,34 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result1 = await (async () => { + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + })(); const checkpoint1 = result1!.flushed_op; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result2 = await (async () => { // This is flushed but not committed (does not advance the checkpoint) - await batch.save({ + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - }); + await writer2.flush(); + return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; + })(); const checkpoint2 = result2!.flushed_op; const checkpointBeforeCompact = await bucketStorage.getCheckpoint(); diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 29c0e669d..2e07d9402 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -39,18 +39,18 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.flush(); const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ heads: { '1': '5/0' }, user_id: 'user1' }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('5/0'); - }); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.keepalive('5/0'); + await writer2.flush(); const result = await iter.next(); expect(result).toMatchObject({ @@ -82,9 +82,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(r.persisted_sync_rules!); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -92,9 +92,9 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('5/0'); - }); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.keepalive('5/0'); + await writer2.flush(); const result = await iter.next(); expect(result).toMatchObject({ @@ -115,9 +115,9 @@ bucket_definitions: // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage. // This is what is effetively triggered with RouteAPI.createReplicationHead(). // MongoDB storage doesn't explicitly need this anymore. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('6/0'); - }); + await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer3.keepalive('6/0'); + await writer3.flush(); let result2 = await iter.next(); if (result2.value?.base?.lsn == '5/0') { @@ -154,9 +154,9 @@ bucket_definitions: const bucketStorage = factory.getInstance(r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -164,14 +164,14 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.addCustomWriteCheckpoint({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.addCustomWriteCheckpoint({ checkpoint: 5n, user_id: 'user1' }); - await batch.flush(); - await batch.keepalive('5/0'); - }); + await writer2.flush(); + await writer2.keepalive('5/0'); + await writer2.flush(); const result = await iter.next(); expect(result).toMatchObject({ @@ -203,9 +203,9 @@ bucket_definitions: const bucketStorage = factory.getInstance(r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -213,17 +213,17 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Flush to clear state - await batch.flush(); + await writer2.flush(); - await batch.addCustomWriteCheckpoint({ + await writer2.addCustomWriteCheckpoint({ checkpoint: 5n, user_id: 'user1' }); - await batch.flush(); - await batch.keepalive('5/0'); - }); + await writer2.flush(); + await writer2.keepalive('5/0'); + await writer2.flush(); const result = await iter.next(); expect(result).toMatchObject({ @@ -255,9 +255,9 @@ bucket_definitions: const bucketStorage = factory.getInstance(r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -265,9 +265,9 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('5/0'); - }); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.keepalive('5/0'); + await writer2.flush(); const result = await iter.next(); expect(result).toMatchObject({ @@ -280,14 +280,14 @@ bucket_definitions: } }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - batch.addCustomWriteCheckpoint({ + await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + writer3.addCustomWriteCheckpoint({ checkpoint: 6n, user_id: 'user1' }); - await batch.flush(); - await batch.keepalive('6/0'); - }); + await writer3.flush(); + await writer3.keepalive('6/0'); + await writer3.flush(); let result2 = await iter.next(); expect(result2).toMatchObject({ @@ -301,14 +301,14 @@ bucket_definitions: } }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - batch.addCustomWriteCheckpoint({ + await using writer4 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + writer4.addCustomWriteCheckpoint({ checkpoint: 7n, user_id: 'user1' }); - await batch.flush(); - await batch.keepalive('7/0'); - }); + await writer4.flush(); + await writer4.keepalive('7/0'); + await writer4.flush(); let result3 = await iter.next(); expect(result3).toMatchObject({ diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 7f4011f35..28f555665 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -39,10 +39,10 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -54,7 +54,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -66,8 +66,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); @@ -96,9 +96,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -107,11 +107,11 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('user1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const checkpoint1 = await bucketStorage.getCheckpoint(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -120,8 +120,8 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('user1') }); - await batch.commit('1/2'); - }); + await writer2.commit('1/2'); + await writer2.flush(); const checkpoint2 = await bucketStorage.getCheckpoint(); const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); @@ -160,10 +160,10 @@ bucket_definitions: const table = test_utils.makeTestTable('todos', ['id', 'list_id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); // Create two todos which initially belong to different lists - await batch.save({ + await writer.save({ sourceTable: table, tag: storage.SaveOperationTag.INSERT, after: { @@ -172,7 +172,7 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('todo1') }); - await batch.save({ + await writer.save({ sourceTable: table, tag: storage.SaveOperationTag.INSERT, after: { @@ -182,12 +182,12 @@ bucket_definitions: afterReplicaId: test_utils.rid('todo2') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Update the second todo item to now belong to list 1 - await batch.save({ + await writer2.save({ sourceTable: table, tag: storage.SaveOperationTag.UPDATE, after: { @@ -197,8 +197,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('todo2') }); - await batch.commit('1/1'); - }); + await writer2.commit('1/1'); + await writer2.flush(); // We specifically request the todo_ids for both lists. // There removal operation for the association of `list2`::`todo2` should not interfere with the new @@ -237,9 +237,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -252,8 +252,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const TEST_PARAMS = { group_id: 'group1' }; @@ -293,9 +293,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -306,7 +306,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -319,8 +319,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('t1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const TEST_PARAMS = { group_id: 'group1' }; @@ -354,9 +354,9 @@ bucket_definitions: const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -365,8 +365,8 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('workspace1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {}); @@ -414,9 +414,9 @@ bucket_definitions: const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -426,7 +426,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace1') }); - await batch.save({ + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -436,7 +436,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace2') }); - await batch.save({ + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -446,8 +446,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace3') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -506,9 +506,9 @@ bucket_definitions: const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -518,7 +518,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace1') }); - await batch.save({ + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -528,7 +528,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace2') }); - await batch.save({ + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -539,7 +539,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace3') }); - await batch.save({ + await writer.save({ sourceTable: WORKSPACE_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -550,8 +550,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace4') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -605,9 +605,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -619,8 +619,8 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await batch.truncate([TEST_TABLE]); - }); + await writer.truncate([TEST_TABLE]); + await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -684,9 +684,9 @@ streams: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -696,8 +696,8 @@ streams: afterReplicaId: test_utils.rid('t1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 0e6106ede..1fbcb7a5c 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -21,9 +21,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -32,7 +32,7 @@ bucket_definitions: afterReplicaId: 't1' }); - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -41,8 +41,8 @@ bucket_definitions: afterReplicaId: 't2' }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); const lookup = ScopedParameterLookup.direct(parameterLookupScope('test', '1'), ['t1']); @@ -50,8 +50,8 @@ bucket_definitions: const parameters1 = await checkpoint1.getParameterSets([lookup]); expect(parameters1).toEqual([{ id: 't1' }]); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, before: { @@ -64,7 +64,7 @@ bucket_definitions: afterReplicaId: 't1' }); - await batch.save({ + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -72,8 +72,8 @@ bucket_definitions: }, beforeReplicaId: 't1' }); - await batch.commit('1/2'); - }); + await writer2.commit('1/2'); + await writer2.flush(); const checkpoint2 = await bucketStorage.getCheckpoint(); const parameters2 = await checkpoint2.getParameterSets([lookup]); expect(parameters2).toEqual([]); @@ -105,9 +105,9 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -117,7 +117,7 @@ bucket_definitions: afterReplicaId: 't1' }); // Interleave with another operation, to evict the other cache entry when compacting. - await batch.save({ + await writer.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.INSERT, after: { @@ -127,11 +127,11 @@ bucket_definitions: afterReplicaId: 't2' }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + await writer.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer2.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.DELETE, before: { @@ -140,11 +140,11 @@ bucket_definitions: }, beforeReplicaId: 't1' }); - await batch.commit('2/1'); - }); + await writer2.commit('2/1'); + await writer2.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ + await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer3.save({ sourceTable: TEST_TABLE, tag: storage.SaveOperationTag.UPDATE, after: { @@ -153,8 +153,8 @@ bucket_definitions: }, afterReplicaId: 't2' }); - await batch.commit('3/1'); - }); + await writer3.commit('3/1'); + await writer3.flush(); const lookup = ScopedParameterLookup.direct(parameterLookupScope('test', '1'), ['u1']); From dc907b37c215d7023a31691b490d16095dde4747 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 6 Mar 2026 08:56:47 +0200 Subject: [PATCH 3/8] Furthe test refactor. --- .../test/src/storage_compacting.test.ts | 32 +- .../src/test-utils/general-utils.ts | 15 + .../src/tests/register-compacting-tests.ts | 42 +- .../register-data-storage-checkpoint-tests.ts | 26 +- .../tests/register-data-storage-data-tests.ts | 1251 ++++++++--------- .../register-data-storage-parameter-tests.ts | 354 ++--- .../register-parameter-compacting-tests.ts | 130 +- .../src/tests/register-sync-tests.ts | 723 +++++----- 8 files changed, 1252 insertions(+), 1321 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 065f75d05..1689c34be 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -14,24 +14,24 @@ describe('Mongo Sync Bucket Storage Compact', () => { await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - owner_id: 'u1' - }, - afterReplicaId: test_utils.rid('t1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + owner_id: 'u1' + }, + afterReplicaId: test_utils.rid('t1') + }); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - owner_id: 'u2' - }, - afterReplicaId: test_utils.rid('t2') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + owner_id: 'u2' + }, + afterReplicaId: test_utils.rid('t2') + }); await writer.commit('1/1'); await writer.flush(); diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 755c9e476..1abe3ebb1 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -33,6 +33,21 @@ export function makeTestTable( }); } +/** + * With incremental reprocessing, we need actual test tables, resolved via the writer. + * + * This prepares for it. + */ +export async function resolveTestTable( + _writer: storage.BucketStorageBatch, + name: string, + replicaIdColumns?: string[] | undefined, + options?: { tableIdStrings: boolean }, + _idIndex: number = 1 +) { + return makeTestTable(name, replicaIdColumns, options); +} + export function getBatchData( batch: utils.SyncBucketData[] | storage.SyncBucketDataChunk[] | storage.SyncBucketDataChunk ) { diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index a9c79420a..8b3a5df1b 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -474,31 +474,31 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: 't2' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: 't2' + }); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' + }); await writer.commit('1/1'); await writer.flush(); diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 2e07d9402..85ac0eadd 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -166,9 +166,9 @@ bucket_definitions: await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer2.addCustomWriteCheckpoint({ - checkpoint: 5n, - user_id: 'user1' - }); + checkpoint: 5n, + user_id: 'user1' + }); await writer2.flush(); await writer2.keepalive('5/0'); await writer2.flush(); @@ -214,13 +214,13 @@ bucket_definitions: [Symbol.asyncIterator](); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - // Flush to clear state + // Flush to clear state await writer2.flush(); await writer2.addCustomWriteCheckpoint({ - checkpoint: 5n, - user_id: 'user1' - }); + checkpoint: 5n, + user_id: 'user1' + }); await writer2.flush(); await writer2.keepalive('5/0'); await writer2.flush(); @@ -282,9 +282,9 @@ bucket_definitions: await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); writer3.addCustomWriteCheckpoint({ - checkpoint: 6n, - user_id: 'user1' - }); + checkpoint: 6n, + user_id: 'user1' + }); await writer3.flush(); await writer3.keepalive('6/0'); await writer3.flush(); @@ -303,9 +303,9 @@ bucket_definitions: await using writer4 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); writer4.addCustomWriteCheckpoint({ - checkpoint: 7n, - user_id: 'user1' - }); + checkpoint: 7n, + user_id: 'user1' + }); await writer4.flush(); await writer4.keepalive('7/0'); await writer4.flush(); diff --git a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts index 489209752..5dcbaaa81 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts @@ -37,8 +37,6 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig) const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); - test('removing row', async () => { await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( @@ -53,27 +51,26 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -123,33 +120,28 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await batch.commit('0/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await writer.commit('0/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('2/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('2/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -196,37 +188,32 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await writer.commit('0/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1' - }, - after: { - id: 'test1', - description: 'test1' - }, - beforeReplicaId: test_utils.rid('test1'), - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('2/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1' + }, + after: { + id: 'test1', + description: 'test1' + }, + beforeReplicaId: test_utils.rid('test1'), + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('2/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -273,27 +260,25 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -340,12 +325,12 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -354,12 +339,12 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -368,20 +353,21 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('test1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + } - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); + { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); - await batch.commit('2/1'); - }); + await writer.commit('2/1'); + } const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -418,44 +404,42 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + client_id: 'client1a', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + client_id: 'client1b', + description: 'test1b' + }, + afterReplicaId: test_utils.rid('test1') + }); - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - client_id: 'client1a', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - client_id: 'client1b', - description: 'test1b' - }, - afterReplicaId: test_utils.rid('test1') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test2', - client_id: 'client2', - description: 'test2' - }, - afterReplicaId: test_utils.rid('test2') - }); - - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test2', + client_id: 'client2', + description: 'test2' + }, + afterReplicaId: test_utils.rid('test2') }); + + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); const batch = await test_utils.fromAsync( bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) @@ -491,43 +475,35 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await writer.commit('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); + await writer.flush(); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -576,88 +552,81 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; + await writer.commit('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.commit('2/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); + await writer.commit('2/1'); + const { checkpoint } = await bucketStorage.getCheckpoint(); const batch = await test_utils.fromAsync( @@ -716,114 +685,112 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); // Pre-setup - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test2', - description: 'test2a' - }, - afterReplicaId: test_utils.rid('test2') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test2', + description: 'test2a' + }, + afterReplicaId: test_utils.rid('test2') }); + const result1 = await writer.flush(); const checkpoint1 = result1?.flushed_op ?? 0n; // Test batch - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - // b - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1b' - }, - afterReplicaId: test_utils.rid('test1') - }); + // b + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1b' + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1' - }, - beforeReplicaId: test_utils.rid('test1'), - after: { - id: 'test2', - description: 'test2b' - }, - afterReplicaId: test_utils.rid('test2') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1' + }, + beforeReplicaId: test_utils.rid('test1'), + after: { + id: 'test2', + description: 'test2b' + }, + afterReplicaId: test_utils.rid('test2') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test2' - }, - beforeReplicaId: test_utils.rid('test2'), - after: { - id: 'test3', - description: 'test3b' - }, + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test2' + }, + beforeReplicaId: test_utils.rid('test2'), + after: { + id: 'test3', + description: 'test3b' + }, - afterReplicaId: test_utils.rid('test3') - }); + afterReplicaId: test_utils.rid('test3') + }); - // c - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test2', - description: 'test2c' - }, - afterReplicaId: test_utils.rid('test2') - }); + // c + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test2', + description: 'test2c' + }, + afterReplicaId: test_utils.rid('test2') + }); - // d - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test4', - description: 'test4d' - }, - afterReplicaId: test_utils.rid('test4') - }); + // d + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test4', + description: 'test4d' + }, + afterReplicaId: test_utils.rid('test4') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test4' - }, - beforeReplicaId: test_utils.rid('test4'), - after: { - id: 'test5', - description: 'test5d' - }, - afterReplicaId: test_utils.rid('test5') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test4' + }, + beforeReplicaId: test_utils.rid('test4'), + after: { + id: 'test5', + description: 'test5d' + }, + afterReplicaId: test_utils.rid('test5') }); + const result2 = await writer.flush(); const checkpoint2 = result2!.flushed_op; @@ -881,55 +848,53 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config); // Pre-setup - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: rid2('test1', 'test1a') - }); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: rid2('test1', 'test1a') }); + const result1 = await writer.flush(); const checkpoint1 = result1?.flushed_op ?? 0n; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Unchanged, but has a before id - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1', - description: 'test1a' - }, - beforeReplicaId: rid2('test1', 'test1a'), - after: { - id: 'test1', - description: 'test1b' - }, - afterReplicaId: rid2('test1', 'test1b') - }); + // Unchanged, but has a before id + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1', + description: 'test1a' + }, + beforeReplicaId: rid2('test1', 'test1a'), + after: { + id: 'test1', + description: 'test1b' + }, + afterReplicaId: rid2('test1', 'test1b') }); - - const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Delete - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 'test1', - description: 'test1b' - }, - beforeReplicaId: rid2('test1', 'test1b'), - after: undefined - }); + const result2 = await writer.flush(); + + // Delete + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 'test1', + description: 'test1b' + }, + beforeReplicaId: rid2('test1', 'test1b'), + after: undefined }); + const result3 = await writer.flush(); const checkpoint3 = result3!.flushed_op; @@ -995,55 +960,53 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config); // Pre-setup - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: rid2('test1', 'test1a') - }); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: rid2('test1', 'test1a') }); + const result1 = await writer.flush(); const checkpoint1 = result1?.flushed_op ?? 0n; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Unchanged, but has a before id - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1', - description: 'test1a' - }, - beforeReplicaId: rid2('test1', 'test1a'), - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: rid2('test1', 'test1a') - }); + // Unchanged, but has a before id + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1', + description: 'test1a' + }, + beforeReplicaId: rid2('test1', 'test1a'), + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: rid2('test1', 'test1a') }); - - const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Delete - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 'test1', - description: 'test1a' - }, - beforeReplicaId: rid2('test1', 'test1a'), - after: undefined - }); + const result2 = await writer.flush(); + + // Delete + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 'test1', + description: 'test1a' + }, + beforeReplicaId: rid2('test1', 'test1a'), + after: undefined }); + const result3 = await writer.flush(); const checkpoint3 = result3!.flushed_op; @@ -1098,57 +1061,55 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + const largeDescription = '0123456789'.repeat(12_000_00); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - const largeDescription = '0123456789'.repeat(12_000_00); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large1', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large1') - }); - - // Large enough to split the returned batch - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large2', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large2') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test3', - description: 'test3' - }, - afterReplicaId: test_utils.rid('test3') - }); + // Large enough to split the returned batch + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') }); + await writer.commit('1/1'); + const { checkpoint } = await bucketStorage.getCheckpoint(); const options: storage.BucketDataBatchOptions = { @@ -1213,25 +1174,23 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - for (let i = 1; i <= 6; i++) { - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `test${i}`, - description: `test${i}` - }, - afterReplicaId: `test${i}` - }); - } + for (let i = 1; i <= 6; i++) { + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: `test${i}`, + description: `test${i}` + }, + afterReplicaId: `test${i}` + }); + } - await batch.commit('1/1'); - }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -1304,26 +1263,24 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - for (let i = 1; i <= 10; i++) { - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `test${i}`, - description: `test${i}`, - bucket: i == 1 ? 'global1' : 'global2' - }, - afterReplicaId: `test${i}` - }); - } + for (let i = 1; i <= 10; i++) { + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: `test${i}`, + description: `test${i}`, + bucket: i == 1 ? 'global1' : 'global2' + }, + afterReplicaId: `test${i}` + }); + } - await batch.commit('1/1'); - }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); const batch = await test_utils.fromAsync( @@ -1458,10 +1415,9 @@ bucket_definitions: const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}')); const storage = f.getInstance(r.persisted_sync_rules!); - await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/0'); - await batch.keepalive('1/0'); - }); + await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/0'); + await writer.keepalive('1/0'); await f.getStorageMetrics(); // We don't care about the specific values here @@ -1487,36 +1443,35 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = test_utils.makeTestTable('test', ['id'], config); - const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id'], config); - - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - // This saves a record to current_data, but not bucket_data. - // This causes a checkpoint to be created without increasing the op_id sequence. - await batch.save({ - sourceTable: sourceTableIgnore, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1); + const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2); + + await writer.markAllSnapshotDone('1/1'); + // This saves a record to current_data, but not bucket_data. + // This causes a checkpoint to be created without increasing the op_id sequence. + await writer.save({ + sourceTable: sourceTableIgnore, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + const result1 = await writer.flush(); const checkpoint1 = result1!.flushed_op; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test2' - }, - afterReplicaId: test_utils.rid('test2') - }); + await writer.save({ + sourceTable: sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test2' + }, + afterReplicaId: test_utils.rid('test2') }); + const result2 = await writer.flush(); const checkpoint2 = result2!.flushed_op; // we expect 0n and 1n, or 1n and 2n. @@ -1539,21 +1494,20 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); const checksums = [ @@ -1586,28 +1540,26 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.commit('1/1'); - - const cp1 = await bucketStorage.getCheckpoint(); - expect(cp1.lsn).toEqual('1/1'); + const cp1 = await bucketStorage.getCheckpoint(); + expect(cp1.lsn).toEqual('1/1'); - await batch.commit('2/1', { createEmptyCheckpoints: true }); - const cp2 = await bucketStorage.getCheckpoint(); - expect(cp2.lsn).toEqual('2/1'); + await writer.commit('2/1', { createEmptyCheckpoints: true }); + const cp2 = await bucketStorage.getCheckpoint(); + expect(cp2.lsn).toEqual('2/1'); - await batch.keepalive('3/1'); - const cp3 = await bucketStorage.getCheckpoint(); - expect(cp3.lsn).toEqual('3/1'); + await writer.keepalive('3/1'); + const cp3 = await bucketStorage.getCheckpoint(); + expect(cp3.lsn).toEqual('3/1'); - // For the last one, we skip creating empty checkpoints - // This means the LSN stays at 3/1. - await batch.commit('4/1', { createEmptyCheckpoints: false }); - const cp4 = await bucketStorage.getCheckpoint(); - expect(cp4.lsn).toEqual('3/1'); - }); + // For the last one, we skip creating empty checkpoints + // This means the LSN stays at 3/1. + await writer.commit('4/1', { createEmptyCheckpoints: false }); + const cp4 = await bucketStorage.getCheckpoint(); + expect(cp4.lsn).toEqual('3/1'); }); test('empty checkpoints (2)', async () => { @@ -1626,40 +1578,38 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); - const sourceTable = TEST_TABLE; - // We simulate two concurrent batches, but nesting is the easiest way to do this. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch1) => { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch2) => { - await batch1.markAllSnapshotDone('1/1'); - await batch1.commit('1/1'); - - await batch1.commit('2/1', { createEmptyCheckpoints: false }); - const cp2 = await bucketStorage.getCheckpoint(); - expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped + // We simulate two concurrent batches, but sequential calls are enough for this test. + await writer1.markAllSnapshotDone('1/1'); + await writer1.commit('1/1'); - await batch2.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - // This simulates what happens on a snapshot processor. - // This may later change to a flush() rather than commit(). - await batch2.commit(test_utils.BATCH_OPTIONS.zeroLSN); + await writer1.commit('2/1', { createEmptyCheckpoints: false }); + const cp2 = await bucketStorage.getCheckpoint(); + expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped + + await writer2.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); + // This simulates what happens on a snapshot processor. + // This may later change to a flush() rather than commit(). + await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN); - const cp3 = await bucketStorage.getCheckpoint(); - expect(cp3.lsn).toEqual('1/1'); // Still unchanged + const cp3 = await bucketStorage.getCheckpoint(); + expect(cp3.lsn).toEqual('1/1'); // Still unchanged - // This now needs to advance the LSN, despite {createEmptyCheckpoints: false} - await batch1.commit('4/1', { createEmptyCheckpoints: false }); - const cp4 = await bucketStorage.getCheckpoint(); - expect(cp4.lsn).toEqual('4/1'); - }); - }); + // This now needs to advance the LSN, despite {createEmptyCheckpoints: false} + await writer1.commit('4/1', { createEmptyCheckpoints: false }); + const cp4 = await bucketStorage.getCheckpoint(); + expect(cp4.lsn).toEqual('4/1'); }); test('empty checkpoints (sync rule activation)', async () => { @@ -1679,29 +1629,24 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const result = await batch.commit('1/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false }); - // Snapshot is only valid once we reach 3/1 - await batch.markAllSnapshotDone('3/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false }); + expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false }); + // Snapshot is only valid once we reach 3/1 + await writer.markAllSnapshotDone('3/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // 2/1 < 3/1 - snapshot not valid yet, block checkpoint - const result = await batch.commit('2/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false }); - }); + // 2/1 < 3/1 - snapshot not valid yet, block checkpoint + const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false }); + expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false }); // No empty checkpoint should be created by the commit above. const cp1 = await bucketStorage.getCheckpoint(); expect(cp1.lsn).toEqual(null); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // After this commit, the snapshot should be valid. - // We specifically check that this is done even if createEmptyCheckpoints: false. - const result = await batch.commit('3/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true }); - }); + // After this commit, the snapshot should be valid. + // We specifically check that this is done even if createEmptyCheckpoints: false. + const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false }); + expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true }); // Now, the checkpoint should advance the sync rules active. const cp2 = await bucketStorage.getCheckpoint(); @@ -1710,11 +1655,9 @@ bucket_definitions: const activeSyncRules = await factory.getActiveSyncRulesContent(); expect(activeSyncRules?.id).toEqual(syncRules.id); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // At this point, it should be a truely empty checkpoint - const result = await batch.commit('4/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false }); - }); + // At this point, it should be a truely empty checkpoint + const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false }); + expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false }); // Unchanged const cp3 = await bucketStorage.getCheckpoint(); @@ -1737,39 +1680,41 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using snapshotWriter = await bucketStorage.createWriter({ + ...test_utils.BATCH_OPTIONS, + skipExistingRows: true + }); + await using streamingWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1); + const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1); - const sourceTable = TEST_TABLE; - // We simulate two concurrent batches, and nesting is the easiest way to do this. + // We simulate two concurrent batches; separate writers are enough for this test. // For this test, we assume that we start with a row "test1", which is picked up by a snapshot // query, right before the delete is streamed. But the snapshot query is only persisted _after_ // the delete is streamed, and we need to ensure that the streamed delete takes precedence. - await bucketStorage.startBatch({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true }, async (snapshotBatch) => { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (streamingBatch) => { - streamingBatch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 'test1' - }, - beforeReplicaId: test_utils.rid('test1') - }); - await streamingBatch.commit('2/1'); - - await snapshotBatch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - await snapshotBatch.markAllSnapshotDone('3/1'); - await snapshotBatch.commit('1/1'); - - await streamingBatch.keepalive('3/1'); - }); + await streamingWriter.save({ + sourceTable: streamingTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 'test1' + }, + beforeReplicaId: test_utils.rid('test1') }); + await streamingWriter.commit('2/1'); + + await snapshotWriter.save({ + sourceTable: snapshotTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); + await snapshotWriter.markAllSnapshotDone('3/1'); + await snapshotWriter.commit('1/1'); + + await streamingWriter.keepalive('3/1'); const cp = await bucketStorage.getCheckpoint(); expect(cp.lsn).toEqual('3/1'); @@ -1805,27 +1750,25 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = test_utils.makeTestTable('test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - for (let u of ['u1', 'u2', 'u3', 'u4']) { - for (let t of ['t1', 't2', 't3', 't4']) { - const id = `${t}_${u}`; - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id, - description: `${t} description`, - user_id: u - }, - afterReplicaId: test_utils.rid(id) - }); - } + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + for (let u of ['u1', 'u2', 'u3', 'u4']) { + for (let t of ['t1', 't2', 't3', 't4']) { + const id = `${t}_${u}`; + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id, + description: `${t} description`, + user_id: u + }, + afterReplicaId: test_utils.rid(id) + }); } - await batch.commit('1/1'); - }); + } + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); bucketStorage.clearChecksumCache(); diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 28f555665..42ea4d944 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -43,28 +43,28 @@ bucket_definitions: await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - id1: 'user3', - id2: 'user4', - group_id: 'group2a' - }, - afterReplicaId: test_utils.rid('t2') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + id1: 'user3', + id2: 'user4', + group_id: 'group2a' + }, + afterReplicaId: test_utils.rid('t2') + }); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - id1: 'user1', - id2: 'user2', - group_id: 'group1a' - }, - afterReplicaId: test_utils.rid('t1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + id1: 'user1', + id2: 'user2', + group_id: 'group1a' + }, + afterReplicaId: test_utils.rid('t1') + }); await writer.commit('1/1'); await writer.flush(); @@ -99,27 +99,27 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - group_id: 'group1' - }, - afterReplicaId: test_utils.rid('user1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + group_id: 'group1' + }, + afterReplicaId: test_utils.rid('user1') + }); await writer.commit('1/1'); await writer.flush(); const checkpoint1 = await bucketStorage.getCheckpoint(); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - group_id: 'group2' - }, - afterReplicaId: test_utils.rid('user1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + group_id: 'group2' + }, + afterReplicaId: test_utils.rid('user1') + }); await writer2.commit('1/2'); await writer2.flush(); const checkpoint2 = await bucketStorage.getCheckpoint(); @@ -162,40 +162,40 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); - // Create two todos which initially belong to different lists + // Create two todos which initially belong to different lists await writer.save({ - sourceTable: table, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'todo1', - list_id: 'list1' - }, - afterReplicaId: test_utils.rid('todo1') - }); + sourceTable: table, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'todo1', + list_id: 'list1' + }, + afterReplicaId: test_utils.rid('todo1') + }); await writer.save({ - sourceTable: table, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'todo2', - list_id: 'list2' - }, - afterReplicaId: test_utils.rid('todo2') - }); + sourceTable: table, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'todo2', + list_id: 'list2' + }, + afterReplicaId: test_utils.rid('todo2') + }); await writer.commit('1/1'); await writer.flush(); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - // Update the second todo item to now belong to list 1 + // Update the second todo item to now belong to list 1 await writer2.save({ - sourceTable: table, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'todo2', - list_id: 'list1' - }, - afterReplicaId: test_utils.rid('todo2') - }); + sourceTable: table, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'todo2', + list_id: 'list1' + }, + afterReplicaId: test_utils.rid('todo2') + }); await writer2.commit('1/1'); await writer2.flush(); @@ -240,17 +240,17 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - group_id: 'group1', - n1: 314n, - f2: 314, - f3: 3.14 - }, - afterReplicaId: test_utils.rid('t1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + group_id: 'group1', + n1: 314n, + f2: 314, + f3: 3.14 + }, + afterReplicaId: test_utils.rid('t1') + }); await writer.commit('1/1'); await writer.flush(); @@ -296,28 +296,28 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - group_id: 'group1', - n1: 1152921504606846976n // 2^60 - }, - afterReplicaId: test_utils.rid('t1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + group_id: 'group1', + n1: 1152921504606846976n // 2^60 + }, + afterReplicaId: test_utils.rid('t1') + }); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - group_id: 'group1', - // Simulate a TOAST value, even though it can't happen for values like this - // in practice. - n1: undefined - }, - afterReplicaId: test_utils.rid('t1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + group_id: 'group1', + // Simulate a TOAST value, even though it can't happen for values like this + // in practice. + n1: undefined + }, + afterReplicaId: test_utils.rid('t1') + }); await writer.commit('1/1'); await writer.flush(); @@ -357,14 +357,14 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace1', - userId: 'u1' - }, - afterReplicaId: test_utils.rid('workspace1') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace1', + userId: 'u1' + }, + afterReplicaId: test_utils.rid('workspace1') + }); await writer.commit('1/1'); await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -417,34 +417,34 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace1', - visibility: 'public' - }, - afterReplicaId: test_utils.rid('workspace1') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace1', + visibility: 'public' + }, + afterReplicaId: test_utils.rid('workspace1') + }); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace2', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace2') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace2', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace2') + }); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace3', - visibility: 'public' - }, - afterReplicaId: test_utils.rid('workspace3') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace3', + visibility: 'public' + }, + afterReplicaId: test_utils.rid('workspace3') + }); await writer.commit('1/1'); await writer.flush(); @@ -509,46 +509,46 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace1', - visibility: 'public' - }, - afterReplicaId: test_utils.rid('workspace1') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace1', + visibility: 'public' + }, + afterReplicaId: test_utils.rid('workspace1') + }); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace2', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace2') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace2', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace2') + }); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace3', - user_id: 'u1', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace3') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace3', + user_id: 'u1', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace3') + }); await writer.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace4', - user_id: 'u2', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace4') - }); + sourceTable: WORKSPACE_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace4', + user_id: 'u2', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace4') + }); await writer.commit('1/1'); await writer.flush(); @@ -608,16 +608,16 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - id1: 'user3', - id2: 'user4', - group_id: 'group2a' - }, - afterReplicaId: test_utils.rid('t2') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + id1: 'user3', + id2: 'user4', + group_id: 'group2a' + }, + afterReplicaId: test_utils.rid('t2') + }); await writer.truncate([TEST_TABLE]); await writer.flush(); @@ -687,14 +687,14 @@ streams: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - baz: 'baz', - bar: 'bar' - }, - afterReplicaId: test_utils.rid('t1') - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + baz: 'baz', + bar: 'bar' + }, + afterReplicaId: test_utils.rid('t1') + }); await writer.commit('1/1'); await writer.flush(); diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 1fbcb7a5c..6d222be46 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -24,22 +24,22 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: 't2' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: 't2' + }); await writer.commit('1/1'); await writer.flush(); @@ -52,26 +52,26 @@ bucket_definitions: await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 't1' - }, - beforeReplicaId: 't1', - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 't1' + }, + beforeReplicaId: 't1', + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' + }); await writer2.commit('1/2'); await writer2.flush(); const checkpoint2 = await bucketStorage.getCheckpoint(); @@ -108,51 +108,51 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - uid: 'u1' - }, - afterReplicaId: 't1' - }); - // Interleave with another operation, to evict the other cache entry when compacting. + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + uid: 'u1' + }, + afterReplicaId: 't1' + }); + // Interleave with another operation, to evict the other cache entry when compacting. await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - uid: 'u1' - }, - afterReplicaId: 't2' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + uid: 'u1' + }, + afterReplicaId: 't2' + }); await writer.commit('1/1'); await writer.flush(); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1', - uid: 'u1' - }, - beforeReplicaId: 't1' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1', + uid: 'u1' + }, + beforeReplicaId: 't1' + }); await writer2.commit('2/1'); await writer2.flush(); await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer3.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2', - uid: 'u2' - }, - afterReplicaId: 't2' - }); + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2', + uid: 'u2' + }, + afterReplicaId: 't2' + }); await writer3.commit('3/1'); await writer3.flush(); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index b71a422c5..50746b5e3 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -71,33 +71,33 @@ export function registerSyncTests( }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); + await writer.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - description: 'Test 2' - }, - afterReplicaId: 't2' - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' + }); - await batch.commit('0/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + description: 'Test 2' + }, + afterReplicaId: 't2' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage: bucketStorage, @@ -134,32 +134,32 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'earlier', - description: 'Test 2' - }, - afterReplicaId: 'earlier' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'earlier', + description: 'Test 2' + }, + afterReplicaId: 'earlier' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -196,33 +196,33 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // Initial data: Add one priority row and 10k low-priority rows. - await batch.save({ - sourceTable: TEST_TABLE, + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + // Initial data: Add one priority row and 10k low-priority rows. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio', + description: 'High priority row' + }, + afterReplicaId: 'highprio' + }); + for (let i = 0; i < 10_000; i++) { + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { - id: 'highprio', - description: 'High priority row' + id: `${i}`, + description: 'low prio' }, - afterReplicaId: 'highprio' + afterReplicaId: `${i}` }); - for (let i = 0; i < 10_000; i++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `${i}`, - description: 'low prio' - }, - afterReplicaId: `${i}` - }); - } + } - await batch.commit('0/1'); - }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -250,20 +250,20 @@ bucket_definitions: if (sentCheckpoints == 1) { // Save new data to interrupt the low-priority sync. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add another high-priority row. This should interrupt the long-running low-priority sync. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio2', - description: 'Another high-priority row' - }, - afterReplicaId: 'highprio2' - }); - - await batch.commit('0/2'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + // Add another high-priority row. This should interrupt the long-running low-priority sync. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio2', + description: 'Another high-priority row' + }, + afterReplicaId: 'highprio2' }); + + await writer.commit('0/2'); } else { // Low-priority sync from the first checkpoint was interrupted. This should not happen before // 1000 low-priority items were synchronized. @@ -307,33 +307,33 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // Initial data: Add one priority row and 10k low-priority rows. - await batch.save({ - sourceTable: TEST_TABLE, + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + // Initial data: Add one priority row and 10k low-priority rows. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio', + description: 'user_one' + }, + afterReplicaId: 'highprio' + }); + for (let i = 0; i < 10_000; i++) { + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { - id: 'highprio', - description: 'user_one' + id: `${i}`, + description: 'low prio' }, - afterReplicaId: 'highprio' + afterReplicaId: `${i}` }); - for (let i = 0; i < 10_000; i++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `${i}`, - description: 'low prio' - }, - afterReplicaId: `${i}` - }); - } + } - await batch.commit('0/1'); - }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -366,20 +366,18 @@ bucket_definitions: if (typeof next === 'object' && next !== null) { if ('partial_checkpoint_complete' in next) { if (sentCheckpoints == 1) { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add a high-priority row that doesn't affect this sync stream. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio2', - description: 'user_two' - }, - afterReplicaId: 'highprio2' - }); - - await batch.commit('0/2'); + // Add a high-priority row that doesn't affect this sync stream. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio2', + description: 'user_two' + }, + afterReplicaId: 'highprio2' }); + + await writer.commit('0/2'); } else { expect(sentCheckpoints).toBe(2); expect(sentRows).toBe(10002); @@ -400,20 +398,18 @@ bucket_definitions: if (completedCheckpoints == 1) { expect(sentRows).toBe(10001); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add a high-priority row that affects this sync stream. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio3', - description: 'user_one' - }, - afterReplicaId: 'highprio3' - }); - - await batch.commit('0/3'); + // Add a high-priority row that affects this sync stream. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio3', + description: 'user_one' + }, + afterReplicaId: 'highprio3' }); + + await writer.commit('0/3'); } } } @@ -449,33 +445,33 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // Initial data: Add one priority row and 10k low-priority rows. - await batch.save({ - sourceTable: TEST_TABLE, + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + // Initial data: Add one priority row and 10k low-priority rows. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio', + description: 'High priority row' + }, + afterReplicaId: 'highprio' + }); + for (let i = 0; i < 2_000; i++) { + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { - id: 'highprio', - description: 'High priority row' + id: `${i}`, + description: 'low prio' }, - afterReplicaId: 'highprio' + afterReplicaId: `${i}` }); - for (let i = 0; i < 2_000; i++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `${i}`, - description: 'low prio' - }, - afterReplicaId: `${i}` - }); - } + } - await batch.commit('0/1'); - }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -512,31 +508,29 @@ bucket_definitions: if (sentRows == 1001) { // Save new data to interrupt the low-priority sync. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add another high-priority row. This should interrupt the long-running low-priority sync. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio2', - description: 'Another high-priority row' - }, - afterReplicaId: 'highprio2' - }); - - // Also add a low-priority row - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: '2001', - description: 'Another low-priority row' - }, - afterReplicaId: '2001' - }); - - await batch.commit('0/2'); + // Add another high-priority row. This should interrupt the long-running low-priority sync. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio2', + description: 'Another high-priority row' + }, + afterReplicaId: 'highprio2' }); + + // Also add a low-priority row + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: '2001', + description: 'Another low-priority row' + }, + afterReplicaId: '2001' + }); + + await writer.commit('0/2'); } if (sentRows >= 1000 && sentRows <= 2001) { @@ -579,20 +573,20 @@ bucket_definitions: content: BASIC_SYNC_RULES }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'sync' - }, - afterReplicaId: 't1' - }); - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'sync' + }, + afterReplicaId: 't1' }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -623,9 +617,7 @@ bucket_definitions: if (receivedCompletions == 1) { // Trigger an empty bucket update. await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.commit('1/0'); - }); + await writer.commit('1/0'); } else { break; } @@ -637,30 +629,30 @@ bucket_definitions: }); test('sync legacy non-raw data', async () => { - const f = await factory(); + await using f = await factory(); const syncRules = await updateSyncRules(f, { content: BASIC_SYNC_RULES }); const bucketStorage = await f.getInstance(syncRules); - - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test\n"string"', - large_num: 12345678901234567890n - }, - afterReplicaId: 't1' - }); - - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test\n"string"', + large_num: 12345678901234567890n + }, + afterReplicaId: 't1' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -716,11 +708,11 @@ bucket_definitions: }); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const stream = sync.streamResponse({ syncContext, @@ -742,36 +734,32 @@ bucket_definitions: expect(await getCheckpointLines(iter)).toMatchSnapshot(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' }); - expect(await getCheckpointLines(iter)).toMatchSnapshot(); + await writer.commit('0/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - description: 'Test 2' - }, - afterReplicaId: 't2' - }); + expect(await getCheckpointLines(iter)).toMatchSnapshot(); - await batch.commit('0/2'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + description: 'Test 2' + }, + afterReplicaId: 't2' }); + await writer.commit('0/2'); + expect(await getCheckpointLines(iter)).toMatchSnapshot(); }); @@ -791,11 +779,10 @@ bucket_definitions: const listsTable = test_utils.makeTestTable('lists', ['id'], config); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const stream = sync.streamResponse({ syncContext, @@ -821,20 +808,18 @@ bucket_definitions: expect(checkpoint1).toMatchSnapshot(); // Add user - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: usersTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'user1' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: usersTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'user1' }); + await writer.commit('0/1'); + const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); const checkpoint2 = await getCheckpointLines(iter); expect( @@ -859,22 +844,21 @@ bucket_definitions: const listsTable = test_utils.makeTestTable('lists', ['id'], config); const bucketStorage = await f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: usersTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'user1' - }); - - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: usersTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'user1' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -898,21 +882,19 @@ bucket_definitions: expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]); expect(checkpoint1).toMatchSnapshot(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: listsTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'list1', - user_id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'list1' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: listsTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'list1', + user_id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'list1' }); + await writer.commit('0/1'); + const checkpoint2 = await getCheckpointLines(iter); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) @@ -936,11 +918,10 @@ bucket_definitions: const listsTable = test_utils.makeTestTable('lists', ['id'], config); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const stream = sync.streamResponse({ syncContext, @@ -963,32 +944,30 @@ bucket_definitions: // Initial empty checkpoint expect(await getCheckpointLines(iter)).toMatchSnapshot(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: listsTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'list1', - user_id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'list1' - }); - - await batch.save({ - sourceTable: usersTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'user1' - }); + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: listsTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'list1', + user_id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'list1' + }); - await batch.commit('0/1'); + await writer.save({ + sourceTable: usersTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'user1' }); + await writer.commit('0/1'); + const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); const checkpoint2 = await getCheckpointLines(iter); expect( @@ -1005,11 +984,10 @@ bucket_definitions: }); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const exp = Date.now() / 1000 + 0.1; @@ -1051,32 +1029,32 @@ bucket_definitions: }); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - description: 'Test 2' - }, - afterReplicaId: 't2' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + description: 'Test 2' + }, + afterReplicaId: 't2' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -1108,31 +1086,29 @@ bucket_definitions: // Now we save additional data AND compact before continuing. // This invalidates the checkpoint we've received above. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - description: 'Test 1b' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2', - description: 'Test 2b' - }, - afterReplicaId: 't2' - }); + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + description: 'Test 1b' + }, + afterReplicaId: 't1' + }); - await batch.commit('0/2'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2', + description: 'Test 2b' + }, + afterReplicaId: 't2' }); + await writer.commit('0/2'); + await bucketStorage.compact({ minBucketChanges: 1, minChangeRatio: 0 @@ -1196,12 +1172,11 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // <= the managed write checkpoint LSN below - await batch.commit('0/1'); - }); + await writer.markAllSnapshotDone('0/1'); + // <= the managed write checkpoint LSN below + await writer.commit('0/1'); const checkpoint = await bucketStorage.createManagedWriteCheckpoint({ user_id: 'test', @@ -1233,11 +1208,9 @@ bucket_definitions: }) }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // must be >= the managed write checkpoint LSN - await batch.commit('1/0'); - }); + await writer.markAllSnapshotDone('0/1'); + // must be >= the managed write checkpoint LSN + await writer.commit('1/0'); // At this point the LSN has advanced, so the write checkpoint should be // included in the next checkpoint message. @@ -1268,20 +1241,20 @@ config: content: rules }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - await batch.commit('0/1'); + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, From e50f1953e085497bfd46494caf0ede616b2f4210 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 6 Mar 2026 09:14:33 +0200 Subject: [PATCH 4/8] Port more test changes from incremental-reprocessing branch. --- .../__snapshots__/storage_sync.test.ts.snap | 48 ++-- .../__snapshots__/storage_sync.test.ts.snap | 48 ++-- .../src/test-utils/general-utils.ts | 38 +++- .../tests/register-data-storage-data-tests.ts | 212 +++++++----------- .../src/tests/register-sync-tests.ts | 42 ++-- 5 files changed, 190 insertions(+), 198 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index 1ad77205f..d59bfb500 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -104,7 +104,7 @@ exports[`sync - mongodb > storage v1 > compacting data - invalidate checkpoint 2 ] `; -exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -159,13 +159,13 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea ] `; -exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -181,7 +181,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -190,7 +190,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -199,7 +199,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a", + "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, @@ -819,7 +819,7 @@ exports[`sync - mongodb > storage v1 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -1026,7 +1026,7 @@ exports[`sync - mongodb > storage v1 > sync updates to parameter query + data 2` "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -1199,7 +1199,7 @@ exports[`sync - mongodb > storage v2 > compacting data - invalidate checkpoint 2 ] `; -exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -1254,13 +1254,13 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea ] `; -exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -1276,7 +1276,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -1285,7 +1285,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -1294,7 +1294,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a", + "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, @@ -1914,7 +1914,7 @@ exports[`sync - mongodb > storage v2 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -2121,7 +2121,7 @@ exports[`sync - mongodb > storage v2 > sync updates to parameter query + data 2` "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -2294,7 +2294,7 @@ exports[`sync - mongodb > storage v3 > compacting data - invalidate checkpoint 2 ] `; -exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - mongodb > storage v3 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -2349,13 +2349,13 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea ] `; -exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - mongodb > storage v3 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -2371,7 +2371,7 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -2380,7 +2380,7 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -2389,7 +2389,7 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a", + "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, @@ -3009,7 +3009,7 @@ exports[`sync - mongodb > storage v3 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -3216,7 +3216,7 @@ exports[`sync - mongodb > storage v3 > sync updates to parameter query + data 2` "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap index d45099f02..f628d9130 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -104,7 +104,7 @@ exports[`sync - postgres > storage v1 > compacting data - invalidate checkpoint ] `; -exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - postgres > storage v1 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -159,13 +159,13 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre ] `; -exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - postgres > storage v1 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -181,7 +181,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -190,7 +190,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -199,7 +199,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", }, ], "has_more": false, @@ -819,7 +819,7 @@ exports[`sync - postgres > storage v1 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -1026,7 +1026,7 @@ exports[`sync - postgres > storage v1 > sync updates to parameter query + data 2 "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -1199,7 +1199,7 @@ exports[`sync - postgres > storage v2 > compacting data - invalidate checkpoint ] `; -exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - postgres > storage v2 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -1254,13 +1254,13 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre ] `; -exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - postgres > storage v2 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -1276,7 +1276,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -1285,7 +1285,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -1294,7 +1294,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", }, ], "has_more": false, @@ -1914,7 +1914,7 @@ exports[`sync - postgres > storage v2 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -2121,7 +2121,7 @@ exports[`sync - postgres > storage v2 > sync updates to parameter query + data 2 "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -2294,7 +2294,7 @@ exports[`sync - postgres > storage v3 > compacting data - invalidate checkpoint ] `; -exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - postgres > storage v3 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -2349,13 +2349,13 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre ] `; -exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - postgres > storage v3 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -2371,7 +2371,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -2380,7 +2380,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -2389,7 +2389,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", }, ], "has_more": false, @@ -3009,7 +3009,7 @@ exports[`sync - postgres > storage v3 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -3216,7 +3216,7 @@ exports[`sync - postgres > storage v3 > sync updates to parameter query + data 2 "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 1abe3ebb1..37c1fe2d2 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -1,4 +1,4 @@ -import { BucketDataRequest, InternalOpId, storage, utils } from '@powersync/service-core'; +import { BucketDataRequest, InternalOpId, JwtPayload, storage, utils } from '@powersync/service-core'; import { GetQuerierOptions, RequestParameters } from '@powersync/service-sync-rules'; import * as bson from 'bson'; @@ -8,7 +8,7 @@ export const PARSE_OPTIONS: storage.ParseSyncRulesOptions = { defaultSchema: 'public' }; -export const BATCH_OPTIONS: storage.StartBatchOptions = { +export const BATCH_OPTIONS: storage.CreateWriterOptions = { ...PARSE_OPTIONS, zeroLSN: ZERO_LSN, storeCurrentData: true @@ -32,7 +32,6 @@ export function makeTestTable( snapshotComplete: true }); } - /** * With incremental reprocessing, we need actual test tables, resolved via the writer. * @@ -41,11 +40,24 @@ export function makeTestTable( export async function resolveTestTable( _writer: storage.BucketStorageBatch, name: string, - replicaIdColumns?: string[] | undefined, - options?: { tableIdStrings: boolean }, - _idIndex: number = 1 + replicaIdColumns: string[] | undefined, + options: { tableIdStrings: boolean }, + idIndex: number = 1 ) { - return makeTestTable(name, replicaIdColumns, options); + const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(',')); + // Generate unique ids per test table (if idIndex is specified), without completely + // breaking all the existing tests. + const idString = '6544e3899293153fa7b383' + (30 + idIndex).toString().padStart(2, '0'); + const id = options.tableIdStrings == false ? new bson.ObjectId(idString) : idString; + return new storage.SourceTable({ + id: id, + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: relId, + schema: 'public', + name: name, + replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })), + snapshotComplete: true + }); } export function getBatchData( @@ -143,3 +155,15 @@ export function querierOptions(globalParameters: RequestParameters): GetQuerierO streams: {} }; } + +export function requestParameters( + jwtPayload: Record, + clientParameters?: Record +): RequestParameters { + return new RequestParameters(new JwtPayload(jwtPayload), clientParameters ?? {}); +} + +export function removeSource(obj: T): Omit { + const { source, ...rest } = obj; + return rest; +} diff --git a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts index 5dcbaaa81..d2533561d 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts @@ -10,7 +10,6 @@ import { import { describe, expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; -import { bucketRequestMap, bucketRequests } from './util.js'; /** * Normalize data from OplogEntries for comparison in tests. @@ -52,12 +51,12 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'test1', @@ -66,7 +65,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('test1') }); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); @@ -74,9 +73,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -93,12 +91,10 @@ bucket_definitions: { op: 'REMOVE', object_id: 'test1', checksum: c2 } ]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: (c1 + c2) & 0xffffffff, count: 2 } @@ -119,13 +115,12 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); @@ -133,7 +128,7 @@ bucket_definitions: await writer.commit('0/1'); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'test1', @@ -145,9 +140,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -160,12 +154,10 @@ bucket_definitions: expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: c1 & 0xffffffff, count: 1 } @@ -187,13 +179,12 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); @@ -201,7 +192,7 @@ bucket_definitions: await writer.commit('0/1'); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.UPDATE, before: { id: 'test1' @@ -217,9 +208,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -232,12 +222,10 @@ bucket_definitions: expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: c1 & 0xffffffff, count: 1 } @@ -261,16 +249,16 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); await writer.save({ - sourceTable, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'test1', @@ -282,9 +270,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -297,12 +284,10 @@ bucket_definitions: expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: c1 & 0xffffffff, count: 1 } @@ -371,9 +356,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]); @@ -442,7 +426,7 @@ bucket_definitions: await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]) ); const data = batch[0].chunkData.data.map((d) => { return { @@ -507,9 +491,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -526,9 +509,7 @@ bucket_definitions: { op: 'REMOVE', object_id: 'test1', checksum: c2 } ]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { bucket: bucketRequest(syncRules, 'global[]').bucket, @@ -629,9 +610,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { @@ -651,9 +631,7 @@ bucket_definitions: { op: 'REMOVE', object_id: 'test1', checksum: c2 } ]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { bucket: bucketRequest(syncRules, 'global[]').bucket, @@ -794,9 +772,8 @@ bucket_definitions: const checkpoint2 = result2!.flushed_op; - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', checkpoint1]])) - ); + const request = bucketRequest(syncRules, 'global[]', checkpoint1); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request])); const data = batch[0].chunkData.data.map((d) => { return { @@ -898,8 +875,9 @@ bucket_definitions: const checkpoint3 = result3!.flushed_op; + const request = bucketRequest(syncRules, 'global[]'); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])) + bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]) ); const data = batch[0].chunkData.data.map((d) => { return { @@ -1010,8 +988,9 @@ bucket_definitions: const checkpoint3 = result3!.flushed_op; + const request = bucketRequest(syncRules, 'global[]'); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])) + bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]) ); const data = batch[0].chunkData.data.map((d) => { return { @@ -1116,9 +1095,8 @@ bucket_definitions: chunkLimitBytes: 16 * 1024 * 1024 }; - const batch1 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options)); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }, { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 } @@ -1132,7 +1110,7 @@ bucket_definitions: const batch2 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]), + [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options ) ); @@ -1149,7 +1127,7 @@ bucket_definitions: const batch3 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]), + [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options ) ); @@ -1194,9 +1172,8 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch1 = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), { limit: 4 }) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 })); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }, @@ -1212,13 +1189,9 @@ bucket_definitions: }); const batch2 = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch( - checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch1.chunkData.next_after)]]), - { - limit: 4 - } - ) + bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], { + limit: 4 + }) ); expect(test_utils.getBatchData(batch2)).toEqual([ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }, @@ -1232,13 +1205,9 @@ bucket_definitions: }); const batch3 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch( - checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch2.chunkData.next_after)]]), - { - limit: 4 - } - ) + bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], { + limit: 4 + }) ); expect(test_utils.getBatchData(batch3)).toEqual([]); @@ -1283,26 +1252,21 @@ bucket_definitions: await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); + const global1Request = bucketRequest(syncRules, 'global1[]', 0n); + const global2Request = bucketRequest(syncRules, 'global2[]', 0n); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch( - checkpoint, - bucketRequestMap(syncRules, [ - ['global1[]', 0n], - ['global2[]', 0n] - ]), - options - ) + bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options) ); - return { syncRules, batch }; + return { batch, global1Request, global2Request }; }; test('batch has_more (1)', async () => { - const { batch, syncRules } = await setup({ limit: 5 }); + const { batch, global1Request, global2Request } = await setup({ limit: 5 }); expect(batch.length).toEqual(2); - expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket); - expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); + expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket); + expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket); expect(test_utils.getBatchData(batch[0])).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } @@ -1329,11 +1293,11 @@ bucket_definitions: }); test('batch has_more (2)', async () => { - const { batch, syncRules } = await setup({ limit: 11 }); + const { batch, global1Request, global2Request } = await setup({ limit: 11 }); expect(batch.length).toEqual(2); - expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket); - expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); + expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket); + expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket); expect(test_utils.getBatchData(batch[0])).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } @@ -1366,12 +1330,12 @@ bucket_definitions: test('batch has_more (3)', async () => { // 50 bytes is more than 1 row, less than 2 rows - const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 }); + const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 }); expect(batch.length).toEqual(3); - expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket); - expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); - expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); + expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket); + expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket); + expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket); expect(test_utils.getBatchData(batch[0])).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } @@ -1510,18 +1474,11 @@ bucket_definitions: await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; - expect(checksums).toEqual([ - { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 } - ]); - const checksums2 = [ - ...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values() - ]; - expect(checksums2).toEqual([ - { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 } - ]); + const request = bucketRequest(syncRules, 'global[]'); + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; + expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]); + const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()]; + expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]); }); testChecksumBatching(config); @@ -1719,7 +1676,7 @@ bucket_definitions: const cp = await bucketStorage.getCheckpoint(); expect(cp.lsn).toEqual('3/1'); const data = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(cp.checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]) ); expect(data).toEqual([]); @@ -1772,14 +1729,17 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); bucketStorage.clearChecksumCache(); - const buckets = bucketRequests(syncRules, ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]']); - const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()]; + const users = ['u1', 'u2', 'u3', 'u4']; + const expectedChecksums = [346204588, 5261081, 134760718, -302639724]; + const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`)); + const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()]; checksums.sort((a, b) => a.bucket.localeCompare(b.bucket)); - expect(checksums).toEqual([ - { bucket: bucketRequest(syncRules, 'user["u1"]').bucket, count: 4, checksum: 346204588 }, - { bucket: bucketRequest(syncRules, 'user["u2"]').bucket, count: 4, checksum: 5261081 }, - { bucket: bucketRequest(syncRules, 'user["u3"]').bucket, count: 4, checksum: 134760718 }, - { bucket: bucketRequest(syncRules, 'user["u4"]').bucket, count: 4, checksum: -302639724 } - ]); + const expected = bucketRequests.map((request, index) => ({ + bucket: request.bucket, + count: 4, + checksum: expectedChecksums[index] + })); + expected.sort((a, b) => a.bucket.localeCompare(b.bucket)); + expect(checksums).toEqual(expected); }); } diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 50746b5e3..04c68a899 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -53,7 +53,6 @@ export function registerSyncTests( maxDataFetchConcurrency: 2 }); - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); const updateSyncRules = (bucketStorageFactory: storage.BucketStorageFactory, updateOptions: { content: string }) => { return bucketStorageFactory.updateSyncRules( updateSyncRulesFromYaml(updateOptions.content, { @@ -775,11 +774,9 @@ bucket_definitions: ` }); - const usersTable = test_utils.makeTestTable('users', ['id'], config); - const listsTable = test_utils.makeTestTable('lists', ['id'], config); - const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); // Activate await writer.markAllSnapshotDone('0/0'); await writer.keepalive('0/0'); @@ -820,8 +817,8 @@ bucket_definitions: await writer.commit('0/1'); - const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); const checkpoint2 = await getCheckpointLines(iter); + const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) ).toEqual([bucket]); @@ -840,11 +837,10 @@ bucket_definitions: ` }); - const usersTable = test_utils.makeTestTable('users', ['id'], config); - const listsTable = test_utils.makeTestTable('lists', ['id'], config); - const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); + const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2); await writer.markAllSnapshotDone('0/1'); await writer.save({ @@ -914,11 +910,10 @@ bucket_definitions: ` }); - const usersTable = test_utils.makeTestTable('users', ['id'], config); - const listsTable = test_utils.makeTestTable('lists', ['id'], config); - const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); + const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2); // Activate await writer.markAllSnapshotDone('0/0'); await writer.keepalive('0/0'); @@ -968,7 +963,7 @@ bucket_definitions: await writer.commit('0/1'); - const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); + const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); const checkpoint2 = await getCheckpointLines(iter); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) @@ -1224,9 +1219,12 @@ bucket_definitions: }); }); - test('encodes sync rules id in buckes for streams', async () => { + test('encodes sync rules id in buckets for streams', async () => { await using f = await factory(); - const rules = ` + // This test relies making an actual update to sync rules to test the different bucket names. + // The actual naming scheme may change, as long as the two buckets have different names. + const rules = [ + ` streams: test: auto_subscribe: true @@ -1234,15 +1232,25 @@ streams: config: edition: 2 -`; +`, + ` +streams: + test2: + auto_subscribe: true + query: SELECT * FROM test WHERE 1; + +config: + edition: 2 +` + ]; for (let i = 0; i < 2; i++) { const syncRules = await updateSyncRules(f, { - content: rules + content: rules[i] }); const bucketStorage = f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1); await writer.markAllSnapshotDone('0/1'); await writer.save({ From 2ed09ebe2ff369f1893d257b93b559ed159c2db4 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 6 Mar 2026 09:19:45 +0200 Subject: [PATCH 5/8] Further test changes. --- .../register-data-storage-parameter-tests.ts | 76 ++++++++----------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 42ea4d944..d9ddc2475 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -18,7 +18,6 @@ import { parameterLookupScope } from './util.js'; export function registerDataStorageParameterTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); const MYBUCKET_1 = parameterLookupScope('mybucket', '1'); test('save and load parameters', async () => { @@ -40,10 +39,11 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2', @@ -55,7 +55,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1', @@ -67,7 +67,6 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); @@ -97,9 +96,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'user1', @@ -108,11 +108,9 @@ bucket_definitions: afterReplicaId: test_utils.rid('user1') }); await writer.commit('1/1'); - await writer.flush(); const checkpoint1 = await bucketStorage.getCheckpoint(); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'user1', @@ -120,8 +118,7 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('user1') }); - await writer2.commit('1/2'); - await writer2.flush(); + await writer.commit('1/2'); const checkpoint2 = await bucketStorage.getCheckpoint(); const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); @@ -158,9 +155,8 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const table = test_utils.makeTestTable('todos', ['id', 'list_id'], config); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config); await writer.markAllSnapshotDone('1/1'); // Create two todos which initially belong to different lists await writer.save({ @@ -183,11 +179,9 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Update the second todo item to now belong to list 1 - await writer2.save({ + await writer.save({ sourceTable: table, tag: storage.SaveOperationTag.UPDATE, after: { @@ -197,8 +191,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('todo2') }); - await writer2.commit('1/1'); - await writer2.flush(); + await writer.commit('1/1'); // We specifically request the todo_ids for both lists. // There removal operation for the association of `list2`::`todo2` should not interfere with the new @@ -238,9 +231,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1', @@ -253,7 +247,6 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); const TEST_PARAMS = { group_id: 'group1' }; @@ -294,9 +287,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1', @@ -307,7 +301,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.UPDATE, after: { id: 't1', @@ -320,7 +314,6 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); const TEST_PARAMS = { group_id: 'group1' }; @@ -333,8 +326,6 @@ bucket_definitions: }); test('save and load parameters with workspaceId', async () => { - const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', ['id'], config); - await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml( @@ -355,9 +346,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace1', @@ -366,7 +358,6 @@ bucket_definitions: afterReplicaId: test_utils.rid('workspace1') }); await writer.commit('1/1'); - await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {}); @@ -393,8 +384,6 @@ bucket_definitions: }); test('save and load parameters with dynamic global buckets', async () => { - const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', undefined, config); - await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml( @@ -415,9 +404,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace1', @@ -427,7 +417,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace2', @@ -437,7 +427,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace3', @@ -447,7 +437,6 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -483,8 +472,6 @@ bucket_definitions: }); test('multiple parameter queries', async () => { - const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', undefined, config); - await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml( @@ -507,9 +494,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace1', @@ -519,7 +507,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace2', @@ -529,7 +517,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace3', @@ -540,7 +528,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: WORKSPACE_TABLE, + sourceTable: workspaceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 'workspace4', @@ -551,7 +539,6 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -606,9 +593,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2', @@ -619,7 +607,7 @@ bucket_definitions: afterReplicaId: test_utils.rid('t2') }); - await writer.truncate([TEST_TABLE]); + await writer.truncate([testTable]); await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); @@ -685,9 +673,10 @@ streams: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { baz: 'baz', @@ -697,7 +686,6 @@ streams: }); await writer.commit('1/1'); - await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ From 02c6291e05ef9e9bdae64252e23cf7d8ab189678 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 6 Mar 2026 09:44:13 +0200 Subject: [PATCH 6/8] Further test changes. --- .../test/src/storage_compacting.test.ts | 20 +- .../test/src/storage_sync.test.ts | 90 ++- .../test/src/storage.test.ts | 20 +- .../test/src/storage_compacting.test.ts | 31 +- .../test/src/storage_sync.test.ts | 98 ++-- .../src/tests/register-compacting-tests.ts | 515 +++++++++--------- .../register-data-storage-checkpoint-tests.ts | 55 +- .../register-parameter-compacting-tests.ts | 50 +- .../src/tests/register-sync-tests.ts | 7 +- 9 files changed, 419 insertions(+), 467 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 1689c34be..86b8f730a 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -5,16 +5,23 @@ import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY); - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); describe('with blank bucket_state', () => { // This can happen when migrating from older service versions, that did not populate bucket_state yet. - const populate = async (bucketStorage: SyncRulesBucketStorage) => { + const populate = async (bucketStorage: SyncRulesBucketStorage, sourceTableIndex: number) => { await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable( + writer, + 'test', + ['id'], + INITIALIZED_MONGO_STORAGE_FACTORY, + sourceTableIndex + ); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1', @@ -24,7 +31,7 @@ describe('Mongo Sync Bucket Storage Compact', () => { }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2', @@ -34,7 +41,6 @@ describe('Mongo Sync Bucket Storage Compact', () => { }); await writer.commit('1/1'); - await writer.flush(); return bucketStorage.getCheckpoint(); }; @@ -50,7 +56,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); - const { checkpoint } = await populate(bucketStorage); + const { checkpoint } = await populate(bucketStorage, 1); return { bucketStorage, checkpoint, factory, syncRules }; }; @@ -102,7 +108,7 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await populate(bucketStorage); + await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); // Default is to small small numbers - should be a no-op diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 3e2c73667..7e53340c8 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -8,8 +8,6 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor storageVersion, tableIdStrings: storageConfig.tableIdStrings }); - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageConfig); - // The split of returned results can vary depending on storage drivers test('large batch (2)', async () => { // Test syncing a batch of data that is small in count, @@ -28,59 +26,57 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor ) ); const bucketStorage = factory.getInstance(syncRules); - const globalBucket = bucketRequest(syncRules, 'global[]'); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - const sourceTable = TEST_TABLE; - const largeDescription = '0123456789'.repeat(2_000_00); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + const largeDescription = '0123456789'.repeat(2_000_00); - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large1', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); - // Large enough to split the returned batch - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large2', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large2') - }); + // Large enough to split the returned batch + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') + }); - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test3', - description: 'test3' - }, - afterReplicaId: test_utils.rid('test3') - }); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + const flushResult = await writer.flush(); - const checkpoint = result!.flushed_op; + const checkpoint = flushResult!.flushed_op; const options: storage.BucketDataBatchOptions = {}; const batch1 = await test_utils.fromAsync( diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index fd14eb01c..446592614 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequestMap, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -39,10 +39,11 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { ) ); const bucketStorage = factory.getInstance(syncRules); + const globalBucket = bucketRequest(syncRules, 'global[]'); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const result = await (async () => { - const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); const largeDescription = '0123456789'.repeat(2_000_00); @@ -86,17 +87,14 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { }, afterReplicaId: test_utils.rid('test3') }); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + return writer.flush(); })(); const checkpoint = result!.flushed_op; const options: storage.BucketDataBatchOptions = {}; - const batch1 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options) - ); + const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options)); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } ]); @@ -109,7 +107,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { const batch2 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]), + [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }], options ) ); @@ -125,7 +123,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { const batch3 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]), + [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }], options ) ); @@ -141,7 +139,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { const batch4 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]), + [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }], options ) ); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts index 5d0be7413..392750bdc 100644 --- a/modules/module-postgres-storage/test/src/storage_compacting.test.ts +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, bucketRequestMap, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js'; import { POSTGRES_STORAGE_FACTORY } from './util.js'; @@ -7,7 +7,6 @@ import { POSTGRES_STORAGE_FACTORY } from './util.js'; describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY)); describe('Postgres Compact - explicit bucket name', () => { - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY); test('compacts a specific bucket by exact name', async () => { await using factory = await POSTGRES_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules( @@ -19,24 +18,25 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const result = await (async () => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.UPDATE, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); await writer.markAllSnapshotDone('1/1'); + const flushed = await writer.flush(); await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + return flushed; })(); const checkpoint = result!.flushed_op; @@ -49,7 +49,7 @@ bucket_definitions: }); const batch = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)]) ); expect(batch.chunkData.data).toMatchObject([ @@ -72,36 +72,37 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); const request = bucketRequest(syncRules, 'global[]'); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const result = await (async () => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't1' }, beforeReplicaId: test_utils.rid('t1') }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2' }, afterReplicaId: test_utils.rid('t2') }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't2' }, beforeReplicaId: test_utils.rid('t2') }); + const flushed = await writer.flush(); await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + return flushed; })(); const checkpoint = result!.flushed_op; diff --git a/modules/module-postgres-storage/test/src/storage_sync.test.ts b/modules/module-postgres-storage/test/src/storage_sync.test.ts index 32b526f06..88965e59b 100644 --- a/modules/module-postgres-storage/test/src/storage_sync.test.ts +++ b/modules/module-postgres-storage/test/src/storage_sync.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, register, resolveTestTable, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -11,7 +11,6 @@ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; function registerStorageVersionTests(storageVersion: number) { describe(`storage v${storageVersion}`, () => { const storageFactory = POSTGRES_STORAGE_FACTORY; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageFactory); register.registerSyncTests(storageFactory.factory, { storageVersion, @@ -38,54 +37,53 @@ function registerStorageVersionTests(storageVersion: number) { const globalBucket = bucketRequest(syncRules, 'global[]'); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - const sourceTable = TEST_TABLE; - - const largeDescription = '0123456789'.repeat(2_000_00); - - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large1', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large1') - }); - - // Large enough to split the returned batch - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large2', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large2') - }); - - await writer.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test3', - description: 'test3' - }, - afterReplicaId: test_utils.rid('test3') - }); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + + const sourceTable = await resolveTestTable(writer, 'test', ['id'], storageFactory); + + const largeDescription = '0123456789'.repeat(2_000_00); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); + + // Large enough to split the returned batch + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') + }); + + const result = await writer.flush(); const checkpoint = result!.flushed_op; diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index 8b3a5df1b..97b71ba79 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -6,7 +6,6 @@ import { bucketRequestMap, bucketRequests } from './util.js'; export function registerCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); test('compacting (1)', async () => { await using factory = await generateStorageFactory(); @@ -20,41 +19,39 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - await writer.markAllSnapshotDone('1/1'); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: test_utils.rid('t1') - }); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: test_utils.rid('t1') + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') + }); - await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + await writer.commit('1/1'); + await writer.flush(); - const checkpoint = result!.flushed_op; + const checkpoint = writer.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); @@ -129,50 +126,48 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - await writer.markAllSnapshotDone('1/1'); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: test_utils.rid('t1') - }); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: test_utils.rid('t1') + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: test_utils.rid('t1') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: test_utils.rid('t1') + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') + }); - await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + await writer.commit('1/1'); + await writer.flush(); - const checkpoint = result!.flushed_op; + const checkpoint = writer.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); @@ -247,59 +242,53 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - await writer.markAllSnapshotDone('1/1'); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: 't2' - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: 't2' + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' + }); - await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + await writer.commit('1/1'); + await writer.flush(); - const checkpoint1 = result!.flushed_op; + const checkpoint1 = writer.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); - const checksumBefore = await bucketStorage.getChecksums(checkpoint1, [request]); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result2 = await (async () => { - await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't2' - }, - beforeReplicaId: 't2' - }); - await writer2.commit('2/1'); - await writer2.flush(); - return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; - })(); - const checkpoint2 = result2!.flushed_op; + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't2' + }, + beforeReplicaId: 't2' + }); + await writer2.commit('2/1'); + await writer2.flush(); + const checkpoint2 = writer2.last_flushed_op!; await bucketStorage.compact({ clearBatchLimit: 2, @@ -341,78 +330,77 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - await writer.markAllSnapshotDone('1/1'); - /** - * Repeatedly create operations which fall into different buckets. - * The bucket operations are purposely interleaved as the op_id increases. - * A large amount of operations are created here. - * The configured window of compacting operations is 100. This means the initial window will - * contain operations from multiple buckets. - */ - for (let count = 0; count < 100; count++) { - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - b: 'b1', - value: 'start' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - b: 'b1', - value: 'intermediate' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - b: 'b2', - value: 'start' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - b: 'b1', - value: 'final' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2', - b: 'b2', - value: 'final' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await writer.commit('1/1'); - } - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + /** + * Repeatedly create operations which fall into different buckets. + * The bucket operations are purposely interleaved as the op_id increases. + * A large amount of operations are created here. + * The configured window of compacting operations is 100. This means the initial window will + * contain operations from multiple buckets. + */ + for (let count = 0; count < 100; count++) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + b: 'b1', + value: 'start' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + b: 'b1', + value: 'intermediate' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + b: 'b2', + value: 'start' + }, + afterReplicaId: test_utils.rid('t2') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + b: 'b1', + value: 'final' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2', + b: 'b2', + value: 'final' + }, + afterReplicaId: test_utils.rid('t2') + }); + + await writer.commit('1/1'); + } + + await writer.flush(); - const checkpoint = result!.flushed_op; + const checkpoint = writer.last_flushed_op!; await bucketStorage.compact({ clearBatchLimit: 100, @@ -472,9 +460,10 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' @@ -483,7 +472,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2' @@ -492,7 +481,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't1' @@ -512,20 +501,18 @@ bucket_definitions: }); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result2 = await (async () => { - await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't2' - }, - beforeReplicaId: 't2' - }); - await writer2.commit('2/1'); - await writer2.flush(); - return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; - })(); - const checkpoint2 = result2!.flushed_op; + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't2' + }, + beforeReplicaId: 't2' + }); + await writer2.commit('2/1'); + await writer2.flush(); + const checkpoint2 = writer2.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); await bucketStorage.clearChecksumCache(); const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]); @@ -551,47 +538,43 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result = await (async () => { - await writer.markAllSnapshotDone('1/1'); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + await writer.commit('1/1'); + await writer.flush(); // Get checksums here just to populate the cache - await bucketStorage.getChecksums(result!.flushed_op, bucketRequests(syncRules, ['global[]'])); + await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRules, ['global[]'])); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result2 = await (async () => { - await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); - await writer2.commit('2/1'); - await writer2.flush(); - return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; - })(); + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' + }); + await writer2.commit('2/1'); + await writer2.flush(); await bucketStorage.compact({ clearBatchLimit: 20, @@ -601,7 +584,7 @@ bucket_definitions: minChangeRatio: 0 }); - const checkpoint2 = result2!.flushed_op; + const checkpoint2 = writer2.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); // Check that the checksum was correctly updated with the clear operation after having a cached checksum const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]); @@ -626,34 +609,30 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result1 = await (async () => { - await writer.markAllSnapshotDone('1/1'); - await writer.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { id: 't1' }, - afterReplicaId: test_utils.rid('t1') - }); - await writer.commit('1/1'); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; - })(); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 't1' }, + afterReplicaId: test_utils.rid('t1') + }); + await writer.commit('1/1'); + await writer.flush(); - const checkpoint1 = result1!.flushed_op; + const checkpoint1 = writer.last_flushed_op!; await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const result2 = await (async () => { - // This is flushed but not committed (does not advance the checkpoint) - await writer2.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { id: 't1' }, - afterReplicaId: test_utils.rid('t1') - }); - await writer2.flush(); - return writer2.last_flushed_op != null ? { flushed_op: writer2.last_flushed_op } : null; - })(); - const checkpoint2 = result2!.flushed_op; + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + // This is flushed but not committed (does not advance the checkpoint) + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.UPDATE, + after: { id: 't1' }, + afterReplicaId: test_utils.rid('t1') + }); + await writer2.flush(); + const checkpoint2 = writer2.last_flushed_op!; const checkpointBeforeCompact = await bucketStorage.getCheckpoint(); expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1); diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 85ac0eadd..3837071a5 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -41,16 +41,13 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); - await writer.flush(); const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ heads: { '1': '5/0' }, user_id: 'user1' }); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.keepalive('5/0'); - await writer2.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -84,7 +81,6 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); - await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -92,9 +88,7 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.keepalive('5/0'); - await writer2.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -115,9 +109,7 @@ bucket_definitions: // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage. // This is what is effetively triggered with RouteAPI.createReplicationHead(). // MongoDB storage doesn't explicitly need this anymore. - await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer3.keepalive('6/0'); - await writer3.flush(); + await writer.keepalive('6/0'); let result2 = await iter.next(); if (result2.value?.base?.lsn == '5/0') { @@ -156,7 +148,6 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); - await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -164,14 +155,12 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.addCustomWriteCheckpoint({ + writer.addCustomWriteCheckpoint({ checkpoint: 5n, user_id: 'user1' }); - await writer2.flush(); - await writer2.keepalive('5/0'); - await writer2.flush(); + await writer.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -205,7 +194,6 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); - await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -213,17 +201,15 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Flush to clear state - await writer2.flush(); + await writer.flush(); - await writer2.addCustomWriteCheckpoint({ + writer.addCustomWriteCheckpoint({ checkpoint: 5n, user_id: 'user1' }); - await writer2.flush(); - await writer2.keepalive('5/0'); - await writer2.flush(); + await writer.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -257,7 +243,6 @@ bucket_definitions: await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); - await writer.flush(); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -265,9 +250,7 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.keepalive('5/0'); - await writer2.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -280,14 +263,12 @@ bucket_definitions: } }); - await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - writer3.addCustomWriteCheckpoint({ + writer.addCustomWriteCheckpoint({ checkpoint: 6n, user_id: 'user1' }); - await writer3.flush(); - await writer3.keepalive('6/0'); - await writer3.flush(); + await writer.flush(); + await writer.keepalive('6/0'); let result2 = await iter.next(); expect(result2).toMatchObject({ @@ -301,14 +282,12 @@ bucket_definitions: } }); - await using writer4 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - writer4.addCustomWriteCheckpoint({ + writer.addCustomWriteCheckpoint({ checkpoint: 7n, user_id: 'user1' }); - await writer4.flush(); - await writer4.keepalive('7/0'); - await writer4.flush(); + await writer.flush(); + await writer.keepalive('7/0'); let result3 = await iter.next(); expect(result3).toMatchObject({ diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 6d222be46..eaa9518c5 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -7,8 +7,6 @@ import { parameterLookupScope } from './util.js'; export function registerParameterCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); - test('compacting parameters', async () => { await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( @@ -20,11 +18,12 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' @@ -33,7 +32,7 @@ bucket_definitions: }); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2' @@ -42,17 +41,15 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); - const lookup = ScopedParameterLookup.direct(parameterLookupScope('test', '1'), ['t1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1', source: null as any }, ['t1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); expect(parameters1).toEqual([{ id: 't1' }]); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.UPDATE, before: { id: 't1' @@ -64,16 +61,15 @@ bucket_definitions: afterReplicaId: 't1' }); - await writer2.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't1' }, beforeReplicaId: 't1' }); - await writer2.commit('1/2'); - await writer2.flush(); + await writer.commit('1/2'); const checkpoint2 = await bucketStorage.getCheckpoint(); const parameters2 = await checkpoint2.getParameterSets([lookup]); expect(parameters2).toEqual([]); @@ -104,11 +100,12 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1', @@ -118,7 +115,7 @@ bucket_definitions: }); // Interleave with another operation, to evict the other cache entry when compacting. await writer.save({ - sourceTable: TEST_TABLE, + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2', @@ -128,11 +125,9 @@ bucket_definitions: }); await writer.commit('1/1'); - await writer.flush(); - await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer2.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't1', @@ -140,12 +135,10 @@ bucket_definitions: }, beforeReplicaId: 't1' }); - await writer2.commit('2/1'); - await writer2.flush(); + await writer.commit('2/1'); - await using writer3 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await writer3.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.UPDATE, after: { id: 't2', @@ -153,10 +146,9 @@ bucket_definitions: }, afterReplicaId: 't2' }); - await writer3.commit('3/1'); - await writer3.flush(); + await writer.commit('3/1'); - const lookup = ScopedParameterLookup.direct(parameterLookupScope('test', '1'), ['u1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1', source: null as any }, ['u1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 04c68a899..eb16ac1c9 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -249,8 +249,6 @@ bucket_definitions: if (sentCheckpoints == 1) { // Save new data to interrupt the low-priority sync. - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); // Add another high-priority row. This should interrupt the long-running low-priority sync. await writer.save({ sourceTable: testTable, @@ -777,6 +775,7 @@ bucket_definitions: const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); + // Activate await writer.markAllSnapshotDone('0/0'); await writer.keepalive('0/0'); @@ -818,6 +817,7 @@ bucket_definitions: await writer.commit('0/1'); const checkpoint2 = await getCheckpointLines(iter); + const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) @@ -875,6 +875,7 @@ bucket_definitions: const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); const checkpoint1 = await getCheckpointLines(iter); + expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]); expect(checkpoint1).toMatchSnapshot(); @@ -964,6 +965,7 @@ bucket_definitions: await writer.commit('0/1'); const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); + const checkpoint2 = await getCheckpointLines(iter); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) @@ -1250,6 +1252,7 @@ config: }); const bucketStorage = f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1); await writer.markAllSnapshotDone('0/1'); From 868baf2bda267f8794065b533172d65e5136ceb5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 6 Mar 2026 10:11:25 +0200 Subject: [PATCH 7/8] Changeset. --- .changeset/mean-impalas-walk.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changeset/mean-impalas-walk.md diff --git a/.changeset/mean-impalas-walk.md b/.changeset/mean-impalas-walk.md new file mode 100644 index 000000000..f0afb1a0a --- /dev/null +++ b/.changeset/mean-impalas-walk.md @@ -0,0 +1,9 @@ +--- +'@powersync/service-module-postgres-storage': patch +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-core-tests': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-core': patch +--- + +[Internal] Add a createWriter() API to replace startBatch(). From ebf61abf402c49a9dd385317a1e863f4af96f56d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 10 Mar 2026 14:39:18 +0200 Subject: [PATCH 8/8] Warn when disposing writers without flushing. --- .../src/storage/implementation/MongoBucketBatch.ts | 6 ++++++ .../src/storage/batch/PostgresBucketBatch.ts | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index fc6c2547c..ffad6e036 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -664,6 +664,12 @@ export class MongoBucketBatch } async [Symbol.asyncDispose]() { + if (this.batch != null || this.write_checkpoint_batch.length > 0) { + // We don't error here, since: + // 1. In error states, this is expected (we can't distinguish between disposing after success or error). + // 2. SuppressedError is messy to deal with. + this.logger.warn('Disposing writer with unflushed changes'); + } await this.session.endSession(); super.clearListeners(); } diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index b39e622b0..8c0a3719d 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -126,6 +126,12 @@ export class PostgresBucketBatch } async [Symbol.asyncDispose]() { + if (this.batch != null || this.write_checkpoint_batch.length > 0) { + // We don't error here, since: + // 1. In error states, this is expected (we can't distinguish between disposing after success or error). + // 2. SuppressedError is messy to deal with. + this.logger.warn('Disposing writer with unflushed changes'); + } super.clearListeners(); }