diff --git a/.changeset/mean-impalas-walk.md b/.changeset/mean-impalas-walk.md new file mode 100644 index 000000000..f0afb1a0a --- /dev/null +++ b/.changeset/mean-impalas-walk.md @@ -0,0 +1,9 @@ +--- +'@powersync/service-module-postgres-storage': patch +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-core-tests': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-core': patch +--- + +[Internal] Add a createWriter() API to replace startBatch(). diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index c523db533..ffad6e036 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -664,10 +664,20 @@ export class MongoBucketBatch } async [Symbol.asyncDispose]() { + if (this.batch != null || this.write_checkpoint_batch.length > 0) { + // We don't error here, since: + // 1. In error states, this is expected (we can't distinguish between disposing after success or error). + // 2. SuppressedError is messy to deal with. + this.logger.warn('Disposing writer with unflushed changes'); + } await this.session.endSession(); super.clearListeners(); } + async dispose() { + await this[Symbol.asyncDispose](); + } + private lastWaitingLogThottled = 0; async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 1a395ee37..ce3e3f549 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -167,10 +167,7 @@ export class MongoSyncBucketStorage }); } - async startBatch( - options: storage.StartBatchOptions, - callback: (batch: storage.BucketStorageBatch) => Promise - ): Promise { + async createWriter(options: storage.CreateWriterOptions): Promise { const doc = await this.db.sync_rules.findOne( { _id: this.group_id @@ -179,7 +176,7 @@ export class MongoSyncBucketStorage ); const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; - await using batch = new MongoBucketBatch({ + const writer = new MongoBucketBatch({ logger: options.logger, db: this.db, syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), @@ -192,15 +189,21 @@ export class MongoSyncBucketStorage skipExistingRows: options.skipExistingRows ?? false, markRecordUnavailable: options.markRecordUnavailable }); - this.iterateListeners((cb) => cb.batchStarted?.(batch)); - - await callback(batch); - await batch.flush(); - if (batch.last_flushed_op != null) { - return { flushed_op: batch.last_flushed_op }; - } else { - return null; - } + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + /** + * @deprecated Use `createWriter()` with `await using` instead. + */ + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } async resolveTable(options: storage.ResolveTableOptions): Promise { diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index 1ad77205f..d59bfb500 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -104,7 +104,7 @@ exports[`sync - mongodb > storage v1 > compacting data - invalidate checkpoint 2 ] `; -exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -159,13 +159,13 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea ] `; -exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -181,7 +181,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -190,7 +190,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -199,7 +199,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckes for strea "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a", + "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, @@ -819,7 +819,7 @@ exports[`sync - mongodb > storage v1 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -1026,7 +1026,7 @@ exports[`sync - mongodb > storage v1 > sync updates to parameter query + data 2` "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -1199,7 +1199,7 @@ exports[`sync - mongodb > storage v2 > compacting data - invalidate checkpoint 2 ] `; -exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -1254,13 +1254,13 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea ] `; -exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -1276,7 +1276,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -1285,7 +1285,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -1294,7 +1294,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckes for strea "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a", + "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, @@ -1914,7 +1914,7 @@ exports[`sync - mongodb > storage v2 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -2121,7 +2121,7 @@ exports[`sync - mongodb > storage v2 > sync updates to parameter query + data 2` "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -2294,7 +2294,7 @@ exports[`sync - mongodb > storage v3 > compacting data - invalidate checkpoint 2 ] `; -exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - mongodb > storage v3 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -2349,13 +2349,13 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea ] `; -exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - mongodb > storage v3 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -2371,7 +2371,7 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -2380,7 +2380,7 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -2389,7 +2389,7 @@ exports[`sync - mongodb > storage v3 > encodes sync rules id in buckes for strea "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "e5aa2ddc-1328-58fa-a000-0b5ed31eaf1a", + "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, @@ -3009,7 +3009,7 @@ exports[`sync - mongodb > storage v3 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, @@ -3216,7 +3216,7 @@ exports[`sync - mongodb > storage v3 > sync updates to parameter query + data 2` "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8", + "subkey": "ae9cbda1-5d8a-5a61-aaa4-366940758339", }, ], "has_more": false, diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 9f797a5d3..86b8f730a 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -5,37 +5,43 @@ import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY); - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); describe('with blank bucket_state', () => { // This can happen when migrating from older service versions, that did not populate bucket_state yet. - const populate = async (bucketStorage: SyncRulesBucketStorage) => { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - owner_id: 'u1' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - owner_id: 'u2' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await batch.commit('1/1'); + const populate = async (bucketStorage: SyncRulesBucketStorage, sourceTableIndex: number) => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable( + writer, + 'test', + ['id'], + INITIALIZED_MONGO_STORAGE_FACTORY, + sourceTableIndex + ); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + owner_id: 'u1' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + owner_id: 'u2' + }, + afterReplicaId: test_utils.rid('t2') }); + await writer.commit('1/1'); + return bucketStorage.getCheckpoint(); }; @@ -50,7 +56,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); - const { checkpoint } = await populate(bucketStorage); + const { checkpoint } = await populate(bucketStorage, 1); return { bucketStorage, checkpoint, factory, syncRules }; }; @@ -102,7 +108,7 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await populate(bucketStorage); + await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); // Default is to small small numbers - should be a no-op diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index b2767fd83..7e53340c8 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -8,8 +8,6 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor storageVersion, tableIdStrings: storageConfig.tableIdStrings }); - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageConfig); - // The split of returned results can vary depending on storage drivers test('large batch (2)', async () => { // Test syncing a batch of data that is small in count, @@ -28,56 +26,57 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor ) ); const bucketStorage = factory.getInstance(syncRules); - const globalBucket = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); - const largeDescription = '0123456789'.repeat(2_000_00); + const largeDescription = '0123456789'.repeat(2_000_00); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large1', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); - // Large enough to split the returned batch - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large2', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large2') - }); + // Large enough to split the returned batch + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test3', - description: 'test3' - }, - afterReplicaId: test_utils.rid('test3') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') }); - const checkpoint = result!.flushed_op; + const flushResult = await writer.flush(); + + const checkpoint = flushResult!.flushed_op; const options: storage.BucketDataBatchOptions = {}; const batch1 = await test_utils.fromAsync( diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 8fc3931fc..854a3c438 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -164,9 +164,9 @@ export class ChangeStreamTestContext { async markSnapshotConsistent() { const checkpoint = await createCheckpoint(this.client, this.db, STANDALONE_CHECKPOINT_ID); - await this.storage!.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive(checkpoint); - }); + await using writer = await this.storage!.createWriter(test_utils.BATCH_OPTIONS); + await writer.keepalive(checkpoint); + await writer.flush(); } startStreaming() { diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 19d1b25bc..ca99ebf1c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -343,10 +343,7 @@ export class PostgresSyncRulesStorage }); } - async startBatch( - options: storage.StartBatchOptions, - callback: (batch: storage.BucketStorageBatch) => Promise - ): Promise { + async createWriter(options: storage.CreateWriterOptions): Promise { const syncRules = await this.db.sql` SELECT last_checkpoint_lsn, @@ -363,7 +360,7 @@ export class PostgresSyncRulesStorage const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null; - const batch = new PostgresBucketBatch({ + const writer = new PostgresBucketBatch({ logger: options.logger ?? framework.logger, db: this.db, sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(), @@ -378,15 +375,21 @@ export class PostgresSyncRulesStorage markRecordUnavailable: options.markRecordUnavailable, storageConfig: this.storageConfig }); - this.iterateListeners((cb) => cb.batchStarted?.(batch)); - - await callback(batch); - await batch.flush(); - if (batch.last_flushed_op != null) { - return { flushed_op: batch.last_flushed_op }; - } else { - return null; - } + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + /** + * @deprecated Use `createWriter()` with `await using` instead. + */ + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } async getParameterSets( diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index a16178daa..8c0a3719d 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -126,9 +126,19 @@ export class PostgresBucketBatch } async [Symbol.asyncDispose]() { + if (this.batch != null || this.write_checkpoint_batch.length > 0) { + // We don't error here, since: + // 1. In error states, this is expected (we can't distinguish between disposing after success or error). + // 2. SuppressedError is messy to deal with. + this.logger.warn('Disposing writer with unflushed changes'); + } super.clearListeners(); } + async dispose() { + await this[Symbol.asyncDispose](); + } + async save(record: storage.SaveOptions): Promise { // TODO maybe share with abstract class const { after, before, sourceTable, tag } = record; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap index d45099f02..f628d9130 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -104,7 +104,7 @@ exports[`sync - postgres > storage v1 > compacting data - invalidate checkpoint ] `; -exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - postgres > storage v1 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -159,13 +159,13 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre ] `; -exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - postgres > storage v1 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -181,7 +181,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -190,7 +190,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -199,7 +199,7 @@ exports[`sync - postgres > storage v1 > encodes sync rules id in buckes for stre "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", }, ], "has_more": false, @@ -819,7 +819,7 @@ exports[`sync - postgres > storage v1 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -1026,7 +1026,7 @@ exports[`sync - postgres > storage v1 > sync updates to parameter query + data 2 "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -1199,7 +1199,7 @@ exports[`sync - postgres > storage v2 > compacting data - invalidate checkpoint ] `; -exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - postgres > storage v2 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -1254,13 +1254,13 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre ] `; -exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - postgres > storage v2 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -1276,7 +1276,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -1285,7 +1285,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -1294,7 +1294,7 @@ exports[`sync - postgres > storage v2 > encodes sync rules id in buckes for stre "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", }, ], "has_more": false, @@ -1914,7 +1914,7 @@ exports[`sync - postgres > storage v2 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -2121,7 +2121,7 @@ exports[`sync - postgres > storage v2 > sync updates to parameter query + data 2 "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -2294,7 +2294,7 @@ exports[`sync - postgres > storage v3 > compacting data - invalidate checkpoint ] `; -exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 1`] = ` +exports[`sync - postgres > storage v3 > encodes sync rules id in buckets for streams 1`] = ` [ { "checkpoint": { @@ -2349,13 +2349,13 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre ] `; -exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for streams 2`] = ` +exports[`sync - postgres > storage v3 > encodes sync rules id in buckets for streams 2`] = ` [ { "checkpoint": { "buckets": [ { - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "checksum": 920318466, "count": 1, "priority": 3, @@ -2371,7 +2371,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre { "errors": [], "is_default": true, - "name": "test", + "name": "test2", }, ], "write_checkpoint": undefined, @@ -2380,7 +2380,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre { "data": { "after": "0", - "bucket": "2#test|0[]", + "bucket": "2#test2|0[]", "data": [ { "checksum": 920318466, @@ -2389,7 +2389,7 @@ exports[`sync - postgres > storage v3 > encodes sync rules id in buckes for stre "object_type": "test", "op": "PUT", "op_id": "2", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", }, ], "has_more": false, @@ -3009,7 +3009,7 @@ exports[`sync - postgres > storage v3 > sync updates to data query only 2`] = ` "object_type": "lists", "op": "PUT", "op_id": "2", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, @@ -3216,7 +3216,7 @@ exports[`sync - postgres > storage v3 > sync updates to parameter query + data 2 "object_type": "lists", "op": "PUT", "op_id": "1", - "subkey": "5ad0aa14-3d5e-5428-ad5b-2c33927d991c", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", }, ], "has_more": false, diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index f918841c2..446592614 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequestMap, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -39,13 +39,15 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { ) ); const bucketStorage = factory.getInstance(syncRules); + const globalBucket = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY); + const result = await (async () => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); const largeDescription = '0123456789'.repeat(2_000_00); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -55,7 +57,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { afterReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -66,7 +68,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { }); // Large enough to split the returned batch - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -76,7 +78,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { afterReplicaId: test_utils.rid('large2') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -85,15 +87,14 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { }, afterReplicaId: test_utils.rid('test3') }); - }); + return writer.flush(); + })(); const checkpoint = result!.flushed_op; const options: storage.BucketDataBatchOptions = {}; - const batch1 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options) - ); + const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options)); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } ]); @@ -106,7 +107,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { const batch2 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]), + [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }], options ) ); @@ -122,7 +123,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { const batch3 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]), + [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }], options ) ); @@ -138,7 +139,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { const batch4 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]), + [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }], options ) ); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts index 78641e758..392750bdc 100644 --- a/modules/module-postgres-storage/test/src/storage_compacting.test.ts +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, bucketRequestMap, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js'; import { POSTGRES_STORAGE_FACTORY } from './util.js'; @@ -7,7 +7,6 @@ import { POSTGRES_STORAGE_FACTORY } from './util.js'; describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY)); describe('Postgres Compact - explicit bucket name', () => { - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], POSTGRES_STORAGE_FACTORY); test('compacts a specific bucket by exact name', async () => { await using factory = await POSTGRES_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules( @@ -19,22 +18,26 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, + const result = await (async () => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.UPDATE, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.markAllSnapshotDone('1/1'); - await batch.commit('1/1'); - }); + await writer.markAllSnapshotDone('1/1'); + const flushed = await writer.flush(); + await writer.commit('1/1'); + return flushed; + })(); const checkpoint = result!.flushed_op; @@ -46,7 +49,7 @@ bucket_definitions: }); const batch = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)]) ); expect(batch.chunkData.data).toMatchObject([ @@ -69,34 +72,38 @@ bucket_definitions: const bucketStorage = factory.getInstance(syncRules); const request = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, + const result = await (async () => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't1' }, afterReplicaId: test_utils.rid('t1') }); - await batch.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't1' }, beforeReplicaId: test_utils.rid('t1') }); - await batch.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { id: 't2' }, afterReplicaId: test_utils.rid('t2') }); - await batch.save({ - sourceTable: TEST_TABLE, + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.DELETE, before: { id: 't2' }, beforeReplicaId: test_utils.rid('t2') }); - await batch.commit('1/1'); - }); + const flushed = await writer.flush(); + await writer.commit('1/1'); + return flushed; + })(); const checkpoint = result!.flushed_op; const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); diff --git a/modules/module-postgres-storage/test/src/storage_sync.test.ts b/modules/module-postgres-storage/test/src/storage_sync.test.ts index 2596cdabe..88965e59b 100644 --- a/modules/module-postgres-storage/test/src/storage_sync.test.ts +++ b/modules/module-postgres-storage/test/src/storage_sync.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, register, resolveTestTable, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -11,7 +11,6 @@ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; function registerStorageVersionTests(storageVersion: number) { describe(`storage v${storageVersion}`, () => { const storageFactory = POSTGRES_STORAGE_FACTORY; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], storageFactory); register.registerSyncTests(storageFactory.factory, { storageVersion, @@ -37,53 +36,55 @@ function registerStorageVersionTests(storageVersion: number) { const bucketStorage = factory.getInstance(syncRules); const globalBucket = bucketRequest(syncRules, 'global[]'); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - - const largeDescription = '0123456789'.repeat(2_000_00); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large1', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large1') - }); - - // Large enough to split the returned batch - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large2', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large2') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test3', - description: 'test3' - }, - afterReplicaId: test_utils.rid('test3') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await resolveTestTable(writer, 'test', ['id'], storageFactory); + + const largeDescription = '0123456789'.repeat(2_000_00); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') }); + // Large enough to split the returned batch + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') + }); + + const result = await writer.flush(); + const checkpoint = result!.flushed_op; const options: storage.BucketDataBatchOptions = {}; diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 755c9e476..37c1fe2d2 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -1,4 +1,4 @@ -import { BucketDataRequest, InternalOpId, storage, utils } from '@powersync/service-core'; +import { BucketDataRequest, InternalOpId, JwtPayload, storage, utils } from '@powersync/service-core'; import { GetQuerierOptions, RequestParameters } from '@powersync/service-sync-rules'; import * as bson from 'bson'; @@ -8,7 +8,7 @@ export const PARSE_OPTIONS: storage.ParseSyncRulesOptions = { defaultSchema: 'public' }; -export const BATCH_OPTIONS: storage.StartBatchOptions = { +export const BATCH_OPTIONS: storage.CreateWriterOptions = { ...PARSE_OPTIONS, zeroLSN: ZERO_LSN, storeCurrentData: true @@ -32,6 +32,33 @@ export function makeTestTable( snapshotComplete: true }); } +/** + * With incremental reprocessing, we need actual test tables, resolved via the writer. + * + * This prepares for it. + */ +export async function resolveTestTable( + _writer: storage.BucketStorageBatch, + name: string, + replicaIdColumns: string[] | undefined, + options: { tableIdStrings: boolean }, + idIndex: number = 1 +) { + const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(',')); + // Generate unique ids per test table (if idIndex is specified), without completely + // breaking all the existing tests. + const idString = '6544e3899293153fa7b383' + (30 + idIndex).toString().padStart(2, '0'); + const id = options.tableIdStrings == false ? new bson.ObjectId(idString) : idString; + return new storage.SourceTable({ + id: id, + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: relId, + schema: 'public', + name: name, + replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })), + snapshotComplete: true + }); +} export function getBatchData( batch: utils.SyncBucketData[] | storage.SyncBucketDataChunk[] | storage.SyncBucketDataChunk @@ -128,3 +155,15 @@ export function querierOptions(globalParameters: RequestParameters): GetQuerierO streams: {} }; } + +export function requestParameters( + jwtPayload: Record, + clientParameters?: Record +): RequestParameters { + return new RequestParameters(new JwtPayload(jwtPayload), clientParameters ?? {}); +} + +export function removeSource(obj: T): Omit { + const { source, ...rest } = obj; + return rest; +} diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index 5b775426b..97b71ba79 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -6,7 +6,6 @@ import { bucketRequestMap, bucketRequests } from './util.js'; export function registerCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); test('compacting (1)', async () => { await using factory = await generateStorageFactory(); @@ -19,39 +18,40 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: test_utils.rid('t1') + }); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') }); - const checkpoint = result!.flushed_op; + await writer.commit('1/1'); + await writer.flush(); + + const checkpoint = writer.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); @@ -125,48 +125,49 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: test_utils.rid('t1') + }); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: test_utils.rid('t1') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') + }); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2' - }, - afterReplicaId: test_utils.rid('t2') - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: test_utils.rid('t1') + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2' + }, + afterReplicaId: test_utils.rid('t2') }); - const checkpoint = result!.flushed_op; + await writer.commit('1/1'); + await writer.flush(); + + const checkpoint = writer.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); @@ -240,54 +241,54 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: 't2' - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: 't2' + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' }); - const checkpoint1 = result!.flushed_op; + await writer.commit('1/1'); + await writer.flush(); + + const checkpoint1 = writer.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); - const checksumBefore = await bucketStorage.getChecksums(checkpoint1, [request]); - - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't2' - }, - beforeReplicaId: 't2' - }); - await batch.commit('2/1'); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't2' + }, + beforeReplicaId: 't2' }); - const checkpoint2 = result2!.flushed_op; + await writer2.commit('2/1'); + await writer2.flush(); + const checkpoint2 = writer2.last_flushed_op!; await bucketStorage.compact({ clearBatchLimit: 2, @@ -328,76 +329,78 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - /** - * Repeatedly create operations which fall into different buckets. - * The bucket operations are purposely interleaved as the op_id increases. - * A large amount of operations are created here. - * The configured window of compacting operations is 100. This means the initial window will - * contain operations from multiple buckets. - */ - for (let count = 0; count < 100; count++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - b: 'b1', - value: 'start' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - b: 'b1', - value: 'intermediate' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - b: 'b2', - value: 'start' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - b: 'b1', - value: 'final' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2', - b: 'b2', - value: 'final' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await batch.commit('1/1'); - } - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + /** + * Repeatedly create operations which fall into different buckets. + * The bucket operations are purposely interleaved as the op_id increases. + * A large amount of operations are created here. + * The configured window of compacting operations is 100. This means the initial window will + * contain operations from multiple buckets. + */ + for (let count = 0; count < 100; count++) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + b: 'b1', + value: 'start' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + b: 'b1', + value: 'intermediate' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + b: 'b2', + value: 'start' + }, + afterReplicaId: test_utils.rid('t2') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + b: 'b1', + value: 'final' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2', + b: 'b2', + value: 'final' + }, + afterReplicaId: test_utils.rid('t2') + }); + + await writer.commit('1/1'); + } - const checkpoint = result!.flushed_op; + await writer.flush(); + + const checkpoint = writer.last_flushed_op!; await bucketStorage.compact({ clearBatchLimit: 100, @@ -456,38 +459,39 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: 't2' - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: 't2' + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' }); + await writer.commit('1/1'); + await writer.flush(); + await bucketStorage.compact({ clearBatchLimit: 2, moveBatchLimit: 1, @@ -496,18 +500,19 @@ bucket_definitions: minChangeRatio: 0 }); - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't2' - }, - beforeReplicaId: 't2' - }); - await batch.commit('2/1'); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't2' + }, + beforeReplicaId: 't2' }); - const checkpoint2 = result2!.flushed_op; + await writer2.commit('2/1'); + await writer2.flush(); + const checkpoint2 = writer2.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); await bucketStorage.clearChecksumCache(); const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]); @@ -532,42 +537,44 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1' + }, + afterReplicaId: 't1' }); + await writer.commit('1/1'); + await writer.flush(); + // Get checksums here just to populate the cache - await bucketStorage.getChecksums(result!.flushed_op, bucketRequests(syncRules, ['global[]'])); - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); - await batch.commit('2/1'); + await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRules, ['global[]'])); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' }); + await writer2.commit('2/1'); + await writer2.flush(); await bucketStorage.compact({ clearBatchLimit: 20, @@ -577,7 +584,7 @@ bucket_definitions: minChangeRatio: 0 }); - const checkpoint2 = result2!.flushed_op; + const checkpoint2 = writer2.last_flushed_op!; const request = bucketRequest(syncRules, 'global[]'); // Check that the checksum was correctly updated with the clear operation after having a cached checksum const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]); @@ -601,29 +608,31 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { id: 't1' }, - afterReplicaId: test_utils.rid('t1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 't1' }, + afterReplicaId: test_utils.rid('t1') }); - - const checkpoint1 = result1!.flushed_op; - - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // This is flushed but not committed (does not advance the checkpoint) - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { id: 't1' }, - afterReplicaId: test_utils.rid('t1') - }); + await writer.commit('1/1'); + await writer.flush(); + + const checkpoint1 = writer.last_flushed_op!; + + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); + // This is flushed but not committed (does not advance the checkpoint) + await writer2.save({ + sourceTable: testTable2, + tag: storage.SaveOperationTag.UPDATE, + after: { id: 't1' }, + afterReplicaId: test_utils.rid('t1') }); - const checkpoint2 = result2!.flushed_op; + await writer2.flush(); + const checkpoint2 = writer2.last_flushed_op!; const checkpointBeforeCompact = await bucketStorage.getCheckpoint(); expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1); diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 29c0e669d..3837071a5 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -39,18 +39,15 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ heads: { '1': '5/0' }, user_id: 'user1' }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('5/0'); - }); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -82,9 +79,8 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(r.persisted_sync_rules!); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -92,9 +88,7 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('5/0'); - }); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -115,9 +109,7 @@ bucket_definitions: // We have to trigger a new keepalive after the checkpoint, at least to cover postgres storage. // This is what is effetively triggered with RouteAPI.createReplicationHead(). // MongoDB storage doesn't explicitly need this anymore. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('6/0'); - }); + await writer.keepalive('6/0'); let result2 = await iter.next(); if (result2.value?.base?.lsn == '5/0') { @@ -154,9 +146,8 @@ bucket_definitions: const bucketStorage = factory.getInstance(r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -164,14 +155,12 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.addCustomWriteCheckpoint({ - checkpoint: 5n, - user_id: 'user1' - }); - await batch.flush(); - await batch.keepalive('5/0'); + writer.addCustomWriteCheckpoint({ + checkpoint: 5n, + user_id: 'user1' }); + await writer.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -203,9 +192,8 @@ bucket_definitions: const bucketStorage = factory.getInstance(r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -213,17 +201,15 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Flush to clear state - await batch.flush(); + // Flush to clear state + await writer.flush(); - await batch.addCustomWriteCheckpoint({ - checkpoint: 5n, - user_id: 'user1' - }); - await batch.flush(); - await batch.keepalive('5/0'); + writer.addCustomWriteCheckpoint({ + checkpoint: 5n, + user_id: 'user1' }); + await writer.flush(); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -255,9 +241,8 @@ bucket_definitions: const bucketStorage = factory.getInstance(r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -265,9 +250,7 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.keepalive('5/0'); - }); + await writer.keepalive('5/0'); const result = await iter.next(); expect(result).toMatchObject({ @@ -280,14 +263,12 @@ bucket_definitions: } }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - batch.addCustomWriteCheckpoint({ - checkpoint: 6n, - user_id: 'user1' - }); - await batch.flush(); - await batch.keepalive('6/0'); + writer.addCustomWriteCheckpoint({ + checkpoint: 6n, + user_id: 'user1' }); + await writer.flush(); + await writer.keepalive('6/0'); let result2 = await iter.next(); expect(result2).toMatchObject({ @@ -301,14 +282,12 @@ bucket_definitions: } }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - batch.addCustomWriteCheckpoint({ - checkpoint: 7n, - user_id: 'user1' - }); - await batch.flush(); - await batch.keepalive('7/0'); + writer.addCustomWriteCheckpoint({ + checkpoint: 7n, + user_id: 'user1' }); + await writer.flush(); + await writer.keepalive('7/0'); let result3 = await iter.next(); expect(result3).toMatchObject({ diff --git a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts index 489209752..d2533561d 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts @@ -10,7 +10,6 @@ import { import { describe, expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; -import { bucketRequestMap, bucketRequests } from './util.js'; /** * Normalize data from OplogEntries for comparison in tests. @@ -37,8 +36,6 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig) const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); - test('removing row', async () => { await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( @@ -53,33 +50,31 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') + }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -96,12 +91,10 @@ bucket_definitions: { op: 'REMOVE', object_id: 'test1', checksum: c2 } ]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: (c1 + c2) & 0xffffffff, count: 2 } @@ -122,40 +115,33 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await writer.commit('0/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('2/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('2/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -168,12 +154,10 @@ bucket_definitions: expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: c1 & 0xffffffff, count: 1 } @@ -195,44 +179,37 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await writer.commit('0/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1' - }, - after: { - id: 'test1', - description: 'test1' - }, - beforeReplicaId: test_utils.rid('test1'), - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('2/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1' + }, + after: { + id: 'test1', + description: 'test1' + }, + beforeReplicaId: test_utils.rid('test1'), + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('2/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -245,12 +222,10 @@ bucket_definitions: expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: c1 & 0xffffffff, count: 1 } @@ -273,33 +248,30 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') + }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -312,12 +284,10 @@ bucket_definitions: expect(data).toEqual([{ op: 'PUT', object_id: 'test1', checksum: c1 }]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { - bucket: bucketRequest(syncRules, 'global[]').bucket, + bucket: request.bucket, checksum: c1 & 0xffffffff, count: 1 } @@ -340,12 +310,12 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -354,12 +324,12 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.INSERT, after: { @@ -368,26 +338,26 @@ bucket_definitions: }, afterReplicaId: test_utils.rid('test1') }); - await batch.commit('1/1'); - }); + await writer.commit('1/1'); + } - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); + { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ + await writer.save({ sourceTable, tag: storage.SaveOperationTag.DELETE, beforeReplicaId: test_utils.rid('test1') }); - await batch.commit('2/1'); - }); + await writer.commit('2/1'); + } const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); expect(reduceBucket(batch[0].chunkData.data).slice(1)).toEqual([]); @@ -418,47 +388,45 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + client_id: 'client1a', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + client_id: 'client1b', + description: 'test1b' + }, + afterReplicaId: test_utils.rid('test1') + }); - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - client_id: 'client1a', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - client_id: 'client1b', - description: 'test1b' - }, - afterReplicaId: test_utils.rid('test1') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test2', - client_id: 'client2', - description: 'test2' - }, - afterReplicaId: test_utils.rid('test2') - }); - - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test2', + client_id: 'client2', + description: 'test2' + }, + afterReplicaId: test_utils.rid('test2') }); + + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]')]) ); const data = batch[0].chunkData.data.map((d) => { return { @@ -491,49 +459,40 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); - - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; + await writer.commit('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); + await writer.flush(); const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { op: d.op, @@ -550,9 +509,7 @@ bucket_definitions: { op: 'REMOVE', object_id: 'test1', checksum: c2 } ]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { bucket: bucketRequest(syncRules, 'global[]').bucket, @@ -576,93 +533,85 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.flush(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; + await writer.commit('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test1', - description: undefined - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - beforeReplicaId: test_utils.rid('test1') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test1', + description: undefined + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.commit('2/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + beforeReplicaId: test_utils.rid('test1') }); + await writer.commit('2/1'); + const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const data = batch[0].chunkData.data.map((d) => { return { @@ -682,9 +631,7 @@ bucket_definitions: { op: 'REMOVE', object_id: 'test1', checksum: c2 } ]); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; expect(checksums).toEqual([ { bucket: bucketRequest(syncRules, 'global[]').bucket, @@ -716,120 +663,117 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); // Pre-setup - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test2', - description: 'test2a' - }, - afterReplicaId: test_utils.rid('test2') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test2', + description: 'test2a' + }, + afterReplicaId: test_utils.rid('test2') }); + const result1 = await writer.flush(); const checkpoint1 = result1?.flushed_op ?? 0n; // Test batch - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const sourceTable = TEST_TABLE; - // b - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1b' - }, - afterReplicaId: test_utils.rid('test1') - }); + // b + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1b' + }, + afterReplicaId: test_utils.rid('test1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1' - }, - beforeReplicaId: test_utils.rid('test1'), - after: { - id: 'test2', - description: 'test2b' - }, - afterReplicaId: test_utils.rid('test2') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1' + }, + beforeReplicaId: test_utils.rid('test1'), + after: { + id: 'test2', + description: 'test2b' + }, + afterReplicaId: test_utils.rid('test2') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test2' - }, - beforeReplicaId: test_utils.rid('test2'), - after: { - id: 'test3', - description: 'test3b' - }, + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test2' + }, + beforeReplicaId: test_utils.rid('test2'), + after: { + id: 'test3', + description: 'test3b' + }, - afterReplicaId: test_utils.rid('test3') - }); + afterReplicaId: test_utils.rid('test3') + }); - // c - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'test2', - description: 'test2c' - }, - afterReplicaId: test_utils.rid('test2') - }); + // c + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'test2', + description: 'test2c' + }, + afterReplicaId: test_utils.rid('test2') + }); - // d - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test4', - description: 'test4d' - }, - afterReplicaId: test_utils.rid('test4') - }); + // d + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test4', + description: 'test4d' + }, + afterReplicaId: test_utils.rid('test4') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test4' - }, - beforeReplicaId: test_utils.rid('test4'), - after: { - id: 'test5', - description: 'test5d' - }, - afterReplicaId: test_utils.rid('test5') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test4' + }, + beforeReplicaId: test_utils.rid('test4'), + after: { + id: 'test5', + description: 'test5d' + }, + afterReplicaId: test_utils.rid('test5') }); + const result2 = await writer.flush(); const checkpoint2 = result2!.flushed_op; - const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', checkpoint1]])) - ); + const request = bucketRequest(syncRules, 'global[]', checkpoint1); + const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint2, [request])); const data = batch[0].chunkData.data.map((d) => { return { @@ -881,60 +825,59 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config); // Pre-setup - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: rid2('test1', 'test1a') - }); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: rid2('test1', 'test1a') }); + const result1 = await writer.flush(); const checkpoint1 = result1?.flushed_op ?? 0n; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Unchanged, but has a before id - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1', - description: 'test1a' - }, - beforeReplicaId: rid2('test1', 'test1a'), - after: { - id: 'test1', - description: 'test1b' - }, - afterReplicaId: rid2('test1', 'test1b') - }); + // Unchanged, but has a before id + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1', + description: 'test1a' + }, + beforeReplicaId: rid2('test1', 'test1a'), + after: { + id: 'test1', + description: 'test1b' + }, + afterReplicaId: rid2('test1', 'test1b') }); - - const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Delete - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 'test1', - description: 'test1b' - }, - beforeReplicaId: rid2('test1', 'test1b'), - after: undefined - }); + const result2 = await writer.flush(); + + // Delete + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 'test1', + description: 'test1b' + }, + beforeReplicaId: rid2('test1', 'test1b'), + after: undefined }); + const result3 = await writer.flush(); const checkpoint3 = result3!.flushed_op; + const request = bucketRequest(syncRules, 'global[]'); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])) + bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]) ); const data = batch[0].chunkData.data.map((d) => { return { @@ -995,60 +938,59 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = test_utils.makeTestTable('test', ['id', 'description'], config); // Pre-setup - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: rid2('test1', 'test1a') - }); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: rid2('test1', 'test1a') }); + const result1 = await writer.flush(); const checkpoint1 = result1?.flushed_op ?? 0n; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Unchanged, but has a before id - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 'test1', - description: 'test1a' - }, - beforeReplicaId: rid2('test1', 'test1a'), - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: rid2('test1', 'test1a') - }); + // Unchanged, but has a before id + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 'test1', + description: 'test1a' + }, + beforeReplicaId: rid2('test1', 'test1a'), + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: rid2('test1', 'test1a') }); - - const result3 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Delete - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 'test1', - description: 'test1a' - }, - beforeReplicaId: rid2('test1', 'test1a'), - after: undefined - }); + const result2 = await writer.flush(); + + // Delete + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 'test1', + description: 'test1a' + }, + beforeReplicaId: rid2('test1', 'test1a'), + after: undefined }); + const result3 = await writer.flush(); const checkpoint3 = result3!.flushed_op; + const request = bucketRequest(syncRules, 'global[]'); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint3, bucketRequestMap(syncRules, [['global[]', checkpoint1]])) + bucketStorage.getBucketDataBatch(checkpoint3, [{ ...request, start: checkpoint1 }]) ); const data = batch[0].chunkData.data.map((d) => { return { @@ -1098,66 +1040,63 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + const largeDescription = '0123456789'.repeat(12_000_00); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - const largeDescription = '0123456789'.repeat(12_000_00); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); - - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large1', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large1') - }); - - // Large enough to split the returned batch - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'large2', - description: largeDescription - }, - afterReplicaId: test_utils.rid('large2') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test3', - description: 'test3' - }, - afterReplicaId: test_utils.rid('test3') - }); + // Large enough to split the returned batch + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); - await batch.commit('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') }); + await writer.commit('1/1'); + const { checkpoint } = await bucketStorage.getCheckpoint(); const options: storage.BucketDataBatchOptions = { chunkLimitBytes: 16 * 1024 * 1024 }; - const batch1 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], options)); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }, { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 454746904 } @@ -1171,7 +1110,7 @@ bucket_definitions: const batch2 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]), + [{ ...request, start: BigInt(batch1[0].chunkData.next_after) }], options ) ); @@ -1188,7 +1127,7 @@ bucket_definitions: const batch3 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]), + [{ ...request, start: BigInt(batch2[0].chunkData.next_after) }], options ) ); @@ -1213,31 +1152,28 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - for (let i = 1; i <= 6; i++) { - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `test${i}`, - description: `test${i}` - }, - afterReplicaId: `test${i}` - }); - } + for (let i = 1; i <= 6; i++) { + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: `test${i}`, + description: `test${i}` + }, + afterReplicaId: `test${i}` + }); + } - await batch.commit('1/1'); - }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const batch1 = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), { limit: 4 }) - ); + const request = bucketRequest(syncRules, 'global[]'); + const batch1 = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request], { limit: 4 })); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }, @@ -1253,13 +1189,9 @@ bucket_definitions: }); const batch2 = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch( - checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch1.chunkData.next_after)]]), - { - limit: 4 - } - ) + bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch1.chunkData.next_after) }], { + limit: 4 + }) ); expect(test_utils.getBatchData(batch2)).toEqual([ { op_id: '5', op: 'PUT', object_id: 'test5', checksum: 3686902721 }, @@ -1273,13 +1205,9 @@ bucket_definitions: }); const batch3 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch( - checkpoint, - bucketRequestMap(syncRules, [['global[]', BigInt(batch2.chunkData.next_after)]]), - { - limit: 4 - } - ) + bucketStorage.getBucketDataBatch(checkpoint, [{ ...request, start: BigInt(batch2.chunkData.next_after) }], { + limit: 4 + }) ); expect(test_utils.getBatchData(batch3)).toEqual([]); @@ -1304,48 +1232,41 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - const sourceTable = TEST_TABLE; - - for (let i = 1; i <= 10; i++) { - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `test${i}`, - description: `test${i}`, - bucket: i == 1 ? 'global1' : 'global2' - }, - afterReplicaId: `test${i}` - }); - } + for (let i = 1; i <= 10; i++) { + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: `test${i}`, + description: `test${i}`, + bucket: i == 1 ? 'global1' : 'global2' + }, + afterReplicaId: `test${i}` + }); + } - await batch.commit('1/1'); - }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); + const global1Request = bucketRequest(syncRules, 'global1[]', 0n); + const global2Request = bucketRequest(syncRules, 'global2[]', 0n); const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch( - checkpoint, - bucketRequestMap(syncRules, [ - ['global1[]', 0n], - ['global2[]', 0n] - ]), - options - ) + bucketStorage.getBucketDataBatch(checkpoint, [global1Request, global2Request], options) ); - return { syncRules, batch }; + return { batch, global1Request, global2Request }; }; test('batch has_more (1)', async () => { - const { batch, syncRules } = await setup({ limit: 5 }); + const { batch, global1Request, global2Request } = await setup({ limit: 5 }); expect(batch.length).toEqual(2); - expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket); - expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); + expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket); + expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket); expect(test_utils.getBatchData(batch[0])).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } @@ -1372,11 +1293,11 @@ bucket_definitions: }); test('batch has_more (2)', async () => { - const { batch, syncRules } = await setup({ limit: 11 }); + const { batch, global1Request, global2Request } = await setup({ limit: 11 }); expect(batch.length).toEqual(2); - expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket); - expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); + expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket); + expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket); expect(test_utils.getBatchData(batch[0])).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } @@ -1409,12 +1330,12 @@ bucket_definitions: test('batch has_more (3)', async () => { // 50 bytes is more than 1 row, less than 2 rows - const { batch, syncRules } = await setup({ limit: 3, chunkLimitBytes: 50 }); + const { batch, global1Request, global2Request } = await setup({ limit: 3, chunkLimitBytes: 50 }); expect(batch.length).toEqual(3); - expect(batch[0].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global1[]').bucket); - expect(batch[1].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); - expect(batch[2].chunkData.bucket).toEqual(bucketRequest(syncRules, 'global2[]').bucket); + expect(batch[0].chunkData.bucket).toEqual(global1Request.bucket); + expect(batch[1].chunkData.bucket).toEqual(global2Request.bucket); + expect(batch[2].chunkData.bucket).toEqual(global2Request.bucket); expect(test_utils.getBatchData(batch[0])).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } @@ -1458,10 +1379,9 @@ bucket_definitions: const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}')); const storage = f.getInstance(r.persisted_sync_rules!); - await storage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/0'); - await batch.keepalive('1/0'); - }); + await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/0'); + await writer.keepalive('1/0'); await f.getStorageMetrics(); // We don't care about the specific values here @@ -1487,36 +1407,35 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = test_utils.makeTestTable('test', ['id'], config); - const sourceTableIgnore = test_utils.makeTestTable('test_ignore', ['id'], config); - - const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - // This saves a record to current_data, but not bucket_data. - // This causes a checkpoint to be created without increasing the op_id sequence. - await batch.save({ - sourceTable: sourceTableIgnore, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1' - }, - afterReplicaId: test_utils.rid('test1') - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1); + const sourceTableIgnore = await test_utils.resolveTestTable(writer, 'test_ignore', ['id'], config, 2); + + await writer.markAllSnapshotDone('1/1'); + // This saves a record to current_data, but not bucket_data. + // This causes a checkpoint to be created without increasing the op_id sequence. + await writer.save({ + sourceTable: sourceTableIgnore, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1' + }, + afterReplicaId: test_utils.rid('test1') }); + const result1 = await writer.flush(); const checkpoint1 = result1!.flushed_op; - const result2 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test2' - }, - afterReplicaId: test_utils.rid('test2') - }); + await writer.save({ + sourceTable: sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test2' + }, + afterReplicaId: test_utils.rid('test2') }); + const result2 = await writer.flush(); const checkpoint2 = result2!.flushed_op; // we expect 0n and 1n, or 1n and 2n. @@ -1539,35 +1458,27 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = TEST_TABLE; - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') }); + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); - const checksums = [ - ...(await bucketStorage.getChecksums(checkpoint, bucketRequests(syncRules, ['global[]']))).values() - ]; - expect(checksums).toEqual([ - { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 } - ]); - const checksums2 = [ - ...(await bucketStorage.getChecksums(checkpoint + 1n, bucketRequests(syncRules, ['global[]']))).values() - ]; - expect(checksums2).toEqual([ - { bucket: bucketRequest(syncRules, 'global[]').bucket, checksum: 1917136889, count: 1 } - ]); + const request = bucketRequest(syncRules, 'global[]'); + const checksums = [...(await bucketStorage.getChecksums(checkpoint, [request])).values()]; + expect(checksums).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]); + const checksums2 = [...(await bucketStorage.getChecksums(checkpoint + 1n, [request])).values()]; + expect(checksums2).toEqual([{ bucket: request.bucket, checksum: 1917136889, count: 1 }]); }); testChecksumBatching(config); @@ -1586,28 +1497,26 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.commit('1/1'); - - const cp1 = await bucketStorage.getCheckpoint(); - expect(cp1.lsn).toEqual('1/1'); + const cp1 = await bucketStorage.getCheckpoint(); + expect(cp1.lsn).toEqual('1/1'); - await batch.commit('2/1', { createEmptyCheckpoints: true }); - const cp2 = await bucketStorage.getCheckpoint(); - expect(cp2.lsn).toEqual('2/1'); + await writer.commit('2/1', { createEmptyCheckpoints: true }); + const cp2 = await bucketStorage.getCheckpoint(); + expect(cp2.lsn).toEqual('2/1'); - await batch.keepalive('3/1'); - const cp3 = await bucketStorage.getCheckpoint(); - expect(cp3.lsn).toEqual('3/1'); + await writer.keepalive('3/1'); + const cp3 = await bucketStorage.getCheckpoint(); + expect(cp3.lsn).toEqual('3/1'); - // For the last one, we skip creating empty checkpoints - // This means the LSN stays at 3/1. - await batch.commit('4/1', { createEmptyCheckpoints: false }); - const cp4 = await bucketStorage.getCheckpoint(); - expect(cp4.lsn).toEqual('3/1'); - }); + // For the last one, we skip creating empty checkpoints + // This means the LSN stays at 3/1. + await writer.commit('4/1', { createEmptyCheckpoints: false }); + const cp4 = await bucketStorage.getCheckpoint(); + expect(cp4.lsn).toEqual('3/1'); }); test('empty checkpoints (2)', async () => { @@ -1626,40 +1535,38 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); - const sourceTable = TEST_TABLE; - // We simulate two concurrent batches, but nesting is the easiest way to do this. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch1) => { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch2) => { - await batch1.markAllSnapshotDone('1/1'); - await batch1.commit('1/1'); - - await batch1.commit('2/1', { createEmptyCheckpoints: false }); - const cp2 = await bucketStorage.getCheckpoint(); - expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped + // We simulate two concurrent batches, but sequential calls are enough for this test. + await writer1.markAllSnapshotDone('1/1'); + await writer1.commit('1/1'); - await batch2.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - // This simulates what happens on a snapshot processor. - // This may later change to a flush() rather than commit(). - await batch2.commit(test_utils.BATCH_OPTIONS.zeroLSN); + await writer1.commit('2/1', { createEmptyCheckpoints: false }); + const cp2 = await bucketStorage.getCheckpoint(); + expect(cp2.lsn).toEqual('1/1'); // checkpoint 2/1 skipped + + await writer2.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); + // This simulates what happens on a snapshot processor. + // This may later change to a flush() rather than commit(). + await writer2.commit(test_utils.BATCH_OPTIONS.zeroLSN); - const cp3 = await bucketStorage.getCheckpoint(); - expect(cp3.lsn).toEqual('1/1'); // Still unchanged + const cp3 = await bucketStorage.getCheckpoint(); + expect(cp3.lsn).toEqual('1/1'); // Still unchanged - // This now needs to advance the LSN, despite {createEmptyCheckpoints: false} - await batch1.commit('4/1', { createEmptyCheckpoints: false }); - const cp4 = await bucketStorage.getCheckpoint(); - expect(cp4.lsn).toEqual('4/1'); - }); - }); + // This now needs to advance the LSN, despite {createEmptyCheckpoints: false} + await writer1.commit('4/1', { createEmptyCheckpoints: false }); + const cp4 = await bucketStorage.getCheckpoint(); + expect(cp4.lsn).toEqual('4/1'); }); test('empty checkpoints (sync rule activation)', async () => { @@ -1679,29 +1586,24 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - const result = await batch.commit('1/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false }); - // Snapshot is only valid once we reach 3/1 - await batch.markAllSnapshotDone('3/1'); - }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false }); + expect(result1).toEqual({ checkpointBlocked: true, checkpointCreated: false }); + // Snapshot is only valid once we reach 3/1 + await writer.markAllSnapshotDone('3/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // 2/1 < 3/1 - snapshot not valid yet, block checkpoint - const result = await batch.commit('2/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: true, checkpointCreated: false }); - }); + // 2/1 < 3/1 - snapshot not valid yet, block checkpoint + const result2 = await writer.commit('2/1', { createEmptyCheckpoints: false }); + expect(result2).toEqual({ checkpointBlocked: true, checkpointCreated: false }); // No empty checkpoint should be created by the commit above. const cp1 = await bucketStorage.getCheckpoint(); expect(cp1.lsn).toEqual(null); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // After this commit, the snapshot should be valid. - // We specifically check that this is done even if createEmptyCheckpoints: false. - const result = await batch.commit('3/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: true }); - }); + // After this commit, the snapshot should be valid. + // We specifically check that this is done even if createEmptyCheckpoints: false. + const result3 = await writer.commit('3/1', { createEmptyCheckpoints: false }); + expect(result3).toEqual({ checkpointBlocked: false, checkpointCreated: true }); // Now, the checkpoint should advance the sync rules active. const cp2 = await bucketStorage.getCheckpoint(); @@ -1710,11 +1612,9 @@ bucket_definitions: const activeSyncRules = await factory.getActiveSyncRulesContent(); expect(activeSyncRules?.id).toEqual(syncRules.id); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // At this point, it should be a truely empty checkpoint - const result = await batch.commit('4/1', { createEmptyCheckpoints: false }); - expect(result).toEqual({ checkpointBlocked: false, checkpointCreated: false }); - }); + // At this point, it should be a truely empty checkpoint + const result4 = await writer.commit('4/1', { createEmptyCheckpoints: false }); + expect(result4).toEqual({ checkpointBlocked: false, checkpointCreated: false }); // Unchanged const cp3 = await bucketStorage.getCheckpoint(); @@ -1737,44 +1637,46 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + await using snapshotWriter = await bucketStorage.createWriter({ + ...test_utils.BATCH_OPTIONS, + skipExistingRows: true + }); + await using streamingWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const snapshotTable = await test_utils.resolveTestTable(snapshotWriter, 'test', ['id'], config, 1); + const streamingTable = await test_utils.resolveTestTable(streamingWriter, 'test', ['id'], config, 1); - const sourceTable = TEST_TABLE; - // We simulate two concurrent batches, and nesting is the easiest way to do this. + // We simulate two concurrent batches; separate writers are enough for this test. // For this test, we assume that we start with a row "test1", which is picked up by a snapshot // query, right before the delete is streamed. But the snapshot query is only persisted _after_ // the delete is streamed, and we need to ensure that the streamed delete takes precedence. - await bucketStorage.startBatch({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true }, async (snapshotBatch) => { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (streamingBatch) => { - streamingBatch.save({ - sourceTable, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 'test1' - }, - beforeReplicaId: test_utils.rid('test1') - }); - await streamingBatch.commit('2/1'); - - await snapshotBatch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'test1', - description: 'test1a' - }, - afterReplicaId: test_utils.rid('test1') - }); - await snapshotBatch.markAllSnapshotDone('3/1'); - await snapshotBatch.commit('1/1'); - - await streamingBatch.keepalive('3/1'); - }); + await streamingWriter.save({ + sourceTable: streamingTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 'test1' + }, + beforeReplicaId: test_utils.rid('test1') }); + await streamingWriter.commit('2/1'); + + await snapshotWriter.save({ + sourceTable: snapshotTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1a' + }, + afterReplicaId: test_utils.rid('test1') + }); + await snapshotWriter.markAllSnapshotDone('3/1'); + await snapshotWriter.commit('1/1'); + + await streamingWriter.keepalive('3/1'); const cp = await bucketStorage.getCheckpoint(); expect(cp.lsn).toEqual('3/1'); const data = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(cp.checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(cp.checkpoint, [bucketRequest(syncRules, 'global[]')]) ); expect(data).toEqual([]); @@ -1805,38 +1707,39 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); - - const sourceTable = test_utils.makeTestTable('test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - for (let u of ['u1', 'u2', 'u3', 'u4']) { - for (let t of ['t1', 't2', 't3', 't4']) { - const id = `${t}_${u}`; - await batch.save({ - sourceTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id, - description: `${t} description`, - user_id: u - }, - afterReplicaId: test_utils.rid(id) - }); - } + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + for (let u of ['u1', 'u2', 'u3', 'u4']) { + for (let t of ['t1', 't2', 't3', 't4']) { + const id = `${t}_${u}`; + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id, + description: `${t} description`, + user_id: u + }, + afterReplicaId: test_utils.rid(id) + }); } - await batch.commit('1/1'); - }); + } + await writer.commit('1/1'); const { checkpoint } = await bucketStorage.getCheckpoint(); bucketStorage.clearChecksumCache(); - const buckets = bucketRequests(syncRules, ['user["u1"]', 'user["u2"]', 'user["u3"]', 'user["u4"]']); - const checksums = [...(await bucketStorage.getChecksums(checkpoint, buckets)).values()]; + const users = ['u1', 'u2', 'u3', 'u4']; + const expectedChecksums = [346204588, 5261081, 134760718, -302639724]; + const bucketRequests = users.map((user) => bucketRequest(syncRules, `user["${user}"]`)); + const checksums = [...(await bucketStorage.getChecksums(checkpoint, bucketRequests)).values()]; checksums.sort((a, b) => a.bucket.localeCompare(b.bucket)); - expect(checksums).toEqual([ - { bucket: bucketRequest(syncRules, 'user["u1"]').bucket, count: 4, checksum: 346204588 }, - { bucket: bucketRequest(syncRules, 'user["u2"]').bucket, count: 4, checksum: 5261081 }, - { bucket: bucketRequest(syncRules, 'user["u3"]').bucket, count: 4, checksum: 134760718 }, - { bucket: bucketRequest(syncRules, 'user["u4"]').bucket, count: 4, checksum: -302639724 } - ]); + const expected = bucketRequests.map((request, index) => ({ + bucket: request.bucket, + count: 4, + checksum: expectedChecksums[index] + })); + expected.sort((a, b) => a.bucket.localeCompare(b.bucket)); + expect(checksums).toEqual(expected); }); } diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 7f4011f35..d9ddc2475 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -18,7 +18,6 @@ import { parameterLookupScope } from './util.js'; export function registerDataStorageParameterTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); const MYBUCKET_1 = parameterLookupScope('mybucket', '1'); test('save and load parameters', async () => { @@ -39,36 +38,36 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - id1: 'user3', - id2: 'user4', - group_id: 'group2a' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - id1: 'user1', - id2: 'user2', - group_id: 'group1a' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + id1: 'user3', + id2: 'user4', + group_id: 'group2a' + }, + afterReplicaId: test_utils.rid('t2') }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + id1: 'user1', + id2: 'user2', + group_id: 'group1a' + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.commit('1/1'); + const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); expect(parameters).toEqual([ @@ -96,32 +95,30 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - group_id: 'group1' - }, - afterReplicaId: test_utils.rid('user1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + group_id: 'group1' + }, + afterReplicaId: test_utils.rid('user1') }); + await writer.commit('1/1'); const checkpoint1 = await bucketStorage.getCheckpoint(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - group_id: 'group2' - }, - afterReplicaId: test_utils.rid('user1') - }); - await batch.commit('1/2'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + group_id: 'group2' + }, + afterReplicaId: test_utils.rid('user1') }); + await writer.commit('1/2'); const checkpoint2 = await bucketStorage.getCheckpoint(); const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); @@ -158,48 +155,44 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - const table = test_utils.makeTestTable('todos', ['id', 'list_id'], config); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - // Create two todos which initially belong to different lists - await batch.save({ - sourceTable: table, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'todo1', - list_id: 'list1' - }, - afterReplicaId: test_utils.rid('todo1') - }); - await batch.save({ - sourceTable: table, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'todo2', - list_id: 'list2' - }, - afterReplicaId: test_utils.rid('todo2') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config); + await writer.markAllSnapshotDone('1/1'); + // Create two todos which initially belong to different lists + await writer.save({ + sourceTable: table, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'todo1', + list_id: 'list1' + }, + afterReplicaId: test_utils.rid('todo1') + }); + await writer.save({ + sourceTable: table, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'todo2', + list_id: 'list2' + }, + afterReplicaId: test_utils.rid('todo2') }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Update the second todo item to now belong to list 1 - await batch.save({ - sourceTable: table, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 'todo2', - list_id: 'list1' - }, - afterReplicaId: test_utils.rid('todo2') - }); - - await batch.commit('1/1'); + await writer.commit('1/1'); + + // Update the second todo item to now belong to list 1 + await writer.save({ + sourceTable: table, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 'todo2', + list_id: 'list1' + }, + afterReplicaId: test_utils.rid('todo2') }); + await writer.commit('1/1'); + // We specifically request the todo_ids for both lists. // There removal operation for the association of `list2`::`todo2` should not interfere with the new // association of `list1`::`todo2` @@ -237,24 +230,24 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - group_id: 'group1', - n1: 314n, - f2: 314, - f3: 3.14 - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + group_id: 'group1', + n1: 314n, + f2: 314, + f3: 3.14 + }, + afterReplicaId: test_utils.rid('t1') }); + await writer.commit('1/1'); + const TEST_PARAMS = { group_id: 'group1' }; const checkpoint = await bucketStorage.getCheckpoint(); @@ -293,35 +286,35 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - group_id: 'group1', - n1: 1152921504606846976n // 2^60 - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - group_id: 'group1', - // Simulate a TOAST value, even though it can't happen for values like this - // in practice. - n1: undefined - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + group_id: 'group1', + n1: 1152921504606846976n // 2^60 + }, + afterReplicaId: test_utils.rid('t1') + }); + + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + group_id: 'group1', + // Simulate a TOAST value, even though it can't happen for values like this + // in practice. + n1: undefined + }, + afterReplicaId: test_utils.rid('t1') }); + await writer.commit('1/1'); + const TEST_PARAMS = { group_id: 'group1' }; const checkpoint = await bucketStorage.getCheckpoint(); @@ -333,8 +326,6 @@ bucket_definitions: }); test('save and load parameters with workspaceId', async () => { - const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', ['id'], config); - await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml( @@ -354,19 +345,19 @@ bucket_definitions: const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace1', - userId: 'u1' - }, - afterReplicaId: test_utils.rid('workspace1') - }); - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace1', + userId: 'u1' + }, + afterReplicaId: test_utils.rid('workspace1') }); + await writer.commit('1/1'); const checkpoint = await bucketStorage.getCheckpoint(); const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {}); @@ -393,8 +384,6 @@ bucket_definitions: }); test('save and load parameters with dynamic global buckets', async () => { - const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', undefined, config); - await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml( @@ -414,41 +403,41 @@ bucket_definitions: const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace1', - visibility: 'public' - }, - afterReplicaId: test_utils.rid('workspace1') - }); - - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace2', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace2') - }); - - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace3', - visibility: 'public' - }, - afterReplicaId: test_utils.rid('workspace3') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace1', + visibility: 'public' + }, + afterReplicaId: test_utils.rid('workspace1') }); + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace2', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace2') + }); + + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace3', + visibility: 'public' + }, + afterReplicaId: test_utils.rid('workspace3') + }); + + await writer.commit('1/1'); + const checkpoint = await bucketStorage.getCheckpoint(); const parameters = new RequestParameters(new JwtPayload({ sub: 'unknown' }), {}); @@ -483,8 +472,6 @@ bucket_definitions: }); test('multiple parameter queries', async () => { - const WORKSPACE_TABLE = test_utils.makeTestTable('workspace', undefined, config); - await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml( @@ -506,53 +493,53 @@ bucket_definitions: const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace1', - visibility: 'public' - }, - afterReplicaId: test_utils.rid('workspace1') - }); - - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace2', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace2') - }); - - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace3', - user_id: 'u1', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace3') - }); - - await batch.save({ - sourceTable: WORKSPACE_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'workspace4', - user_id: 'u2', - visibility: 'private' - }, - afterReplicaId: test_utils.rid('workspace4') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace1', + visibility: 'public' + }, + afterReplicaId: test_utils.rid('workspace1') }); + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace2', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace2') + }); + + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace3', + user_id: 'u1', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace3') + }); + + await writer.save({ + sourceTable: workspaceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'workspace4', + user_id: 'u2', + visibility: 'private' + }, + afterReplicaId: test_utils.rid('workspace4') + }); + + await writer.commit('1/1'); + const checkpoint = await bucketStorage.getCheckpoint(); const parameters = new RequestParameters(new JwtPayload({ sub: 'u1' }), {}); @@ -605,23 +592,24 @@ bucket_definitions: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - id1: 'user3', - id2: 'user4', - group_id: 'group2a' - }, - afterReplicaId: test_utils.rid('t2') - }); - - await batch.truncate([TEST_TABLE]); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + id1: 'user3', + id2: 'user4', + group_id: 'group2a' + }, + afterReplicaId: test_utils.rid('t2') }); + await writer.truncate([testTable]); + await writer.flush(); + const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); @@ -684,21 +672,21 @@ streams: ); const bucketStorage = factory.getInstance(syncRules); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - baz: 'baz', - bar: 'bar' - }, - afterReplicaId: test_utils.rid('t1') - }); - - await batch.commit('1/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + baz: 'baz', + bar: 'bar' + }, + afterReplicaId: test_utils.rid('t1') }); + await writer.commit('1/1'); + const checkpoint = await bucketStorage.getCheckpoint(); const parameters = await checkpoint.getParameterSets([ ScopedParameterLookup.direct(parameterLookupScope('lookup', '0'), ['baz']) diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 0e6106ede..eaa9518c5 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -7,8 +7,6 @@ import { parameterLookupScope } from './util.js'; export function registerParameterCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); - test('compacting parameters', async () => { await using factory = await generateStorageFactory(); const syncRules = await factory.updateSyncRules( @@ -20,60 +18,58 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2' - }, - afterReplicaId: 't2' - }); - - await batch.commit('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2' + }, + afterReplicaId: 't2' }); - const lookup = ScopedParameterLookup.direct(parameterLookupScope('test', '1'), ['t1']); + await writer.commit('1/1'); + + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1', source: null as any }, ['t1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); expect(parameters1).toEqual([{ id: 't1' }]); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - before: { - id: 't1' - }, - beforeReplicaId: 't1', - after: { - id: 't1' - }, - afterReplicaId: 't1' - }); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + before: { + id: 't1' + }, + beforeReplicaId: 't1', + after: { + id: 't1' + }, + afterReplicaId: 't1' + }); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1' - }, - beforeReplicaId: 't1' - }); - await batch.commit('1/2'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1' + }, + beforeReplicaId: 't1' }); + await writer.commit('1/2'); const checkpoint2 = await bucketStorage.getCheckpoint(); const parameters2 = await checkpoint2.getParameterSets([lookup]); expect(parameters2).toEqual([]); @@ -104,59 +100,55 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('1/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - uid: 'u1' - }, - afterReplicaId: 't1' - }); - // Interleave with another operation, to evict the other cache entry when compacting. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - uid: 'u1' - }, - afterReplicaId: 't2' - }); - - await batch.commit('1/1'); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + uid: 'u1' + }, + afterReplicaId: 't1' }); + // Interleave with another operation, to evict the other cache entry when compacting. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + uid: 'u1' + }, + afterReplicaId: 't2' + }); + + await writer.commit('1/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.DELETE, - before: { - id: 't1', - uid: 'u1' - }, - beforeReplicaId: 't1' - }); - await batch.commit('2/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.DELETE, + before: { + id: 't1', + uid: 'u1' + }, + beforeReplicaId: 't1' }); + await writer.commit('2/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2', - uid: 'u2' - }, - afterReplicaId: 't2' - }); - await batch.commit('3/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2', + uid: 'u2' + }, + afterReplicaId: 't2' }); + await writer.commit('3/1'); - const lookup = ScopedParameterLookup.direct(parameterLookupScope('test', '1'), ['u1']); + const lookup = ScopedParameterLookup.direct({ lookupName: 'test', queryId: '1', source: null as any }, ['u1']); const checkpoint1 = await bucketStorage.getCheckpoint(); const parameters1 = await checkpoint1.getParameterSets([lookup]); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index b71a422c5..eb16ac1c9 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -53,7 +53,6 @@ export function registerSyncTests( maxDataFetchConcurrency: 2 }); - const TEST_TABLE = test_utils.makeTestTable('test', ['id'], config); const updateSyncRules = (bucketStorageFactory: storage.BucketStorageFactory, updateOptions: { content: string }) => { return bucketStorageFactory.updateSyncRules( updateSyncRulesFromYaml(updateOptions.content, { @@ -71,33 +70,33 @@ export function registerSyncTests( }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); + await writer.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - description: 'Test 2' - }, - afterReplicaId: 't2' - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' + }); - await batch.commit('0/1'); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + description: 'Test 2' + }, + afterReplicaId: 't2' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage: bucketStorage, @@ -134,32 +133,32 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'earlier', - description: 'Test 2' - }, - afterReplicaId: 'earlier' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'earlier', + description: 'Test 2' + }, + afterReplicaId: 'earlier' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -196,33 +195,33 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // Initial data: Add one priority row and 10k low-priority rows. - await batch.save({ - sourceTable: TEST_TABLE, + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + // Initial data: Add one priority row and 10k low-priority rows. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio', + description: 'High priority row' + }, + afterReplicaId: 'highprio' + }); + for (let i = 0; i < 10_000; i++) { + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { - id: 'highprio', - description: 'High priority row' + id: `${i}`, + description: 'low prio' }, - afterReplicaId: 'highprio' + afterReplicaId: `${i}` }); - for (let i = 0; i < 10_000; i++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `${i}`, - description: 'low prio' - }, - afterReplicaId: `${i}` - }); - } + } - await batch.commit('0/1'); - }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -250,20 +249,18 @@ bucket_definitions: if (sentCheckpoints == 1) { // Save new data to interrupt the low-priority sync. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add another high-priority row. This should interrupt the long-running low-priority sync. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio2', - description: 'Another high-priority row' - }, - afterReplicaId: 'highprio2' - }); - - await batch.commit('0/2'); + // Add another high-priority row. This should interrupt the long-running low-priority sync. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio2', + description: 'Another high-priority row' + }, + afterReplicaId: 'highprio2' }); + + await writer.commit('0/2'); } else { // Low-priority sync from the first checkpoint was interrupted. This should not happen before // 1000 low-priority items were synchronized. @@ -307,33 +304,33 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // Initial data: Add one priority row and 10k low-priority rows. - await batch.save({ - sourceTable: TEST_TABLE, + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + // Initial data: Add one priority row and 10k low-priority rows. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio', + description: 'user_one' + }, + afterReplicaId: 'highprio' + }); + for (let i = 0; i < 10_000; i++) { + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { - id: 'highprio', - description: 'user_one' + id: `${i}`, + description: 'low prio' }, - afterReplicaId: 'highprio' + afterReplicaId: `${i}` }); - for (let i = 0; i < 10_000; i++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `${i}`, - description: 'low prio' - }, - afterReplicaId: `${i}` - }); - } + } - await batch.commit('0/1'); - }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -366,20 +363,18 @@ bucket_definitions: if (typeof next === 'object' && next !== null) { if ('partial_checkpoint_complete' in next) { if (sentCheckpoints == 1) { - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add a high-priority row that doesn't affect this sync stream. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio2', - description: 'user_two' - }, - afterReplicaId: 'highprio2' - }); - - await batch.commit('0/2'); + // Add a high-priority row that doesn't affect this sync stream. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio2', + description: 'user_two' + }, + afterReplicaId: 'highprio2' }); + + await writer.commit('0/2'); } else { expect(sentCheckpoints).toBe(2); expect(sentRows).toBe(10002); @@ -400,20 +395,18 @@ bucket_definitions: if (completedCheckpoints == 1) { expect(sentRows).toBe(10001); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add a high-priority row that affects this sync stream. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio3', - description: 'user_one' - }, - afterReplicaId: 'highprio3' - }); - - await batch.commit('0/3'); + // Add a high-priority row that affects this sync stream. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio3', + description: 'user_one' + }, + afterReplicaId: 'highprio3' }); + + await writer.commit('0/3'); } } } @@ -449,33 +442,33 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // Initial data: Add one priority row and 10k low-priority rows. - await batch.save({ - sourceTable: TEST_TABLE, + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + // Initial data: Add one priority row and 10k low-priority rows. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio', + description: 'High priority row' + }, + afterReplicaId: 'highprio' + }); + for (let i = 0; i < 2_000; i++) { + await writer.save({ + sourceTable: testTable, tag: storage.SaveOperationTag.INSERT, after: { - id: 'highprio', - description: 'High priority row' + id: `${i}`, + description: 'low prio' }, - afterReplicaId: 'highprio' + afterReplicaId: `${i}` }); - for (let i = 0; i < 2_000; i++) { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: `${i}`, - description: 'low prio' - }, - afterReplicaId: `${i}` - }); - } + } - await batch.commit('0/1'); - }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -512,31 +505,29 @@ bucket_definitions: if (sentRows == 1001) { // Save new data to interrupt the low-priority sync. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - // Add another high-priority row. This should interrupt the long-running low-priority sync. - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'highprio2', - description: 'Another high-priority row' - }, - afterReplicaId: 'highprio2' - }); - - // Also add a low-priority row - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: '2001', - description: 'Another low-priority row' - }, - afterReplicaId: '2001' - }); - - await batch.commit('0/2'); + // Add another high-priority row. This should interrupt the long-running low-priority sync. + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'highprio2', + description: 'Another high-priority row' + }, + afterReplicaId: 'highprio2' + }); + + // Also add a low-priority row + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: '2001', + description: 'Another low-priority row' + }, + afterReplicaId: '2001' }); + + await writer.commit('0/2'); } if (sentRows >= 1000 && sentRows <= 2001) { @@ -579,20 +570,20 @@ bucket_definitions: content: BASIC_SYNC_RULES }); const bucketStorage = f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'sync' - }, - afterReplicaId: 't1' - }); - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'sync' + }, + afterReplicaId: 't1' }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, @@ -623,9 +614,7 @@ bucket_definitions: if (receivedCompletions == 1) { // Trigger an empty bucket update. await bucketStorage.createManagedWriteCheckpoint({ user_id: '', heads: { '1': '1/0' } }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.commit('1/0'); - }); + await writer.commit('1/0'); } else { break; } @@ -637,30 +626,30 @@ bucket_definitions: }); test('sync legacy non-raw data', async () => { - const f = await factory(); + await using f = await factory(); const syncRules = await updateSyncRules(f, { content: BASIC_SYNC_RULES }); const bucketStorage = await f.getInstance(syncRules); - - const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test\n"string"', - large_num: 12345678901234567890n - }, - afterReplicaId: 't1' - }); - - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test\n"string"', + large_num: 12345678901234567890n + }, + afterReplicaId: 't1' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -716,11 +705,11 @@ bucket_definitions: }); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const stream = sync.streamResponse({ syncContext, @@ -742,36 +731,32 @@ bucket_definitions: expect(await getCheckpointLines(iter)).toMatchSnapshot(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' }); - expect(await getCheckpointLines(iter)).toMatchSnapshot(); + await writer.commit('0/1'); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - description: 'Test 2' - }, - afterReplicaId: 't2' - }); + expect(await getCheckpointLines(iter)).toMatchSnapshot(); - await batch.commit('0/2'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + description: 'Test 2' + }, + afterReplicaId: 't2' }); + await writer.commit('0/2'); + expect(await getCheckpointLines(iter)).toMatchSnapshot(); }); @@ -787,15 +772,13 @@ bucket_definitions: ` }); - const usersTable = test_utils.makeTestTable('users', ['id'], config); - const listsTable = test_utils.makeTestTable('lists', ['id'], config); - const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); + // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const stream = sync.streamResponse({ syncContext, @@ -821,22 +804,21 @@ bucket_definitions: expect(checkpoint1).toMatchSnapshot(); // Add user - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: usersTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'user1' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: usersTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'user1' }); - const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); + await writer.commit('0/1'); + const checkpoint2 = await getCheckpointLines(iter); + + const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) ).toEqual([bucket]); @@ -855,26 +837,24 @@ bucket_definitions: ` }); - const usersTable = test_utils.makeTestTable('users', ['id'], config); - const listsTable = test_utils.makeTestTable('lists', ['id'], config); - const bucketStorage = await f.getInstance(syncRules); - - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: usersTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'user1' - }); - - await batch.commit('0/1'); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); + const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: usersTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'user1' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -895,24 +875,23 @@ bucket_definitions: const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); const checkpoint1 = await getCheckpointLines(iter); + expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]); expect(checkpoint1).toMatchSnapshot(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: listsTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'list1', - user_id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'list1' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: listsTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'list1', + user_id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'list1' }); + await writer.commit('0/1'); + const checkpoint2 = await getCheckpointLines(iter); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) @@ -932,15 +911,13 @@ bucket_definitions: ` }); - const usersTable = test_utils.makeTestTable('users', ['id'], config); - const listsTable = test_utils.makeTestTable('lists', ['id'], config); - const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); + const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const stream = sync.streamResponse({ syncContext, @@ -963,33 +940,32 @@ bucket_definitions: // Initial empty checkpoint expect(await getCheckpointLines(iter)).toMatchSnapshot(); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: listsTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'list1', - user_id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'list1' - }); - - await batch.save({ - sourceTable: usersTable, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'user1', - name: 'User 1' - }, - afterReplicaId: 'user1' - }); + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: listsTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'list1', + user_id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'list1' + }); - await batch.commit('0/1'); + await writer.save({ + sourceTable: usersTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'user1', + name: 'User 1' + }, + afterReplicaId: 'user1' }); - const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); + await writer.commit('0/1'); + + const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); + const checkpoint2 = await getCheckpointLines(iter); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) @@ -1005,11 +981,10 @@ bucket_definitions: }); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Activate - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/0'); - await batch.keepalive('0/0'); - }); + await writer.markAllSnapshotDone('0/0'); + await writer.keepalive('0/0'); const exp = Date.now() / 1000 + 0.1; @@ -1051,32 +1026,32 @@ bucket_definitions: }); const bucketStorage = await f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' + }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't2', - description: 'Test 2' - }, - afterReplicaId: 't2' - }); - - await batch.commit('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't2', + description: 'Test 2' + }, + afterReplicaId: 't2' }); + await writer.commit('0/1'); + const stream = sync.streamResponse({ syncContext, bucketStorage, @@ -1108,31 +1083,29 @@ bucket_definitions: // Now we save additional data AND compact before continuing. // This invalidates the checkpoint we've received above. - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't1', - description: 'Test 1b' - }, - afterReplicaId: 't1' - }); - - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.UPDATE, - after: { - id: 't2', - description: 'Test 2b' - }, - afterReplicaId: 't2' - }); + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't1', + description: 'Test 1b' + }, + afterReplicaId: 't1' + }); - await batch.commit('0/2'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { + id: 't2', + description: 'Test 2b' + }, + afterReplicaId: 't2' }); + await writer.commit('0/2'); + await bucketStorage.compact({ minBucketChanges: 1, minChangeRatio: 0 @@ -1196,12 +1169,11 @@ bucket_definitions: }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // <= the managed write checkpoint LSN below - await batch.commit('0/1'); - }); + await writer.markAllSnapshotDone('0/1'); + // <= the managed write checkpoint LSN below + await writer.commit('0/1'); const checkpoint = await bucketStorage.createManagedWriteCheckpoint({ user_id: 'test', @@ -1233,11 +1205,9 @@ bucket_definitions: }) }); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - // must be >= the managed write checkpoint LSN - await batch.commit('1/0'); - }); + await writer.markAllSnapshotDone('0/1'); + // must be >= the managed write checkpoint LSN + await writer.commit('1/0'); // At this point the LSN has advanced, so the write checkpoint should be // included in the next checkpoint message. @@ -1251,9 +1221,12 @@ bucket_definitions: }); }); - test('encodes sync rules id in buckes for streams', async () => { + test('encodes sync rules id in buckets for streams', async () => { await using f = await factory(); - const rules = ` + // This test relies making an actual update to sync rules to test the different bucket names. + // The actual naming scheme may change, as long as the two buckets have different names. + const rules = [ + ` streams: test: auto_subscribe: true @@ -1261,27 +1234,38 @@ streams: config: edition: 2 -`; +`, + ` +streams: + test2: + auto_subscribe: true + query: SELECT * FROM test WHERE 1; + +config: + edition: 2 +` + ]; for (let i = 0; i < 2; i++) { const syncRules = await updateSyncRules(f, { - content: rules + content: rules[i] }); const bucketStorage = f.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.markAllSnapshotDone('0/1'); - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 't1', - description: 'Test 1' - }, - afterReplicaId: 't1' - }); - await batch.commit('0/1'); + await writer.markAllSnapshotDone('0/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 't1', + description: 'Test 1' + }, + afterReplicaId: 't1' }); + await writer.commit('0/1'); const stream = sync.streamResponse({ syncContext, diff --git a/packages/service-core/src/storage/BucketStorageBatch.ts b/packages/service-core/src/storage/BucketStorageBatch.ts index 7023fab05..2c5a195e4 100644 --- a/packages/service-core/src/storage/BucketStorageBatch.ts +++ b/packages/service-core/src/storage/BucketStorageBatch.ts @@ -12,6 +12,16 @@ export const DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS: ResolvedBucketBatchCommitOptio }; export interface BucketStorageBatch extends ObserverClient, AsyncDisposable { + /** + * Alias for [Symbol.asyncDispose] + */ + dispose(): Promise; + + /** + * Last written op, if any. This may not reflect a consistent checkpoint. + */ + last_flushed_op: InternalOpId | null; + /** * Save an op, and potentially flush. * diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 9be697f21..8a623747a 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -30,10 +30,17 @@ export interface SyncRulesBucketStorage resolveTable(options: ResolveTableOptions): Promise; /** - * Use this to get access to update storage data. + * Create a new writer. + * + * The writer must be flushed and disposed when done. + */ + createWriter(options: CreateWriterOptions): Promise; + + /** + * @deprecated Use `createWriter()` with `await using` instead. */ startBatch( - options: StartBatchOptions, + options: CreateWriterOptions, callback: (batch: BucketStorageBatch) => Promise ): Promise; @@ -162,7 +169,7 @@ export interface ResolveTableResult { dropTables: SourceTable[]; } -export interface StartBatchOptions extends ParseSyncRulesOptions { +export interface CreateWriterOptions extends ParseSyncRulesOptions { zeroLSN: string; /** * Whether or not to store a copy of the current data. @@ -192,6 +199,11 @@ export interface StartBatchOptions extends ParseSyncRulesOptions { logger?: Logger; } +/** + * @deprecated Use `CreateWriterOptions`. + */ +export interface StartBatchOptions extends CreateWriterOptions {} + export interface CompactOptions { /** * Heap memory limit for the compact process.