Improve large Content database performance

templates/content/actions/_batch-utils.ts

@@ -1,3 +1,5 @@
+import { getDialect, type Dialect } from "@agent-native/core/db";
+
 export function chunks<T>(items: T[], size: number): T[][] {
   if (items.length === 0) return [];
   const out: T[][] = [];
@@ -6,3 +8,13 @@ export function chunks<T>(items: T[], size: number): T[][] {
   }
   return out;
 }
+
+export function bulkChunkSizeForColumnCount(
+  columnCount: number,
+  dialect: Dialect = getDialect(),
+) {
+  // D1 rejects statements with more than 100 bound params, so derive every
+  // bulk chunk from the statement's column count instead of a fixed row count.
+  const budget = dialect === "d1" ? 90 : 900;
+  return Math.max(1, Math.floor(budget / Math.max(1, columnCount)));
+}

templates/content/actions/_content-files.ts

@@ -4,6 +4,7 @@ import { accessFilter } from "@agent-native/core/sharing";
 import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
 
 import { schema } from "../server/db/index.js";
+import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js";
 import {
   listContentOrganizationMemberships,
   normalizeContentSpaceEmail,
@@ -32,23 +33,29 @@ export function contentFilesItemId(databaseId: string, documentId: string) {
 
 async function remapItemReferences(db: Db, replacements: Map<string, string>) {
   if (!replacements.size) return;
-  const duplicateIds = [...replacements.keys()];
-  const remap = (column: any) => {
-    let expression = sql`CASE ${column}`;
-    for (const [duplicateId, canonicalId] of replacements)
-      expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`;
-    return sql`${expression} ELSE ${column} END`;
-  };
-  for (const table of [
-    schema.contentSpaceCatalogItems,
-    schema.contentDatabaseBodyHydrationQueue,
-    schema.contentDatabaseSourceRows,
-    schema.contentDatabaseSourceChangeSets,
-  ]) {
-    await db
-      .update(table)
-      .set({ databaseItemId: remap(table.databaseItemId) })
-      .where(inArray(table.databaseItemId, duplicateIds));
+  for (const replacementBatch of chunks(
+    [...replacements.entries()],
+    bulkChunkSizeForColumnCount(3),
+  )) {
+    const batchReplacements = new Map(replacementBatch);
+    const duplicateIds = [...batchReplacements.keys()];
+    const remap = (column: any) => {
+      let expression = sql`CASE ${column}`;
+      for (const [duplicateId, canonicalId] of batchReplacements)
+        expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`;
+      return sql`${expression} ELSE ${column} END`;
+    };
+    for (const table of [
+      schema.contentSpaceCatalogItems,
+      schema.contentDatabaseBodyHydrationQueue,
+      schema.contentDatabaseSourceRows,
+      schema.contentDatabaseSourceChangeSets,
+    ]) {
+      await db
+        .update(table)
+        .set({ databaseItemId: remap(table.databaseItemId) })
+        .where(inArray(table.databaseItemId, duplicateIds));
+    }
   }
 }
 
@@ -170,15 +177,18 @@ async function reconcileDocuments(args: {
     }
   }
   await remapItemReferences(args.db, replacements);
-  if (deleteIds.size) {
+  for (const deleteBatch of chunks(
+    [...deleteIds],
+    bulkChunkSizeForColumnCount(1),
+  )) {
     await args.db
       .delete(schema.contentDatabaseItems)
-      .where(inArray(schema.contentDatabaseItems.id, [...deleteIds]));
+      .where(inArray(schema.contentDatabaseItems.id, deleteBatch));
   }
-  if (inserts.length) {
+  for (const insertBatch of chunks(inserts, bulkChunkSizeForColumnCount(8))) {
     await args.db
       .insert(schema.contentDatabaseItems)
-      .values(inserts)
+      .values(insertBatch)
       .onConflictDoNothing();
   }
   return { inserted: inserts.length, removed: deleteIds.size };

templates/content/actions/_database-source-utils.test.ts

@@ -51,6 +51,7 @@ import {
   serializeSourceMetadataRecord,
   sourceSnapshotValuesJsonProjectionSql,
   sourceSnapshotDocumentSelection,
+  sourceSnapshotPageDocumentIds,
   sourceValuesForSnapshot,
   sourceValuesForSeededSourceRow,
   sourceChangeSetKey,
@@ -103,6 +104,52 @@ function item(id: string, title: string): ContentDatabaseItem {
 }
 
 describe("database source helpers", () => {
+  it("page-scopes primary Builder rows without truncating secondary federation", () => {
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "builder-cms",
+        metadataJson: "{}",
+        documentIds: ["doc-1", "doc-2"],
+      }),
+    ).toEqual(["doc-1", "doc-2"]);
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "builder-cms",
+        metadataJson: "{}",
+        documentIds: [],
+      }),
+    ).toEqual([]);
+
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "builder-cms",
+        metadataJson: JSON.stringify({
+          federation: {
+            role: "secondary",
+            keyField: "slug",
+            normalizationFormula: "lower(trim(value))",
+            join: {
+              kind: "identity",
+              collection: null,
+              localExpr: "{Slug}",
+              remoteKeyField: "slug",
+              normalizationFormula: "lower(trim(value))",
+            },
+          },
+        }),
+        documentIds: ["doc-1"],
+      }),
+    ).toBeUndefined();
+
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "local-table",
+        metadataJson: "{}",
+        documentIds: ["doc-1"],
+      }),
+    ).toBeUndefined();
+  });
+
   it("keeps the provider-bound property when duplicate local labels collide", () => {
     const existingFields = [
       {