Improve large Content database performance

.changeset/tidy-action-query-objects.md

@@ -0,0 +1,5 @@
+---
+"@agent-native/core": patch
+---
+
+Preserve nested object parameters when browser clients call GET actions.

packages/core/src/client/use-action.spec.ts

@@ -54,6 +54,29 @@ describe("serializeActionQueryParams", () => {
     expect(params.has("empty")).toBe(false);
     expect(params.has("none")).toBe(false);
   });
+
+  it("serializes nested object GET params as JSON", () => {
+    const tableQuery = {
+      filters: [
+        {
+          propertyId: "status",
+          operator: "equals",
+          value: "published",
+        },
+      ],
+      sorts: [{ propertyId: "date", direction: "desc" }],
+    };
+
+    const query = serializeActionQueryParams({
+      documentId: "doc-1",
+      tableQuery,
+    });
+
+    const params = new URLSearchParams(query);
+    expect(params.get("documentId")).toBe("doc-1");
+    expect(params.get("tableQuery")).toBe(JSON.stringify(tableQuery));
+    expect(query).not.toContain("%5Bobject+Object%5D");
+  });
 });
 
 describe("callAction", () => {

packages/core/src/client/use-action.ts

@@ -207,6 +207,12 @@ function appendActionQueryParam(
     }
     return;
   }
+  if (typeof value === "object") {
+    // defineAction restores JSON strings when the schema expects an object.
+    // Preserve nested GET params instead of collapsing them to "[object Object]".
+    qs.append(key, JSON.stringify(value));
+    return;
+  }
   qs.append(key, String(value));
 }
 

packages/core/src/server/action-routes.spec.ts

@@ -823,6 +823,23 @@ describe("mountActionRoutes", () => {
         baseCurrency: z.string().optional(),
         includeSeries: z.boolean().optional(),
         limit: z.number().optional(),
+        tableQuery: z
+          .object({
+            filters: z.array(
+              z.object({
+                propertyId: z.string(),
+                operator: z.string(),
+                value: z.string(),
+              }),
+            ),
+            sorts: z.array(
+              z.object({
+                propertyId: z.string(),
+                direction: z.enum(["asc", "desc"]),
+              }),
+            ),
+          })
+          .optional(),
       }),
       run: async (params: any) => ({ params }),
     });
@@ -836,7 +853,24 @@ describe("mountActionRoutes", () => {
     const result = await mounted[0].handler({
       _method: "GET",
       req: {
-        url: "http://app.test/_agent-native/actions/instrument-overview?portfolioId=p1&isin=US67066G1040&includeSeries=true&limit=5",
+        url: `http://app.test/_agent-native/actions/instrument-overview?${new URLSearchParams(
+          {
+            portfolioId: "p1",
+            isin: "US67066G1040",
+            includeSeries: "true",
+            limit: "5",
+            tableQuery: JSON.stringify({
+              filters: [
+                {
+                  propertyId: "status",
+                  operator: "equals",
+                  value: "published",
+                },
+              ],
+              sorts: [{ propertyId: "date", direction: "desc" }],
+            }),
+          },
+        )}`,
       },
     });
 
@@ -846,6 +880,16 @@ describe("mountActionRoutes", () => {
         isin: "US67066G1040",
         includeSeries: true,
         limit: 5,
+        tableQuery: {
+          filters: [
+            {
+              propertyId: "status",
+              operator: "equals",
+              value: "published",
+            },
+          ],
+          sorts: [{ propertyId: "date", direction: "desc" }],
+        },
       },
     });
   });

templates/content/actions/_batch-utils.ts

@@ -1,3 +1,5 @@
+import { getDialect, type Dialect } from "@agent-native/core/db";
+
 export function chunks<T>(items: T[], size: number): T[][] {
   if (items.length === 0) return [];
   const out: T[][] = [];
@@ -6,3 +8,13 @@ export function chunks<T>(items: T[], size: number): T[][] {
   }
   return out;
 }
+
+export function bulkChunkSizeForColumnCount(
+  columnCount: number,
+  dialect: Dialect = getDialect(),
+) {
+  // D1 rejects statements with more than 100 bound params, so derive every
+  // bulk chunk from the statement's column count instead of a fixed row count.
+  const budget = dialect === "d1" ? 90 : 900;
+  return Math.max(1, Math.floor(budget / Math.max(1, columnCount)));
+}

templates/content/actions/_builder-cms-read-client.test.ts

@@ -428,6 +428,7 @@ describe("Builder CMS read client", () => {
           "data.customModelField",
           "data.Status",
           "data.status",
+          "data.optionalField",
           "data.blocks",
         ],
         fetchImpl: fetchImpl as unknown as typeof fetch,
@@ -448,6 +449,7 @@ describe("Builder CMS read client", () => {
             "data.customModelField": "Preserved",
             "data.Status": "Editorial",
             "data.status": "published",
+            "data.optionalField": null,
           },
         },
       ],

templates/content/actions/_builder-cms-read-client.ts

@@ -218,6 +218,36 @@ function normalizeBuilderCmsListFieldPath(fieldPath: string) {
   return normalized;
 }
 
+function preserveProjectedBuilderFieldAbsence(
+  read: BuilderCmsReadResult,
+  fieldPaths: readonly string[] | undefined,
+  rawData: boolean | undefined,
+): BuilderCmsReadResult {
+  if (read.state !== "live" || rawData === true || !fieldPaths?.length) {
+    return read;
+  }
+  const projectedFieldPaths = [
+    ...new Set(
+      fieldPaths
+        .map(normalizeBuilderCmsListFieldPath)
+        .filter((fieldPath): fieldPath is string => fieldPath !== null),
+    ),
+  ];
+  if (projectedFieldPaths.length === 0) return read;
+  return {
+    ...read,
+    entries: read.entries.map((entry) => {
+      const sourceValues = { ...entry.sourceValues };
+      for (const fieldPath of projectedFieldPaths) {
+        if (!Object.prototype.hasOwnProperty.call(sourceValues, fieldPath)) {
+          sourceValues[fieldPath] = null;
+        }
+      }
+      return { ...entry, sourceValues };
+    }),
+  };
+}
+
 export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) {
   const fields = new Map<string, string>();
   for (const fieldPath of [
@@ -1273,22 +1303,32 @@ export async function readBuilderCmsContentEntries(args: {
       publicKey,
       privateKey: privateKey ?? undefined,
     });
-    if (contentApiRead.state === "live") return contentApiRead;
+    if (contentApiRead.state === "live") {
+      return preserveProjectedBuilderFieldAbsence(
+        contentApiRead,
+        args.fieldPaths,
+        args.rawData,
+      );
+    }
     if (!privateKey) return contentApiRead;
   }
 
   if (privateKey) {
     try {
-      return await readBuilderCmsContentEntriesViaMcp({
-        model: args.model,
-        fieldPaths: args.fieldPaths,
-        rawData: args.rawData,
-        limit: args.limit,
-        maxPages: args.maxPages,
-        offset: args.offset,
-        fetchImpl,
-        privateKey,
-      });
+      return preserveProjectedBuilderFieldAbsence(
+        await readBuilderCmsContentEntriesViaMcp({
+          model: args.model,
+          fieldPaths: args.fieldPaths,
+          rawData: args.rawData,
+          limit: args.limit,
+          maxPages: args.maxPages,
+          offset: args.offset,
+          fetchImpl,
+          privateKey,
+        }),
+        args.fieldPaths,
+        args.rawData,
+      );
     } catch (error) {
       return {
         state: "error",

templates/content/actions/_content-files.ts

@@ -4,6 +4,7 @@ import { accessFilter } from "@agent-native/core/sharing";
 import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
 
 import { schema } from "../server/db/index.js";
+import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js";
 import {
   listContentOrganizationMemberships,
   normalizeContentSpaceEmail,
@@ -32,23 +33,29 @@ export function contentFilesItemId(databaseId: string, documentId: string) {
 
 async function remapItemReferences(db: Db, replacements: Map<string, string>) {
   if (!replacements.size) return;
-  const duplicateIds = [...replacements.keys()];
-  const remap = (column: any) => {
-    let expression = sql`CASE ${column}`;
-    for (const [duplicateId, canonicalId] of replacements)
-      expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`;
-    return sql`${expression} ELSE ${column} END`;
-  };
-  for (const table of [
-    schema.contentSpaceCatalogItems,
-    schema.contentDatabaseBodyHydrationQueue,
-    schema.contentDatabaseSourceRows,
-    schema.contentDatabaseSourceChangeSets,
-  ]) {
-    await db
-      .update(table)
-      .set({ databaseItemId: remap(table.databaseItemId) })
-      .where(inArray(table.databaseItemId, duplicateIds));
+  for (const replacementBatch of chunks(
+    [...replacements.entries()],
+    bulkChunkSizeForColumnCount(3),
+  )) {
+    const batchReplacements = new Map(replacementBatch);
+    const duplicateIds = [...batchReplacements.keys()];
+    const remap = (column: any) => {
+      let expression = sql`CASE ${column}`;
+      for (const [duplicateId, canonicalId] of batchReplacements)
+        expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`;
+      return sql`${expression} ELSE ${column} END`;
+    };
+    for (const table of [
+      schema.contentSpaceCatalogItems,
+      schema.contentDatabaseBodyHydrationQueue,
+      schema.contentDatabaseSourceRows,
+      schema.contentDatabaseSourceChangeSets,
+    ]) {
+      await db
+        .update(table)
+        .set({ databaseItemId: remap(table.databaseItemId) })
+        .where(inArray(table.databaseItemId, duplicateIds));
+    }
   }
 }
 
@@ -170,15 +177,18 @@ async function reconcileDocuments(args: {
     }
   }
   await remapItemReferences(args.db, replacements);
-  if (deleteIds.size) {
+  for (const deleteBatch of chunks(
+    [...deleteIds],
+    bulkChunkSizeForColumnCount(1),
+  )) {
     await args.db
       .delete(schema.contentDatabaseItems)
-      .where(inArray(schema.contentDatabaseItems.id, [...deleteIds]));
+      .where(inArray(schema.contentDatabaseItems.id, deleteBatch));
   }
-  if (inserts.length) {
+  for (const insertBatch of chunks(inserts, bulkChunkSizeForColumnCount(8))) {
     await args.db
       .insert(schema.contentDatabaseItems)
-      .values(inserts)
+      .values(insertBatch)
       .onConflictDoNothing();
   }
   return { inserted: inserts.length, removed: deleteIds.size };

templates/content/actions/_database-source-utils.test.ts

@@ -51,6 +51,7 @@ import {
   serializeSourceMetadataRecord,
   sourceSnapshotValuesJsonProjectionSql,
   sourceSnapshotDocumentSelection,
+  sourceSnapshotPageDocumentIds,
   sourceValuesForSnapshot,
   sourceValuesForSeededSourceRow,
   sourceChangeSetKey,
@@ -103,6 +104,52 @@ function item(id: string, title: string): ContentDatabaseItem {
 }
 
 describe("database source helpers", () => {
+  it("page-scopes primary Builder rows without truncating secondary federation", () => {
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "builder-cms",
+        metadataJson: "{}",
+        documentIds: ["doc-1", "doc-2"],
+      }),
+    ).toEqual(["doc-1", "doc-2"]);
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "builder-cms",
+        metadataJson: "{}",
+        documentIds: [],
+      }),
+    ).toEqual([]);
+
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "builder-cms",
+        metadataJson: JSON.stringify({
+          federation: {
+            role: "secondary",
+            keyField: "slug",
+            normalizationFormula: "lower(trim(value))",
+            join: {
+              kind: "identity",
+              collection: null,
+              localExpr: "{Slug}",
+              remoteKeyField: "slug",
+              normalizationFormula: "lower(trim(value))",
+            },
+          },
+        }),
+        documentIds: ["doc-1"],
+      }),
+    ).toBeUndefined();
+
+    expect(
+      sourceSnapshotPageDocumentIds({
+        sourceType: "local-table",
+        metadataJson: "{}",
+        documentIds: ["doc-1"],
+      }),
+    ).toBeUndefined();
+  });
+
   it("keeps the provider-bound property when duplicate local labels collide", () => {
     const existingFields = [
       {