diff --git a/.changeset/tidy-action-query-objects.md b/.changeset/tidy-action-query-objects.md new file mode 100644 index 0000000000..223a5801e2 --- /dev/null +++ b/.changeset/tidy-action-query-objects.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Preserve nested object parameters when browser clients call GET actions. diff --git a/packages/core/src/client/use-action.spec.ts b/packages/core/src/client/use-action.spec.ts index 8e4e32aeea..03b7fa186d 100644 --- a/packages/core/src/client/use-action.spec.ts +++ b/packages/core/src/client/use-action.spec.ts @@ -54,6 +54,29 @@ describe("serializeActionQueryParams", () => { expect(params.has("empty")).toBe(false); expect(params.has("none")).toBe(false); }); + + it("serializes nested object GET params as JSON", () => { + const tableQuery = { + filters: [ + { + propertyId: "status", + operator: "equals", + value: "published", + }, + ], + sorts: [{ propertyId: "date", direction: "desc" }], + }; + + const query = serializeActionQueryParams({ + documentId: "doc-1", + tableQuery, + }); + + const params = new URLSearchParams(query); + expect(params.get("documentId")).toBe("doc-1"); + expect(params.get("tableQuery")).toBe(JSON.stringify(tableQuery)); + expect(query).not.toContain("%5Bobject+Object%5D"); + }); }); describe("callAction", () => { diff --git a/packages/core/src/client/use-action.ts b/packages/core/src/client/use-action.ts index f1cd36c4e2..af99ec43be 100644 --- a/packages/core/src/client/use-action.ts +++ b/packages/core/src/client/use-action.ts @@ -207,6 +207,12 @@ function appendActionQueryParam( } return; } + if (typeof value === "object") { + // defineAction restores JSON strings when the schema expects an object. + // Preserve nested GET params instead of collapsing them to "[object Object]". + qs.append(key, JSON.stringify(value)); + return; + } qs.append(key, String(value)); } diff --git a/packages/core/src/server/action-routes.spec.ts b/packages/core/src/server/action-routes.spec.ts index f3268d141f..0a5b437343 100644 --- a/packages/core/src/server/action-routes.spec.ts +++ b/packages/core/src/server/action-routes.spec.ts @@ -823,6 +823,23 @@ describe("mountActionRoutes", () => { baseCurrency: z.string().optional(), includeSeries: z.boolean().optional(), limit: z.number().optional(), + tableQuery: z + .object({ + filters: z.array( + z.object({ + propertyId: z.string(), + operator: z.string(), + value: z.string(), + }), + ), + sorts: z.array( + z.object({ + propertyId: z.string(), + direction: z.enum(["asc", "desc"]), + }), + ), + }) + .optional(), }), run: async (params: any) => ({ params }), }); @@ -836,7 +853,24 @@ describe("mountActionRoutes", () => { const result = await mounted[0].handler({ _method: "GET", req: { - url: "http://app.test/_agent-native/actions/instrument-overview?portfolioId=p1&isin=US67066G1040&includeSeries=true&limit=5", + url: `http://app.test/_agent-native/actions/instrument-overview?${new URLSearchParams( + { + portfolioId: "p1", + isin: "US67066G1040", + includeSeries: "true", + limit: "5", + tableQuery: JSON.stringify({ + filters: [ + { + propertyId: "status", + operator: "equals", + value: "published", + }, + ], + sorts: [{ propertyId: "date", direction: "desc" }], + }), + }, + )}`, }, }); @@ -846,6 +880,16 @@ describe("mountActionRoutes", () => { isin: "US67066G1040", includeSeries: true, limit: 5, + tableQuery: { + filters: [ + { + propertyId: "status", + operator: "equals", + value: "published", + }, + ], + sorts: [{ propertyId: "date", direction: "desc" }], + }, }, }); }); diff --git a/plans/content-blog-table-performance-research.md b/plans/content-blog-table-performance-research.md new file mode 100644 index 0000000000..72432751ab --- /dev/null +++ b/plans/content-blog-table-performance-research.md @@ -0,0 +1,1200 @@ +# Why did the Content blog table feel unusably slow? + +Date: 2026-07-28 + +## Answer + +The 16:01 Clips recording separates four different kinds of delay that Rewind had blurred together: + +- creating the replacement Blog database left a titled but empty surface on screen for **at least 25 seconds**, and the usable empty table only appeared after Alice navigated away and back; +- attaching the Builder source produced the first useful imported rows in roughly **5-10 seconds**, but truthful background source/body hydration continued for roughly **3 minutes 15 seconds**; +- sorting by Date replaced the populated table with `Loading database` for roughly **5-10 seconds**; +- opening Builder's review surface spent roughly **1 minute 40 seconds** on `Loading the complete Builder diff` before showing the review. + +Opening an imported row was materially better but still staged: useful article content appeared in roughly **5 seconds**, with its property metadata settling roughly **10-15 seconds later**. Property-picker dwell was long, but the clip shows Alice browsing and choosing fields during much of it, so it is not valid to label the whole picker-open interval as application latency. + +The current code makes those delays plausible. Opening a database is a composed read with more than a dozen visible SQL phases. Even though the UI initially asks for only 100 rows, it still loads complete attached-source snapshots and filters them to the visible page afterward. Several mutations also return a complete, sometimes unpaginated database snapshot and then invalidate queries that read the database again. Search, filter, sort, grouping, calendar, and timeline modes may expand the request from 100 rows to as many as 5,000, which the table renders without row virtualization. + +The implementation shape is now frozen. Work should first capture exact browser and server spans for the representative workflow, then deliver four bounded changes: a page-bounded table/source projection, server-bounded table constraints, delta-shaped mutations with exact cache updates, and progressive Builder review/body hydration. Each slice must be accepted against the same visual boundaries and approved budgets without weakening source-review correctness, sharing, cross-tab sync, or Yjs row-body behavior. + +## Rewind Evidence + +### Coverage and limits + +- Screen Memory was enabled and not paused, but status reported `segmentCount: 0`. +- The chapter search returned no matching current chapter and carried a stale `generatedAt` value from 2026-07-21. +- A 32-minute local OCR search covered 10:33:25-10:52:04 EDT and reported 66 Blog-matching observations; its broad result returned the newest 50 and omitted 16. Targeted searches recovered the important earlier transitions. +- Four five-minute contact-sheet requests covering 10:34-10:54 and an exact-frame request at 10:50:44 all failed with `No clean retained Rewind frames`. +- Therefore the observations below are OCR screen-state samples, normally ten seconds apart. They measure visible-state dwell, not exact click-to-response latency. Deliberate user pauses cannot be distinguished from an unresponsive UI unless a start and completion state are both visible. + +### Observed timeline + +| Local time | Visible state | Measurement | Confidence | +| ----------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ------------------------- | +| 10:37:05 | Blog table with `Sort Descending` visible | Operation boundary unknown | Low | +| 10:37:15 | Blog table with `Duplicate property` visible | Picker/menu changed within about 10 seconds | Medium | +| 10:37:25 | Only `Blog (ALPHA TESTING)` remained visible | Later coverage gap prevents a latency claim | Low | +| 10:42:44-10:43:14 | `Questions to Ask Beth...` visible under Blog | State persisted for at least 30 seconds; no click boundary | Low | +| 10:43:24-10:43:34 | `Agent-Native Plan` title/shell only | Body not yet visible | Medium | +| 10:43:44 | Agent-Native Plan body visible | About 20 seconds title-to-body | Medium; not click-to-body | +| 10:43:44-10:44:04 | Delete-page, then delete-database UI, then body | Each visible transition happened within one 10-second sample | Medium | +| 10:44:14 | `Blog` shell/title only | Start of clearest table-load interval | High | +| 10:44:24-10:44:34 | Blog table frame/title, no useful rows detected | Still waiting | High | +| 10:44:54 | First useful row detected | Roughly 30-40 seconds shell-to-first-row | Medium-high | +| 10:45:04 | `100 changes / ready` and one row detected | Intermediate state | Medium | +| 10:45:34 | Multiple rows detected | Roughly 50-80 seconds shell-to-populated-table because 10:45:14/24 samples did not match the query | Medium | +| 10:45:34-10:47:44 | Same populated table state | Stable view; not operation latency | High | +| 10:49:14-10:49:44 | Property search/picker visible | 30-second dwell; user choice versus UI delay unknown | Low | +| 10:49:54 | Picker closed and table visible | Completion occurred within the next sample | Medium | +| 10:50:24-10:51:04 | Property picker visible again with changing choices | At least 40 seconds of interaction; latency cannot be isolated | Low | +| 10:51:14 | KPMG row body visible | Start click unknown | Low | +| 10:51:24 | Back at table | Visible round trip completed within about 10 seconds | Medium | +| 10:51:34-10:52:04 | Property picker visible | At least 30 seconds; no completion captured | Low | + +## Clips Evidence + +### Coverage and method + +- Public recording: `https://clips.agent-native.com/share/BfEnyRiC4Pu7?ref=clip_share`. +- Recording duration: 16:00.905, 1280x830, captured in Safari on 2026-07-28. +- The Clips transcript was still `pending`, so timings come from the video clock and visible UI states. +- The full recording was sampled every five seconds, then the operation boundaries below were checked against the neighboring frames. Durations are therefore reported as ranges, normally with a +/- 2.5-second boundary tolerance, rather than invented millisecond precision. +- `First useful` means Alice can see or act on real rows/content. `Settled` means the visible loading/sync state for that operation has completed. A persistent background sync is reported separately rather than pretending the earlier usable state was still blank. + +### Measured operation ledger + +| Clip time | User operation | First useful / completion | Measured wait | What was visibly happening | Confidence | +| ----------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------ | ------------------------------------------------------ | +| 00:17-00:22 | Permanently delete the old `Blog (ALPHA TESTING)` database | Dialog disappears and deletion toast appears | <=5s | Destructive mutation | Medium-high | +| 00:27-00:42 | Create a new page/database and name it `Blog (ALPHA TEST)` | Titled database shell appears | ~15s to shell | Page/database creation | Medium | +| 00:42-01:07 | Wait on the new database shell | No usable table appears before Alice leaves for Notion | >=25s unresolved | Empty/titled surface with no table controls or rows | High for visible dwell; root cause unknown | +| 01:42-02:06 | Return to Content, navigate away from the stuck Blog page, then back | Empty Blog table with Name and Content columns appears | ~24s end-to-end; includes navigation | Recovery/navigation plus database read | Medium; not a single request | +| 02:32-02:52 | Find the Builder Blog model and choose Attach | Attach command issued | ~20s user interaction | Search/browse/selection, not system latency | High; exclude from performance budget | +| 02:52-03:01 | Attach Builder source | First imported rows and source-progress UI appear | ~5-10s to useful rows | Source attach and initial projection | Medium-high | +| 03:01-06:17 | Let attached source finish fetching/hydrating | Progress advances from 24/100 through 584 rows and body sync; normal table remains available | ~3m15s to settled | Background row fetch/body hydration | High; table was usable during most of it | +| 06:37-06:42 | Open `You Optimized the Wrong 0.6%` | Article title/body visible in side panel | ~5s to useful body | Imported-row/document read | Medium-high | +| 06:52-07:16 | Add/reveal Author property | Author column appears populated | ~24s total picker-to-column | Mostly property browsing plus a final mutation/read | Low as latency; exact choice click needs request trace | +| 07:37-07:57 | Add/reveal Date property | Date column appears populated | ~20s total picker-to-column | Mostly property browsing plus a final mutation/read | Low as latency; exact choice click needs request trace | +| 08:07-08:16 | Sort Date descending | Sorted populated table returns | ~5-10s | Whole table replaced by `Loading database` | High | +| 08:22-08:27 | Reorder Date and Author columns | New column order visible | <=5s | Local/view configuration | Medium-high | +| 08:52-09:02 | Rename database to `Blog Test` | New title stable | ~5-10s including typing | Title mutation | Medium | +| 09:47-09:57 | Open `The Future of SaaS Is Cloneable` | Useful article body visible | ~5-10s | Row/document read | Medium-high | +| 09:57-10:12 | Wait for row metadata to settle | Kind, Parent, and Source fields become visible | ~10-15s after body | Property/metadata read after useful content | Medium-high | +| 10:52-10:57 | Open database settings | Settings panel visible | <=5s | Local settings surface | Medium-high | +| 11:37-13:17 | Open Review Builder update | Complete diff finally appears | ~1m40s | Modal remains on `Loading the complete Builder diff` / `Preparing full review` | High | +| 14:42-14:52 | Prepare/review one selected Builder update | Validation error becomes visible | ~5-10s | `Preparing` ends in typed author-model validation failure | Medium-high | +| 15:27-15:32 | Close the review modal | Table visible again | <=5s | Local modal close | High | + +### Longest confirmed waits + +1. Attached-source fetch/body hydration: approximately **3m15s** to settle, although useful rows appeared after 5-10 seconds. +2. Complete Builder review diff: approximately **1m40s** before review content appeared. +3. New database shell with no usable table: **at least 25s**, unresolved until Alice navigated away. +4. Return/recovery to the empty table: approximately **24s**, including Alice's navigation rather than one isolated request. +5. Date sort: approximately **5-10s** with a blocking whole-table loading state. + +The clip therefore disproves a single "table takes a minute to load" story. The worst elapsed operation is background hydration; the worst blocking routine table operation captured is sort; and the longest blocking source-review operation is diff preparation. + +## Current-Code Evidence + +### Database open carries too much serial work + +- The editor deliberately requires an authoritative `get-document` read; a seeded database snapshot does not satisfy it (`templates/content/app/hooks/use-documents.ts`, `templates/content/app/components/editor/DocumentEditor.tsx`). +- The database page mounts another document read and independently starts `get-content-database`, personal-view, and content-space reads (`templates/content/app/components/editor/DocumentDatabase.tsx`, `templates/content/app/components/editor/database/DatabaseView.tsx`). +- `get-content-database` resolves the database, checks access, builds the composed response, reloads the backing document, and computes context (`templates/content/actions/get-content-database.ts`). +- `getContentDatabaseResponse` serially reads the database, backing document, count, items, documents, favorites, shares, row properties, schema properties, hydration queue, attached sources, federation, and context (`templates/content/actions/_database-utils.ts`). +- Property values are batched, but batch chunks and rollups can still run serially. Rollups can grow with rows times rollup properties (`templates/content/actions/_property-utils.ts`). + +### Pagination does not bound attached-source work + +- The UI initially requests 100 rows (`DatabaseView.tsx`). +- Every attached source snapshot is nevertheless loaded serially (`templates/content/actions/_database-source-utils.ts`). +- Each snapshot reads complete source rows and can repeat a consistency sequence of marker -> rows -> marker up to three times. +- Only after complete source snapshots exist does `getContentDatabaseResponse` filter their rows to the visible page (`_database-utils.ts`). +- This is the strongest code-backed explanation for a Builder-backed Blog table taking tens of seconds while an ordinary page is much lighter. It remains an inference until exact server spans identify the dominant phase. + +### Several writes perform avoidable full reads + +- `create-content-database` and `add-database-item` return a complete database response after the write (`templates/content/actions/create-content-database.ts`, `add-database-item.ts`). +- Add, move, duplicate, bulk delete, and bulk duplicate omit the page limit when rebuilding the response, so the returned snapshot can be unpaginated. +- Successful mutations broadly invalidate active action queries unless a hook opts out. Several database hooks then issue their own targeted invalidations (`packages/core/src/client/use-action.ts`, `templates/content/app/hooks/use-content-database.ts`). +- The result can be: perform write -> rebuild full snapshot -> return it -> invalidate -> fetch again. +- Property configuration is narrower, but it still reconstructs row properties and then invalidates properties, document, and database reads (`templates/content/actions/configure-document-property.ts`, `templates/content/app/hooks/use-document-properties.ts`). + +### Large constrained views move the problem into the browser too + +- Search, filter, sort, grouping, calendar, and timeline constraints can expand from the initial 100 rows to the complete dataset, capped at 5,000 (`DatabaseView.tsx`). +- Table rows are rendered with an unvirtualized `items.map`. +- This can combine a larger server response, more source/federation work, more transferred bytes, and a large React commit. + +### Existing protections that should not be blamed or removed + +- Ordinary client navigation no longer revalidates the root locale loader (`templates/content/app/root.tsx`); prior independent slow-network QA measured a 65.3 ms route commit under an artificial eight-second delay. Route commitment is not the same as useful table content. +- Core `useDbSync` ignores the current tab's own action echo, coalesces invalidations, does not cancel and restart matching reads already in flight, and suppresses broad invalidation for source refresh/body hydration events (`packages/core/src/client/use-db-sync.ts`, `templates/content/app/hooks/use-db-sync.ts`). +- Property values and several document metadata edits already use optimistic cache patches with rollback. Preserve that direction. + +## Existing Observability + +The Core action client already emits `action.response` for every action taking at least one second and for a sample of faster successes. The event includes action name, request ID, total duration, time to first byte, body-read duration, server duration, network overhead, framework startup wait, database wall time/connect time/slowest operation, response bytes, outcome, and status (`packages/core/src/client/use-action.ts`). + +This is useful but not sufficient. It does not identify which phase inside `get-content-database` consumed the server time, how many SQL round trips ran, how many source rows/change sets were scanned, how many duplicate client requests followed one gesture, or when the interface first became useful. + +## Inferences + +Ranked hypotheses to test: + +1. **Complete source snapshots are blocking basic table reads.** This best matches a Builder-backed Blog table and the 30-80 second observed load. +2. **Writes rebuild an unbounded snapshot and then cause another read.** This best explains why ordinary row/property operations can feel as slow as opening the table. +3. **A constrained view expands to thousands of rows and renders them all.** This would amplify both server and browser cost after sort/filter/search/group changes. +4. **Authoritative row reads and Builder/Yjs hydration delay useful/editable row content.** This matches the observed title-before-body interval. +5. **Rollup evaluation can produce row-by-property query growth.** Relevant only if this Blog schema uses rollups; not established from Rewind. + +## Uncertainties + +- Clip boundaries are accurate to roughly one five-second sampling interval, not milliseconds; matching request IDs are still needed for exact action latency. +- The video distinguishes picker deliberation from blocking states, but the exact property-choice click falls between sampled frames. +- No matching production analytics query capability is available in this task, so the existing `action.response` events for Alice's exact session were not retrieved. +- The clip proves one attached Builder source and a 584-row imported table. Payload sizes, rollups, database latency, source-snapshot retry count, and React commit duration remain unmeasured. +- It is not yet known which server phase dominated the stuck new-database shell, Date sort, complete-diff preparation, or the delayed row metadata. + +## Architecture Constraints + +### Demonstrated caller and request + +- Alice, using the production Agent Native Content interface, opened and edited the Builder-backed `Blog (ALPHA TESTING)` database and experienced routine operations as unusably slow. +- The required workflow is ordinary table use: open the database, reveal/add a property, edit a value, open a row, and return. + +### Existing primitives and ownership boundaries + +- UI operations already use Content actions and React Query hooks; they should continue to do so. +- Core owns action transport, Server-Timing parsing, `action.response` telemetry, request-source identity, and database-sync invalidation behavior. +- Content owns database read composition, source snapshot serialization, pagination, source review state, optimistic cache updates, and row/document hydration. +- Builder is a source provider. Basic table visibility should consume persisted SQL projections; live provider work must not become an implicit prerequisite for every interaction. + +### Legacy contracts that must remain unchanged + +- Access checks and ownable-data scoping. +- Multi-source federation and exact source-review/change-set semantics. +- Agent/script/other-tab changes still refresh the current UI. +- Current-tab optimistic edits roll back on failure. +- Open documents still use an authoritative body path and preserve Yjs collaboration. +- Initial table pagination, source status, body hydration status, and review counts remain truthful rather than being replaced by plausible empty values. + +### Smallest compatible delta + +Do not begin with a broad cache rewrite. First capture one exact representative trace and split `get-content-database` into named timing phases. Then make the basic 100-row table projection independent of complete source-review snapshots, returning only page-bounded source overlay/status data needed to render those rows. Load full review/change-set/audit state only for the review surface that consumes it. In parallel, replace full post-write snapshots with bounded deltas or page-bounded responses and remove invalidations made redundant by an exact optimistic patch. + +### Deferred capabilities + +- A general query planner for every Content database shape. +- Infinite scrolling or a redesigned pagination model. +- A complete rollup engine rewrite. +- A new sync transport. +- Provider live reads during ordinary table render. + +### Direct evidence versus inference + +- Direct: OCR state timestamps, frame-retention failures, current action/hook/read paths, existing telemetry fields, current 100/5,000 limits, serial source-snapshot loading, post-read page filtering, full-snapshot mutation responses, broad-plus-targeted invalidations, unvirtualized row rendering. +- Inference: which phase dominated Alice's exact session and the relative contribution of cold start, database network latency, source snapshots, client refetches, and React rendering. + +## Recommendation + +### First Work slice: exact trace and critical-path separation + +1. Reproduce the now-defined blocking cases separately: create/open the new database, sort Date, open an imported row, and open the complete Builder review diff. The supplied 16-minute clip is the UX baseline; no replacement clip is required. +2. Capture each reproduction's browser network waterfall and Core `action.response` request IDs. +3. Add server spans inside `get-content-database` for access, count/items, documents, properties/rollups, per-source snapshots and consistency retries, federation, context, and serialization. Attach query count, sequential critical-path round trips, rows scanned/returned, source/change-set/execution counts, response bytes, and cold-start/connect timing. +4. Record browser marks for intent -> route/shell -> first useful rows -> settled table, plus intent -> optimistic paint -> action acknowledgement -> cache settled. +5. Compare an unsourced table with the same row count against the Blog Builder source, then 0/100/1,000 rows, no rollups versus rollups, and unconstrained versus filtered/sorted/grouped views. + +### Likely second Work slice, if the trace confirms hypothesis 1 + +- Keep the initial 100-row database projection page-bounded end to end. +- Read only the source rows/overlays needed for visible documents. +- Return lightweight source status/counts with the table; fetch full change sets, reviews, executions, and consistency material when the review surface opens. +- Preserve a typed unavailable/error state; never turn a failed source snapshot into an empty successful table. + +### Likely third Work slice, if the trace confirms hypothesis 2 or 3 + +- Make row/property mutations optimistically useful immediately and return bounded deltas instead of complete unpaginated snapshots. +- Suppress default broad invalidation where the hook performs an exact targeted update; keep other-tab and agent refresh semantics. +- Keep constrained reads server-side and paginated rather than expanding silently to 5,000 client-rendered rows; virtualize only if a legitimately large client result remains part of the accepted product contract. + +## Proposed Acceptance Story + +On the production-like Builder-backed Blog fixture, Alice can open the table, reveal/add a property, edit one value, open a row, and return without losing context or waiting through a blank/stale interface. + +Required assertions: + +- Route/shell responds to every gesture within 100 ms. +- Warm table open shows first useful rows within 1 second; cold open within 2 seconds, excluding an explicitly measured and separately reported platform cold start. +- Property picker opens locally within 100 ms. +- Property/value changes paint optimistically within 100 ms, acknowledge within 1 second warm / 2 seconds cold, roll back visibly on failure, and do not blank the table. +- Row shell opens within 100 ms; authoritative body appears within 1 second warm / 2 seconds cold; editability reports Yjs or Builder hydration separately instead of holding the entire page in an undifferentiated wait. +- One gesture produces a bounded, asserted action/request count with no full unpaginated snapshot or redundant table refetch. +- Initial table read work is proportional to the requested page and attached sources needed for that page, not the complete provider corpus or review history. +- 100-, 1,000-, and 5,000-row fixtures meet explicit server, payload, and React-commit budgets; constrained views do not silently force an unvirtualized 5,000-row client render. +- Sharing, multi-source federation, source-review truth, other-tab/agent refresh, optimistic rollback, Yjs collaboration, and typed failure states remain correct. + +### Approved operation budgets + +Alice approved these values on 2026-07-29 as the acceptance target for this lane: + +| Operation | Accepted target | +| --------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| Delete database | Immediate removal; server acknowledgement within 1 second | +| Create and name database | Shell within 100 ms; usable empty table within 1 second warm / 2 seconds cold | +| Recover or return to table | Route shell within 100 ms; useful table within 1 second warm / 2 seconds cold | +| Open Builder model picker | Picker within 100 ms; search results within 300 ms | +| Attach Builder source | Feedback within 100 ms; first useful rows within 2 seconds | +| Hydrate a 584-row attached source | Never block table interaction; metadata within 10 seconds and bodies within 30-60 seconds in the background | +| Open imported row | Shell within 100 ms; useful body within 1 second warm / 2 seconds cold | +| Reveal/add a property | Picker within 100 ms; selected column paints within 100 ms; acknowledgement within 1 second | +| Sort table | Existing rows remain visible; sorted result within 500 ms warm / 1 second cold | +| Reorder columns | Within 100 ms | +| Rename database | Optimistic title within 100 ms; acknowledgement within 1 second | +| Load row metadata | Within 1 second warm / 2 seconds cold, without delaying the body | +| Open database settings | Within 100 ms | +| Open Builder review | Shell within 100 ms; first changes within 2 seconds; full diff within 10 seconds or progressively loaded | +| Prepare selected Builder update | Success or typed validation error within 2 seconds | +| Close review surface | Within 100 ms | + +## Hosted acceptance ledger — 2026-07-30 + +The Builder connection was restored and the production-like 584-row source was +tested on PR #2522's exact Netlify deploy previews. The writable fixture was a +task-owned Content database; the Builder model remained read-only. + +### Runtime improvements verified + +- Before the source-field projection fix, revealing Author took **15.49s**. + The bounded SQL projection reduced that to **969ms**; the later single-scan + path produced three hosted acknowledgements of **1,087ms**, **977ms**, and + **745ms** (median **977ms**) while the column painted optimistically in + **34.2ms** and the existing 100 rows never disappeared. +- Sort retained the existing rows immediately. The hosted background request + was **1.49-1.59s** cold, so visual continuity passes while the cold + acknowledgement remains above the 1s target. +- Opening the Builder review showed progressive changes within **465ms**. The + complete review request remained slow at **29.97s** (`app=26.14s`, + `db=3.81s`), so progressive acceptance passes but eager full completion does + not. +- Prefetching Builder models when Sources opens reduced the measured + space-to-model paint from **1,263ms** to **43ms**. + +### Fresh 584-row attach measurements + +Exact deploy `6a6b7e943e99e3000899a44a` at commit `83c45829f`: + +- feedback: **93ms**; +- first 100 useful rows: **5.56s**; +- attach request: **5.40s** (`app=5.20s`, `db=2.46s`); +- one complete continuation replaced five client-driven page continuations, + but still took **22.61s** (`app=22.35s`, `db=11.15s`); +- complete metadata was therefore about **28.1s**, improved from the earlier + run that was still at 300 rows after **59.5s** and only known complete by + **182s**. + +Exact deploy `6a6b8100dac828000830b1de` at commit `46ffed6a3` reused the +persisted row cursor and field schema: + +- feedback: **78ms**; +- first 100 useful rows: **6.18s**; +- attach request: **5.94s** (`app=5.64s`, `db=2.97s`); +- remaining-row continuation: **20.30s** (`app=20.07s`, `db=10.29s`); +- complete metadata: **29.38s** from pointer intent. + +The model-picker and immediate-feedback budgets now pass. First useful rows, +complete metadata, and background bodies do not: the prior completed run +reached 584 metadata rows between **59.5s and 182s**, then reported 582/584 +usable bodies ready by **280.5s**. The single-continuation implementation makes +metadata substantially faster, but the accepted **2s / 10s / 30-60s** attach +ladder is not yet met. + +### Verification and cleanup + +- Focused UI tests: **64 passed**. +- Focused Builder resync tests: **4 passed**, including explicit full refresh, + continuation convergence, a 597-row snapshot, and typed model-field failure. +- Content typecheck passed (with the existing Node 22.21/react-router version + warning). +- PR fast lanes, Content DB tests, typecheck, build, security, and parity were + green at `e717fdb3b`; the unrelated standalone Chat template E2E was red. +- All five task-owned hosted pages/databases were moved to Trash and permanently + deleted. The final Personal and Trash snapshots contained no + `__an_content_perf_2522_` marker. No Builder provider rows were written or + deleted. + +### Land decision + +Do not merge yet. The remaining ingestion work needs a separate bounded slice: +parallelize the Builder list-page reads, reduce hosted SQL round trips while +importing the remaining 484 rows, and bulk/lazily schedule body conversion +without making metadata acknowledgement await the full hydration queue. + +## Shaped implementation plan for the four remaining failures + +Shaped on 2026-07-30 after hosted acceptance at `46ffed6a3`. This section is +the governing repair plan for PR #2522. It does not change the approved +operation budgets or permit a code-ready-only merge. + +### Refresher and current truth + +Routine interaction is no longer the large-table problem: Builder model +results paint in 43ms, attach feedback in 78-93ms, source columns in 34.2ms, +and progressive review in 465ms. Four assertions still fail: + +| Boundary | Current | Required | +| -------------------------------------- | ---------------------------------: | -------: | +| First useful Builder rows after Attach | 6.18s | <=2s | +| Complete metadata for 584 rows | 29.38s | <=10s | +| Usable bodies for the 584-row source | <=280.5s in the last completed run | 30-60s | +| Cold Date-sort acknowledgement | 1.49-1.59s | <=1s | + +The first three failures share one ingestion path. The initial attach waits on +provider discovery, a projected first page, SQL import, source-row seeding, and +hydration-queue creation. The continuation then reads Builder pages serially +and repeats database setup reads before scheduling bodies. Body hydration +claims 25 jobs at a time but persists each converted row through its own +multi-statement transaction. Sort is separate: it retains rows visually, but a +constraint change still recomposes database/schema/source/context data when it +only needs a newly ordered item page. + +### Frozen implementation boundary + +- **Outcome:** every approved operation budget passes on the read-only + `Agent Native Blog Article Test` model with more than 500 rows. +- **Shipping surface:** `BuilderIO/agent-native`, Content database UI and + actions, for any Content user attaching and operating a large provider-backed + database; durable destination is the existing PR #2522 merged to `main`. +- **Governing architecture:** actions remain the only data-operation surface; + SQL remains authoritative; provider reads are projected and bounded; useful + UI is optimistic/progressive; unfinished body work remains durable and typed. +- **Acceptance story:** Alice attaches the 584-row Builder model, sees real + rows within 2s, receives complete metadata within 10s and bodies within 60s, + then sorts within 1s without blanking or losing the table. +- **Risk strategy:** system-ready. No feature flag and no merge until all four + assertions and the already-passing interaction assertions pass on the exact + PR head. + +Architecture grounding is required because the repair touches the shared +Content action contract, provider pagination, SQL import transactions, and the +durable hydration queue. Existing primitives to preserve are +`defineAction`, `useActionQuery`/`useActionMutation`, the Builder read client, +stable Builder import identities, refresh claims/cursors, the hydration queue, +targeted cache invalidation, and the current row-body conflict/CAS rules. No +custom REST route, provider write, direct UI fetch, new scheduler, large SQL +blob, or client-trusted provider snapshot belongs in this repair. + +### Slice 1 — make sort a page read, not a database re-open + +Add a page-shaped action, `query-content-database-items`, that reuses the +existing access filter, server-side constraint evaluator, item serializer, and +page-bounded source overlay but returns only: + +- the ordered/filtered 100-row item page; +- pagination/count and `tableQueryMode`; +- property values and source overlays needed by those rows. + +It must not reload database metadata, schema properties, full source status, +context path, review state, or hydration state unrelated to the returned page. +`DatabaseView` keeps the last good page visible with placeholder data and swaps +only the page cache when a sort changes. Other-tab/agent invalidation must still +refresh this action key. + +Instrumentation for this slice separates constraint planning, item IDs, +documents, property values, overlay, and serialization, aligned to +pointer-to-paint and request acknowledgement. Remove or gate temporary phase +logging before completion. + +Acceptance: + +- existing rows never disappear; +- sorted paint and acknowledgement are <=500ms warm / <=1s cold across five + hosted samples; +- exactly one page action follows one sort gesture; +- the response contains at most 100 rows and no document bodies/full source + snapshots; +- access, filters, multi-sort, null ordering, pagination, federation, and + other-tab refresh tests remain green. + +Stop and reshape if the existing server constraint evaluator cannot produce a +page without first materializing all documents/properties in application +memory; that would require a broader SQL query-planner decision. + +### Slice 2 — show truthful rows before attachment persistence settles + +Add a read-only `preview-content-database-source-attach` action and query it +when the user opens a Builder model leaf. It returns the same projected first +100 provider rows the attach path consumes, without bodies or writes. On +Attach, the hook paints those real provider rows immediately in a typed +`attaching` state. They can be opened as read-only previews but are not +represented as persisted/editable until the attach action succeeds. Failure +rolls the preview back and leaves the original database intact. + +In parallel, make the server attach read model fields and the first projected +content page concurrently, then batch the minimal document/item/source-identity +writes. The server must independently read and validate Builder data; it must +not trust provider rows echoed back by the browser. The successful response +reconciles the optimistic preview to stable document/item IDs without reflow. + +Acceptance: + +- feedback remains <=100ms; +- real first rows paint <=2s even when Attach is clicked before prefetch + completes; +- rows are visibly typed as attaching/read-only until persistence succeeds; +- success preserves selection/order and failure removes only the preview; +- no provider body or credential enters the client cache or SQL list rows; +- attach idempotency, duplicate-title identity, access, and rollback tests pass. + +Stop and reshape if a useful row must be editable before stable SQL identity +exists. The approved plan treats early rows as truthful read-only provider +previews, not fictional local rows. + +### Slice 3 — finish all metadata in one bounded provider/SQL pass + +Keep the current persisted continuation cursor, then change the complete read +to a bounded parallel window: + +- fetch offsets 100-500 with at most four concurrent Builder requests; +- preserve offset order and stable-ID deduplication; +- stop at the first short page; +- retry transient page failures through the existing retry policy; +- report a typed incomplete/error result if any required page fails rather + than treating a hole as completion. + +On the SQL side, make `importBuilderCmsEntriesAsDatabaseItems` return the +minimal serialized imported rows it just created. Use that result to avoid the +second `sourceSetupPayload` and full existing-row reread. In one bounded +transaction, bulk-write documents, memberships, database items, source rows, +materialized values for already-bound properties, the compact hydration queue +identity/version envelope, and final source progress. Do not reseed unchanged +source fields. Body conversion is explicitly outside the metadata +acknowledgement; only durable queue creation belongs inside it. + +Temporary spans must separately report provider-page wall time, import SQL +round trips, source-row/value writes, queue insertion, and final metadata +publication. The intended hosted budget allocation is <=3s provider, +<=5s SQL, and <=2s framework/network margin. + +Acceptance: + +- 584 metadata rows are complete <=10s from Attach across five hosted runs; +- the initial 100 rows remain visible throughout; +- the final active-source identity set is exactly 584 and stale rows prune only + after complete coverage; +- 584/597/1,000-row deterministic fixtures prove ordering, cursor completion, + duplicate IDs, short final pages, retries, suspicious-empty preservation, + concurrent refresh claims, and provider failure; +- SQL/query-count assertions prove no second full setup read and no per-row + write loop. + +Stop and reduce provider concurrency if Builder returns 429s or the existing +retry envelope cannot bound the window safely. Do not trade correctness for an +optimistic completed count. + +### Slice 4 — bulk body conversion without per-row database chatter + +Retain the durable hydration queue and open-row priority, but refactor one pump +into three explicit phases: + +1. claim and preload a bounded batch in bulk; +2. convert bodies with measured bounded concurrency; +3. persist successful, unavailable, superseded, and failed results with + chunked CASE/CAS updates rather than one multi-query transaction per row. + +The compare-and-swap protections remain mandatory: a local/Yjs edit, a newer +queue payload, or a changed source baseline must win over stale hydration. Use +portable Drizzle/shared SQL helpers and reviewed `getDbExec().execute()` only if +the shared query builder cannot express the chunked CAS. Queue payloads remain +compact identity/version/source references; raw bodies, screenshots, or large +Builder entries do not move into SQL. + +Start with 50 jobs per pump and conversion concurrency 8, then tune from hosted +phase spans rather than raising both blindly. The client immediately requests +the next batch from the returned queue count without interleaving a complete +database/source refetch. Existing `useDbSync` suppression and targeted +invalidation remain in force. Two genuinely bodyless rows may end in typed +`unavailable`; they count as settled, not hydrated. + +Acceptance: + +- the table remains interactive for the entire run; +- 582 hydrated plus two typed unavailable rows settle <=60s in five hosted + runs, with first-open row priority still <=2s; +- queue depth is monotonic except for explicit newer-version replacement; +- closing/reopening the page resumes durable work without duplication; +- supersession, local-edit conflict, retry cap, empty-body, bodyless-row, + source-change, and partial-batch failure tests pass; +- query-count evidence shows O(chunks) writes, not O(rows) transactions. + +Stop and reshape toward the existing framework job substrate only if the +durable client-driven pump cannot meet 60s while the table remains open. A new +background scheduler is deliberately deferred from the first repair. + +### Cross-slice verification and Land gate + +Use one deterministic local 584-row fixture for regression speed and one +task-owned hosted Content database attached read-only to +`Agent Native Blog Article Test`. Every hosted run records visual marks, +action request IDs, Server-Timing phases, request counts, row/body counts, and +cleanup proof. Instrumentation must align each server request with the visible +state transition it caused. + +Before Land, rerun all previously passing budgets as non-regression gates: +43ms model results, <=100ms attach feedback, <=100ms source-column paint, +<=1s median property acknowledgement with no >2s cold sample, progressive +review <=2s, row shell/body targets, and no table blanking. Run Content DB, +source/resync/hydration, hook/cache, parity, typecheck, format, build, security, +and independent H1-H5 browser QA against the exact PR head. Permanently delete +every task-owned hosted fixture and independently prove its marker absent. + +Land remains blocked until all four numbers and every preserved assertion pass. +The next authorized stage, after Alice approves this shaped plan, is `/work` +against this section of the existing brief. + +The acceptance run must measure visible intent-to-paint and action completion separately. Background completion cannot be reported as blocking latency, and an optimistic paint cannot be reported as success before acknowledgement or rollback is observed. + +## Local Performance Environment + +### Current preflight, 2026-07-29 + +The repository contains the intended local database-mode path, but this worktree is not currently test-ready: + +- repo root pins `pnpm@10.29.1`; +- `templates/content/package.json` still routes `dev:database` through `scripts/check-native-deps.mjs` and `scripts/dev-database.mjs`; +- Node is v22.21.1, ABI 127, arm64; +- repository `node_modules` is absent; +- `templates/content/.env.local` and repo-root `.env` are absent; +- no Content dev server is running; +- the native preflight fails loudly with `better-sqlite3 is not installed`. + +The canonical checkout at `/Users/alicemoore/Developer/agent-native` does have repository dependencies plus both relevant env files. Their values were not read or printed. Work can use that existing local setup as the bootstrap source after verifying key presence and checkout compatibility, so no Alice-owned setup step is currently known. + +This is environment absence, not evidence about Content runtime performance. Shape does not repair it. Work should restore dependencies with the repo-pinned package manager, provision the local env without printing secrets, cold-start `dev:database`, and verify the root, database backend, and auth shape before taking a baseline. + +### Work proving ground + +Use a task-owned local database and deterministic fixtures through Content's normal action surface. The primary fixture should reproduce the captured table shape without production/customer data: + +- 584 imported-looking rows; +- Name, Content, Author, and Date properties; +- one attached Builder-shaped source using fixture/read-only adapter data; +- queued body hydration with independently observable progress; +- representative review changes large enough to exercise complete-diff preparation; +- imported rows with useful body content plus later metadata; +- stable ownership marker and an explicit local cleanup/absence check. + +Run the real local UI for create/open, attach, reveal property, sort, reorder, rename, row open, settings, and review. Measure browser intent -> first paint -> useful state -> action acknowledgement -> settled background state. Pair those marks with `action.response` request IDs and named server phases. Repeat cold and warm cases after every material optimization so the lane works the measured waits downward rather than relying on one favorable run. + +Local acceptance proves application behavior under a controlled database/source fixture. It does not by itself prove production database latency, cold platform startup, or live Builder provider behavior; those remain separate destination evidence for Land. + +### Correlated instrumentation protocol + +Every performance run must produce one joined record from the user's gesture through the action/server path to the visible UI result. A fast server log without a fast screen is not a pass; a fast optimistic paint without acknowledgement or truthful settlement is not a pass either. + +For each operation, capture the same correlation identity across: + +1. **Intent:** pointer/keyboard gesture and browser monotonic timestamp. +2. **Immediate visual response:** shell, optimistic paint, retained rows, or explicit progress state. +3. **Client transport:** action name, request ID, request start, time to first byte, response completion, response bytes, and invalidations/refetches caused by that gesture. +4. **Server phases:** access, database connection, count/items, documents, properties/rollups, each attached-source snapshot and consistency retry, federation, review-diff preparation, serialization, and total duration. +5. **Useful visual state:** the semantic UI condition Alice can act on, such as real rows visible, sorted order visible, article body present, or first review changes present. +6. **Acknowledgement:** mutation success or typed failure, including rollback when applicable. +7. **Settled background state:** source/body hydration or complete review work, reported separately from the earlier useful state. + +Use the existing Core `action.response` telemetry and Server-Timing/request IDs before adding new machinery. Temporary probes may add named phase durations and query/row/retry counts where the existing event is too coarse. Browser marks must be paired with semantic UI assertions and timestamped screenshots or video checkpoints from the same run. The visual clock and log clock must be joined through the interaction/request identity and a recorded run-start offset; do not correlate by eyeballing two unrelated timestamps. + +The measurement report for one gesture should therefore read as a single waterfall, for example: + +`gesture -> optimistic/shell paint -> request -> server phases -> response -> useful UI -> acknowledgement -> background settled` + +The acceptance budget is judged on the relevant visual boundary. Logs diagnose which phase consumed that time; they do not redefine success. + +### Temporary-probe hygiene + +- Gate verbose phase logging to the local performance environment and task-owned fixture. +- Record identifiers, durations, counts, status, and coarse error classes; do not log document bodies, source payloads, credentials, or customer data. +- Keep an explicit inventory of every temporary probe and its purpose. +- Before the Work artifact is handed to Land, either remove each probe or deliberately promote a low-overhead, content-free measurement into the existing observability surface with tests and documentation. +- Verify the final diff contains no forgotten debug logging, local-only flags, fixture IDs, or accidental timing behavior. +- Re-run the clean production-like workflow after temporary probes are removed or disabled, because instrumentation overhead can otherwise become the last performance bug. The stopwatch has occasionally eaten the runner. + +## Frozen Implementation Plan + +### Exact acceptance fixture + +The live provider fixture is the isolated Builder collection Alice used in the clip: + +- visible Builder model label: **`Agent Native Blog Article Test`**; +- canonical Builder model name: **`agent-native-blog-article-test`**; +- expected scale: the clip showed **584 rows**; +- direct code corroboration: Builder model discovery deliberately prioritizes the canonical name `agent-native-blog-article-test` in `actions/_builder-cms-read-client.ts`; +- authority: Alice explicitly identified this as the disconnected test collection and authorized writes because it does not feed another product surface. + +The display label or model name alone is not mutation authority. At Work activation, call the read-only `list-builder-cms-models` action and require exactly one result whose `name` and `displayName` match the pair above. Persist its returned Builder model `id`, row count, and a marker-absence baseline in the run ledger. A missing, duplicate, renamed, or differently identified model fails closed before any Builder write. + +Existing collection rows are a read-only scale corpus. Provider mutations are limited to task-created rows whose title contains a run-unique marker such as `__an_content_perf___`. Work must never bulk-edit or delete the existing 584-row corpus, never delete the model, and never use an unmarked row for mutation acceptance. + +Use two additional task-owned local fixtures through Content actions: + +- a **1,000-row** SQL database with the same Name, Content, Author, and Date shape; +- a **5,000-row** SQL database with deterministic values and enough matching/non-matching rows to exercise search, filter, and sort boundaries. + +These local fixtures establish scaling independently of live Builder/provider variance. They contain synthetic text only and are deleted with an independent absence check before Work completes. + +### Delivery slices + +#### Slice 0 — restore the proving ground and capture the old path + +1. Use the current worktree and repo-pinned `pnpm@10.29.1`; restore dependencies without changing branches or printing secrets. +2. Provision the Content local database environment from the compatible canonical checkout configuration without copying values into logs or source. +3. Start the real `dev:database` UI, verify the local database/auth/provider shape, resolve the exact Builder model identity read-only, and create the task-owned local database plus one marked Builder row only after the Work resource gate is active. +4. Run the complete operation ledger once on the unchanged path. Save a joined browser/server waterfall, semantic screenshot checkpoints, action request IDs, response sizes, SQL/query counts, and the old visual durations. This is the comparison baseline, not an acceptance pass. + +#### Slice 1 — correlated instrumentation + +1. Add a Content-local timing collector for `get-content-database` and reuse the existing Builder timing collector for review/execute paths. +2. Instrument `actions/_database-utils.ts` with named spans for database/access resolution, count/page items, list-document projection, shares/favorites, properties/rollups, hydration status, source summaries/page overlays, federation, context, and serialization. +3. Instrument `actions/_database-source-utils.ts` with source ID/type, consistency attempt count, source rows scanned/returned, change-set/execution counts, body-field inclusion, and per-source duration. +4. Extend `actions/prepare-builder-source-review.ts` / `preview-builder-source-review.ts` timing around candidate discovery, authoritative target-row loading, body conversion/diff construction, validation, reconciliation, and response construction. +5. Add browser marks in `DatabaseView.tsx` at the actual intent handlers and semantic UI boundaries: shell/optimistic paint, first useful rows, stable sorted order, article body, first review changes, acknowledgement/rollback, and background settled. +6. Join browser marks to the existing Core action request ID and `action.response`/Server-Timing event. Change Core transport only if the existing request identity cannot be surfaced to the Content mark; do not invent a second unrelated trace ID. + +The Work ledger must inventory each temporary probe. Logging remains local/preview gated and content-free. Remove it before the clean final replay, except for low-overhead counts/timings deliberately promoted into existing telemetry with tests. + +#### Slice 2 — make the basic table read page-bounded end to end + +1. Split the ordinary table projection from the complete source snapshot in `getContentDatabaseResponse`. +2. Query source metadata/status plus only the source rows needed for the returned document IDs. Do not call `getAllContentDatabaseSourceSnapshots` and then filter a complete corpus after the fact. +3. Keep full change sets, execution history, heavy Builder body values, consistency/reconciliation material, and full review payloads off the table-open critical path. Load them only through the source/review action that consumes them. +4. Parallelize independent list reads after access and visible item IDs are known, while retaining typed failures. An unavailable source projection must remain distinguishable from a valid source with zero rows. +5. Return and assert page diagnostics in local timing only: requested limit, database rows scanned/returned, source rows scanned/returned, attached source count, query count, response bytes, and sequential critical-path database round trips. + +Primary files: `actions/_database-utils.ts`, `actions/_database-source-utils.ts`, `actions/get-content-database.ts`, `shared/api.ts`, and focused database/source tests. + +#### Slice 3 — stop sort/filter/search from becoming a 5,000-row client fetch + +1. Extend the existing `get-content-database` action contract with typed table-query inputs for search, active filters, sort order, and page cursor/limit. Keep this on the action surface; do not add a REST twin. +2. Apply table constraints in portable Drizzle queries and return one bounded result page plus truthful total/more state. Add only additive indexes demonstrated by query plans on hot membership/property/sort columns; no destructive schema changes or dialect-specific app calls. +3. Remove the `DatabaseView.tsx` behavior that expands a constrained table request to as many as 5,000 rows and maps the entire result into React. +4. Retain the current rows while a changed query is in flight. A sort gesture immediately updates the control and either reorders cached-complete data or keeps the prior rows visible until the bounded server result commits; it never swaps the table for `Loading database`. +5. Preserve the legacy behavior for board/calendar/timeline semantics unless the same portable query contract can represent them without changing product meaning. They must not regress, but the frozen latency target is the large **table** workflow shown in the clip. + +Primary files: `actions/get-content-database.ts`, `actions/_database-utils.ts`, schema/query helpers and additive migrations only if traces require them, `app/hooks/use-content-database.ts`, `DatabaseView.tsx`, and view/query unit plus database tests. + +#### Slice 4 — make routine writes bounded and optimistic + +1. Inventory every operation in the approved ledger and record its current mutation response shape and invalidation set. +2. For create/name, reveal property, edit value, reorder columns, rename database, and row operations, paint the exact optimistic delta within 100 ms with rollback state retained. +3. Replace complete or unpaginated post-write database reconstruction with the smallest typed delta or the caller's current bounded page. Do not return a plausible empty/success state when reconstruction fails. +4. Set `skipActionQueryInvalidation` only where the hook applies an exact cache update. Keep Core other-tab, agent/script, and external-source invalidation behavior intact. +5. Assert one gesture's action count, cache writes, invalidations, and follow-up reads so a write cannot regress into `write -> full response -> broad invalidate -> full refetch`. + +Primary files: the relevant database mutation actions, `app/hooks/use-content-database.ts`, `app/hooks/use-document-properties.ts`, and Core `use-action` only if a framework-level invalidation defect is directly demonstrated. + +#### Slice 5 — make attach, hydration, row open, and Builder review progressive + +1. Attach returns the source binding and first page of lightweight metadata quickly; continuation fetches remaining metadata in bounded pages without blocking table interaction. +2. Keep heavy Builder body fields in the existing durable hydration queue. Expose queued/progress/error counts truthfully and process bounded batches until all 584 bodies settle or a typed partial failure is visible. +3. Opening an imported row renders the shell immediately and loads its one authoritative body independently from later metadata/Yjs readiness. +4. Opening Builder review renders a local shell immediately, requests only the first review page/candidate summaries, and progressively loads full body diffs for visible or selected changes. `prepare-builder-source-review` must not rebuild an unpaginated database response after validation. +5. Preserve exact Builder target identity, change-set state, publication transition checks, dry-run/live-write gates, and reconciliation behavior. Performance is not permission to make review truth approximate. + +Primary files: `actions/_database-source-utils.ts`, refresh/hydration actions, `preview-builder-source-review.ts`, `prepare-builder-source-review.ts`, `app/hooks/use-content-database.ts`, and the Builder review/hydration portions of `DatabaseView.tsx`. + +### Measurement and acceptance loop + +Each slice follows the same loop: clean baseline -> one bounded change -> focused tests -> real browser replay -> joined waterfall -> compare against the prior run. Work continues until every operation in the approved budget table passes or a material architecture/acceptance change returns the lane to Shape. + +For warm operations, run ten repetitions after one discarded warm-up and require the approved target at p95. For cold table/row opens, run three fresh browser-context repetitions and require each to meet the 2-second target, with framework startup recorded separately rather than silently subtracted. Attach/hydration/review are repeated three times because they mutate durable local state; each run gets a fresh task-owned local database and clean source binding. + +Every measurement row contains: + +- fixture identity and row count; +- cold/warm classification; +- gesture and browser monotonic time; +- request ID and action name; +- shell/optimistic, first-useful, acknowledgement, and settled timestamps; +- server phase durations, query/row/retry counts, response bytes, and action/refetch count; +- semantic assertion and same-run screenshot/video checkpoint; +- pass/fail against the approved operation budget. + +Acceptance must cover the exact 584-row Builder fixture and synthetic 1,000/5,000-row SQL fixtures. A fast synthetic fixture cannot substitute for the Builder run; provider time cannot excuse a table path that reads the entire provider corpus. Conversely, the live collection is not used to manufacture 5,000 remote records. + +### Verification gates + +Work is complete only when all of these are current on the exact artifact: + +- focused unit/database tests prove page-bounded source reads, portable table constraints, typed failure states, bounded mutation responses, exact invalidation counts, hydration continuation, and progressive review scoping; +- `pnpm test` for Content passes; +- `pnpm typecheck` and `pnpm build` pass; +- modified source is formatted with oxfmt; +- the real local UI passes the operation ledger at the approved numbers; +- a production-like branch preview repeats the large-table acceptance with the new path and no temporary instrumentation overhead; +- an off-path/regression pass covers sharing/access, multi-source federation, other-tab/agent refresh, optimistic rollback, source-review identity/state, Builder validation failures, and Yjs row editing; +- the final diff contains no bodies/secrets, hard-coded returned Builder IDs, fixture markers, forgotten debug logging, or local-only performance switches; +- every task-created local and Builder resource is independently proven absent. + +No production performance claim may be made from unit tests, server logs alone, or a visually fast optimistic paint without acknowledgement. Local and preview evidence are reported separately. + +### Risk and rollback strategy + +The frozen strategy is **system-ready**, not a permanent production feature flag. This change preserves an existing public action/product contract and must be fully accepted before merge; merging an unaccepted dormant alternative would only move the uncertainty downstream. During Work, an explicitly local, temporary benchmark switch may compare old and new read implementations against the same fixture, but it is not a rollout mechanism and must be removed before Work completion. + +Keep each slice reversible in review: timing only, table projection, constrained queries, mutation deltas, then progressive source/review. If a slice fails correctness or its acceptance budget, revert that slice's implementation while preserving its measurements and continue from the last passing boundary. No migration may require rollback by dropping or renaming data. + +Stop and return to Shape if meeting the target requires changing the action architecture, source-review truth model, public table semantics, named shipping surface, accepted budgets, or system-ready risk strategy. Fail closed before provider writes if the Builder model identity or task-owned marker boundary cannot be proven. + +## Sources + +- Local Clips Screen Memory status, chapter search, recent-context OCR, contact-sheet attempts, and exact-frame attempt on 2026-07-28. +- Public 16:01 Clips recording `BfEnyRiC4Pu7`, inspected at five-second intervals with key visible state boundaries recorded above. +- Current checkout `9d16c580b` under `templates/content` and `packages/core`. +- `templates/content/actions/get-content-database.ts` +- `templates/content/actions/_database-utils.ts` +- `templates/content/actions/_database-source-utils.ts` +- `templates/content/actions/_property-utils.ts` +- `templates/content/actions/create-content-database.ts` +- `templates/content/actions/add-database-item.ts` +- `templates/content/actions/configure-document-property.ts` +- `templates/content/app/components/editor/database/DatabaseView.tsx` +- `templates/content/app/hooks/use-content-database.ts` +- `templates/content/app/hooks/use-document-properties.ts` +- `packages/core/src/client/use-action.ts` +- `packages/core/src/client/use-db-sync.ts` +- Prior Content slow-network route and property-save isolation research, revalidated against the current checkout before use. + +## Lifecycle + +```yaml +stage: work +ledger-revision: work-2026-07-30-four-failures-v1 +authority-source: >- + Alice invoked $work on 2026-07-30 after approving the four remaining + operation budgets and reconnecting the PR #2522 deployment to the isolated + Builder test collection used in clip BfEnyRiC4Pu7 +authorized-scope: + repositories: + - /Users/alicemoore/.codex/worktrees/8c63/agent-native + product-surfaces: + - Agent Native Content large database table UI + - Builder-backed Content source attach, hydration, row, and review surfaces + outcome: >- + Make ordinary Content table operations on Builder-backed and SQL databases + over 500 rows meet Alice's approved visual and acknowledgement budgets +allowed-mutations: + - artifact-write + - ephemeral-test-resource + - branch + - commit + - push + - pull-request + - deploy + - schema +write-targets: + artifacts: + - /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md +execution-lane: + worktree: /Users/alicemoore/.codex/worktrees/8c63/agent-native + branch: codex/content-large-database-performance + refreshed-base: origin/main@58bdd10a7f60ba08c01fa425f836732c73c5f5b6 + remote-default-branch: origin/main + calling-task-id: unavailable from current host surface + parent-task-id: none + descendants: + - /root/hot_path_inventory + - /root/performance_technical_review +work-progress: + completed-slices: + - page-scoped primary Builder rows, change sets, reviews, and executions + - bounded table search, filter, and sort response pages with retained rows + - bounded routine mutation responses with explicit created and duplicated rows + - immediate page-scoped Builder review while authoritative review loads + - exact federated-field fallback without expanding ordinary large tables + - source-free setup projections and a 100-row fast path for fresh Builder attachment + - personal sort persistence that preserves existing visibility rules without source snapshots + - correlated loopback-only server and visible-row timing replay, with probes removed afterward + - exact imported-document reload for Builder source binding and hydration + - exact identity-based created and duplicated mutation response projections + verification: + - shared table-query TypeScript compilation passed + - shared table-query focused Vitest passed + - oxfmt and git diff whitespace checks passed + - live PR review found four correctness defects; exact Builder binding and + identity-based mutation response repairs were implemented + - Content typecheck and 273 focused action, database, batching, table, + sidebar, personal-view, source, and query tests passed on Node 26 + - independent repair review found no remaining P1 or P2 in the four paths + pending: + - exact read-only Builder model identity and marker-absence baseline + - exact Builder-backed attach, hydration, property, and progressive-review acceptance + - independent human QA on a host that exposes the in-app browser + - final repair verification, exact-head PR prose receipt, review, and CI +temporary-probes: + - id: content-four-failure-action-phases + status: active + gate: PR #2522 preview and isolated local performance runtime only + records: >- + content-free phase names and durations for Builder attach preview, + primary attach, metadata continuation, body hydration batches, and + constrained table pages; action responses carry the same timing array + so browser network evidence can be joined to the visible boundary + removal-trigger: after the final joined preview acceptance replay + - id: content-four-failure-browser-network-waterfall + status: active + gate: browser developer timing capture during the frozen H1-H5 replay + records: >- + pointer intent, semantic row/progress state, action URL, request and + response timestamps, x-agent-native-request-id, status, and response + timing phases; no bodies, credentials, or provider payloads + removal-trigger: after the final joined preview acceptance replay + - id: content-database-response-trace + status: removed + gate: >- + CONTENT_DATABASE_PERFORMANCE_TRACE=1 on a verified loopback action request + records: >- + trace ID, phase durations, requested and returned row counts, source row + and change-set counts, per-source duration, consistency attempts, review + count, and execution count; no titles, bodies, provider payloads, or secrets + removal-trigger: after the final joined acceptance replay + - id: content-database-visible-row-trace + status: removed + gate: response carries content-database-response-trace + records: >- + response trace ID, local interaction ID, intent-to-row-commit duration, + server duration, and returned item count + removal-trigger: after the final joined acceptance replay +test-resources: + - id: builder-perf-entry-agent-native-blog-article-test + kind: record + surface: >- + Builder model name agent-native-blog-article-test, expected display name + Agent Native Blog Article Test, exact returned model ID recorded read-only + before activation + ownership-marker: __an_content_perf___ in every task-created title + baseline: >- + list-builder-cms-models returns exactly one matching model and a provider + list query returns zero entries carrying the run marker before creation + allowed-actions: + - create + - update + - exercise + - delete + cleanup-trigger: after each live-provider acceptance run and before Work completion + cleanup-method: >- + delete every exact returned Builder entry ID created with the run marker; + never delete the model or pre-existing corpus + cleanup-proof: >- + independent reads show every returned entry ID absent and zero entries + matching the run marker + shared-impact: none + isolation: isolated-test-surface + ownership: task-created + production-data: false + customer-data: false + cost: none + boundary-evidence: + - Alice identified this disconnected collection as freely writable test data + - code recognizes canonical model name agent-native-blog-article-test + - trusted read-only model discovery must bind the exact returned provider ID + max-lifetime-minutes: 1440 + declared-at: 2026-07-29T19:00:06Z + expires-at: 2026-07-30T19:00:06Z + status: declared + phase: work + - id: content-large-database-local-fixtures + kind: database + surface: >- + task-owned local Content databases marked __an_content_perf___ + with deterministic 584, 1000, and 5000 row fixtures + ownership-marker: __an_content_perf___ database and row titles + baseline: no local database or document carries the run marker + allowed-actions: + - create + - update + - exercise + - delete + cleanup-trigger: after each destructive acceptance run and before Work completion + cleanup-method: delete task-created databases through Content actions + cleanup-proof: >- + list/search actions and scoped database reads independently report no + database, document, or row carrying the run marker + shared-impact: none + isolation: local-runtime + ownership: task-created + production-data: false + customer-data: false + cost: none + boundary-evidence: + - local database runtime and unique task marker + - creation result IDs retained in the resource ledger + max-lifetime-minutes: 1440 + declared-at: 2026-07-29T19:00:06Z + expires-at: 2026-07-30T19:00:06Z + status: cleaned + active-run-id: 20260729_1715 + active-resources: + - kind: sqlite-runtime + path: /tmp/agent-native-content-perf.GnFBS0/app.db + state: removed-after-marker-free-proof + - kind: content-database + database-id: TYhMNt9YVgjW + document-id: YXN4WiFhHCo0 + title: __an_content_perf_20260729_1715__ 1000 rows + intended-row-count: 1000 + state: permanently-deleted + - kind: content-database + database-id: pyehc2Pq3FT1 + document-id: 4h6N49c1GNkj + title: __an_content_perf_20260729_1715__ 5000 rows + intended-row-count: 5000 + state: permanently-deleted + - kind: content-database + database-id: FLeIYkBrKXeC + document-id: 4UiIEVdifuyF + title: __an_content_perf_20260729_1715__ 584 rows + intended-row-count: 584 + state: permanently-deleted + cleanup-completed-at: 2026-07-30T03:58:00Z + cleanup-observation: + - >- + Content action-backed UI soft-deleted database IDs TYhMNt9YVgjW, + pyehc2Pq3FT1, and FLeIYkBrKXeC with their exact root documents. + - >- + permanently-delete-document removed root IDs YXN4WiFhHCo0, + 4h6N49c1GNkj, and 4UiIEVdifuyF with 1001, 5001, and 585 subtree + documents respectively. + - >- + list-content-databases and search-documents returned zero marker + matches; get-content-database returned typed not_found for all three + exact database IDs; read-only SQL corroborated zero marker documents, + fixture databases, and fixture roots. + - >- + The marker-free SQLite runtime and generated task-local Content env + file were moved to macOS Trash and their original paths verified absent. + phase: work +governing-artifact: + path: /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md + revision: shape-blog-table-latency-r5 +architecture-fingerprint: + outcome: >- + Large Content tables remain immediately usable and meet the approved + operation budgets at 584, 1000, and 5000 rows + shipping-surfaces: + - id: agent-native-content-large-table-performance + repository: /Users/alicemoore/.codex/worktrees/8c63/agent-native + product-surface: Agent Native Content database table and Builder source workflow + constituency: >- + source-blind developers and Content users working with databases over + 500 rows, including Builder-backed tables + durable-destination: merged Agent Native public repository implementation + integration-action: merge + governing-architecture: >- + Content actions remain the single UI/agent data surface; ordinary table reads + use persisted SQL page projections, provider/body/review work stays progressive, + and client caches apply bounded optimistic deltas with truthful rollback + acceptance-story: + id: content-large-table-approved-budgets-v1 + summary: >- + On the exact isolated 584-row Builder model and deterministic 1000/5000-row + SQL fixtures, Alice can create/open a table, attach the source, reveal/edit + properties, sort, open a row, return, and review Builder changes without a + blank interface and within the approved budget table in this artifact + required-assertions: + - gesture-to-shell or optimistic response is at most 100 ms where specified + - warm reads and acknowledgements meet 1 second and cold paths meet 2 seconds where specified + - sort retains rows and settles within 500 ms warm or 1 second cold + - attach shows first useful rows within 2 seconds and hydration never blocks interaction + - 584-row metadata settles within 10 seconds and bodies within 30-60 seconds in the background + - Builder review shows first changes within 2 seconds and full or progressive diff within 10 seconds + - every gesture has one correlated visual/client/server waterfall and bounded action/refetch count + - work scales with requested page or selected review scope rather than complete corpus size + - typed failure, access, sharing, federation, sync, rollback, Yjs, and Builder review truth remain correct + - all temporary probes and task-created resources are removed with current independent proof + risk-strategy: + kind: system-ready + production-validation-after-merge: false +architecture-grounding: + applicability: required + reason: >- + The work changes shared Content action response composition, source projection, + query pagination, cache invalidation, and Builder review boundaries + status: grounded + demonstrated-callers: + - Alice's 584-row Builder-backed Blog table workflow in clip BfEnyRiC4Pu7 + - get-content-database called by DatabaseView with a 100-row initial limit + existing-primitives: + - defineAction plus useActionQuery/useActionMutation + - persisted Content SQL source rows and body hydration queue + - Core action.response request IDs and Server-Timing + - Builder source timing collector and review actions + - Core useDbSync current-tab echo suppression and targeted invalidation hooks + ownership-boundaries: + - Core owns action transport, request identity, timing headers, and DB-sync mechanics + - Content owns database projection, pagination, source/review state, hydration, and optimistic cache policy + - Builder owns provider content; only the exact isolated test model is writable in acceptance + legacy-contracts: + - action surface parity for UI and agent callers + - ownable-data access and sharing scope + - truthful multi-source and review/change-set state + - other-tab and agent/script refresh + - optimistic rollback and authoritative Yjs document behavior + shared-vocabulary: + - first useful + - acknowledgement + - settled background state + - page-bounded table projection + - progressive Builder review + smallest-compatible-delta: >- + Page-bound get-content-database source overlays and table constraints before + changing mutation/review response shapes; no broad cache or sync rewrite + deferred-capabilities: + - general query planner for every database view type + - redesigned infinite scrolling + - rollup engine rewrite + - new real-time transport + - provider live reads on ordinary table render + reversibility: >- + Six ordered slices isolate environment/baseline, timing, read projection, + constraints, mutations, and progressive provider work; all schema changes are additive and each slice + can be removed without deleting user data + direct-evidence: + - clip operation ledger and approved budgets in this artifact + - getContentDatabaseResponse loads complete source snapshots before page filtering + - DatabaseView expands constrained reads to 5000 and renders items with map + - Builder review timing already names snapshot/diff and reconciliation phases + - model discovery prioritizes canonical name agent-native-blog-article-test + inferences: + - relative contribution of source snapshots, query waterfalls, payload, and React commit before joined baseline + unresolved-owner-questions: [] +delegation-ceiling: + - read-only + - artifact-write +product-boundary-gates: + agent-native-public-constituency: >- + The implementation and synthetic fixtures work without Alice's vault or + credentials; her isolated Builder model supplies provider acceptance only, + not a runtime dependency or hard-coded identity + bowerbird-product-boundary: not-applicable +acceptance-state: + status: blocked + summary: >- + Local implementation, deterministic large-database acceptance, repair + verification, independent technical review, and resource cleanup are + complete. Exact Builder-provider acceptance and independent human QA remain + unproved, so the system-ready risk strategy prohibits merge and closure. + local-evidence: + - >- + 5000-row warm open, ten settled repetitions: first useful React commit + 146-189 ms; correlated server read 5-12 ms; 100 of 5000 rows returned + - >- + 5000-row Date sort, ten repetitions: existing rows remained visible and + the sorted paint completed in 272-285 ms + - >- + representative 1000-row and 584-row opens committed 100 useful rows in + 217 ms and 210 ms respectively, with 12 ms and 6 ms server reads + - >- + 584-row row preview showed shell, metadata, and synthetic authoritative + body within 526 ms including browser-control overhead + - >- + Final local replay retained 100 of 5000 rows through Date sort reversal + and opened the first row preview with Author, Date, and authoritative body + - >- + final ABI-matched verification: Content typecheck and 228 focused action, + batching, table, sidebar, personal-view, and query tests passed + - >- + post-review repair verification: Content typecheck and 273 focused tests + passed, including exact imported Builder binding, sparse-position + duplicate projections, and concurrent exact created-item responses + blockers: + - >- + The isolated local runtime has no Builder vault credential, so the exact + Agent Native Test Blog model identity, attach/hydration, and progressive + review budgets cannot yet be accepted against the provider fixture. + - >- + Independent human QA preflight was blocked before H1-H5 because the tester + host did not expose the in-app browser; no frozen-test action was executed. + Exact-head PR prose, review, and CI must refresh after the repair commit. + cleanup-evidence: + - >- + The first file-only cleanup was audited and rejected because the restored + database still contained every marker row. The exact fixture was then + recovered and cleaned through Content actions: all three database roots + were soft-deleted, permanently deleted by exact document ID, and proved + absent through list/search/not-found action reads plus corroborating SQL. + - >- + After action-level marker-absence proof, the stopped marker-free runtime + and regenerated task-local Content env file were moved to macOS Trash and + their original paths were verified absent; ambiguous older Trash env files + were preserved untouched. + - >- + Removed the environment-gated server phase trace, source snapshot trace, + response trace fields, and correlated browser console timing after the + final joined replay; no temporary Content performance probe remains. + last-land-packet: + observed-at: 2026-07-30T04:13:17Z + artifact-revision: land-audit-blocked-r8 + passed: >- + deterministic 584/1000/5000-row browser budgets; Content typecheck and + 273 focused tests; exact imported-row and identity-projection repairs; + independent no-P1/P2 repair review; 28 exact-head CI checks with zero + failures at observation time; action-level fixture marker-absence proof + missing-acceptance-evidence: + - >- + exact isolated Builder model identity, attach, hydration, property, + row, and progressive-review budgets on a credentialed runtime + - >- + frozen H1-H5 independent human QA on a host exposing the in-app browser + - >- + two CI jobs were still pending at the observation time, and the + external Review Agent skipped the repair rerun + feature-flags: >- + none; the frozen system-ready strategy has no code-ready-only or off-state + merge fallback + repository-governance: not-satisfied + independent-technical-review: evidenced + acceptance-story: not-satisfied + merge-permitted: false + enablement-permitted: false + may-call-shipped: false + task-may-close: false + readiness: blocked +status: active +``` + +Frozen-five status: **frozen at `shape-blog-table-latency-r5`**. + +Output: `/Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md` + +Authority: Land under `land-blog-table-latency-r8`. Bare Land carries the frozen +PR merge action, but the persisted no-go packet prohibits exercising it. No +Builder record, branch operation, deployment, merge, cleanup, or archival was +performed by Land. + +Invalidated by: a change to the approved outcome/budgets, named Agent Native Content shipping surface or constituency, action/SQL/progressive-provider architecture, acceptance story, or system-ready risk strategy. + +Task attention: `autonomous`; the connected PR preview and in-app browser are +available for the four-failure Work loop. + +Next: deploy the exact Work artifact to PR #2522, run correlated exact-provider +acceptance, remove temporary probes, and replay the clean artifact before Land. + +## Work result — 2026-08-01 + +The credentialed local replay now satisfies all four frozen performance budgets +against the read-only `Agent Native Blog Article Test` Builder collection: + +- first useful rows after Attach: **88 ms** (target ≤2 s) +- complete 584-row metadata acknowledgment: **8.66 s** (target ≤10 s) +- terminal body synchronization: **30.20 s** (target ≤60 s; 582 hydrated, + two genuinely bodyless rows resolved as unavailable, no leased repair jobs) +- cold Date sort acknowledgment: **434 ms** (target ≤1 s; 100 of 584 rows + retained and sorted result painted) + +The final clean artifact contains no temporary timing probes. Content typecheck, +102 focused database/cache tests, the full 2,003-test suite (including three +expected failures and five skips), and the production build pass locally. Exact +PR-preview deployment and independent hosted H1-H5 acceptance remain the next +Land gates; these local results do not substitute for them. diff --git a/templates/content/actions/_batch-utils.test.ts b/templates/content/actions/_batch-utils.test.ts new file mode 100644 index 0000000000..c46b20b412 --- /dev/null +++ b/templates/content/actions/_batch-utils.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; + +import { processWithConcurrency } from "./_batch-utils.js"; + +describe("processWithConcurrency", () => { + it("starts the next item as soon as one worker becomes available", async () => { + const releases = new Map void>(); + const started: number[] = []; + const run = processWithConcurrency([1, 2, 3, 4], 2, async (item) => { + started.push(item); + await new Promise((resolve) => releases.set(item, resolve)); + }); + + await expect.poll(() => started).toEqual([1, 2]); + releases.get(1)?.(); + await expect.poll(() => started).toEqual([1, 2, 3]); + releases.get(3)?.(); + await expect.poll(() => started).toEqual([1, 2, 3, 4]); + releases.get(2)?.(); + releases.get(4)?.(); + + await run; + }); +}); diff --git a/templates/content/actions/_batch-utils.ts b/templates/content/actions/_batch-utils.ts index f4ff5ded55..fde23f11ae 100644 --- a/templates/content/actions/_batch-utils.ts +++ b/templates/content/actions/_batch-utils.ts @@ -1,3 +1,5 @@ +import { getDialect, type Dialect } from "@agent-native/core/db"; + export function chunks(items: T[], size: number): T[][] { if (items.length === 0) return []; const out: T[][] = []; @@ -6,3 +8,34 @@ export function chunks(items: T[], size: number): T[][] { } return out; } + +export function bulkChunkSizeForColumnCount( + columnCount: number, + dialect: Dialect = getDialect(), +) { + // D1 rejects statements with more than 100 bound params, so derive every + // bulk chunk from the statement's column count instead of a fixed row count. + const budget = dialect === "d1" ? 90 : 900; + return Math.max(1, Math.floor(budget / Math.max(1, columnCount))); +} + +export async function processWithConcurrency( + items: readonly T[], + concurrency: number, + worker: (item: T) => Promise, +) { + let nextIndex = 0; + const workerCount = Math.min( + items.length, + Math.max(1, Math.floor(concurrency)), + ); + await Promise.all( + Array.from({ length: workerCount }, async () => { + while (nextIndex < items.length) { + const item = items[nextIndex]; + nextIndex += 1; + if (item !== undefined) await worker(item); + } + }), + ); +} diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index f6bb60f401..4663110853 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -428,6 +428,7 @@ describe("Builder CMS read client", () => { "data.customModelField", "data.Status", "data.status", + "data.optionalField", "data.blocks", ], fetchImpl: fetchImpl as unknown as typeof fetch, @@ -448,12 +449,93 @@ describe("Builder CMS read client", () => { "data.customModelField": "Preserved", "data.Status": "Editorial", "data.status": "published", + "data.optionalField": null, }, }, ], }); }); + it("allows the read-only attach preview to use Builder's cached projection", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + expect(input.searchParams.get("noCache")).toBeNull(); + expect(input.searchParams.get("cachebust")).toBeNull(); + expect(input.searchParams.get("includeUnpublished")).toBe("true"); + expect(input.searchParams.get("fields")).toContain("data.title"); + return new Response( + JSON.stringify({ + results: [ + { + id: "builder-preview-entry", + data: { title: "Cached preview" }, + }, + ], + }), + { status: 200 }, + ); + }); + + await expect( + readBuilderCmsContentEntries({ + model: "blog_article", + fieldPaths: ["data.title"], + allowCached: true, + maxPages: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toMatchObject({ + state: "live", + entries: [{ id: "builder-preview-entry", title: "Cached preview" }], + progress: { partial: false }, + }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("can project Builder bodies in paged list reads for bulk hydration", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + expect(input.searchParams.get("enrich")).toBe("true"); + expect(input.searchParams.get("fields")?.split(",")).toEqual( + expect.arrayContaining([ + "data.title", + "data.blocks", + "data.blocksString", + ]), + ); + return new Response( + JSON.stringify({ + results: [ + { + id: "builder-entry-with-body", + data: { + title: "Builder body", + blocks: [{ id: "block-1" }], + }, + }, + ], + }), + { status: 200 }, + ); + }); + + const result = await readBuilderCmsContentEntries({ + model: "blog-article", + includeBodies: true, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(result.entries[0]?.rawEntry?.data).toMatchObject({ + blocks: [{ id: "block-1" }], + }); + }); + it("performs authenticated, cachebusted raw fidelity reads without projection or enrichment", async () => { process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; resolveBuilderCredentialMock.mockImplementation(async (key) => { @@ -944,7 +1026,7 @@ describe("Builder CMS read client", () => { }); expect(requests.map((request) => request.offset)).toEqual([ - 0, 100, 200, 300, 400, 500, + 0, 100, 200, 300, 400, 500, 600, 700, 800, ]); expect(requests.every((request) => request.limit > 0)).toBe(true); expect(result.entries).toHaveLength(597); @@ -956,6 +1038,167 @@ describe("Builder CMS read client", () => { }); }); + it("reads projected Content API pages in bounded parallel windows and preserves offset order", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + let activeRequests = 0; + let maxActiveRequests = 0; + const resolvers = new Map void>(); + const fetchImpl = vi.fn(async (input: URL) => { + const offset = Number(input.searchParams.get("offset")); + activeRequests += 1; + maxActiveRequests = Math.max(maxActiveRequests, activeRequests); + return await new Promise((resolve) => { + resolvers.set(offset, () => { + resolvers.delete(offset); + activeRequests -= 1; + resolve( + new Response( + JSON.stringify({ + results: Array.from({ length: 100 }, (_, index) => ({ + id: `builder-entry-${offset + index + 1}`, + data: { title: `Builder title ${offset + index + 1}` }, + })), + }), + { status: 200 }, + ), + ); + }); + }); + }); + + const read = readBuilderCmsContentEntries({ + model: "blog_article", + limit: 900, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await vi.waitFor(() => expect(resolvers.size).toBe(1)); + resolvers.get(0)?.(); + await vi.waitFor(() => expect(resolvers.size).toBe(8)); + expect(maxActiveRequests).toBe(8); + for (const offset of [800, 700, 600, 500, 400, 300, 200, 100]) { + resolvers.get(offset)?.(); + } + + const result = await read; + + expect(result.entries.map((entry) => entry.id)).toEqual( + Array.from({ length: 900 }, (_, index) => `builder-entry-${index + 1}`), + ); + expect(maxActiveRequests).toBeLessThanOrEqual(8); + }); + + it("stops at the first short projected page and ignores later in-flight pages", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const requestedOffsets: number[] = []; + const fetchImpl = vi.fn(async (input: URL) => { + const offset = Number(input.searchParams.get("offset")); + requestedOffsets.push(offset); + const resultCount = offset === 100 ? 10 : 100; + return new Response( + JSON.stringify({ + results: Array.from({ length: resultCount }, (_, index) => ({ + id: `builder-entry-${offset + index + 1}`, + data: { title: `Builder title ${offset + index + 1}` }, + })), + }), + { status: 200 }, + ); + }); + + const result = await readBuilderCmsContentEntries({ + model: "blog_article", + limit: 400, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(requestedOffsets).toEqual([0, 100, 200, 300]); + expect(result.entries.map((entry) => entry.id)).toEqual( + Array.from({ length: 110 }, (_, index) => `builder-entry-${index + 1}`), + ); + expect(result.progress).toMatchObject({ + nextOffset: 110, + fetchedEntryCount: 110, + hasMore: false, + partial: false, + }); + }); + + it("keeps stable-ID deduplication across projected Content API windows", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + const offset = Number(input.searchParams.get("offset")); + const ids = + offset === 0 + ? Array.from({ length: 100 }, (_, index) => index + 1) + : offset === 100 + ? Array.from({ length: 100 }, (_, index) => index + 51) + : Array.from({ length: 50 }, (_, index) => index + 151); + return new Response( + JSON.stringify({ + results: ids.map((id) => ({ + id: `builder-entry-${id}`, + data: { title: `Builder title ${id}` }, + })), + }), + { status: 200 }, + ); + }); + + const result = await readBuilderCmsContentEntries({ + model: "blog_article", + limit: 200, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(result.entries.map((entry) => entry.id)).toEqual( + Array.from({ length: 200 }, (_, index) => `builder-entry-${index + 1}`), + ); + }); + + it("returns a typed error when a required projected page fails", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + if (input.searchParams.get("offset") === "100") { + return new Response("bad request", { status: 400 }); + } + return new Response( + JSON.stringify({ + results: Array.from({ length: 100 }, (_, index) => ({ + id: `builder-entry-${index + 1}`, + data: { title: `Builder title ${index + 1}` }, + })), + }), + { status: 200 }, + ); + }); + + await expect( + readBuilderCmsContentEntries({ + model: "blog_article", + limit: 400, + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toMatchObject({ + state: "error", + entries: [], + message: "Builder CMS read failed with HTTP 400.", + progress: { readMode: "builder-api" }, + }); + }); + it("continues a 597-entry MCP source from offset 500", async () => { resolveBuilderCredentialMock.mockImplementation(async (key) => key === "BUILDER_PRIVATE_KEY" ? "private-key" : null, diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index 04287d0c08..155524820d 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -163,6 +163,7 @@ type BuilderMcpToolResult = { const BUILDER_CMS_DEFAULT_READ_LIMIT = 500; const BUILDER_CMS_MAX_READ_LIMIT = 10_000; const BUILDER_CMS_PAGE_SIZE = 100; +const BUILDER_CMS_PROJECTED_READ_CONCURRENCY = 8; const BUILDER_CMS_READ_RETRIES = 2; const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [ "id", @@ -192,6 +193,10 @@ const BUILDER_CMS_HEAVY_BODY_FIELD_PATHS = [ const BUILDER_CMS_FIELD_PATH_PATTERN = /^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/; +type BuilderCmsContentPageResult = + | { pageLimit: number; pageEntries: BuilderCmsSourceEntry[] } + | { pageLimit: number; error: string }; + function normalizeBuilderCmsListFieldPath(fieldPath: string) { const trimmed = fieldPath.trim(); if (!trimmed || !BUILDER_CMS_FIELD_PATH_PATTERN.test(trimmed)) return null; @@ -218,6 +223,36 @@ function normalizeBuilderCmsListFieldPath(fieldPath: string) { return normalized; } +function preserveProjectedBuilderFieldAbsence( + read: BuilderCmsReadResult, + fieldPaths: readonly string[] | undefined, + rawData: boolean | undefined, +): BuilderCmsReadResult { + if (read.state !== "live" || rawData === true || !fieldPaths?.length) { + return read; + } + const projectedFieldPaths = [ + ...new Set( + fieldPaths + .map(normalizeBuilderCmsListFieldPath) + .filter((fieldPath): fieldPath is string => fieldPath !== null), + ), + ]; + if (projectedFieldPaths.length === 0) return read; + return { + ...read, + entries: read.entries.map((entry) => { + const sourceValues = { ...entry.sourceValues }; + for (const fieldPath of projectedFieldPaths) { + if (!Object.prototype.hasOwnProperty.call(sourceValues, fieldPath)) { + sourceValues[fieldPath] = null; + } + } + return { ...entry, sourceValues }; + }), + }; +} + export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) { const fields = new Map(); for (const fieldPath of [ @@ -752,6 +787,7 @@ async function initializeBuilderMcp(args: { async function readBuilderCmsContentEntriesViaMcp(args: { model: string; fieldPaths?: readonly string[]; + includeBodies?: boolean; rawData?: boolean; limit?: number; maxPages?: number; @@ -775,7 +811,9 @@ async function readBuilderCmsContentEntriesViaMcp(args: { const contentEntries: BuilderCmsSourceEntry[] = []; const fields = args.rawData ? undefined - : builderCmsListEntryFields(args.fieldPaths); + : args.includeBodies + ? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}` + : builderCmsListEntryFields(args.fieldPaths); const seenContentIds = new Set(); let pagesRead = 0; let hasMore = false; @@ -941,6 +979,8 @@ async function readBuilderCmsContentEntriesViaMcp(args: { async function readBuilderCmsContentEntriesViaContentApi(args: { model: string; fieldPaths?: readonly string[]; + includeBodies?: boolean; + allowCached?: boolean; rawData?: boolean; limit?: number; maxPages?: number; @@ -959,14 +999,21 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { // Fidelity reads intentionally do neither: Builder's raw data object is the // clone contract, including unresolved references and every nested field. url.searchParams.set("enrich", args.rawData === true ? "false" : "true"); - url.searchParams.set("noCache", "true"); - url.searchParams.set( - "cachebust", - args.rawData === true ? String(Date.now()) : "true", - ); + if (args.allowCached !== true || args.rawData === true) { + url.searchParams.set("noCache", "true"); + url.searchParams.set( + "cachebust", + args.rawData === true ? String(Date.now()) : "true", + ); + } url.searchParams.set("includeUnpublished", "true"); if (args.rawData !== true) { - url.searchParams.set("fields", builderCmsListEntryFields(args.fieldPaths)); + url.searchParams.set( + "fields", + args.includeBodies + ? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}` + : builderCmsListEntryFields(args.fieldPaths), + ); } const limit = readLimit(args.limit); @@ -978,73 +1025,103 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { const seenIds = new Set(); let pagesRead = 0; let hasMore = false; - for ( - let offset = startOffset; - entries.length < limit; - offset += BUILDER_CMS_PAGE_SIZE - ) { - const pageUrl = new URL(url); - const pageLimit = readPageLimit(limit - entries.length); - pageUrl.searchParams.set("limit", String(pageLimit)); - pageUrl.searchParams.set("offset", String(offset)); + const concurrency = + args.rawData === true ? 1 : BUILDER_CMS_PROJECTED_READ_CONCURRENCY; + const errorResult = (message: string): BuilderCmsReadResult => ({ + state: "error", + entries: [], + fetchedAt, + message, + progress: { + requestedLimit: limit, + pageSize: BUILDER_CMS_PAGE_SIZE, + startOffset, + nextOffset: startOffset + entries.length, + fetchedEntryCount: startOffset + entries.length, + hasMore, + partial: Boolean(args.maxPages) && hasMore, + readMode: "builder-api", + }, + }); - let response: Response; - try { - response = await fetchBuilderContentPage({ - fetchImpl: args.fetchImpl, - url: pageUrl, - privateKey: args.privateKey, - }); - } catch (error) { - return { - state: "error", - entries: [], - fetchedAt, - message: - error instanceof Error - ? `Builder CMS read failed: ${error.message}` - : "Builder CMS read failed.", - progress: { - requestedLimit: limit, - pageSize: BUILDER_CMS_PAGE_SIZE, - startOffset, - nextOffset: startOffset + entries.length, - fetchedEntryCount: startOffset + entries.length, - hasMore, - partial: Boolean(args.maxPages) && hasMore, - readMode: "builder-api", + while (entries.length < limit) { + const pagesRemaining = args.maxPages + ? args.maxPages - pagesRead + : Number.POSITIVE_INFINITY; + if (pagesRemaining <= 0) break; + const pageCount = Math.min( + pagesRead === 0 ? 1 : concurrency, + pagesRemaining, + Math.ceil((limit - entries.length) / BUILDER_CMS_PAGE_SIZE), + ); + const pageRequests = Array.from({ length: pageCount }, (_, index) => { + const pageUrl = new URL(url); + const pageLimit = readPageLimit( + limit - entries.length - index * BUILDER_CMS_PAGE_SIZE, + ); + pageUrl.searchParams.set("limit", String(pageLimit)); + pageUrl.searchParams.set( + "offset", + String(startOffset + (pagesRead + index) * BUILDER_CMS_PAGE_SIZE), + ); + return { pageLimit, pageUrl }; + }); + const pageResults = await Promise.all( + pageRequests.map( + async ({ + pageLimit, + pageUrl, + }): Promise => { + try { + const response = await fetchBuilderContentPage({ + fetchImpl: args.fetchImpl, + url: pageUrl, + privateKey: args.privateKey, + }); + if (!response.ok) { + return { + pageLimit, + error: `Builder CMS read failed with HTTP ${response.status}.`, + }; + } + const json = (await response.json()) as unknown; + const pageEntries = entryArrayFromResponse(json) + .map((entry) => normalizeBuilderCmsApiEntry(entry, args.model)) + .filter((entry): entry is BuilderCmsSourceEntry => + Boolean(entry), + ); + return { pageLimit, pageEntries }; + } catch (error) { + return { + pageLimit, + error: + error instanceof Error + ? `Builder CMS read failed: ${error.message}` + : "Builder CMS read failed.", + }; + } }, - }; - } + ), + ); - if (!response.ok) { - return { - state: "error", - entries: [], - fetchedAt, - message: `Builder CMS read failed with HTTP ${response.status}.`, - progress: { - requestedLimit: limit, - pageSize: BUILDER_CMS_PAGE_SIZE, - startOffset, - nextOffset: startOffset + entries.length, - fetchedEntryCount: startOffset + entries.length, - hasMore, - partial: Boolean(args.maxPages) && hasMore, - readMode: "builder-api", - }, - }; + let stoppedOnShortPage = false; + for (const pageResult of pageResults) { + if ("error" in pageResult) return errorResult(pageResult.error); + const appended = appendUniqueBuilderEntries( + entries, + seenIds, + pageResult.pageEntries, + ); + pagesRead += 1; + hasMore = + pageResult.pageEntries.length >= pageResult.pageLimit && appended > 0; + if (args.maxPages && pagesRead >= args.maxPages) break; + if (!hasMore) { + stoppedOnShortPage = true; + break; + } } - - const json = (await response.json()) as unknown; - const pageEntries = entryArrayFromResponse(json) - .map((entry) => normalizeBuilderCmsApiEntry(entry, args.model)) - .filter((entry): entry is BuilderCmsSourceEntry => Boolean(entry)); - const appended = appendUniqueBuilderEntries(entries, seenIds, pageEntries); - pagesRead += 1; - hasMore = pageEntries.length >= pageLimit && appended > 0; - if (args.maxPages && pagesRead >= args.maxPages) break; - if (!hasMore) break; + if (stoppedOnShortPage || !hasMore) break; } return { @@ -1231,6 +1308,8 @@ export async function readBuilderCmsModelFields(args: { export async function readBuilderCmsContentEntries(args: { model: string; fieldPaths?: readonly string[]; + includeBodies?: boolean; + allowCached?: boolean; rawData?: boolean; requirePrivateKey?: boolean; limit?: number; @@ -1265,6 +1344,8 @@ export async function readBuilderCmsContentEntries(args: { const contentApiRead = await readBuilderCmsContentEntriesViaContentApi({ model: args.model, fieldPaths: args.fieldPaths, + includeBodies: args.includeBodies, + allowCached: args.allowCached, rawData: args.rawData, limit: args.limit, maxPages: args.maxPages, @@ -1273,22 +1354,33 @@ export async function readBuilderCmsContentEntries(args: { publicKey, privateKey: privateKey ?? undefined, }); - if (contentApiRead.state === "live") return contentApiRead; + if (contentApiRead.state === "live") { + return preserveProjectedBuilderFieldAbsence( + contentApiRead, + args.fieldPaths, + args.rawData, + ); + } if (!privateKey) return contentApiRead; } if (privateKey) { try { - return await readBuilderCmsContentEntriesViaMcp({ - model: args.model, - fieldPaths: args.fieldPaths, - rawData: args.rawData, - limit: args.limit, - maxPages: args.maxPages, - offset: args.offset, - fetchImpl, - privateKey, - }); + return preserveProjectedBuilderFieldAbsence( + await readBuilderCmsContentEntriesViaMcp({ + model: args.model, + fieldPaths: args.fieldPaths, + includeBodies: args.includeBodies, + rawData: args.rawData, + limit: args.limit, + maxPages: args.maxPages, + offset: args.offset, + fetchImpl, + privateKey, + }), + args.fieldPaths, + args.rawData, + ); } catch (error) { return { state: "error", diff --git a/templates/content/actions/_content-files.ts b/templates/content/actions/_content-files.ts index a82e0fbb42..68d43fbd1d 100644 --- a/templates/content/actions/_content-files.ts +++ b/templates/content/actions/_content-files.ts @@ -4,6 +4,7 @@ import { accessFilter } from "@agent-native/core/sharing"; import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { schema } from "../server/db/index.js"; +import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js"; import { listContentOrganizationMemberships, normalizeContentSpaceEmail, @@ -32,23 +33,29 @@ export function contentFilesItemId(databaseId: string, documentId: string) { async function remapItemReferences(db: Db, replacements: Map) { if (!replacements.size) return; - const duplicateIds = [...replacements.keys()]; - const remap = (column: any) => { - let expression = sql`CASE ${column}`; - for (const [duplicateId, canonicalId] of replacements) - expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`; - return sql`${expression} ELSE ${column} END`; - }; - for (const table of [ - schema.contentSpaceCatalogItems, - schema.contentDatabaseBodyHydrationQueue, - schema.contentDatabaseSourceRows, - schema.contentDatabaseSourceChangeSets, - ]) { - await db - .update(table) - .set({ databaseItemId: remap(table.databaseItemId) }) - .where(inArray(table.databaseItemId, duplicateIds)); + for (const replacementBatch of chunks( + [...replacements.entries()], + bulkChunkSizeForColumnCount(3), + )) { + const batchReplacements = new Map(replacementBatch); + const duplicateIds = [...batchReplacements.keys()]; + const remap = (column: any) => { + let expression = sql`CASE ${column}`; + for (const [duplicateId, canonicalId] of batchReplacements) + expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`; + return sql`${expression} ELSE ${column} END`; + }; + for (const table of [ + schema.contentSpaceCatalogItems, + schema.contentDatabaseBodyHydrationQueue, + schema.contentDatabaseSourceRows, + schema.contentDatabaseSourceChangeSets, + ]) { + await db + .update(table) + .set({ databaseItemId: remap(table.databaseItemId) }) + .where(inArray(table.databaseItemId, duplicateIds)); + } } } @@ -170,15 +177,18 @@ async function reconcileDocuments(args: { } } await remapItemReferences(args.db, replacements); - if (deleteIds.size) { + for (const deleteBatch of chunks( + [...deleteIds], + bulkChunkSizeForColumnCount(1), + )) { await args.db .delete(schema.contentDatabaseItems) - .where(inArray(schema.contentDatabaseItems.id, [...deleteIds])); + .where(inArray(schema.contentDatabaseItems.id, deleteBatch)); } - if (inserts.length) { + for (const insertBatch of chunks(inserts, bulkChunkSizeForColumnCount(8))) { await args.db .insert(schema.contentDatabaseItems) - .values(inserts) + .values(insertBatch) .onConflictDoNothing(); } return { inserted: inserts.length, removed: deleteIds.size }; diff --git a/templates/content/actions/_database-source-utils.test.ts b/templates/content/actions/_database-source-utils.test.ts index 738397bcbe..ff68253d36 100644 --- a/templates/content/actions/_database-source-utils.test.ts +++ b/templates/content/actions/_database-source-utils.test.ts @@ -33,6 +33,7 @@ import { builderBodyHydrationCanAdoptSameVersionVariant, builderBodyBaselineHasSameVersionConflict, builderAuthoritativeRawBodyHash, + builderBodyHydrationBulkChunkLimit, bulkChunkSizeForColumnCount, builderCmsEntryAlreadyRepresented, builderCmsSourceContinuationIsCurrent, @@ -51,6 +52,7 @@ import { serializeSourceMetadataRecord, sourceSnapshotValuesJsonProjectionSql, sourceSnapshotDocumentSelection, + sourceSnapshotPageDocumentIds, sourceValuesForSnapshot, sourceValuesForSeededSourceRow, sourceChangeSetKey, @@ -103,6 +105,52 @@ function item(id: string, title: string): ContentDatabaseItem { } describe("database source helpers", () => { + it("page-scopes primary Builder rows without truncating secondary federation", () => { + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "builder-cms", + metadataJson: "{}", + documentIds: ["doc-1", "doc-2"], + }), + ).toEqual(["doc-1", "doc-2"]); + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "builder-cms", + metadataJson: "{}", + documentIds: [], + }), + ).toEqual([]); + + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "builder-cms", + metadataJson: JSON.stringify({ + federation: { + role: "secondary", + keyField: "slug", + normalizationFormula: "lower(trim(value))", + join: { + kind: "identity", + collection: null, + localExpr: "{Slug}", + remoteKeyField: "slug", + normalizationFormula: "lower(trim(value))", + }, + }, + }), + documentIds: ["doc-1"], + }), + ).toBeUndefined(); + + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "local-table", + metadataJson: "{}", + documentIds: ["doc-1"], + }), + ).toBeUndefined(); + }); + it("keeps the provider-bound property when duplicate local labels collide", () => { const existingFields = [ { @@ -246,6 +294,12 @@ describe("database source helpers", () => { expect(bulkChunkSizeForColumnCount(15, "postgres")).toBe(60); }); + it("uses one fewer transaction for the 584-row Postgres hydration case", () => { + expect(builderBodyHydrationBulkChunkLimit("postgres")).toBe(200); + expect(builderBodyHydrationBulkChunkLimit("sqlite")).toBe(112); + expect(builderBodyHydrationBulkChunkLimit("d1")).toBe(11); + }); + it("serializes queued Builder body hydration with an unset item status as pending", () => { expect( serializeBodyHydration( diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 08b3bdd4d3..42ef43a29a 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { getDialect, type Dialect } from "@agent-native/core/db"; -import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, lt, or, sql } from "drizzle-orm"; import { getDb, schema } from "../server/db/index.js"; import type { @@ -55,7 +55,12 @@ import { type DocumentPropertyOptionColor, } from "../shared/properties.js"; import { sanitizeNormalizationFormula } from "../shared/properties.js"; -import { chunks } from "./_batch-utils.js"; +import { + bulkChunkSizeForColumnCount, + chunks, + processWithConcurrency, +} from "./_batch-utils.js"; +export { bulkChunkSizeForColumnCount } from "./_batch-utils.js"; import { readBuilderCmsContentEntry, readBuilderCmsContentEntries, @@ -123,6 +128,24 @@ function stableBuilderImportId( .slice(0, 24)}`; } +export function builderCmsImportIds(args: { + ownerEmail: string; + databaseId: string; + sourceTable: string; + entryId: string; +}) { + const identity = [ + args.ownerEmail, + args.databaseId, + args.sourceTable, + args.entryId, + ]; + return { + documentId: stableBuilderImportId("builder-doc", identity), + itemId: stableBuilderImportId("builder-item", identity), + }; +} + const DEFAULT_SOURCE_CAPABILITIES: ContentDatabaseSourceCapabilities = { canRefresh: true, canCreateChangeSets: true, @@ -958,38 +981,22 @@ function builderBodyUsesCurrentMediaConverter( const BUILDER_BODY_HYDRATION_BACKGROUND_PRIORITY = 10; const BUILDER_BODY_HYDRATION_OPEN_PRIORITY = 0; -const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 25; -const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 6; +const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 600; +const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 72; +const BUILDER_BODY_HYDRATION_BULK_PRELOAD_MIN_JOBS = 20; const BUILDER_BODY_HYDRATION_MAX_ATTEMPTS = 5; +const BUILDER_BODY_HYDRATION_CLAIM_LEASE_MS = 2 * 60 * 1000; +const BUILDER_BODY_HYDRATION_POSTGRES_BULK_LIMIT = 200; +const BUILDER_BODY_HYDRATION_MAX_BOUND_PARAMS_PER_ROW = 8; const BUILDER_BODY_HYDRATION_CODEC_VERSION = "readable-native-images-authoritative-raw-baseline-v9"; const BUILDER_CMS_REFRESH_INITIAL_PAGES = 1; const BUILDER_BODY_NOT_AVAILABLE_ERROR = "body not yet available from Builder"; -export function bulkChunkSizeForColumnCount( - columnCount: number, - dialect: Dialect = getDialect(), -) { - // D1 rejects statements with more than 100 bound params, so derive every - // bulk chunk from the statement's column count instead of a fixed row count. - const budget = dialect === "d1" ? 90 : 900; - return Math.max(1, Math.floor(budget / Math.max(1, columnCount))); -} - function idChunkSize() { return bulkChunkSizeForColumnCount(1); } -async function processInBatches( - items: T[], - concurrency: number, - worker: (item: T) => Promise, -) { - for (const batch of chunks(items, concurrency)) { - await Promise.all(batch.map((item) => worker(item))); - } -} - function builderBodyHydrationDelayMs() { if ( process.env.NODE_ENV === "production" || @@ -1081,7 +1088,10 @@ function normalizeHydrationLimit(limit: number | null | undefined) { if (typeof limit !== "number" || !Number.isFinite(limit)) { return BUILDER_BODY_HYDRATION_BATCH_LIMIT; } - return Math.max(1, Math.min(Math.floor(limit), 50)); + return Math.max( + 1, + Math.min(Math.floor(limit), BUILDER_BODY_HYDRATION_BATCH_LIMIT), + ); } function builderBodyIsRawPlaceholderOnly(content: string | null | undefined) { @@ -1531,6 +1541,7 @@ export async function enqueueBuilderBodyHydrationForItems(args: { items: ContentDatabaseItem[]; builderEntriesByDocumentId: Map | undefined; now: string; + processInBackground?: boolean; }) { if (!args.builderEntriesByDocumentId?.size) return; const persistedStateByDocumentId = new Map< @@ -1626,10 +1637,12 @@ export async function enqueueBuilderBodyHydrationForItems(args: { }); } const queuedJobs = await enqueueBuilderBodyHydrations(requests); + if (args.processInBackground === false) return; void processBuilderBodyHydrationQueue({ sourceId: args.sourceId, limit: BUILDER_BODY_HYDRATION_BATCH_LIMIT, preloadedJobs: queuedJobs, + preloadBodies: true, }).catch((error) => { console.error("Builder body hydration background kick failed", error); }); @@ -1732,6 +1745,7 @@ async function processBuilderBodyHydrationJob( sourceRow?: ContentDatabaseSourceRecordRowDb | null; documentContent?: string | null; bodyHydrationVersion?: string | null; + bodyEntry?: BuilderCmsSourceEntry | null; }, ) { const db = getDb(); @@ -1741,9 +1755,19 @@ async function processBuilderBodyHydrationJob( entry.sourceValues, BUILDER_CMS_BODY_BLOCKS_HASH_KEY, ); + const bodyEntry = + preloaded?.bodyEntry?.id === entry.id + ? { + ...preloaded.bodyEntry, + sourceValues: { + ...entry.sourceValues, + ...preloaded.bodyEntry.sourceValues, + }, + } + : entry; let activeSourceEntryJson = row.sourceEntryJson; let entryWithBody = await refreshBuilderBodySourceValuesFromStoredLossless( - await withBuilderBodySourceValues(entry), + await withBuilderBodySourceValues(bodyEntry), ); const sourceRow = preloaded?.sourceRow != null @@ -1873,6 +1897,10 @@ async function processBuilderBodyHydrationJob( schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, activeSourceEntryJson, ), + eq( + schema.contentDatabaseBodyHydrationQueue.attempts, + row.attempts, + ), ), ) .returning({ id: schema.contentDatabaseBodyHydrationQueue.id }); @@ -1912,6 +1940,10 @@ async function processBuilderBodyHydrationJob( schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, activeSourceEntryJson, ), + eq( + schema.contentDatabaseBodyHydrationQueue.attempts, + row.attempts, + ), ), ) .returning({ id: schema.contentDatabaseBodyHydrationQueue.id }); @@ -1926,12 +1958,9 @@ async function processBuilderBodyHydrationJob( if (!nextContent.trim()) { const attempts = row.attempts; await db.transaction(async (tx) => { - const queueRowCas = and( - eq(schema.contentDatabaseBodyHydrationQueue.id, row.id), - eq( - schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, - activeSourceEntryJson, - ), + const queueRowCas = builderBodyHydrationQueueOwnershipFilter( + row, + activeSourceEntryJson, ); const markPendingIfReplaced = async () => { const [replacedByNewerJob] = await tx @@ -1974,6 +2003,7 @@ async function processBuilderBodyHydrationJob( const [stillOwnsQueueRow] = await tx .update(schema.contentDatabaseBodyHydrationQueue) .set({ + lastAttemptedAt: null, lastError: BUILDER_BODY_NOT_AVAILABLE_ERROR, updatedAt: now, }) @@ -2021,12 +2051,9 @@ async function processBuilderBodyHydrationJob( }); let wroteBody = false; await db.transaction(async (tx) => { - const queueRowCas = and( - eq(schema.contentDatabaseBodyHydrationQueue.id, row.id), - eq( - schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, - activeSourceEntryJson, - ), + const queueRowCas = builderBodyHydrationQueueOwnershipFilter( + row, + activeSourceEntryJson, ); const markPendingIfReplaced = async () => { const [replacedByNewerJob] = await tx @@ -2185,6 +2212,276 @@ async function processBuilderBodyHydrationJob( // repersist the empty state over the newly hydrated Builder body. } +type PreparedPristineBuilderBodyHydration = { + job: ContentDatabaseBodyHydrationQueueRowDb; + sourceRow: ContentDatabaseSourceRecordRowDb; + documentContent: string; + content: string; + sourceValuesJson: string; + lastSourceUpdatedAt: string; + bodyHydrationVersion: string; +}; + +class PristineBuilderBodyHydrationCasMiss extends Error {} + +function hydrationCaseSql( + idColumn: unknown, + fallbackColumn: unknown, + values: Array<{ id: string; value: T }>, +) { + const cases = values.map(({ id, value }) => sql`WHEN ${id} THEN ${value}`); + return sql`CASE ${idColumn} ${sql.join(cases, sql` `)} ELSE ${fallbackColumn} END`; +} + +function builderBodyHydrationQueueOwnershipFilter( + job: ContentDatabaseBodyHydrationQueueRowDb, + sourceEntryJson = job.sourceEntryJson, +) { + return and( + eq(schema.contentDatabaseBodyHydrationQueue.id, job.id), + eq( + schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, + sourceEntryJson, + ), + eq(schema.contentDatabaseBodyHydrationQueue.attempts, job.attempts), + ); +} + +async function preparePristineBuilderBodyHydration(args: { + job: ContentDatabaseBodyHydrationQueueRowDb; + sourceRow: ContentDatabaseSourceRecordRowDb | null; + documentContent: string | null | undefined; + bodyHydrationVersion: string | null; + bodyEntry: BuilderCmsSourceEntry | null; +}): Promise { + const { job, sourceRow, documentContent, bodyHydrationVersion, bodyEntry } = + args; + if ( + job.attempts !== 1 || + !sourceRow || + !bodyEntry || + bodyHydrationVersion !== null || + !isEffectivelyEmptyDocumentContent(documentContent ?? "") || + sourceRow.sourceRowId !== job.sourceRowId || + bodyEntry.id !== job.sourceRowId + ) { + return null; + } + const queuedEntry = parseHydrationEntry(job); + if (!queuedEntry || queuedEntry.id !== bodyEntry.id) return null; + const sourceValues = + parseObject>( + sourceRow.sourceValuesJson, + ) ?? {}; + if ( + stringSourceValue(sourceValues, BUILDER_CMS_BODY_CONTENT_KEY)?.trim() || + stringSourceValue(sourceValues, BUILDER_CMS_BODY_BLOCKS_HASH_KEY) + ) { + return null; + } + const queuedLastUpdated = + stringSourceValue(queuedEntry.sourceValues, "lastUpdated") ?? + queuedEntry.updatedAt; + const bodyLastUpdated = + stringSourceValue(bodyEntry.sourceValues, "lastUpdated") ?? + bodyEntry.updatedAt; + if ( + queuedLastUpdated && + bodyLastUpdated && + queuedLastUpdated !== bodyLastUpdated + ) { + return null; + } + const entryWithBody = await refreshBuilderBodySourceValuesFromStoredLossless( + await withBuilderBodySourceValues({ + ...bodyEntry, + sourceValues: { + ...queuedEntry.sourceValues, + ...bodyEntry.sourceValues, + }, + }), + ); + const nextValues = { + ...sourceValues, + ...entryWithBody.sourceValues, + }; + const content = + stringSourceValue(nextValues, BUILDER_CMS_BODY_CONTENT_KEY) ?? ""; + if (!content.trim()) return null; + return { + job, + sourceRow, + documentContent: documentContent ?? "", + content, + sourceValuesJson: JSON.stringify(nextValues), + lastSourceUpdatedAt: entryWithBody.updatedAt ?? new Date().toISOString(), + bodyHydrationVersion: builderBodyHydrationVersion(entryWithBody), + }; +} + +async function persistPristineBuilderBodyHydrationsInBulk( + prepared: PreparedPristineBuilderBodyHydration[], + now: string, +) { + const db = getDb(); + const persistedJobIds = new Set(); + const chunkLimit = builderBodyHydrationBulkChunkLimit(); + for (const batch of chunks(prepared, chunkLimit)) { + try { + await db.transaction(async (tx) => { + const queueOwnership = batch.map(({ job }) => + builderBodyHydrationQueueOwnershipFilter(job), + ); + + const updatedDocuments = await tx + .update(schema.documents) + .set({ + content: hydrationCaseSql( + schema.documents.id, + schema.documents.content, + batch.map((row) => ({ + id: row.job.documentId, + value: row.content, + })), + ), + updatedAt: now, + }) + .where( + or( + ...batch.map((row) => + and( + eq(schema.documents.id, row.job.documentId), + eq(schema.documents.content, row.documentContent), + ), + ), + ), + ) + .returning({ id: schema.documents.id }); + if (updatedDocuments.length !== batch.length) { + throw new PristineBuilderBodyHydrationCasMiss( + "Builder body hydration document changed.", + ); + } + + const updatedSourceRows = await tx + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: hydrationCaseSql( + schema.contentDatabaseSourceRows.id, + schema.contentDatabaseSourceRows.sourceValuesJson, + batch.map((row) => ({ + id: row.sourceRow.id, + value: row.sourceValuesJson, + })), + ), + lastSyncedAt: now, + lastSourceUpdatedAt: hydrationCaseSql( + schema.contentDatabaseSourceRows.id, + schema.contentDatabaseSourceRows.lastSourceUpdatedAt, + batch.map((row) => ({ + id: row.sourceRow.id, + value: row.lastSourceUpdatedAt, + })), + ), + updatedAt: now, + }) + .where( + or( + ...batch.map((row) => + and( + eq(schema.contentDatabaseSourceRows.id, row.sourceRow.id), + eq( + schema.contentDatabaseSourceRows.sourceId, + row.job.sourceId, + ), + eq( + schema.contentDatabaseSourceRows.databaseItemId, + row.job.databaseItemId, + ), + eq( + schema.contentDatabaseSourceRows.sourceValuesJson, + row.sourceRow.sourceValuesJson, + ), + ), + ), + ), + ) + .returning({ id: schema.contentDatabaseSourceRows.id }); + if (updatedSourceRows.length !== batch.length) { + throw new PristineBuilderBodyHydrationCasMiss( + "Builder body hydration source row changed.", + ); + } + + const deletedQueueRows = await tx + .delete(schema.contentDatabaseBodyHydrationQueue) + .where(or(...queueOwnership)) + .returning({ + id: schema.contentDatabaseBodyHydrationQueue.id, + }); + if (deletedQueueRows.length !== batch.length) { + // This guarded delete is the queue-ownership CAS for the whole + // transaction; a miss rolls back the preceding document/source writes. + throw new PristineBuilderBodyHydrationCasMiss( + "Builder body hydration queue changed.", + ); + } + + const updatedItems = await tx + .update(schema.contentDatabaseItems) + .set({ + bodyHydrationStatus: "hydrated", + bodyHydrationAttemptedAt: now, + bodyHydrationError: null, + bodyHydrationVersion: hydrationCaseSql( + schema.contentDatabaseItems.id, + schema.contentDatabaseItems.bodyHydrationVersion, + batch.map((row) => ({ + id: row.job.databaseItemId, + value: row.bodyHydrationVersion, + })), + ), + updatedAt: now, + }) + .where( + and( + inArray( + schema.contentDatabaseItems.id, + batch.map((row) => row.job.databaseItemId), + ), + eq(schema.contentDatabaseItems.bodyHydrationStatus, "hydrating"), + ), + ) + .returning({ id: schema.contentDatabaseItems.id }); + if (updatedItems.length !== batch.length) { + throw new PristineBuilderBodyHydrationCasMiss( + "Builder body hydration item changed.", + ); + } + }); + for (const row of batch) persistedJobIds.add(row.job.id); + } catch (error) { + if (!(error instanceof PristineBuilderBodyHydrationCasMiss)) throw error; + // The transaction rolled the whole chunk back. The existing per-row path + // below reloads its conflict-sensitive state and preserves the safer + // edit, migration, retry, and unavailable-body semantics. + } + } + return persistedJobIds; +} + +export function builderBodyHydrationBulkChunkLimit( + dialect: Dialect = getDialect(), +) { + const portableLimit = bulkChunkSizeForColumnCount( + BUILDER_BODY_HYDRATION_MAX_BOUND_PARAMS_PER_ROW, + dialect, + ); + return dialect === "postgres" + ? Math.max(portableLimit, BUILDER_BODY_HYDRATION_POSTGRES_BULK_LIMIT) + : portableLimit; +} + async function enqueueStaleBuilderBodyHydrationForOpenDocument(args: { sourceId: string; documentId: string; @@ -2263,6 +2560,7 @@ export async function processBuilderBodyHydrationQueue(args: { documentId?: string | null; limit?: number | null; preloadedJobs?: ContentDatabaseBodyHydrationQueueRowDb[]; + preloadBodies?: boolean; }) { const db = getDb(); const limit = normalizeHydrationLimit(args.limit); @@ -2329,11 +2627,17 @@ export async function processBuilderBodyHydrationQueue(args: { }); })() : persistedJobs(limit)); + const claimLeaseCutoff = new Date( + Date.now() - BUILDER_BODY_HYDRATION_CLAIM_LEASE_MS, + ).toISOString(); let succeeded = 0; let failed = 0; const claimedJobs: ContentDatabaseBodyHydrationQueueRowDb[] = []; - for (const jobChunk of chunks(jobs, bulkChunkSizeForColumnCount(2))) { + for (const jobChunk of chunks( + jobs, + bulkChunkSizeForColumnCount(args.preloadBodies === true ? 5 : 3), + )) { const claimFilters = jobChunk.map((job) => and( eq(schema.contentDatabaseBodyHydrationQueue.id, job.id), @@ -2341,6 +2645,28 @@ export async function processBuilderBodyHydrationQueue(args: { schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, job.sourceEntryJson, ), + eq(schema.contentDatabaseBodyHydrationQueue.attempts, job.attempts), + args.preloadBodies === true + ? and( + job.lastAttemptedAt + ? eq( + schema.contentDatabaseBodyHydrationQueue.lastAttemptedAt, + job.lastAttemptedAt, + ) + : isNull( + schema.contentDatabaseBodyHydrationQueue.lastAttemptedAt, + ), + or( + isNull( + schema.contentDatabaseBodyHydrationQueue.lastAttemptedAt, + ), + lt( + schema.contentDatabaseBodyHydrationQueue.lastAttemptedAt, + claimLeaseCutoff, + ), + ), + ) + : undefined, ), ); if (claimFilters.length === 0) continue; @@ -2446,8 +2772,64 @@ export async function processBuilderBodyHydrationQueue(args: { const documentContentById = new Map( documents.map((document) => [document.id, document.content]), ); - await processInBatches( - claimedJobs, + const claimedSourceRowIds = new Set( + claimedJobs.map((job) => job.sourceRowId), + ); + const bodyEntryById = new Map(); + const bulkPreloadBodies = + args.preloadBodies === true && + claimedJobs.length >= BUILDER_BODY_HYDRATION_BULK_PRELOAD_MIN_JOBS; + if (!args.documentId && bulkPreloadBodies && claimedJobs.length > 0) { + const sourceTables = Array.from( + new Set(claimedJobs.map((job) => job.sourceTable)), + ); + const bodyReads = await Promise.all( + sourceTables.map((model) => + readBuilderCmsContentEntries({ + model, + includeBodies: true, + limit: 10_000, + }), + ), + ); + for (const read of bodyReads) { + if (read.state !== "live") continue; + for (const entry of read.entries) { + if (claimedSourceRowIds.has(entry.id)) { + bodyEntryById.set(entry.id, entry); + } + } + } + } + const preparedPristineHydrations: PreparedPristineBuilderBodyHydration[] = []; + if (!args.documentId && bulkPreloadBodies) { + await processWithConcurrency( + claimedJobs, + BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY, + async (job) => { + try { + const prepared = await preparePristineBuilderBodyHydration({ + job, + sourceRow: sourceRowsByItemId.get(job.databaseItemId) ?? null, + documentContent: documentContentById.get(job.documentId), + bodyHydrationVersion: + bodyHydrationVersionByItemId.get(job.databaseItemId) ?? null, + bodyEntry: bodyEntryById.get(job.sourceRowId) ?? null, + }); + if (prepared) preparedPristineHydrations.push(prepared); + } catch { + // The established per-row path below owns conversion and retry errors. + } + }, + ); + } + const bulkPersistedJobIds = await persistPristineBuilderBodyHydrationsInBulk( + preparedPristineHydrations, + now, + ); + succeeded += bulkPersistedJobIds.size; + await processWithConcurrency( + claimedJobs.filter((job) => !bulkPersistedJobIds.has(job.id)), BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY, async (job) => { const attemptNow = new Date().toISOString(); @@ -2458,6 +2840,7 @@ export async function processBuilderBodyHydrationQueue(args: { sourceRow: sourceRowsByItemId.get(job.databaseItemId) ?? null, bodyHydrationVersion: bodyHydrationVersionByItemId.get(job.databaseItemId) ?? null, + bodyEntry: bodyEntryById.get(job.sourceRowId) ?? null, documentContent: documentContentById.has(job.documentId) ? (documentContentById.get(job.documentId) ?? null) : undefined, @@ -2467,13 +2850,7 @@ export async function processBuilderBodyHydrationQueue(args: { failed += 1; const message = error instanceof Error ? error.message : String(error); const attempts = job.attempts; - const queueRowCas = and( - eq(schema.contentDatabaseBodyHydrationQueue.id, job.id), - eq( - schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, - job.sourceEntryJson, - ), - ); + const queueRowCas = builderBodyHydrationQueueOwnershipFilter(job); const markPendingIfReplaced = async () => { const [replacedByNewerJob] = await db .select({ id: schema.contentDatabaseBodyHydrationQueue.id }) @@ -2514,7 +2891,7 @@ export async function processBuilderBodyHydrationQueue(args: { .update(schema.contentDatabaseBodyHydrationQueue) .set({ attempts, - lastAttemptedAt: attemptNow, + lastAttemptedAt: null, lastError: message, priority: job.priority + 10, updatedAt: attemptNow, @@ -3514,6 +3891,7 @@ export async function getContentDatabaseSourceSnapshotForReview( */ export async function getAllContentDatabaseSourceSnapshots( database: ContentDatabaseRow | ContentDatabase, + options: { documentIds?: string[] } = {}, ): Promise { if ("deletedAt" in database && database.deletedAt) { throw new Error(`Database "${database.id}" not found`); @@ -3527,15 +3905,31 @@ export async function getAllContentDatabaseSourceSnapshots( asc(schema.contentDatabaseSources.createdAt), asc(schema.contentDatabaseSources.id), ); - const snapshots: ContentDatabaseSource[] = []; - for (const source of sources) { - snapshots.push( - await loadSourceSnapshot(source, database, { + return Promise.all( + sources.map((source) => { + return loadSourceSnapshot(source, database, { includeHeavyBuilderBodyValues: false, - }), - ); - } - return snapshots; + documentIds: sourceSnapshotPageDocumentIds({ + sourceType: source.sourceType, + metadataJson: source.metadataJson, + documentIds: options.documentIds, + }), + }); + }), + ); +} + +export function sourceSnapshotPageDocumentIds(args: { + sourceType: string; + metadataJson: string | null; + documentIds?: string[]; +}) { + const metadata = parseObject(args.metadataJson); + const federation = normalizeSourceFederation(metadata?.federation); + return normalizeSourceType(args.sourceType) === "builder-cms" && + federation?.role !== "secondary" + ? args.documentIds + : undefined; } async function readSourceSnapshotRowsOnce(args: { @@ -3546,9 +3940,7 @@ async function readSourceSnapshotRowsOnce(args: { documentIds?: string[]; }) { const db = getDb(); - const documentScope = args.documentIds?.length - ? new Set(args.documentIds) - : null; + const documentScope = args.documentIds ? new Set(args.documentIds) : null; const rowRows = await db .select( sourceSnapshotRowSelection({ @@ -3660,9 +4052,7 @@ async function sourceSnapshotConsistencyMarker(args: { documentIds?: string[]; }) { const db = getDb(); - const documentScope = args.documentIds?.length - ? new Set(args.documentIds) - : null; + const documentScope = args.documentIds ? new Set(args.documentIds) : null; const [rows] = await db .select({ count: sql`COUNT(*)`, @@ -3737,9 +4127,14 @@ async function loadSourceSnapshotRowsOptimistically(args: { const before = await sourceSnapshotConsistencyMarker(args); latest = await readSourceSnapshotRowsOnce(args); const after = await sourceSnapshotConsistencyMarker(args); - if (sourceSnapshotConsistencyMarkersEqual(before, after)) return latest; + if (sourceSnapshotConsistencyMarkersEqual(before, after)) { + return { ...latest, consistencyAttempts: attempt + 1 }; + } } - return latest ?? (await readSourceSnapshotRowsOnce(args)); + return { + ...(latest ?? (await readSourceSnapshotRowsOnce(args))), + consistencyAttempts: 3, + }; } async function loadSourceSnapshot( @@ -3748,50 +4143,76 @@ async function loadSourceSnapshot( options: { includeHeavyBuilderBodyValues: boolean; documentIds?: string[] }, ): Promise { const db = getDb(); - const [fieldRows, changeRows, reviewRows, executionRows, propertyDefs] = - await Promise.all([ - db - .select() - .from(schema.contentDatabaseSourceFields) - .where(eq(schema.contentDatabaseSourceFields.sourceId, source.id)) - .orderBy(asc(schema.contentDatabaseSourceFields.createdAt)), - db - .select() - .from(schema.contentDatabaseSourceChangeSets) - .where( - options.documentIds?.length - ? and( - eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), - inArray( - schema.contentDatabaseSourceChangeSets.documentId, - options.documentIds, + const [fieldRows, changeRows, propertyDefs] = await Promise.all([ + db + .select() + .from(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.sourceId, source.id)) + .orderBy(asc(schema.contentDatabaseSourceFields.createdAt)), + db + .select() + .from(schema.contentDatabaseSourceChangeSets) + .where( + options.documentIds !== undefined + ? and( + eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), + inArray( + schema.contentDatabaseSourceChangeSets.documentId, + options.documentIds, + ), + ) + : eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), + ) + .orderBy(asc(schema.contentDatabaseSourceChangeSets.createdAt)), + db + .select({ + id: schema.documentPropertyDefinitions.id, + name: schema.documentPropertyDefinitions.name, + type: schema.documentPropertyDefinitions.type, + optionsJson: schema.documentPropertyDefinitions.optionsJson, + }) + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)), + ]); + const scopedChangeSetIds = changeRows.map((changeSet) => changeSet.id); + const [reviewRows, executionRows] = + options.documentIds !== undefined && scopedChangeSetIds.length === 0 + ? [[], []] + : await Promise.all([ + db + .select() + .from(schema.contentDatabaseSourceChangeReviews) + .where( + and( + eq( + schema.contentDatabaseSourceChangeReviews.sourceId, + source.id, ), - ) - : eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), - ) - .orderBy(asc(schema.contentDatabaseSourceChangeSets.createdAt)), - db - .select() - .from(schema.contentDatabaseSourceChangeReviews) - .where( - eq(schema.contentDatabaseSourceChangeReviews.sourceId, source.id), - ) - .orderBy(asc(schema.contentDatabaseSourceChangeReviews.createdAt)), - db - .select() - .from(schema.contentDatabaseSourceExecutions) - .where(eq(schema.contentDatabaseSourceExecutions.sourceId, source.id)) - .orderBy(asc(schema.contentDatabaseSourceExecutions.createdAt)), - db - .select({ - id: schema.documentPropertyDefinitions.id, - name: schema.documentPropertyDefinitions.name, - type: schema.documentPropertyDefinitions.type, - optionsJson: schema.documentPropertyDefinitions.optionsJson, - }) - .from(schema.documentPropertyDefinitions) - .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)), - ]); + options.documentIds !== undefined + ? inArray( + schema.contentDatabaseSourceChangeReviews.changeSetId, + scopedChangeSetIds, + ) + : undefined, + ), + ) + .orderBy(asc(schema.contentDatabaseSourceChangeReviews.createdAt)), + db + .select() + .from(schema.contentDatabaseSourceExecutions) + .where( + and( + eq(schema.contentDatabaseSourceExecutions.sourceId, source.id), + options.documentIds !== undefined + ? inArray( + schema.contentDatabaseSourceExecutions.changeSetId, + scopedChangeSetIds, + ) + : undefined, + ), + ) + .orderBy(asc(schema.contentDatabaseSourceExecutions.createdAt)), + ]); const propertyNameById = new Map( propertyDefs.map((row) => [row.id, row.name]), @@ -3844,6 +4265,7 @@ async function loadSourceSnapshot( allDocumentIds, rowDocuments, propertyValueRows, + consistencyAttempts, } = await loadSourceSnapshotRowsOptimistically({ source, database, @@ -4224,6 +4646,10 @@ async function loadSourceSnapshot( fields, rows, changeSets, + projection: + options.documentIds !== undefined + ? { rows: "page", changeSets: "page" } + : { rows: "complete", changeSets: "complete" }, bodyHydration, }; } @@ -5640,10 +6066,11 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { }): Promise<{ imported: number; importedEntriesByDocumentId: Map; + importedItems: ContentDatabaseItem[]; }> { const importedEntriesByDocumentId = new Map(); if (args.entries.length === 0) { - return { imported: 0, importedEntriesByDocumentId }; + return { imported: 0, importedEntriesByDocumentId, importedItems: [] }; } const db = getDb(); const [databaseDocument] = await db @@ -5732,6 +6159,9 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { .filter((row) => !representedDocumentIds.has(row.document.id)) .map((row) => row.document.title.trim().toLowerCase()), ); + const currentItemByDocumentId = new Map( + currentItems.map((row) => [row.document.id, row.item]), + ); // Reads MAX(position) for both `documents` and `content_database_items` // then batch-inserts at MAX+1.. — serialize the whole read-through-write @@ -5777,26 +6207,29 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { continue; } + const { documentId, itemId } = builderCmsImportIds({ + ownerEmail: args.database.ownerEmail, + databaseId: args.database.id, + sourceTable: args.sourceTable, + entryId: entry.id, + }); + const existingDeterministicItem = + currentItemByDocumentId.get(documentId); + if (existingDeterministicItem?.id === itemId) { + // A prior attach can commit the deterministic document/item and + // fail before linking its source row. Treat that pair as the + // same Builder identity so refresh repairs the missing link + // without synthesizing a duplicate response item. + importedEntriesByDocumentId.set(documentId, entry); + continue; + } + const title = entry.title.trim() || entry.id; const titleKey = title.toLowerCase(); if (!args.skipTitleDedup && existingUnlinkedTitles.has(titleKey)) { continue; } - const importIdentity = [ - args.database.ownerEmail, - args.database.id, - args.sourceTable, - entry.id, - ]; - const documentId = stableBuilderImportId( - "builder-doc", - importIdentity, - ); - const itemId = stableBuilderImportId( - "builder-item", - importIdentity, - ); documentRows.push({ id: documentId, spaceId: databaseSpaceId, @@ -5827,6 +6260,7 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { }); importedEntriesByDocumentId.set(documentId, entry); } + const insertedItemIds = new Set(); await db.transaction(async (tx) => { for (const chunk of chunks( documentRows, @@ -5841,10 +6275,14 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { itemRows, bulkChunkSizeForColumnCount(10), )) { - await tx + const insertedItems = await tx .insert(schema.contentDatabaseItems) .values(chunk) - .onConflictDoNothing(); + .onConflictDoNothing() + .returning({ id: schema.contentDatabaseItems.id }); + for (const item of insertedItems) { + insertedItemIds.add(item.id); + } } await ensureDocumentsFilesMembership( tx, @@ -5854,7 +6292,47 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { ); }); - return { imported: itemRows.length, importedEntriesByDocumentId }; + const candidateDocumentById = new Map( + documentRows.map((document) => [document.id!, document]), + ); + const importedItems: ContentDatabaseItem[] = itemRows + .filter((item) => insertedItemIds.has(item.id!)) + .map((item) => { + const document = candidateDocumentById.get(item.documentId)!; + return { + id: item.id!, + databaseId: args.database.id, + document: { + id: document.id!, + parentId: document.parentId ?? null, + title: document.title ?? "Untitled", + content: "", + icon: document.icon ?? null, + position: document.position ?? 0, + isFavorite: false, + hideFromSearch: Boolean(document.hideFromSearch), + visibility: document.visibility ?? "private", + accessRole: "owner", + canEdit: true, + canManage: true, + createdAt: document.createdAt!, + updatedAt: document.updatedAt!, + }, + position: item.position ?? 0, + properties: [], + bodyHydration: { + status: "pending", + attemptedAt: null, + error: null, + version: null, + }, + }; + }); + return { + imported: insertedItemIds.size, + importedEntriesByDocumentId, + importedItems, + }; }, ), ); @@ -5865,9 +6343,10 @@ export async function resyncBuilderCmsSourceSnapshot(args: { source: ContentDatabaseSourceRowDb; now: string; runFullRefresh?: boolean; + finishPagination?: boolean; refreshClaimId?: string; }) { - let { properties, response } = await sourceSetupPayload(args.database.id); + const setupPromise = sourceSetupPayload(args.database.id); const db = getDb(); const sourceMetadata = parseObject(args.source.metadataJson) ?? {}; @@ -5877,7 +6356,7 @@ export async function resyncBuilderCmsSourceSnapshot(args: { ? sourceMetadata.activeReadSourceRowIds : []; const continueOffset = - !args.runFullRefresh && + (!args.runFullRefresh || args.finishPagination) && sourceMetadata.sourceFetchState === "fetching" && sourceMetadata.lastReadHasMore === true && activeReadSourceRowIds.length > 0 && @@ -5891,25 +6370,33 @@ export async function resyncBuilderCmsSourceSnapshot(args: { .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id)); let builderModelFields: BuilderCmsModelFieldSummary[] | undefined; let builderModelFieldsReadFailed = false; - try { - builderModelFields = await readBuilderCmsModelFields({ - model: args.source.sourceTable, - }); - builderModelFields = mergeBuilderCmsModelFieldsPreservingReferenceModels({ - existing: sourceMetadata.builderModelFields, - refreshed: builderModelFields, - }); - } catch (error) { - builderModelFieldsReadFailed = true; - const message = error instanceof Error ? error.message : String(error); - console.warn( - `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`, - ); + if (continueOffset > 0 && sourceMetadata.builderModelFields?.length) { + builderModelFields = sourceMetadata.builderModelFields; + } else { + try { + builderModelFields = await readBuilderCmsModelFields({ + model: args.source.sourceTable, + }); + builderModelFields = mergeBuilderCmsModelFieldsPreservingReferenceModels({ + existing: sourceMetadata.builderModelFields, + refreshed: builderModelFields, + }); + } catch (error) { + builderModelFieldsReadFailed = true; + const message = error instanceof Error ? error.message : String(error); + console.warn( + `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`, + ); + } } const projectionModelFields = builderModelFields && builderModelFields.length > 0 ? builderModelFields : (sourceMetadata.builderModelFields ?? []); + const existingRowsPromise = db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); const builderRead = await readBuilderCmsContentEntries({ model: args.source.sourceTable, fieldPaths: [ @@ -5945,10 +6432,9 @@ export async function resyncBuilderCmsSourceSnapshot(args: { "Builder source refresh claim was lost before snapshot mutation.", ); } - let existingRows = await db - .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); + const { properties, response: initialResponse } = await setupPromise; + let response = initialResponse; + const existingRows = await existingRowsPromise; const readStartOffset = builderRead.progress?.startOffset ?? 0; const activeReadSourceRowIdSet = new Set(activeReadSourceRowIds); const suspiciousEmptyRead = @@ -5995,11 +6481,14 @@ export async function resyncBuilderCmsSourceSnapshot(args: { }); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; if (importResult.imported > 0) { - ({ properties, response } = await sourceSetupPayload(args.database.id)); - existingRows = await db - .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); + const existingItemIds = new Set(response.items.map((item) => item.id)); + const importedItems = importResult.importedItems.filter( + (item) => !existingItemIds.has(item.id), + ); + response = { + ...response, + items: [...response.items, ...importedItems], + }; } } const builderEntriesByDocumentId = @@ -6971,15 +7460,24 @@ export async function getSourceRows(sourceId: string) { .where(eq(schema.contentDatabaseSourceRows.sourceId, sourceId)); } -export async function listDatabasePropertiesAndItems(databaseId: string) { +export async function listDatabasePropertiesAndItems( + databaseId: string, + options: { limit?: number; offset?: number; documentIds?: string[] } = {}, +) { const { getContentDatabaseResponse } = await import("./_database-utils.js"); - return getContentDatabaseResponse(databaseId); + return getContentDatabaseResponse(databaseId, { + ...options, + includeSources: false, + }); } -export async function sourceSetupPayload(databaseId: string) { +export async function sourceSetupPayload( + databaseId: string, + options: { limit?: number; offset?: number; documentIds?: string[] } = {}, +) { const [properties, response] = await Promise.all([ listPropertiesForDatabase(databaseId), - listDatabasePropertiesAndItems(databaseId), + listDatabasePropertiesAndItems(databaseId, options), ]); return { properties, response }; } diff --git a/templates/content/actions/_database-utils.ts b/templates/content/actions/_database-utils.ts index 6ec1470fdf..11e938a9a4 100644 --- a/templates/content/actions/_database-utils.ts +++ b/templates/content/actions/_database-utils.ts @@ -5,9 +5,11 @@ import { import { accessFilter, ROLE_RANK, + resolveAccess, type ShareRole, } from "@agent-native/core/sharing"; import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; import { getDocumentContextPath } from "../server/lib/document-context.js"; @@ -18,13 +20,29 @@ import { } from "../server/lib/documents.js"; import type { ContentDatabaseBodyHydration, + ContentDatabaseItem, ContentDatabaseMembership, ContentDatabaseResponse, + ContentDatabaseTableQuery, + DocumentProperty, } from "../shared/api.js"; +import { + applyContentDatabaseTableQuery, + contentDatabaseTableQueryUsesProperties, +} from "../shared/database-query.js"; +import { + evaluatePropertyFormula, + isBlocksPropertyType, + isComputedPropertyType, + isPrimaryBlocksField, + parsePropertyValue, + type DocumentPropertyType, +} from "../shared/properties.js"; import { favoriteDocumentIds } from "./_content-favorites.js"; import { listContentOrganizationMemberships, normalizeContentSpaceEmail, + resolveContentSpaceAccess, } from "./_content-space-access.js"; import { getAllContentDatabaseSourceSnapshots } from "./_database-source-utils.js"; import { @@ -44,6 +62,135 @@ export { getDocumentContextPath }; export const CONTENT_DATABASE_MAX_READ_LIMIT = 5_000; +const QUERY_PROJECTION_UNSUPPORTED_PROPERTY_TYPES = + new Set(["rollup"]); + +function boundedTableQueryProjectionPropertyIds( + query: ContentDatabaseTableQuery, + properties: DocumentProperty[], +) { + let propertyIds = new Set( + query.search.trim() + ? properties.map((property) => property.definition.id) + : [...query.filters, ...query.sorts] + .map((constraint) => constraint.key) + .filter((key) => key !== "name"), + ); + if ( + properties.some( + (property) => + propertyIds.has(property.definition.id) && + property.definition.type === "formula", + ) + ) { + propertyIds = new Set(properties.map((property) => property.definition.id)); + } + for (const property of properties) { + if ( + propertyIds.has(property.definition.id) && + QUERY_PROJECTION_UNSUPPORTED_PROPERTY_TYPES.has(property.definition.type) + ) { + return null; + } + } + return propertyIds; +} + +function projectedComputedPropertyValue( + type: DocumentPropertyType, + document: { + ownerEmail: string; + createdAt: string; + updatedAt: string; + }, +) { + if (type === "created_time") return document.createdAt; + if (type === "created_by" || type === "last_edited_by") { + return document.ownerEmail; + } + if (type === "last_edited_time") return document.updatedAt; + return null; +} + +export const contentDatabaseTableQuerySchema = z + .object({ + search: z.string().max(500), + filters: z + .array( + z.object({ + key: z.string(), + label: z.string(), + operator: z.enum([ + "contains", + "equals", + "does_not_equal", + "greater_than", + "less_than", + "before", + "after", + "between", + "is_checked", + "is_unchecked", + "is_empty", + "is_not_empty", + ]), + value: z.string(), + filterGroupId: z.string().optional(), + parentFilterGroupId: z.string().optional(), + }), + ) + .max(50), + sorts: z + .array( + z.object({ + key: z.string(), + label: z.string(), + direction: z.enum(["asc", "desc"]), + }), + ) + .max(20), + filterMode: z.enum(["and", "or"]), + }) + .optional(); + +async function contentDatabaseTableQueryMode( + databaseId: string, + query: ContentDatabaseTableQuery | undefined, +) { + if (!query) return undefined; + const sourceFields = await getDb() + .select({ + metadataJson: schema.contentDatabaseSources.metadataJson, + propertyId: schema.contentDatabaseSourceFields.propertyId, + }) + .from(schema.contentDatabaseSourceFields) + .innerJoin( + schema.contentDatabaseSources, + eq( + schema.contentDatabaseSources.id, + schema.contentDatabaseSourceFields.sourceId, + ), + ) + .where(eq(schema.contentDatabaseSources.databaseId, databaseId)); + const secondaryPropertyIds = new Set(); + for (const field of sourceFields) { + let role: unknown = null; + try { + role = JSON.parse(field.metadataJson || "{}").federation?.role; + } catch { + role = null; + } + if (role === "secondary" && field.propertyId) { + secondaryPropertyIds.add(field.propertyId); + } + } + const usesSecondaryField = contentDatabaseTableQueryUsesProperties( + query, + secondaryPropertyIds, + ); + return usesSecondaryField ? "client-required" : "server"; +} + function canManageRole(role: string) { return role === "owner" || role === "admin"; } @@ -277,33 +424,122 @@ function serializeDocument( }; } -export async function getContentDatabaseResponse( - databaseId: string, - options: { limit?: number; offset?: number } = {}, -): Promise { +export type ContentDatabasePageResponse = Pick< + ContentDatabaseResponse, + "items" | "source" | "sources" | "pagination" | "tableQueryMode" +>; + +export type ContentDatabaseReadResolution = + | { + available: true; + database: typeof schema.contentDatabases.$inferSelect; + } + | { + available: false; + reason: "not_found" | "deleted"; + databaseId: string; + documentId: string | null; + deletedAt?: string | null; + message: string; + }; + +export async function resolveContentDatabaseRead(args: { + databaseId?: string; + documentId?: string; +}): Promise { const db = getDb(); + let databaseId = args.databaseId; + if (!databaseId && args.documentId) { + const [database] = await db + .select({ id: schema.contentDatabases.id }) + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.documentId, args.documentId)); + databaseId = database?.id; + } + if (!databaseId) { + throw new Error("Either databaseId or documentId is required."); + } + const [database] = await db .select() .from(schema.contentDatabases) .where(eq(schema.contentDatabases.id, databaseId)); + if (!database) { + return { + available: false, + reason: "not_found", + databaseId, + documentId: args.documentId ?? null, + message: `Database "${databaseId}" not found`, + }; + } + + let canRead = Boolean(await resolveAccess("document", database.documentId)); + if (!canRead && database.systemRole === "files" && database.spaceId) { + try { + await resolveContentSpaceAccess(database.spaceId); + canRead = true; + } catch { + canRead = false; + } + } + if (!canRead) throw new Error(`Database "${databaseId}" not found`); + + if (database.deletedAt) { + return { + available: false, + reason: "deleted", + databaseId: database.id, + documentId: database.documentId, + deletedAt: database.deletedAt, + message: `Database "${database.id}" has been deleted`, + }; + } + + return { available: true, database }; +} + +type ContentDatabasePageBuild = ContentDatabasePageResponse & { + databaseRecord: typeof schema.contentDatabases.$inferSelect; + properties: ContentDatabaseResponse["properties"]; + hydratedItemCount: number; +}; + +export async function getContentDatabasePageResponse( + databaseId: string, + options: { + limit?: number; + offset?: number; + tableQuery?: ContentDatabaseTableQuery; + includeSources?: boolean; + documentIds?: string[]; + database?: typeof schema.contentDatabases.$inferSelect; + } = {}, +): Promise { + const db = getDb(); + const database = + options.database ?? + ( + await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, databaseId)) + )[0]; if (!database || database.deletedAt) { throw new Error(`Database "${databaseId}" not found`); } - const [databaseDocument] = await db - .select({ - id: schema.documents.id, - parentId: schema.documents.parentId, - description: schema.documents.description, - }) - .from(schema.documents) - .where(eq(schema.documents.id, database.documentId)); - // PURE read: the primary "Content" Blocks field is seeded at create time and // by the one-time startup repair — never here. Reading a database (including a // shared one a viewer is opening) must not mutate schema. const { limit, offset } = normalizeContentDatabasePageOptions(options); + const tableQuery = options.tableQuery; + const tableQueryMode = await contentDatabaseTableQueryMode( + databaseId, + tableQuery, + ); + const serverTableQuery = tableQueryMode === "server" ? tableQuery : undefined; const userEmail = getRequestUserEmail(); const normalizedUserEmail = userEmail ? normalizeContentSpaceEmail(userEmail) @@ -401,6 +637,11 @@ export async function getContentDatabaseResponse( : undefined; const visibleItemFilter = and( eq(schema.contentDatabaseItems.databaseId, databaseId), + options.documentIds !== undefined + ? options.documentIds.length > 0 + ? inArray(schema.contentDatabaseItems.documentId, options.documentIds) + : sql`1 = 0` + : undefined, sql`exists ( select 1 from ${schema.documents} where ${schema.documents.id} = ${schema.contentDatabaseItems.documentId} @@ -428,17 +669,198 @@ export async function getContentDatabaseResponse( .select({ count: sql`COUNT(*)` }) .from(schema.contentDatabaseItems) .where(visibleItemFilter); + const totalVisibleItems = Number(itemCount?.count ?? 0); + if (tableQuery && totalVisibleItems > CONTENT_DATABASE_MAX_READ_LIMIT) { + throw new Error( + `Table constraints support up to ${CONTENT_DATABASE_MAX_READ_LIMIT} rows; this database has ${totalVisibleItems}.`, + ); + } + + const databaseProperties = await listPropertiesForDatabase(databaseId); + const boundedProjectionPropertyIds = + serverTableQuery && !database.systemRole + ? boundedTableQueryProjectionPropertyIds( + serverTableQuery, + databaseProperties, + ) + : null; let itemsQuery = db .select() .from(schema.contentDatabaseItems) .where(visibleItemFilter) - .orderBy(asc(schema.contentDatabaseItems.position)) + .orderBy( + asc(schema.contentDatabaseItems.position), + asc(schema.contentDatabaseItems.createdAt), + asc(schema.contentDatabaseItems.id), + ) .$dynamic(); - if (limit !== null) { + if (serverTableQuery) { + itemsQuery = itemsQuery.limit(CONTENT_DATABASE_MAX_READ_LIMIT); + } else if (limit !== null) { itemsQuery = itemsQuery.limit(limit).offset(offset); } - const items = await itemsQuery; + let items = await itemsQuery; + let boundedTableQueryTotal: number | null = null; + if (serverTableQuery && boundedProjectionPropertyIds) { + const candidateDocuments = await db + .select({ + id: schema.documents.id, + title: schema.documents.title, + ownerEmail: schema.documents.ownerEmail, + createdAt: schema.documents.createdAt, + updatedAt: schema.documents.updatedAt, + }) + .from(schema.documents) + .innerJoin( + schema.contentDatabaseItems, + eq(schema.contentDatabaseItems.documentId, schema.documents.id), + ) + .where( + and( + visibleItemFilter, + eq(schema.documents.ownerEmail, database.ownerEmail), + ), + ); + const candidateDocumentById = new Map( + candidateDocuments.map((document) => [document.id, document]), + ); + const candidateValues = + boundedProjectionPropertyIds.size > 0 + ? await db + .select({ + documentId: schema.documentPropertyValues.documentId, + propertyId: schema.documentPropertyValues.propertyId, + valueJson: schema.documentPropertyValues.valueJson, + }) + .from(schema.documentPropertyValues) + .innerJoin( + schema.contentDatabaseItems, + eq( + schema.contentDatabaseItems.documentId, + schema.documentPropertyValues.documentId, + ), + ) + .where( + and( + visibleItemFilter, + inArray(schema.documentPropertyValues.propertyId, [ + ...boundedProjectionPropertyIds, + ]), + ), + ) + : []; + const candidateValueByDocumentAndProperty = new Map( + candidateValues.map((value) => [ + `${value.documentId}\0${value.propertyId}`, + parsePropertyValue(value.valueJson), + ]), + ); + const queryProperties = databaseProperties.filter((property) => + boundedProjectionPropertyIds.has(property.definition.id), + ); + const additionalBlocksPropertyIds = queryProperties.flatMap((property) => + isBlocksPropertyType(property.definition.type) && + !isPrimaryBlocksField(property.definition.options) + ? [property.definition.id] + : [], + ); + const candidateBlockContents = + additionalBlocksPropertyIds.length > 0 + ? await db + .select({ + documentId: schema.documentBlockFieldContents.documentId, + propertyId: schema.documentBlockFieldContents.propertyId, + content: schema.documentBlockFieldContents.content, + }) + .from(schema.documentBlockFieldContents) + .innerJoin( + schema.contentDatabaseItems, + eq( + schema.contentDatabaseItems.documentId, + schema.documentBlockFieldContents.documentId, + ), + ) + .where( + and( + visibleItemFilter, + inArray( + schema.documentBlockFieldContents.propertyId, + additionalBlocksPropertyIds, + ), + ), + ) + : []; + const candidateBlockContentByDocumentAndProperty = new Map( + candidateBlockContents.map((row) => [ + `${row.documentId}\0${row.propertyId}`, + row.content ?? "", + ]), + ); + const candidateItems = items.flatMap((item, itemIndex) => { + const document = candidateDocumentById.get(item.documentId); + if (!document) return []; + const properties = queryProperties.map((property) => { + const type = property.definition.type; + const value = + type === "id" + ? itemIndex + 1 + : isBlocksPropertyType(type) + ? isPrimaryBlocksField(property.definition.options) + ? "" + : (candidateBlockContentByDocumentAndProperty.get( + `${document.id}\0${property.definition.id}`, + ) ?? "") + : isComputedPropertyType(type) + ? projectedComputedPropertyValue(type, document) + : (candidateValueByDocumentAndProperty.get( + `${document.id}\0${property.definition.id}`, + ) ?? null); + return { ...property, value }; + }); + const valuesByName = Object.fromEntries( + properties + .filter((property) => property.definition.type !== "formula") + .map((property) => [property.definition.name, property.value]), + ); + return [ + { + id: item.id, + databaseId: item.databaseId, + document: { title: document.title }, + properties: properties.map((property) => + property.definition.type === "formula" + ? { + ...property, + value: evaluatePropertyFormula( + property.definition.options.formula, + valuesByName, + ), + } + : property, + ), + } as ContentDatabaseItem, + ]; + }); + const constrainedCandidates = applyContentDatabaseTableQuery( + candidateItems, + databaseProperties, + serverTableQuery, + ); + boundedTableQueryTotal = constrainedCandidates.length; + const pageItemIds = new Set( + limit === null + ? constrainedCandidates.map((item) => item.id) + : constrainedCandidates + .slice(offset, offset + limit) + .map((item) => item.id), + ); + const itemById = new Map(items.map((item) => [item.id, item])); + items = [...pageItemIds].flatMap((itemId) => { + const item = itemById.get(itemId); + return item ? [item] : []; + }); + } const documents = items.length > 0 @@ -542,7 +964,6 @@ export async function getContentDatabaseResponse( // every document field it consumes except the deliberately omitted body. documents as Array, ); - const databaseProperties = await listPropertiesForDatabase(databaseId); const filesProjection = await filesSystemPropertyProjection({ database, documents, @@ -586,12 +1007,12 @@ export async function getContentDatabaseResponse( ) : new Set(); - const serializedItems = []; + const serializedCandidateItems = []; for (const item of items) { const document = documentById.get(item.documentId); if (!document) continue; const bodyHydrationQueued = queuedBodyHydrationItemIds.has(item.id); - serializedItems.push({ + serializedCandidateItems.push({ id: item.id, databaseId: item.databaseId, document: serializeDocument( @@ -612,7 +1033,30 @@ export async function getContentDatabaseResponse( }); } - const sourceSnapshots = await getAllContentDatabaseSourceSnapshots(database); + const constrainedItems = + serverTableQuery && boundedTableQueryTotal === null + ? applyContentDatabaseTableQuery( + serializedCandidateItems, + responseProperties, + serverTableQuery, + ) + : serializedCandidateItems; + const serializedItems = + boundedTableQueryTotal !== null + ? serializedCandidateItems + : serverTableQuery && limit !== null + ? constrainedItems.slice(offset, offset + limit) + : constrainedItems; + + const serializedDocumentIds = new Set( + serializedItems.map((item) => item.document.id), + ); + const sourceSnapshots = + options.includeSources === false + ? [] + : await getAllContentDatabaseSourceSnapshots(database, { + documentIds: limit !== null ? [...serializedDocumentIds] : undefined, + }); const organizationVisibleDocumentIds = organizationFilesItemFilter ? new Set( ( @@ -631,25 +1075,26 @@ export async function getContentDatabaseResponse( ), ) : sourceSnapshots; - const serializedDocumentIds = new Set( - serializedItems.map((item) => item.document.id), - ); - // When paginating, scope every DOCUMENT-BACKED source's rows to the visible - // page, plus the small set referenced by actionable reviews. The dialog gets - // change sets independently of the item page and needs those rows to retain - // the linked provider target instead of misclassifying an off-page update as - // a create. Federated join rows carry no document (empty documentId), so - // they're kept intact — only matched ones overlay anyway. + // Keep the returned source overlay aligned to the visible item page. + // Secondary federation sources stay complete until their join-key lookup can + // be bounded independently; only matched rows overlay the returned items. const pagedSources = limit !== null - ? sources.map((source) => ({ - ...source, - rows: filterContentDatabaseSourceRowsForPage({ + ? sources.map((source) => { + const rows = filterContentDatabaseSourceRowsForPage({ rows: source.rows, changeSets: source.changeSets, visibleDocumentIds: serializedDocumentIds, - }), - })) + }); + return { + ...source, + rows, + projection: { + rows: "page" as const, + changeSets: source.projection?.changeSets ?? "complete", + }, + }; + }) : sources; const pagedPrimary = pagedSources[0] ?? null; @@ -660,12 +1105,8 @@ export async function getContentDatabaseResponse( // Opt-in federated columns (a secondary field the user added via the picker) // get their per-row values from the matched overlay at read time. const itemsWithOverlay = applyFederatedOverlayValues(federatedItems); - return { - database: serializeDatabase(database, databaseDocument?.description ?? ""), - contextPath: databaseDocument - ? await getDocumentContextPath(databaseDocument) - : [], + databaseRecord: database, properties: responseProperties, items: itemsWithOverlay, source: pagedPrimary, @@ -675,12 +1116,65 @@ export async function getContentDatabaseResponse( ? { offset, limit, - totalItems: Number(itemCount?.count ?? 0), + totalItems: + boundedTableQueryTotal ?? + (serverTableQuery ? constrainedItems.length : totalVisibleItems), returnedItems: serializedItems.length, hasMore: - offset + serializedItems.length < Number(itemCount?.count ?? 0), + offset + serializedItems.length < + (boundedTableQueryTotal ?? + (serverTableQuery + ? constrainedItems.length + : totalVisibleItems)), } : undefined, + tableQueryMode, + hydratedItemCount: documents.length, + }; +} + +export async function getContentDatabaseResponse( + databaseId: string, + options: { + limit?: number; + offset?: number; + tableQuery?: ContentDatabaseTableQuery; + includeSources?: boolean; + documentIds?: string[]; + database?: typeof schema.contentDatabases.$inferSelect; + } = {}, +): Promise { + const page = await getContentDatabasePageResponse(databaseId, options); + const db = getDb(); + const [databaseDocument] = await db + .select({ + id: schema.documents.id, + parentId: schema.documents.parentId, + description: schema.documents.description, + }) + .from(schema.documents) + .where( + and( + eq(schema.documents.id, page.databaseRecord.documentId), + accessFilter(schema.documents, schema.documentShares), + ), + ); + const contextPath = databaseDocument + ? await getDocumentContextPath(databaseDocument) + : []; + + return { + database: serializeDatabase( + page.databaseRecord, + databaseDocument?.description ?? "", + ), + contextPath, + properties: page.properties, + items: page.items, + source: page.source, + sources: page.sources, + pagination: page.pagination, + tableQueryMode: page.tableQueryMode, }; } diff --git a/templates/content/actions/add-content-database-source-field-property.ts b/templates/content/actions/add-content-database-source-field-property.ts index 3b56927751..89edc80300 100644 --- a/templates/content/actions/add-content-database-source-field-property.ts +++ b/templates/content/actions/add-content-database-source-field-property.ts @@ -1,4 +1,5 @@ import { defineAction } from "@agent-native/core"; +import { getDialect } from "@agent-native/core/db"; import { assertAccess } from "@agent-native/core/sharing"; import { and, eq, isNull, sql } from "drizzle-orm"; import { z } from "zod"; @@ -37,6 +38,12 @@ const BUILDER_FIELD_REFRESH_MINIMUM_LIMIT = 500; type SourceRow = typeof schema.contentDatabaseSourceRows.$inferSelect; +class MissingBuilderFieldValuesError extends Error { + constructor(readonly rowCount: number) { + super("Builder source field values are missing from the local snapshot."); + } +} + function parseSourceValues( value: string | null | undefined, ): Record { @@ -61,6 +68,32 @@ function hasSourceFieldValue( ); } +function sourceFieldValueJsonProjection(sourceFieldKey: string) { + const sourceValuesJson = schema.contentDatabaseSourceRows.sourceValuesJson; + if (getDialect() === "postgres") { + return sql< + string | null + >`(${sourceValuesJson}::jsonb -> ${sourceFieldKey})::text`; + } + + const path = `$."${sourceFieldKey}"`; + const type = sql`json_type(${sourceValuesJson}, ${path})`; + return sql`CASE + WHEN ${type} IS NULL THEN NULL + WHEN ${type} = 'true' THEN 'true' + WHEN ${type} = 'false' THEN 'false' + ELSE json_quote(json_extract(${sourceValuesJson}, ${path})) + END`; +} + +function compactSourceValuesJson( + sourceFieldKey: string, + sourceFieldValueJson: string, +) { + const escapedKey = sourceFieldKey.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + return `{"${escapedKey}":${sourceFieldValueJson}}`; +} + function mergeBuilderFieldIntoSourceRows(args: { rows: SourceRow[]; entries: BuilderCmsSourceEntry[]; @@ -383,266 +416,284 @@ export default defineAction({ } const isSecondary = federationRole === "secondary"; - const initialSourceRows = isSecondary - ? [] - : await db - .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); - const shouldRefreshBuilderField = - !isSecondary && - source.sourceType === "builder-cms" && - initialSourceRows.some( - (row) => !hasSourceFieldValue(row, field.sourceFieldKey), - ); - let builderEntries: BuilderCmsSourceEntry[] | null = null; - if (shouldRefreshBuilderField) { - // Read before writing so a Builder outage cannot leave behind a cleanly - // reported but empty property. Start at zero even when the source's - // broader refresh is incremental: the stored rows we are backfilling - // begin with that first page. - const builderRead = await readBuilderCmsContentEntries({ - model: source.sourceTable, - fieldPaths: [field.sourceFieldKey], - limit: Math.max( - BUILDER_FIELD_REFRESH_MINIMUM_LIMIT, - initialSourceRows.length, - ), - offset: 0, - }); - if (builderRead.state !== "live") { - throw new Error( - builderRead.message ?? - "Builder CMS could not refresh this field. No property was created.", - ); - } - if (initialSourceRows.length > 0 && builderRead.entries.length === 0) { - throw new Error( - "Builder CMS returned no entries for this source. No property was created; refresh the source and try again.", - ); - } - builderEntries = builderRead.entries; - } - // Keep the local snapshot, property definition, mapping, and materialized - // values atomic. Re-read rows after the provider request so a concurrent - // source refresh cannot be overwritten by the older pre-request snapshot. - return await db.transaction(async (tx) => { - const now = new Date().toISOString(); - const visibility = normalizePropertyVisibility(undefined); - const propertyId = nanoid(); - const [currentField] = await tx - .select() - .from(schema.contentDatabaseSourceFields) - .where(eq(schema.contentDatabaseSourceFields.id, field.id)); - if (!currentField) throw new Error("Source field not found."); - if (currentField.propertyId) { - throw new Error("Source field is already mapped to a property."); - } - if (currentField.mappingType === "title") { - throw new Error("The title source field is already mapped to Name."); - } - if (currentField.sourceId !== source.id) { - throw new Error("Source field does not belong to this database."); - } - const [currentSource] = await tx - .select() - .from(schema.contentDatabaseSources) - .where( - and( - eq(schema.contentDatabaseSources.id, source.id), - eq(schema.contentDatabaseSources.databaseId, database.id), - ), - ); - if (!currentSource) { - throw new Error("Source field does not belong to this database."); - } + // values atomic. The common complete-snapshot path needs one projected row + // read. If values are missing, that transaction exits before writes, the + // provider read happens outside it, and a second transaction re-reads the + // rows so concurrent source refreshes cannot be overwritten. + const materializeProperty = ( + builderEntries: BuilderCmsSourceEntry[] | null, + ) => + db.transaction(async (tx) => { + const now = new Date().toISOString(); + const visibility = normalizePropertyVisibility(undefined); + const propertyId = nanoid(); + const [currentField] = await tx + .select() + .from(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.id, field.id)); + if (!currentField) throw new Error("Source field not found."); + if (currentField.propertyId) { + throw new Error("Source field is already mapped to a property."); + } + if (currentField.mappingType === "title") { + throw new Error("The title source field is already mapped to Name."); + } + if (currentField.sourceId !== source.id) { + throw new Error("Source field does not belong to this database."); + } + const [currentSource] = await tx + .select() + .from(schema.contentDatabaseSources) + .where( + and( + eq(schema.contentDatabaseSources.id, source.id), + eq(schema.contentDatabaseSources.databaseId, database.id), + ), + ); + if (!currentSource) { + throw new Error("Source field does not belong to this database."); + } - let sourceRows = isSecondary - ? [] - : await tx + let sourceRows: Array<{ + databaseItemId: string; + documentId: string; + sourceValuesJson: string | null; + }> = []; + if (builderEntries) { + const currentSourceRows = await tx .select() .from(schema.contentDatabaseSourceRows) .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); - if (builderEntries) { - const merged = mergeBuilderFieldIntoSourceRows({ - rows: sourceRows, - entries: builderEntries, - sourceTable: source.sourceTable, - sourceFieldKey: currentField.sourceFieldKey, - now, - }); - if (merged.matchedMissingRows < merged.missingRows) { - throw new Error( - "Builder CMS did not return entries for every stored source row. No property was created; refresh the source and try again.", - ); - } - for (let index = 0; index < merged.rows.length; index += 1) { - const currentRow = sourceRows[index]; - const mergedRow = merged.rows[index]; - if ( - !currentRow || - !mergedRow || - currentRow.sourceValuesJson === mergedRow.sourceValuesJson - ) { - continue; + const merged = mergeBuilderFieldIntoSourceRows({ + rows: currentSourceRows, + entries: builderEntries, + sourceTable: source.sourceTable, + sourceFieldKey: currentField.sourceFieldKey, + now, + }); + if (merged.matchedMissingRows < merged.missingRows) { + throw new Error( + "Builder CMS did not return entries for every stored source row. No property was created; refresh the source and try again.", + ); } - const [updatedRow] = await tx - .update(schema.contentDatabaseSourceRows) - .set({ - sourceValuesJson: mergedRow.sourceValuesJson, - updatedAt: now, - }) - .where( - and( - eq(schema.contentDatabaseSourceRows.id, currentRow.id), - eq( - schema.contentDatabaseSourceRows.sourceValuesJson, - currentRow.sourceValuesJson, + for (let index = 0; index < merged.rows.length; index += 1) { + const currentRow = currentSourceRows[index]; + const mergedRow = merged.rows[index]; + if ( + !currentRow || + !mergedRow || + currentRow.sourceValuesJson === mergedRow.sourceValuesJson + ) { + continue; + } + const [updatedRow] = await tx + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: mergedRow.sourceValuesJson, + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseSourceRows.id, currentRow.id), + eq( + schema.contentDatabaseSourceRows.sourceValuesJson, + currentRow.sourceValuesJson, + ), ), + ) + .returning({ id: schema.contentDatabaseSourceRows.id }); + if (!updatedRow) { + throw new Error( + "The Builder source changed while adding this property. No property was created; try again.", + ); + } + } + sourceRows = merged.rows; + } else if (!isSecondary) { + const projectedSourceRows = await tx + .select({ + databaseItemId: schema.contentDatabaseSourceRows.databaseItemId, + documentId: schema.contentDatabaseSourceRows.documentId, + sourceFieldValueJson: sourceFieldValueJsonProjection( + currentField.sourceFieldKey, ), - ) - .returning({ id: schema.contentDatabaseSourceRows.id }); - if (!updatedRow) { - throw new Error( - "The Builder source changed while adding this property. No property was created; try again.", + }) + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); + if ( + source.sourceType === "builder-cms" && + projectedSourceRows.some((row) => row.sourceFieldValueJson === null) + ) { + throw new MissingBuilderFieldValuesError( + projectedSourceRows.length, ); } + sourceRows = projectedSourceRows.flatMap((row) => + row.sourceFieldValueJson === null + ? [] + : [ + { + databaseItemId: row.databaseItemId, + documentId: row.documentId, + sourceValuesJson: compactSourceValuesJson( + currentField.sourceFieldKey, + row.sourceFieldValueJson, + ), + }, + ], + ); } - sourceRows = merged.rows; - } else if ( - source.sourceType === "builder-cms" && - sourceRows.some( - (row) => !hasSourceFieldValue(row, currentField.sourceFieldKey), - ) - ) { - throw new Error( - "The Builder source changed while adding this property. No property was created; try again.", + + const builderMetadata = builderMetadataForSourceField({ + sourceFieldKey: currentField.sourceFieldKey, + sourceMetadataJson: currentSource.metadataJson, + }); + const type = propertyTypeForSourceField( + currentField.sourceFieldType, + builderMetadata, ); - } + const options = sourceFieldPropertyOptions({ + type, + metadata: builderMetadata, + rows: sourceRows, + sourceFieldKey: currentField.sourceFieldKey, + }); + const [maxPos] = await tx + .select({ + max: sql`COALESCE(MAX(position), -1)`, + }) + .from(schema.documentPropertyDefinitions) + .where( + and( + eq( + schema.documentPropertyDefinitions.ownerEmail, + database.ownerEmail, + ), + eq(schema.documentPropertyDefinitions.databaseId, database.id), + ), + ); + const position = (maxPos?.max ?? -1) + 1; + + await tx.insert(schema.documentPropertyDefinitions).values({ + id: propertyId, + ownerEmail: database.ownerEmail, + orgId: database.orgId ?? null, + databaseId: database.id, + name: currentField.sourceFieldLabel, + type, + visibility, + optionsJson: serializePropertyOptions(options), + position, + createdAt: now, + updatedAt: now, + }); - const builderMetadata = builderMetadataForSourceField({ - sourceFieldKey: currentField.sourceFieldKey, - sourceMetadataJson: currentSource.metadataJson, - }); - const type = propertyTypeForSourceField( - currentField.sourceFieldType, - builderMetadata, - ); - const options = sourceFieldPropertyOptions({ - type, - metadata: builderMetadata, - rows: sourceRows, - sourceFieldKey: currentField.sourceFieldKey, - }); - const [maxPos] = await tx - .select({ - max: sql`COALESCE(MAX(position), -1)`, - }) - .from(schema.documentPropertyDefinitions) - .where( - and( - eq( - schema.documentPropertyDefinitions.ownerEmail, - database.ownerEmail, + const [mappedField] = await tx + .update(schema.contentDatabaseSourceFields) + .set({ + propertyId, + localFieldKey: propertyId, + mappingType: "property", + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseSourceFields.id, currentField.id), + isNull(schema.contentDatabaseSourceFields.propertyId), ), - eq(schema.documentPropertyDefinitions.databaseId, database.id), - ), - ); - const position = (maxPos?.max ?? -1) + 1; - - await tx.insert(schema.documentPropertyDefinitions).values({ - id: propertyId, - ownerEmail: database.ownerEmail, - orgId: database.orgId ?? null, - databaseId: database.id, - name: currentField.sourceFieldLabel, - type, - visibility, - optionsJson: serializePropertyOptions(options), - position, - createdAt: now, - updatedAt: now, - }); + ) + .returning({ id: schema.contentDatabaseSourceFields.id }); + if (!mappedField) { + throw new Error("Source field is already mapped to a property."); + } - const [mappedField] = await tx - .update(schema.contentDatabaseSourceFields) - .set({ - propertyId, - localFieldKey: propertyId, - mappingType: "property", - updatedAt: now, - }) - .where( - and( - eq(schema.contentDatabaseSourceFields.id, currentField.id), - isNull(schema.contentDatabaseSourceFields.propertyId), - ), - ) - .returning({ id: schema.contentDatabaseSourceFields.id }); - if (!mappedField) { - throw new Error("Source field is already mapped to a property."); - } + await tx + .update(schema.contentDatabaseSources) + .set({ updatedAt: now }) + .where(eq(schema.contentDatabaseSources.id, currentSource.id)); - await tx - .update(schema.contentDatabaseSources) - .set({ updatedAt: now }) - .where(eq(schema.contentDatabaseSources.id, currentSource.id)); + const itemValues = sourceFieldPropertyValuesFromRows( + sourceRows, + currentField.sourceFieldKey, + type, + options, + ); + if (itemValues.length > 0) { + for (const chunk of chunks(itemValues, 200)) { + await tx.insert(schema.documentPropertyValues).values( + chunk.map((row) => ({ + id: nanoid(), + ownerEmail: database.ownerEmail, + documentId: row.documentId, + propertyId, + valueJson: serializePropertyValue(row.value), + createdAt: now, + updatedAt: now, + })), + ); + } + } - const itemValues = sourceFieldPropertyValuesFromRows( - sourceRows, - currentField.sourceFieldKey, - type, - options, - ); - if (itemValues.length > 0) { - for (const chunk of chunks(itemValues, 200)) { - await tx.insert(schema.documentPropertyValues).values( - chunk.map((row) => ({ - id: nanoid(), - ownerEmail: database.ownerEmail, - documentId: row.documentId, - propertyId, - valueJson: serializePropertyValue(row.value), + const sourceField = serializeSourceField( + { + ...currentField, + propertyId, + localFieldKey: propertyId, + mappingType: "property", + updatedAt: now, + }, + currentField.sourceFieldLabel, + ); + + return { + databaseId: database.id, + documentId: database.documentId, + property: { + definition: { + id: propertyId, + databaseId: database.id, + name: currentField.sourceFieldLabel, + type, + visibility, + options, + position, createdAt: now, updatedAt: now, - })), - ); - } - } + }, + value: null, + editable: true, + }, + sourceField, + itemValues, + }; + }); - const sourceField = serializeSourceField( - { - ...currentField, - propertyId, - localFieldKey: propertyId, - mappingType: "property", - updatedAt: now, - }, - currentField.sourceFieldLabel, - ); + try { + return await materializeProperty(null); + } catch (error) { + if (!(error instanceof MissingBuilderFieldValuesError)) throw error; - return { - databaseId: database.id, - documentId: database.documentId, - property: { - definition: { - id: propertyId, - databaseId: database.id, - name: currentField.sourceFieldLabel, - type, - visibility, - options, - position, - createdAt: now, - updatedAt: now, - }, - value: null, - editable: true, - }, - sourceField, - itemValues, - }; - }); + // Read before retrying the transaction so a Builder outage cannot leave + // behind a cleanly reported but empty property. + const builderRead = await readBuilderCmsContentEntries({ + model: source.sourceTable, + fieldPaths: [field.sourceFieldKey], + limit: Math.max(BUILDER_FIELD_REFRESH_MINIMUM_LIMIT, error.rowCount), + offset: 0, + }); + if (builderRead.state !== "live") { + throw new Error( + builderRead.message ?? + "Builder CMS could not refresh this field. No property was created.", + ); + } + if (error.rowCount > 0 && builderRead.entries.length === 0) { + throw new Error( + "Builder CMS returned no entries for this source. No property was created; refresh the source and try again.", + ); + } + return await materializeProperty(builderRead.entries); + } }, }); diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 1ef8642638..0cc29cb55e 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -213,8 +213,22 @@ export default defineAction({ // SQLite writer briefly blocks this best-effort refresh hint. }); + const response = await getContentDatabaseResponse(databaseId, { + limit: 100, + offset: 0, + }); + const createdItem = + response.items.find((item) => item.id === itemId) ?? + ( + await getContentDatabaseResponse(databaseId, { + limit: 1, + offset: 0, + documentIds: [documentId], + }) + ).items.find((item) => item.id === itemId); return { - ...(await getContentDatabaseResponse(databaseId)), + ...response, + createdItem, createdItemId: itemId, createdDocumentId: documentId, createdDocumentUpdatedAt: now, diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index e038791a83..61316cce68 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -1,10 +1,12 @@ import { defineAction } from "@agent-native/core"; import { assertAccess } from "@agent-native/core/sharing"; +import { eq } from "drizzle-orm"; import { z } from "zod"; +import { getDb, schema } from "../server/db/index.js"; import type { BuilderCmsModelFieldSummary, - ContentDatabaseResponse, + ContentDatabaseSourceAttachmentResult, ContentDatabaseSourceFederation, ContentDatabaseSourceType, } from "../shared/api.js"; @@ -41,11 +43,13 @@ import { readLocalTableEntries, resolveReadableLocalTableSource, } from "./_local-table-source.js"; +import { listPropertiesForDatabase } from "./_property-utils.js"; const sourceTypeSchema = z .enum(["mock-local", "builder-cms", "local-table", "notion-database"]) .default("mock-local"); const BUILDER_CMS_ATTACH_INITIAL_PAGES = 1; +const BUILDER_CMS_ATTACH_METADATA_LIMIT = 10_000; export async function readInitialBuilderCmsAttachEntries( sourceTable: string, @@ -64,24 +68,43 @@ export async function readInitialBuilderCmsAttachSource( dependencies: { readModelFields?: typeof readBuilderCmsModelFields; readEntries?: typeof readBuilderCmsContentEntries; + fieldPaths?: readonly string[]; } = {}, ) { const readModelFields = dependencies.readModelFields ?? readBuilderCmsModelFields; const readEntries = dependencies.readEntries ?? readBuilderCmsContentEntries; - let modelFields: BuilderCmsModelFieldSummary[] = []; - let modelFieldsError: unknown = null; - try { - modelFields = await readModelFields({ model: sourceTable }); - } catch (error) { - modelFieldsError = error; - } - const read = await readInitialBuilderCmsAttachEntries( - sourceTable, - readEntries, - modelFields.map((field) => `data.${field.name}`), - ); - if (modelFieldsError) throw modelFieldsError; + const [modelFields, read] = await Promise.all([ + readModelFields({ model: sourceTable }), + readInitialBuilderCmsAttachEntries( + sourceTable, + readEntries, + dependencies.fieldPaths, + ), + ]); + return { read, modelFields }; +} + +export async function readCompleteBuilderCmsAttachSource( + sourceTable: string, + dependencies: { + readModelFields?: typeof readBuilderCmsModelFields; + readEntries?: typeof readBuilderCmsContentEntries; + fieldPaths?: readonly string[]; + } = {}, +) { + const readModelFields = + dependencies.readModelFields ?? readBuilderCmsModelFields; + const readEntries = dependencies.readEntries ?? readBuilderCmsContentEntries; + const [modelFields, read] = await Promise.all([ + readModelFields({ model: sourceTable }), + readEntries({ + model: sourceTable, + fieldPaths: dependencies.fieldPaths, + allowCached: true, + limit: BUILDER_CMS_ATTACH_METADATA_LIMIT, + }), + ]); return { read, modelFields }; } @@ -122,6 +145,32 @@ export function builderCmsAttachReadMetadata(read: BuilderCmsReadResult) { }; } +export function initialBuilderAttachmentSetupOptions(args: { + builderRead: BuilderCmsReadResult | null; + importedEntriesByDocumentId: ReadonlyMap; +}) { + if ( + args.builderRead?.state !== "live" || + args.importedEntriesByDocumentId.size !== args.builderRead.entries.length + ) { + return undefined; + } + const documentIds = [...args.importedEntriesByDocumentId.keys()]; + return { + documentIds, + limit: Math.max(1, documentIds.length), + offset: 0, + }; +} + +export function builderAttachDurableItemCount( + builderEntriesByDocumentId: + | ReadonlyMap + | undefined, +) { + return builderEntriesByDocumentId?.size ?? 0; +} + // Per-source key mapping the UI commits after the canonical-key confirm step. const normalizationFormulaSchema = z .string() @@ -211,6 +260,7 @@ export default defineAction({ .string() .optional() .describe("Source table/model name, for example content_items."), + builderFieldPaths: z.array(z.string().max(500)).max(200).optional(), relationshipMode: z .enum(["items", "details"]) .optional() @@ -231,7 +281,7 @@ export default defineAction({ limit: z.coerce.number().int().min(1).max(500).default(100), offset: z.coerce.number().int().min(0).default(0), }), - run: async (args): Promise => { + run: async (args): Promise => { const database = await resolveDatabaseForSourceMutation(args); if (!database) throw new Error("Database not found."); await assertAccess("document", database.documentId, "editor"); @@ -445,11 +495,14 @@ export default defineAction({ sourceTable, now, }); - // Snapshot existing items BEFORE importing so we can bind the new source - // to ONLY the rows it imports — never the primary's existing rows. - const beforeSetup = await sourceSetupPayload(database.id); + // Snapshot membership IDs before importing so the new source binds only + // its own rows without serializing the existing database. + const priorItems = await getDb() + .select({ documentId: schema.contentDatabaseItems.documentId }) + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, database.id)); const priorDocumentIds = new Set( - beforeSetup.response.items.map((item) => item.document.id), + priorItems.map((item) => item.documentId), ); let importedEntriesByDocumentId = new Map< string, @@ -466,7 +519,12 @@ export default defineAction({ }); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; } - const additionalSetup = await sourceSetupPayload(database.id); + const importedDocumentIds = [...importedEntriesByDocumentId.keys()]; + const additionalSetup = await sourceSetupPayload(database.id, { + documentIds: importedDocumentIds, + limit: Math.max(1, importedDocumentIds.length), + offset: 0, + }); // Only the items this collection just created — exclude the primary's. const importedItems = additionalSetup.response.items.filter( (item) => !priorDocumentIds.has(item.document.id), @@ -550,7 +608,9 @@ export default defineAction({ : []; const builderInitial = sourceType === "builder-cms" - ? await readInitialBuilderCmsAttachSource(sourceTable) + ? await readCompleteBuilderCmsAttachSource(sourceTable, { + fieldPaths: args.builderFieldPaths, + }) : null; const sourceId = await replaceSourceMetadata({ database, @@ -562,9 +622,15 @@ export default defineAction({ }); const builderModelFields = builderInitial?.modelFields ?? []; const builderRead = builderInitial?.read ?? null; + const builderPropertiesPromise = builderRead + ? listPropertiesForDatabase(database.id) + : null; const builderEntries = builderRead?.state === "live" ? builderRead.entries : []; let importedEntriesByDocumentId = new Map(); + let importedItems: Awaited< + ReturnType + >["importedItems"] = []; if (builderRead?.state === "live") { const importResult = await importBuilderCmsEntriesAsDatabaseItems({ database, @@ -574,14 +640,29 @@ export default defineAction({ existingSourceRows, }); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; + importedItems = importResult.importedItems; } - - const refreshedSetup = await sourceSetupPayload(database.id); + const canUseImportedBuilderItems = + builderRead?.state === "live" && + importedItems.length === builderRead.entries.length && + importedItems.length === importedEntriesByDocumentId.size; + const refreshedSetup = canUseImportedBuilderItems + ? null + : await sourceSetupPayload( + database.id, + initialBuilderAttachmentSetupOptions({ + builderRead, + importedEntriesByDocumentId, + }), + ); + const refreshedProperties = + refreshedSetup?.properties ?? (await builderPropertiesPromise) ?? []; + const refreshedItems = refreshedSetup?.response.items ?? importedItems; const builderEntriesByDocumentId = builderRead?.state === "live" ? mapBuilderCmsEntriesToLocalItems({ entries: builderEntries, - items: refreshedSetup.response.items, + items: refreshedItems, sourceTable, now, existingRows: existingSourceRows, @@ -591,48 +672,65 @@ export default defineAction({ builderEntriesByDocumentId?.set(documentId, entry); } - await seedMockSourceFields({ - sourceId, - ownerEmail: database.ownerEmail, - sourceType, - properties: refreshedSetup.properties, - builderModelFields, - builderSampleEntries: builderEntries, - now, - }); - await seedMockSourceRows({ - sourceId, - ownerEmail: database.ownerEmail, - sourceType, - sourceTable, - items: refreshedSetup.response.items, - now, - builderEntriesByDocumentId, - }); - if (sourceType === "builder-cms" && builderRead?.state === "live") { - await enqueueBuilderBodyHydrationForItems({ + await (async () => { + await seedMockSourceFields({ sourceId, ownerEmail: database.ownerEmail, - orgId: database.orgId, - sourceTable, - items: refreshedSetup.response.items, - builderEntriesByDocumentId, + sourceType, + properties: refreshedProperties, + builderModelFields, + builderSampleEntries: builderEntries, now, }); - } - if (sourceType === "builder-cms" && builderRead) { - await updateBuilderCmsSourceReadMetadata({ + await seedMockSourceRows({ sourceId, + ownerEmail: database.ownerEmail, + sourceType, sourceTable, - readState: builderRead.state, - entryCount: builderRead.entries.length, - matchedRowCount: builderEntriesByDocumentId?.size ?? 0, - fetchedAt: builderRead.fetchedAt, + items: refreshedItems, now, - message: builderRead.message, - builderModelFields, - ...builderCmsAttachReadMetadata(builderRead), + builderEntriesByDocumentId, }); + if (sourceType === "builder-cms" && builderRead?.state === "live") { + await enqueueBuilderBodyHydrationForItems({ + sourceId, + ownerEmail: database.ownerEmail, + orgId: database.orgId, + sourceTable, + items: refreshedItems, + builderEntriesByDocumentId, + now, + processInBackground: false, + }); + } + if (sourceType === "builder-cms" && builderRead) { + await updateBuilderCmsSourceReadMetadata({ + sourceId, + sourceTable, + readState: builderRead.state, + entryCount: builderRead.entries.length, + matchedRowCount: builderEntriesByDocumentId?.size ?? 0, + fetchedAt: builderRead.fetchedAt, + now, + message: builderRead.message, + builderModelFields, + ...builderCmsAttachReadMetadata(builderRead), + }); + } + })(); + if (sourceType === "builder-cms") { + return { + responseProjection: "ack", + databaseId: database.id, + documentId: database.documentId, + sourceId, + sourceType, + sourceTable, + importedItemCount: builderAttachDurableItemCount( + builderEntriesByDocumentId, + ), + fetchedAt: builderRead?.fetchedAt ?? now, + }; } return getContentDatabaseResponse(database.id, { diff --git a/templates/content/actions/bind-content-database-source-field.db.test.ts b/templates/content/actions/bind-content-database-source-field.db.test.ts index d9d67212be..ee26a0696f 100644 --- a/templates/content/actions/bind-content-database-source-field.db.test.ts +++ b/templates/content/actions/bind-content-database-source-field.db.test.ts @@ -469,6 +469,139 @@ describe("bind-content-database-source-field (row-union)", () => { }); describe("add-content-database-source-field-property Builder refresh", () => { + it("creates an explicitly empty projected field without rereading Builder", async () => { + const f = await seedStaleBuilderTopicsSnapshot(); + const db = getDb(); + const rows = await db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, f.sourceId)); + await Promise.all( + rows.map((row) => + db + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: JSON.stringify({ + ...JSON.parse(row.sourceValuesJson), + "data.topics": null, + }), + }) + .where(eq(schema.contentDatabaseSourceRows.id, row.id)), + ), + ); + const readBuilderEntries = vi.spyOn( + await import("./_builder-cms-read-client.js"), + "readBuilderCmsContentEntries", + ); + + const result = await asOwner(() => + addSourceFieldPropertyAction.run({ + documentId: f.databaseDocId, + sourceFieldId: f.fieldId, + }), + ); + + expect(readBuilderEntries).not.toHaveBeenCalled(); + expect(result.itemValues).toEqual([]); + const properties = await db + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, f.databaseId)); + expect(properties).toHaveLength(1); + }); + + it("materializes a projected field without returning unrelated source content", async () => { + const f = await seedStaleBuilderTopicsSnapshot(); + const db = getDb(); + const rows = await db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, f.sourceId)); + await Promise.all( + rows.map((row, index) => + db + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: JSON.stringify({ + ...JSON.parse(row.sourceValuesJson), + "data.topics": [ + index === 0 ? "Agent Native" : "Developer Experience", + ], + "_builder.bodyContent": "unrelated".repeat(5_000), + }), + }) + .where(eq(schema.contentDatabaseSourceRows.id, row.id)), + ), + ); + const readBuilderEntries = vi.spyOn( + await import("./_builder-cms-read-client.js"), + "readBuilderCmsContentEntries", + ); + + const result = await asOwner(() => + addSourceFieldPropertyAction.run({ + documentId: f.databaseDocId, + sourceFieldId: f.fieldId, + }), + ); + + expect(readBuilderEntries).not.toHaveBeenCalled(); + expect(result.itemValues).toEqual([ + { + itemId: f.rows[0].itemId, + documentId: f.rows[0].documentId, + value: ["agent-native"], + }, + { + itemId: f.rows[1].itemId, + documentId: f.rows[1].documentId, + value: ["developer-experience"], + }, + ]); + expect(JSON.stringify(result)).not.toContain("_builder.bodyContent"); + expect(JSON.stringify(result)).not.toContain("unrelated"); + }); + + it("materializes 584 already-projected rows without rereading Builder", async () => { + const f = await seedStaleBuilderTopicsSnapshot(584); + const db = getDb(); + await db + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: JSON.stringify({ + "data.topics": ["Agent Native"], + "_builder.bodyContent": "unrelated".repeat(500), + }), + }) + .where(eq(schema.contentDatabaseSourceRows.sourceId, f.sourceId)); + const readBuilderEntries = vi.spyOn( + await import("./_builder-cms-read-client.js"), + "readBuilderCmsContentEntries", + ); + + const result = await asOwner(() => + addSourceFieldPropertyAction.run({ + documentId: f.databaseDocId, + sourceFieldId: f.fieldId, + }), + ); + + expect(readBuilderEntries).not.toHaveBeenCalled(); + expect(result.itemValues).toHaveLength(584); + expect(result.itemValues[0]?.value).toEqual(["agent-native"]); + expect(JSON.stringify(result)).not.toContain("_builder.bodyContent"); + const propertyValues = await db + .select() + .from(schema.documentPropertyValues) + .where( + eq( + schema.documentPropertyValues.propertyId, + result.property.definition.id, + ), + ); + expect(propertyValues).toHaveLength(584); + }); + it("refreshes a stale Builder snapshot before creating and populating a Topics property", async () => { const f = await seedStaleBuilderTopicsSnapshot(); const readBuilderEntries = vi diff --git a/templates/content/actions/bind-content-database-source-field.ts b/templates/content/actions/bind-content-database-source-field.ts index 305e474e44..e706be76af 100644 --- a/templates/content/actions/bind-content-database-source-field.ts +++ b/templates/content/actions/bind-content-database-source-field.ts @@ -105,7 +105,7 @@ export default defineAction({ .update(schema.contentDatabaseSources) .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); } // ── Bind to an existing column ───────────────────────────────────────── @@ -261,6 +261,6 @@ export default defineAction({ } } - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/builder-source-review-gates.db.test.ts b/templates/content/actions/builder-source-review-gates.db.test.ts index 9e05c8e1e2..7e4c85b3f9 100644 --- a/templates/content/actions/builder-source-review-gates.db.test.ts +++ b/templates/content/actions/builder-source-review-gates.db.test.ts @@ -689,7 +689,7 @@ describe("Builder source review execution gates", () => { expect(heavySnapshotReads).toMatchObject({ review: 1, target: 1, - allSources: 1, + allSources: 0, }); expect(persistedChangeSet.state).toBe("approved"); expect(response.review.rows[0]?.fieldChanges).toEqual(persistedFields); diff --git a/templates/content/actions/cancel-prepared-builder-source-update.ts b/templates/content/actions/cancel-prepared-builder-source-update.ts index e993f0b362..b6ab409134 100644 --- a/templates/content/actions/cancel-prepared-builder-source-update.ts +++ b/templates/content/actions/cancel-prepared-builder-source-update.ts @@ -295,7 +295,10 @@ export default defineAction({ }); return { - ...(await getContentDatabaseResponse(database.id)), + ...(await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + })), cancellation: { sourceId: args.sourceId, changeSetId: args.changeSetId, diff --git a/templates/content/actions/content-database-lifecycle.db.test.ts b/templates/content/actions/content-database-lifecycle.db.test.ts index f18914fe17..f138baa2bb 100644 --- a/templates/content/actions/content-database-lifecycle.db.test.ts +++ b/templates/content/actions/content-database-lifecycle.db.test.ts @@ -21,6 +21,7 @@ let moveDocumentAction: typeof import("./move-document.js").default; let deleteContentDatabaseAction: typeof import("./delete-content-database.js").default; let restoreContentDatabaseAction: typeof import("./restore-content-database.js").default; let getContentDatabaseAction: typeof import("./get-content-database.js").default; +let queryContentDatabaseItemsAction: typeof import("./query-content-database-items.js").default; let listDocumentsAction: typeof import("./list-documents.js").default; let listTrashedContentDatabasesAction: typeof import("./list-trashed-content-databases.js").default; let getDocumentAction: typeof import("./get-document.js").default; @@ -53,6 +54,9 @@ beforeAll(async () => { .default; getContentDatabaseAction = (await import("./get-content-database.js")) .default; + queryContentDatabaseItemsAction = ( + await import("./query-content-database-items.js") + ).default; listDocumentsAction = (await import("./list-documents.js")).default; listTrashedContentDatabasesAction = ( await import("./list-trashed-content-databases.js") @@ -864,6 +868,14 @@ describe("content database soft-delete actions and reads", () => { reason: "deleted", databaseId, }); + const pageResponse = await runWithRequestContext({ userEmail: OWNER }, () => + queryContentDatabaseItemsAction.run({ databaseId }), + ); + expect(pageResponse).toMatchObject({ + available: false, + reason: "deleted", + databaseId, + }); const listResponse = await runWithRequestContext({ userEmail: OWNER }, () => listDocumentsAction.run({}), @@ -874,6 +886,186 @@ describe("content database soft-delete actions and reads", () => { expect(listedIds.has(rowDocumentId)).toBe(false); }); + it("returns only the ordered, filtered database page and preserves read access", async () => { + const { databaseId, databaseDocumentId } = await createDatabase({}); + const db = getDb(); + const now = new Date().toISOString(); + const rows = await Promise.all( + [ + { title: "Zebra", position: 0 }, + { title: "Apricot", position: 1 }, + { title: "Banana", position: 2 }, + ].map(async ({ title, position }) => { + const documentId = await createDocument({ + parentId: databaseDocumentId, + title, + }); + const id = nextId("item"); + await db.insert(schema.contentDatabaseItems).values({ + id, + ownerEmail: OWNER, + databaseId, + documentId, + position, + createdAt: now, + updatedAt: now, + }); + return { id, documentId }; + }), + ); + + const response = await runWithRequestContext({ userEmail: OWNER }, () => + queryContentDatabaseItemsAction.run({ + databaseId, + limit: 1, + offset: 1, + tableQuery: { + search: "a", + filters: [], + sorts: [{ key: "name", label: "Name", direction: "asc" }], + filterMode: "and", + }, + }), + ); + expect(response).toMatchObject({ + tableQueryMode: "server", + pagination: { + offset: 1, + limit: 1, + totalItems: 3, + returnedItems: 1, + hasMore: true, + }, + }); + expect(response.items.map((item) => item.document.title)).toEqual([ + "Banana", + ]); + expect(response).not.toHaveProperty("database"); + expect(response).not.toHaveProperty("contextPath"); + expect(response).not.toHaveProperty("properties"); + expect(response.items[0].id).toBe(rows[2].id); + + await expect( + runWithRequestContext({ userEmail: COLLABORATOR }, () => + queryContentDatabaseItemsAction.run({ databaseId }), + ), + ).rejects.toThrow(`Database "${databaseId}" not found`); + }); + + it("keeps a 584-row Date sort page-bounded before document and property hydration", async () => { + const { databaseId, databaseDocumentId } = await createDatabase({}); + const db = getDb(); + const now = new Date().toISOString(); + const datePropertyId = nextId("date_property"); + await db.insert(schema.documentPropertyDefinitions).values({ + id: datePropertyId, + ownerEmail: OWNER, + databaseId, + name: "Date", + type: "date", + visibility: "always_show", + optionsJson: "{}", + position: 0, + createdAt: now, + updatedAt: now, + }); + + const rows = Array.from({ length: 584 }, (_, index) => { + const documentId = nextId("date_row_doc"); + return { + documentId, + itemId: nextId("date_row_item"), + valueId: nextId("date_row_value"), + index, + date: new Date(Date.UTC(2024, 0, 1) + index * 86_400_000) + .toISOString() + .slice(0, 10), + }; + }); + await db.insert(schema.documents).values( + rows.map((row) => ({ + id: row.documentId, + ownerEmail: OWNER, + parentId: databaseDocumentId, + title: `Dated row ${row.index}`, + content: "", + position: row.index, + visibility: "private" as const, + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.contentDatabaseItems).values( + rows.map((row) => ({ + id: row.itemId, + ownerEmail: OWNER, + databaseId, + documentId: row.documentId, + position: row.index, + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.documentPropertyValues).values( + rows.map((row) => ({ + id: row.valueId, + ownerEmail: OWNER, + documentId: row.documentId, + propertyId: datePropertyId, + valueJson: JSON.stringify({ start: row.date, includeTime: false }), + createdAt: now, + updatedAt: now, + })), + ); + + const tableQuery = { + search: "", + filters: [], + sorts: [ + { + key: datePropertyId, + label: "Date", + direction: "desc" as const, + }, + ], + filterMode: "and" as const, + }; + const startedAt = performance.now(); + const response = await runWithRequestContext({ userEmail: OWNER }, () => + queryContentDatabaseItemsAction.run({ + databaseId, + limit: 1, + offset: 100, + tableQuery, + }), + ); + const durationMs = performance.now() - startedAt; + + expect(response.items.map((item) => item.document.title)).toEqual([ + "Dated row 483", + ]); + expect(response.pagination).toEqual({ + offset: 100, + limit: 1, + totalItems: 584, + returnedItems: 1, + hasMore: true, + }); + expect(durationMs).toBeLessThan(1_000); + + const { getContentDatabasePageResponse } = + await import("./_database-utils.js"); + const page = await runWithRequestContext({ userEmail: OWNER }, () => + getContentDatabasePageResponse(databaseId, { + limit: 1, + offset: 100, + tableQuery, + includeSources: false, + }), + ); + expect(page.hydratedItemCount).toBe(1); + }); + it("blocks direct document and property reads for soft-deleted database pages", async () => { const deletedAt = new Date().toISOString(); const { databaseId, databaseDocumentId } = await createDatabase({ diff --git a/templates/content/actions/content-database-source-actions.test.ts b/templates/content/actions/content-database-source-actions.test.ts index 3539a5ecbf..06bfff38f1 100644 --- a/templates/content/actions/content-database-source-actions.test.ts +++ b/templates/content/actions/content-database-source-actions.test.ts @@ -9,7 +9,10 @@ import addSourceFieldProperty, { sourceFieldPropertyValuesFromRows, } from "./add-content-database-source-field-property"; import attachSource, { + builderAttachDurableItemCount, builderCmsAttachReadMetadata, + initialBuilderAttachmentSetupOptions, + readCompleteBuilderCmsAttachSource, readInitialBuilderCmsAttachEntries, readInitialBuilderCmsAttachSource, } from "./attach-content-database-source"; @@ -28,6 +31,7 @@ import prepareReview, { reviewPreparePriority, } from "./prepare-builder-source-review"; import previewReview from "./preview-builder-source-review"; +import previewSourceAttach from "./preview-content-database-source-attach"; import refreshSource from "./refresh-content-database-source"; import reviewChangeSet from "./review-content-database-source-change-set"; import setWriteMode from "./set-content-database-source-write-mode"; @@ -36,6 +40,20 @@ import stageBulkUpdate from "./stage-builder-source-bulk-update"; import validateExecution from "./validate-builder-source-execution"; describe("content database source actions", () => { + it("bounds Builder attachment previews to one database and safe field paths", () => { + expect( + previewSourceAttach.schema.parse({ + documentId: "database-page", + sourceTable: "agent-native-blog-article-test", + fieldPaths: ["topics", "data.author"], + }), + ).toEqual({ + documentId: "database-page", + sourceTable: "agent-native-blog-article-test", + fieldPaths: ["topics", "data.author"], + }); + }); + it("accepts database or document IDs for source status reads", () => { expect(getSource.schema.parse({ documentId: "database-page" })).toEqual({ documentId: "database-page", @@ -254,6 +272,47 @@ describe("content database source actions", () => { ]); }); + it("reads complete projected metadata for the canonical Builder attachment", async () => { + const calls: Array<{ + model: string; + limit?: number; + maxPages?: number; + fieldPaths?: readonly string[]; + }> = []; + await readCompleteBuilderCmsAttachSource("blog-article", { + readModelFields: async () => [], + readEntries: async (args) => { + calls.push(args); + return { + state: "live", + entries: [], + fetchedAt: "2026-01-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: args.limit ?? 500, + pageSize: 100, + startOffset: 0, + nextOffset: 0, + fetchedEntryCount: 0, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + }, + fieldPaths: ["topics", "tags"], + }); + + expect(calls).toEqual([ + { + allowCached: true, + model: "blog-article", + fieldPaths: ["topics", "tags"], + limit: 10_000, + }, + ]); + }); + it("fails Builder attachment preparation before durable source mutation when model discovery fails", async () => { const calls: string[] = []; await expect( @@ -286,6 +345,56 @@ describe("content database source actions", () => { expect(calls).toEqual(["model-fields", "entries"]); }); + it("starts Builder field discovery and the projected first page together", async () => { + const calls: string[] = []; + let releaseFields!: () => void; + let releaseEntries!: () => void; + const fieldsReady = new Promise((resolve) => { + releaseFields = resolve; + }); + const entriesReady = new Promise((resolve) => { + releaseEntries = resolve; + }); + const pending = readInitialBuilderCmsAttachSource("blog-article", { + fieldPaths: ["topics", "data.author"], + readModelFields: async () => { + calls.push("model-fields"); + await fieldsReady; + return []; + }, + readEntries: async (args) => { + calls.push("entries"); + expect(args.fieldPaths).toEqual(["topics", "data.author"]); + await entriesReady; + return { + state: "live", + entries: [], + fetchedAt: "2026-01-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset: 0, + nextOffset: 0, + fetchedEntryCount: 0, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + }, + }); + + await Promise.resolve(); + expect(calls).toEqual(["model-fields", "entries"]); + releaseEntries(); + releaseFields(); + await expect(pending).resolves.toMatchObject({ + read: { state: "live" }, + modelFields: [], + }); + }); + it("fails role-change preparation before mappings can be rewritten when model discovery fails", async () => { const calls: string[] = []; const existingMappings = ["data.topics", "data.tags"]; @@ -376,6 +485,54 @@ describe("content database source actions", () => { }); }); + it("binds a fully imported initial Builder page by exact document identity", () => { + const read = { + state: "live", + entries: [{ id: "entry-1" }, { id: "entry-2" }], + } as BuilderCmsReadResult; + + expect( + initialBuilderAttachmentSetupOptions({ + builderRead: read, + importedEntriesByDocumentId: new Map([ + ["document-2", read.entries[1]!], + ["document-1", read.entries[0]!], + ]), + }), + ).toEqual({ + documentIds: ["document-2", "document-1"], + limit: 2, + offset: 0, + }); + }); + + it("keeps complete setup when an initial Builder entry was not imported", () => { + const read = { + state: "live", + entries: [{ id: "entry-1" }, { id: "entry-2" }], + } as BuilderCmsReadResult; + + expect( + initialBuilderAttachmentSetupOptions({ + builderRead: read, + importedEntriesByDocumentId: new Map([ + ["document-1", read.entries[0]!], + ]), + }), + ).toBeUndefined(); + }); + + it("reports all durably matched Builder rows when reattachment imports none", () => { + expect( + builderAttachDurableItemCount( + new Map([ + ["document-1", { id: "entry-1" }], + ["document-2", { id: "entry-2" }], + ]), + ), + ).toBe(2); + }); + it("rejects unsafe source federation normalization formulas", () => { expect(() => attachSource.schema.parse({ diff --git a/templates/content/actions/content-spaces.db.test.ts b/templates/content/actions/content-spaces.db.test.ts index 2f77151788..296c4e827d 100644 --- a/templates/content/actions/content-spaces.db.test.ts +++ b/templates/content/actions/content-spaces.db.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { getDbExec } from "@agent-native/core/db"; import { runWithRequestContext } from "@agent-native/core/server"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; const TEST_DB_PATH = join( @@ -40,6 +40,7 @@ const OWNER = "owner@example.com"; const MEMBER = "member@example.com"; const OUTSIDER = "outsider@example.com"; const WORKSPACE_OWNER = "workspace-owner@example.com"; +const LARGE_DATABASE_OWNER = "large-database-owner@example.com"; beforeAll(async () => { process.env.DATABASE_URL = `file:${TEST_DB_PATH}`; @@ -112,6 +113,49 @@ async function addMember( } describe("Content space provisioning", () => { + it("batches Files reconciliation for databases with 5,000 row documents", async () => { + const provisioned = await runWithRequestContext( + { userEmail: LARGE_DATABASE_OWNER }, + () => provisionContentSpaces(getDb(), LARGE_DATABASE_OWNER), + ); + const now = new Date().toISOString(); + const documents = Array.from({ length: 5_000 }, (_, index) => ({ + id: `large-database-document-${index}`, + ownerEmail: LARGE_DATABASE_OWNER, + orgId: null, + spaceId: provisioned.personalSpaceId, + parentId: null, + title: `Large database row ${index}`, + content: "Synthetic row", + position: index, + visibility: "private" as const, + createdAt: now, + updatedAt: now, + })); + for (let start = 0; start < documents.length; start += 250) { + await getDb() + .insert(schema.documents) + .values(documents.slice(start, start + 250)); + } + + await expect( + runWithRequestContext({ userEmail: LARGE_DATABASE_OWNER }, () => + ensureContentSpacesAction.run({}), + ), + ).resolves.toBeDefined(); + + const [membershipCount] = await getDb() + .select({ count: sql`count(*)` }) + .from(schema.contentDatabaseItems) + .where( + eq( + schema.contentDatabaseItems.databaseId, + provisioned.personalFilesDatabaseId, + ), + ); + expect(Number(membershipCount.count)).toBe(5_000); + }); + it("creates an idempotent private named workspace with canonical Files", async () => { const provisioned = await runWithRequestContext( { userEmail: WORKSPACE_OWNER }, diff --git a/templates/content/actions/database-row-batch-actions.db.test.ts b/templates/content/actions/database-row-batch-actions.db.test.ts index 37d9cd976a..c5f3491289 100644 --- a/templates/content/actions/database-row-batch-actions.db.test.ts +++ b/templates/content/actions/database-row-batch-actions.db.test.ts @@ -239,6 +239,9 @@ describe("database row batch actions", () => { ).resolves.toEqual([{ spaceId }, { spaceId }]); expect(result.duplicatedItemId).toBe(result.duplicatedItemIds?.[0]); expect(result.duplicatedDocumentId).toBe(result.duplicatedDocumentIds?.[0]); + expect(result.duplicatedItems?.map((item) => item.id)).toEqual( + result.duplicatedItemIds, + ); expect(result.sourceItemIds).toEqual([rows[1].itemId, rows[2].itemId]); expect(result.sourceDocumentIds).toEqual([ rows[1].documentId, @@ -315,6 +318,49 @@ describe("database row batch actions", () => { expect(await orderedRows(second.databaseId)).toHaveLength(1); }); + it("returns exact duplicate projections when stored positions are sparse", async () => { + const db = getDb(); + const { databaseId, rows } = await createDatabaseWithRows(3); + await Promise.all( + rows.map((row, index) => + db + .update(schema.contentDatabaseItems) + .set({ position: [10, 30, 50][index] }) + .where(eq(schema.contentDatabaseItems.id, row.itemId)), + ), + ); + await Promise.all( + rows.map((row, index) => + db + .update(schema.documents) + .set({ position: [10, 30, 50][index] }) + .where(eq(schema.documents.id, row.documentId)), + ), + ); + + const single = await runWithRequestContext({ userEmail: OWNER }, () => + duplicateDatabaseItemAction.run({ itemId: rows[0].itemId }), + ); + expect(single.duplicatedItems).toHaveLength(1); + expect(single.duplicatedItems?.[0]).toMatchObject({ + id: single.duplicatedItemId, + document: { id: single.duplicatedDocumentId }, + }); + + const batch = await runWithRequestContext({ userEmail: OWNER }, () => + duplicateDatabaseItemsAction.run({ + databaseId, + itemIds: [rows[1].itemId, rows[2].itemId], + }), + ); + expect(batch.duplicatedItems?.map((item) => item.id)).toEqual( + batch.duplicatedItemIds, + ); + expect(batch.duplicatedItems?.map((item) => item.document.id)).toEqual( + batch.duplicatedDocumentIds, + ); + }); + it("deletes selected rows recursively in one batch and renumbers survivors", async () => { const db = getDb(); const { databaseId, databaseDocumentId, rows } = @@ -479,6 +525,10 @@ describe("database row batch actions", () => { const createdItemIds = results.map((result) => result.createdItemId); expect(new Set(createdItemIds).size).toBe(concurrentAdds); for (const result of results) { + expect(result.createdItem).toMatchObject({ + id: result.createdItemId, + document: { id: result.createdDocumentId }, + }); const [createdDocument] = await getDb() .select({ updatedAt: schema.documents.updatedAt }) .from(schema.documents) diff --git a/templates/content/actions/delete-database-items.ts b/templates/content/actions/delete-database-items.ts index a059abdd63..f8e65f45dd 100644 --- a/templates/content/actions/delete-database-items.ts +++ b/templates/content/actions/delete-database-items.ts @@ -38,7 +38,10 @@ export default defineAction({ }); await writeAppState("refresh-signal", { ts: Date.now() }); return { - ...(await getContentDatabaseResponse(database.id)), + ...(await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + })), deletedItemIds: removedItemIds, deletedDocumentIds: [], deletedCount: 0, @@ -74,7 +77,10 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); return { - ...(await getContentDatabaseResponse(database.id)), + ...(await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + })), deletedItemIds, deletedDocumentIds, deletedCount: deletedItemIds.length, diff --git a/templates/content/actions/disconnect-content-database-source.ts b/templates/content/actions/disconnect-content-database-source.ts index a552b22448..092bdd5abd 100644 --- a/templates/content/actions/disconnect-content-database-source.ts +++ b/templates/content/actions/disconnect-content-database-source.ts @@ -71,7 +71,7 @@ export default defineAction({ ), ); if (target) await deleteSourceRecords(target.id); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); } const source = await getExistingSource(database.id); @@ -79,6 +79,6 @@ export default defineAction({ await deleteSourceRecords(source.id); } - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/duplicate-database-item.ts b/templates/content/actions/duplicate-database-item.ts index 5f28ecb52d..9bcb548120 100644 --- a/templates/content/actions/duplicate-database-item.ts +++ b/templates/content/actions/duplicate-database-item.ts @@ -176,8 +176,22 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); + const response = await getContentDatabaseResponse(row.item.databaseId, { + limit: 100, + offset: 0, + }); + const duplicatedItem = + response.items.find((item) => item.id === nextItemId) ?? + ( + await getContentDatabaseResponse(row.item.databaseId, { + limit: 1, + offset: 0, + documentIds: [nextDocumentId], + }) + ).items.find((item) => item.id === nextItemId); return { - ...(await getContentDatabaseResponse(row.item.databaseId)), + ...response, + duplicatedItems: duplicatedItem ? [duplicatedItem] : [], duplicatedItemId: nextItemId, duplicatedDocumentId: nextDocumentId, }; diff --git a/templates/content/actions/duplicate-database-items.ts b/templates/content/actions/duplicate-database-items.ts index 40d4cda9a8..ae6f00d713 100644 --- a/templates/content/actions/duplicate-database-items.ts +++ b/templates/content/actions/duplicate-database-items.ts @@ -186,8 +186,26 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); + const response = await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + }); + const duplicatedPage = await getContentDatabaseResponse(database.id, { + limit: duplicates.length, + offset: 0, + documentIds: duplicates.map( + (duplicate) => duplicate.duplicatedDocumentId, + ), + }); + const duplicatedItemIds = new Set( + duplicates.map((duplicate) => duplicate.duplicatedItemId), + ); + const duplicatedItems = duplicatedPage.items.filter((item) => + duplicatedItemIds.has(item.id), + ); return { - ...(await getContentDatabaseResponse(database.id)), + ...response, + duplicatedItems, duplicatedItemId: duplicates[0]?.duplicatedItemId, duplicatedDocumentId: duplicates[0]?.duplicatedDocumentId, duplicatedItemIds: duplicates.map( diff --git a/templates/content/actions/execute-builder-source-execution.ts b/templates/content/actions/execute-builder-source-execution.ts index 26125d15de..f90d57b7fd 100644 --- a/templates/content/actions/execute-builder-source-execution.ts +++ b/templates/content/actions/execute-builder-source-execution.ts @@ -854,7 +854,8 @@ export function realExecutionDeps( executeWrite: (args) => executeBuilderCmsWrite(args), readLiveEntry: (args) => readBuilderCmsEntryLiveState(args), reconcileWrite: reconcileBuilderCmsWrite, - getResponse: (databaseId) => getContentDatabaseResponse(databaseId), + getResponse: (databaseId) => + getContentDatabaseResponse(databaseId, { limit: 100, offset: 0 }), lookupSafeModelIntent: lookupBuilderCmsSafeModelIntent, }; } diff --git a/templates/content/actions/get-content-database.ts b/templates/content/actions/get-content-database.ts index 3f183d7a71..d31f4b605e 100644 --- a/templates/content/actions/get-content-database.ts +++ b/templates/content/actions/get-content-database.ts @@ -1,18 +1,15 @@ import { defineAction } from "@agent-native/core"; -import { resolveAccess } from "@agent-native/core/sharing"; -import { eq } from "drizzle-orm"; import { z } from "zod"; -import { getDb, schema } from "../server/db/index.js"; import type { ContentDatabaseResponse, ContentDatabaseUnavailableResponse, } from "../shared/api.js"; -import { resolveContentSpaceAccess } from "./_content-space-access.js"; import { CONTENT_DATABASE_MAX_READ_LIMIT, + contentDatabaseTableQuerySchema, getContentDatabaseResponse, - getDocumentContextPath, + resolveContentDatabaseRead, } from "./_database-utils.js"; export default defineAction({ @@ -28,6 +25,7 @@ export default defineAction({ .max(CONTENT_DATABASE_MAX_READ_LIMIT) .optional(), offset: z.coerce.number().int().min(0).optional(), + tableQuery: contentDatabaseTableQuerySchema, }), http: { method: "GET" }, readOnly: true, @@ -36,69 +34,19 @@ export default defineAction({ documentId, limit, offset, + tableQuery, }): Promise => { - const db = getDb(); - let resolvedDatabaseId = databaseId; - - if (!resolvedDatabaseId && documentId) { - const [database] = await db - .select() - .from(schema.contentDatabases) - .where(eq(schema.contentDatabases.documentId, documentId)); - resolvedDatabaseId = database?.id; - } - - if (!resolvedDatabaseId) { - throw new Error("Either databaseId or documentId is required."); - } - - const [database] = await db - .select() - .from(schema.contentDatabases) - .where(eq(schema.contentDatabases.id, resolvedDatabaseId)); - if (!database) { - return { - available: false, - reason: "not_found", - databaseId: resolvedDatabaseId, - documentId: documentId ?? null, - message: `Database "${resolvedDatabaseId}" not found`, - }; - } - - let canRead = Boolean(await resolveAccess("document", database.documentId)); - if (!canRead && database.systemRole === "files" && database.spaceId) { - try { - await resolveContentSpaceAccess(database.spaceId); - canRead = true; - } catch { - canRead = false; - } - } - if (!canRead) throw new Error(`Database "${resolvedDatabaseId}" not found`); - - if (database.deletedAt) { - return { - available: false, - reason: "deleted", - databaseId: database.id, - documentId: database.documentId, - deletedAt: database.deletedAt, - message: `Database "${database.id}" has been deleted`, - }; - } + const resolved = await resolveContentDatabaseRead({ + databaseId, + documentId, + }); + if (!resolved.available) return resolved; - const response = await getContentDatabaseResponse(resolvedDatabaseId, { + return getContentDatabaseResponse(resolved.database.id, { limit, offset, + tableQuery, + database: resolved.database, }); - const [document] = await db - .select() - .from(schema.documents) - .where(eq(schema.documents.id, database.documentId)); - return { - ...response, - contextPath: document ? await getDocumentContextPath(document) : [], - }; }, }); diff --git a/templates/content/actions/move-database-item.ts b/templates/content/actions/move-database-item.ts index 90052bea6e..751c8de55a 100644 --- a/templates/content/actions/move-database-item.ts +++ b/templates/content/actions/move-database-item.ts @@ -135,6 +135,9 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); - return getContentDatabaseResponse(row.item.databaseId); + return getContentDatabaseResponse(row.item.databaseId, { + limit: 100, + offset: 0, + }); }, }); diff --git a/templates/content/actions/prepare-builder-source-execution.ts b/templates/content/actions/prepare-builder-source-execution.ts index d1359e7430..6606dbea45 100644 --- a/templates/content/actions/prepare-builder-source-execution.ts +++ b/templates/content/actions/prepare-builder-source-execution.ts @@ -155,7 +155,7 @@ export default defineAction({ timing.record("gate_preparation_and_dry_run_validation", gateStartedAt); const response = await timing.measure("response_load", () => - getContentDatabaseResponse(database.id), + getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }), ); const result = { ...response, timings: timing.finish() }; timing.log("succeeded"); diff --git a/templates/content/actions/prepare-builder-source-review.ts b/templates/content/actions/prepare-builder-source-review.ts index 03909f0d30..687ccc9539 100644 --- a/templates/content/actions/prepare-builder-source-review.ts +++ b/templates/content/actions/prepare-builder-source-review.ts @@ -34,7 +34,6 @@ import { serializeSourceRowRecord, sourceChangeSetKey, } from "./_database-source-utils.js"; -import { getContentDatabaseResponse } from "./_database-utils.js"; export const BUILDER_SOURCE_REVIEW_PREPARE_LIMIT = 100; @@ -763,18 +762,16 @@ export default defineAction({ approvalStartedAt, ); - const { reviewedSnapshot, response } = await timing.measure( - "reconciliation_and_response_load", - async () => ({ - reviewedSnapshot: await getContentDatabaseSourceSnapshotForWrite( + const reviewedSnapshot = await timing.measure( + "reconciliation_snapshot_load", + () => + getContentDatabaseSourceSnapshotForWrite( database, args.sourceId, reviewableChanges.every((changeSet) => changeSet.documentId) ? reviewableChanges.map((changeSet) => changeSet.documentId!) : args.documentIds, ), - response: await getContentDatabaseResponse(database.id), - }), ); if (!reviewedSnapshot) throw new Error("Builder source disappeared."); // Build the review payload from the TARGET source snapshot, not @@ -797,7 +794,6 @@ export default defineAction({ review.preparedRowLimit = reviewableChanges.length; const result = { - ...response, review, preparedChangeSetMappings, timings: timing.finish(), diff --git a/templates/content/actions/preview-content-database-source-attach.ts b/templates/content/actions/preview-content-database-source-attach.ts new file mode 100644 index 0000000000..4aaf33845f --- /dev/null +++ b/templates/content/actions/preview-content-database-source-attach.ts @@ -0,0 +1,92 @@ +import { defineAction } from "@agent-native/core"; +import { assertAccess } from "@agent-native/core/sharing"; +import { z } from "zod"; + +import type { + BuilderCmsAttachPreviewResponse, + ContentDatabaseItem, +} from "../shared/api.js"; +import { readBuilderCmsContentEntries } from "./_builder-cms-read-client.js"; +import { + builderCmsImportIds, + resolveDatabaseForSourceMutation, +} from "./_database-source-utils.js"; +import { getContentDatabaseResponse } from "./_database-utils.js"; + +export default defineAction({ + description: + "Preview the first projected Builder rows for a database source attachment without writing local or provider data.", + schema: z.object({ + databaseId: z.string().optional(), + documentId: z.string().optional(), + sourceTable: z.string().min(1).max(500), + fieldPaths: z.array(z.string().max(500)).max(200).optional(), + }), + http: { method: "GET" }, + readOnly: true, + run: async (args): Promise => { + const database = await resolveDatabaseForSourceMutation(args); + if (!database) throw new Error("Database not found."); + await assertAccess("document", database.documentId, "editor"); + + const [read, base] = await Promise.all([ + readBuilderCmsContentEntries({ + model: args.sourceTable, + fieldPaths: args.fieldPaths, + allowCached: true, + maxPages: 1, + }), + getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }), + ]); + if (read.state !== "live") { + throw new Error( + read.message ?? "Builder rows are not available for preview.", + ); + } + + const items: ContentDatabaseItem[] = read.entries.map((entry, index) => { + const ids = builderCmsImportIds({ + ownerEmail: database.ownerEmail, + databaseId: database.id, + sourceTable: args.sourceTable, + entryId: entry.id, + }); + return { + id: ids.itemId, + databaseId: database.id, + document: { + id: ids.documentId, + parentId: database.documentId, + title: entry.title.trim() || entry.id, + content: "", + icon: null, + position: index, + isFavorite: false, + hideFromSearch: false, + accessRole: "viewer", + canEdit: false, + canManage: false, + createdAt: entry.updatedAt, + updatedAt: entry.updatedAt, + }, + position: index, + properties: [], + bodyHydration: { + status: "pending", + attemptedAt: null, + error: null, + version: null, + }, + }; + }); + return { + databaseId: database.id, + documentId: database.documentId, + sourceTable: args.sourceTable, + base, + items, + fetchedAt: read.fetchedAt, + hasMore: read.progress.hasMore, + }; + }, +}); diff --git a/templates/content/actions/process-builder-body-hydration.ts b/templates/content/actions/process-builder-body-hydration.ts index 678f673654..24d22013b0 100644 --- a/templates/content/actions/process-builder-body-hydration.ts +++ b/templates/content/actions/process-builder-body-hydration.ts @@ -15,7 +15,7 @@ export default defineAction({ schema: z.object({ sourceId: z.string(), documentId: z.string().optional(), - limit: z.number().int().positive().max(50).optional(), + limit: z.number().int().positive().max(600).optional(), }), agentTool: false, run: async ( @@ -42,6 +42,7 @@ export default defineAction({ sourceId: args.sourceId, documentId: args.documentId, limit: args.limit, + preloadBodies: !args.documentId, }); }, }); diff --git a/templates/content/actions/query-content-database-items.ts b/templates/content/actions/query-content-database-items.ts new file mode 100644 index 0000000000..777619db1a --- /dev/null +++ b/templates/content/actions/query-content-database-items.ts @@ -0,0 +1,64 @@ +import { defineAction } from "@agent-native/core"; +import { z } from "zod"; + +import type { + ContentDatabaseItemsPageResponse, + ContentDatabaseUnavailableResponse, +} from "../shared/api.js"; +import { + CONTENT_DATABASE_MAX_READ_LIMIT, + contentDatabaseTableQuerySchema, + getContentDatabasePageResponse, + resolveContentDatabaseRead, +} from "./_database-utils.js"; + +export default defineAction({ + description: + "Query one ordered and filtered page of a content database without reopening its metadata.", + schema: z.object({ + databaseId: z.string().optional().describe("Database ID"), + documentId: z.string().optional().describe("Database document/page ID"), + limit: z.coerce + .number() + .int() + .min(1) + .max(CONTENT_DATABASE_MAX_READ_LIMIT) + .optional(), + offset: z.coerce.number().int().min(0).optional(), + tableQuery: contentDatabaseTableQuerySchema, + }), + http: { method: "GET" }, + readOnly: true, + agentTool: false, + run: async ({ + databaseId, + documentId, + limit, + offset, + tableQuery, + }): Promise< + ContentDatabaseItemsPageResponse | ContentDatabaseUnavailableResponse + > => { + const resolved = await resolveContentDatabaseRead({ + databaseId, + documentId, + }); + if (!resolved.available) return resolved; + + const page = await getContentDatabasePageResponse(resolved.database.id, { + // This action is the bounded table replacement path; unlike the legacy + // database response, an omitted limit must not turn it into a full read. + limit: limit ?? 100, + offset, + tableQuery, + database: resolved.database, + }); + return { + items: page.items, + source: page.source, + sources: page.sources, + pagination: page.pagination, + tableQueryMode: page.tableQueryMode, + }; + }, +}); diff --git a/templates/content/actions/refresh-content-database-source.ts b/templates/content/actions/refresh-content-database-source.ts index 5d576815be..1c9f2f4ea9 100644 --- a/templates/content/actions/refresh-content-database-source.ts +++ b/templates/content/actions/refresh-content-database-source.ts @@ -35,6 +35,12 @@ export default defineAction({ .describe( "For paginated Builder CMS or Notion sources, read a bounded multi-page snapshot in this refresh.", ), + finishBuilderPagination: z + .boolean() + .optional() + .describe( + "Internal Builder continuation mode. Read every remaining page from the persisted continuation offset.", + ), expectedBuilderContinuationOffset: z .number() .int() @@ -75,6 +81,7 @@ export default defineAction({ source: claimedSource.source, now, runFullRefresh: args.fullRefresh === true, + finishPagination: args.finishBuilderPagination === true, refreshClaimId: claimedSource.claimId, }).catch(async (error: unknown) => { await releaseBuilderCmsSourceRefreshClaim({ diff --git a/templates/content/actions/resync-content-database-source.db.test.ts b/templates/content/actions/resync-content-database-source.db.test.ts index a5cd253ea4..a23a29b34b 100644 --- a/templates/content/actions/resync-content-database-source.db.test.ts +++ b/templates/content/actions/resync-content-database-source.db.test.ts @@ -40,6 +40,7 @@ const builderReadMock = vi.hoisted(() => ({ fieldPaths?: readonly string[]; maxPages?: number; offset?: number; + includeBodies?: boolean; }>, modelFieldsErrorFor: null as string | null, singleEntryCalls: [] as Array<{ model: string; entryId: string }>, @@ -184,13 +185,77 @@ vi.mock("./_builder-cms-read-client.js", async () => { fieldPaths, maxPages, offset, + includeBodies, }: { model: string; fieldPaths?: readonly string[]; maxPages?: number; offset?: number; + includeBodies?: boolean; }) => { - builderReadMock.calls.push({ model, fieldPaths, maxPages, offset }); + builderReadMock.calls.push({ + model, + fieldPaths, + maxPages, + offset, + includeBodies, + }); + if (model === "collection-bulk-pristine") { + const entries = Array.from({ length: 120 }, (_, index) => { + const position = index + 1; + const id = `entry-bulk-pristine-${position}`; + const title = `Bulk pristine ${position}`; + return { + id, + model, + title, + urlPath: `/bulk-pristine-${position}`, + updatedAt: "2026-01-01T00:00:00.000Z", + sourceValues: { + "data.title": title, + "data.url": `/bulk-pristine-${position}`, + lastUpdated: "2026-01-01T00:00:00.000Z", + }, + rawEntry: { + id, + model, + name: title, + lastUpdated: "2026-01-01T00:00:00.000Z", + data: { + title, + url: `/bulk-pristine-${position}`, + blocks: [ + { + "@type": "@builder.io/sdk:Element", + "@version": 2, + id: `text-bulk-pristine-${position}`, + component: { + name: "Text", + options: { text: `

Bulk body ${position}

` }, + }, + }, + ], + }, + }, + }; + }); + return { + state: "live", + entries, + fetchedAt: "2026-01-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 10_000, + pageSize: 100, + startOffset: 0, + nextOffset: entries.length, + fetchedEntryCount: entries.length, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + } if (model === "collection-suspicious-empty") { return { state: "live", @@ -1951,6 +2016,22 @@ it("records freshly imported Builder row identities even when title and URL keys expect(concurrentRetryDocuments).toHaveLength(2); expect(concurrentRetryItems).toHaveLength(2); + const interruptedAttachRecovery = await importBuilderEntries({ + database, + entries, + now, + sourceTable: "collection-duplicates", + existingSourceRows: [], + skipTitleDedup: true, + }); + expect(interruptedAttachRecovery.imported).toBe(0); + expect(interruptedAttachRecovery.importedItems).toEqual([]); + expect( + Array.from(interruptedAttachRecovery.importedEntriesByDocumentId.values()) + .map((entry) => entry.id) + .sort(), + ).toEqual(["entry-dup-1", "entry-dup-2"]); + const importedItems = concurrentRetryItems.map( (item: { id: string; documentId: string }, index: number) => ({ id: item.id, @@ -1983,6 +2064,104 @@ it("records freshly imported Builder row identities even when title and URL keys expect(concurrentSourceRows).toHaveLength(2); }); +it("repairs a deterministic Builder item whose source-row link is missing", async () => { + builderReadMock.mode = "full"; + builderReadMock.calls = []; + const db = getDb(); + const now = new Date().toISOString(); + const databaseId = "db_interrupted_builder_attach"; + const databaseDocId = "doc_db_interrupted_builder_attach"; + const sourceId = "src_interrupted_builder_attach"; + const sourceTable = "collection-metadata-only"; + const entryId = "entry-metadata-only-1"; + const { documentId, itemId } = ( + await import("./_database-source-utils.js") + ).builderCmsImportIds({ + ownerEmail: OWNER, + databaseId, + sourceTable, + entryId, + }); + + await db.insert(schema.documents).values([ + { + id: databaseDocId, + spaceId: IMPORT_SPACE_ID, + ownerEmail: OWNER, + title: "Interrupted Builder attach", + createdAt: now, + updatedAt: now, + }, + { + id: documentId, + spaceId: IMPORT_SPACE_ID, + ownerEmail: OWNER, + parentId: databaseDocId, + title: "Metadata-only hydration", + content: "", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + spaceId: IMPORT_SPACE_ID, + ownerEmail: OWNER, + documentId: databaseDocId, + title: "Interrupted Builder attach", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: itemId, + ownerEmail: OWNER, + databaseId, + documentId, + position: 0, + bodyHydrationStatus: "pending", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: OWNER, + databaseId, + sourceType: "builder-cms", + sourceName: sourceTable, + sourceTable, + createdAt: now, + updatedAt: now, + }); + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, databaseId)); + const [source] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, sourceId)); + + await resync({ database, source, now, runFullRefresh: true }); + + const response = await ( + await import("./_database-utils.js") + ).getContentDatabaseResponse(databaseId); + const sourceRows = await db + .select({ + databaseItemId: schema.contentDatabaseSourceRows.databaseItemId, + documentId: schema.contentDatabaseSourceRows.documentId, + sourceRowId: schema.contentDatabaseSourceRows.sourceRowId, + }) + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, sourceId)); + + expect(response.items.map((item) => item.id)).toEqual([itemId]); + expect(sourceRows).toEqual([ + { databaseItemId: itemId, documentId, sourceRowId: entryId }, + ]); +}); + it("repairs a legacy organization database into its organization space", async () => { builderReadMock.mode = "full"; builderReadMock.calls = []; @@ -2330,7 +2509,11 @@ it("resync advances Builder partial reads with a cursor and converges on the fin 'prop-author:"Grace Hopper"', ]), ); - expect(builderReadMock.calls.map((call) => call.offset ?? 0)).toEqual([0, 1]); + expect( + builderReadMock.calls + .filter((call) => call.includeBodies !== true) + .map((call) => call.offset ?? 0), + ).toEqual([0, 1]); }); it("full Builder refresh reads every page in one resync call", async () => { @@ -2408,11 +2591,13 @@ it("full Builder refresh reads every page in one resync call", async () => { rows.map((row: { sourceRowId: string }) => row.sourceRowId).sort(), ).toEqual(["entry-a1", "entry-a2"]); expect( - builderReadMock.calls.map(({ model, maxPages, offset }) => ({ - model, - maxPages, - offset, - })), + builderReadMock.calls + .filter((call) => call.includeBodies !== true) + .map(({ model, maxPages, offset }) => ({ + model, + maxPages, + offset, + })), ).toEqual([{ model: "collection-a", maxPages: undefined, offset: 0 }]); }); @@ -3035,7 +3220,10 @@ it("continues a 597-row snapshot past offset 500 without pruning or restarting", expect(partialRowCounts).toEqual([597, 597, 597, 597, 597, 597]); expect( builderReadMock.calls - .filter((call) => call.model === "collection-large-597") + .filter( + (call) => + call.model === "collection-large-597" && call.includeBodies !== true, + ) .map((call) => call.offset), ).toEqual([0, 100, 200, 300, 400, 500]); const [finishedSource] = await db @@ -3457,7 +3645,6 @@ it("does not let open-row hydration promotion downgrade a queued full Builder bo ), ) .where(eq(schema.documents.id, documentId)); - expect(after.content).toBe(fullBody); expect(after.status).toBe("hydrated"); expect(after.queued).toBeNull(); @@ -4009,6 +4196,315 @@ it("rebuilds an empty queued Builder body from the current source row before giv expect(after.attempts).toBeNull(); }); +it("lets only one overlapping worker claim and finish the same Builder body job", async () => { + const db = getDb(); + const now = new Date().toISOString(); + const databaseId = "db_hydration_overlapping_claim"; + const databaseDocId = "doc_db_hydration_overlapping_claim"; + const documentId = "doc_hydration_overlapping_claim"; + const itemId = "item_hydration_overlapping_claim"; + const sourceId = "src_hydration_overlapping_claim"; + const sourceRowId = "entry_hydration_overlapping_claim"; + const queueId = "queue_hydration_overlapping_claim"; + const body = "Only the worker holding the queue claim may write this body."; + const entry = { + id: sourceRowId, + model: "collection-hydration-overlapping-claim", + title: "Hydration overlapping claim", + urlPath: "/blog/hydration-overlapping-claim", + updatedAt: "2026-01-01T00:00:00.000Z", + sourceValues: { + "data.title": "Hydration overlapping claim", + "data.url": "/blog/hydration-overlapping-claim", + lastUpdated: "2026-01-01T00:00:00.000Z", + [BUILDER_CMS_BODY_CONTENT_KEY]: body, + }, + }; + + await db.insert(schema.documents).values([ + { + id: databaseDocId, + ownerEmail: OWNER, + title: "DB hydration overlapping claim", + createdAt: now, + updatedAt: now, + }, + { + id: documentId, + ownerEmail: OWNER, + parentId: databaseDocId, + title: "Hydration overlapping claim", + content: "", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + ownerEmail: OWNER, + documentId: databaseDocId, + title: "DB hydration overlapping claim", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: OWNER, + databaseId, + sourceType: "builder-cms", + sourceName: "collection-hydration-overlapping-claim", + sourceTable: "collection-hydration-overlapping-claim", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: itemId, + ownerEmail: OWNER, + databaseId, + documentId, + position: 0, + bodyHydrationStatus: "pending", + bodyHydrationError: null, + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSourceRows).values({ + id: "row_hydration_overlapping_claim", + ownerEmail: OWNER, + sourceId, + databaseItemId: itemId, + documentId, + sourceRowId, + sourceQualifiedId: `builder-cms://collection-hydration-overlapping-claim/${sourceRowId}`, + sourceDisplayKey: "Hydration overlapping claim", + sourceValuesJson: JSON.stringify(entry.sourceValues), + provenance: "Builder CMS read adapter", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseBodyHydrationQueue).values({ + id: queueId, + ownerEmail: OWNER, + sourceId, + databaseItemId: itemId, + documentId, + sourceRowId, + sourceTable: "collection-hydration-overlapping-claim", + sourceEntryJson: JSON.stringify(entry), + priority: 0, + attempts: 0, + createdAt: now, + updatedAt: now, + }); + const [queuedJob] = await db + .select() + .from(schema.contentDatabaseBodyHydrationQueue) + .where(eq(schema.contentDatabaseBodyHydrationQueue.id, queueId)); + + const results = await Promise.all([ + hydrateQueuedBodies({ sourceId, limit: 1, preloadedJobs: [queuedJob] }), + hydrateQueuedBodies({ sourceId, limit: 1, preloadedJobs: [queuedJob] }), + ]); + + const [after] = await db + .select({ + content: schema.documents.content, + status: schema.contentDatabaseItems.bodyHydrationStatus, + error: schema.contentDatabaseItems.bodyHydrationError, + queued: schema.contentDatabaseBodyHydrationQueue.id, + }) + .from(schema.documents) + .innerJoin( + schema.contentDatabaseItems, + eq(schema.contentDatabaseItems.documentId, schema.documents.id), + ) + .leftJoin( + schema.contentDatabaseBodyHydrationQueue, + eq( + schema.contentDatabaseBodyHydrationQueue.databaseItemId, + schema.contentDatabaseItems.id, + ), + ) + .where(eq(schema.documents.id, documentId)); + const remainingQueue = await db + .select({ id: schema.contentDatabaseBodyHydrationQueue.id }) + .from(schema.contentDatabaseBodyHydrationQueue) + .where(eq(schema.contentDatabaseBodyHydrationQueue.sourceId, sourceId)); + + expect(results.map((result) => result.processed).sort()).toEqual([0, 1]); + expect(results.map((result) => result.succeeded).sort()).toEqual([0, 1]); + expect(results.every((result) => result.failed === 0)).toBe(true); + expect(after.content).toBe(body); + expect(after.status).toBe("hydrated"); + expect(after.error).toBeNull(); + expect(after.queued).toBeNull(); + expect(remainingQueue).toEqual([]); +}); + +it("bulk-persists pristine first-attempt Builder bodies without per-row reads", async () => { + builderReadMock.calls = []; + builderReadMock.singleEntryCalls = []; + const db = getDb(); + const now = new Date().toISOString(); + const databaseId = "db_hydration_bulk_pristine"; + const databaseDocId = "doc_db_hydration_bulk_pristine"; + const sourceId = "src_hydration_bulk_pristine"; + const count = 120; + + await db.insert(schema.documents).values({ + id: databaseDocId, + ownerEmail: OWNER, + title: "DB hydration bulk pristine", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + ownerEmail: OWNER, + documentId: databaseDocId, + title: "DB hydration bulk pristine", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: OWNER, + databaseId, + sourceType: "builder-cms", + sourceName: "collection-bulk-pristine", + sourceTable: "collection-bulk-pristine", + createdAt: now, + updatedAt: now, + }); + + const positions = Array.from({ length: count }, (_, index) => index + 1); + for (let offset = 0; offset < positions.length; offset += 40) { + const batch = positions.slice(offset, offset + 40); + await db.insert(schema.documents).values( + batch.map((position) => ({ + id: `doc_hydration_bulk_pristine_${position}`, + ownerEmail: OWNER, + parentId: databaseDocId, + title: `Bulk pristine ${position}`, + content: "", + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.contentDatabaseItems).values( + batch.map((position) => ({ + id: `item_hydration_bulk_pristine_${position}`, + ownerEmail: OWNER, + databaseId, + documentId: `doc_hydration_bulk_pristine_${position}`, + position, + bodyHydrationStatus: "pending", + bodyHydrationError: null, + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.contentDatabaseSourceRows).values( + batch.map((position) => ({ + id: `row_hydration_bulk_pristine_${position}`, + ownerEmail: OWNER, + sourceId, + databaseItemId: `item_hydration_bulk_pristine_${position}`, + documentId: `doc_hydration_bulk_pristine_${position}`, + sourceRowId: `entry-bulk-pristine-${position}`, + sourceQualifiedId: `builder-cms://collection-bulk-pristine/entry-bulk-pristine-${position}`, + sourceDisplayKey: `Bulk pristine ${position}`, + sourceValuesJson: JSON.stringify({ + "data.title": `Bulk pristine ${position}`, + "data.url": `/bulk-pristine-${position}`, + lastUpdated: "2026-01-01T00:00:00.000Z", + }), + provenance: "Builder CMS read adapter", + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.contentDatabaseBodyHydrationQueue).values( + batch.map((position) => ({ + id: `queue_hydration_bulk_pristine_${position}`, + ownerEmail: OWNER, + sourceId, + databaseItemId: `item_hydration_bulk_pristine_${position}`, + documentId: `doc_hydration_bulk_pristine_${position}`, + sourceRowId: `entry-bulk-pristine-${position}`, + sourceTable: "collection-bulk-pristine", + sourceEntryJson: JSON.stringify({ + id: `entry-bulk-pristine-${position}`, + model: "collection-bulk-pristine", + title: `Bulk pristine ${position}`, + urlPath: `/bulk-pristine-${position}`, + updatedAt: "2026-01-01T00:00:00.000Z", + sourceValues: { + "data.title": `Bulk pristine ${position}`, + "data.url": `/bulk-pristine-${position}`, + lastUpdated: "2026-01-01T00:00:00.000Z", + }, + }), + priority: 10, + attempts: 0, + createdAt: now, + updatedAt: now, + })), + ); + } + + const result = await hydrateQueuedBodies({ + sourceId, + limit: count, + preloadBodies: true, + }); + const hydrated = await db + .select({ + content: schema.documents.content, + status: schema.contentDatabaseItems.bodyHydrationStatus, + version: schema.contentDatabaseItems.bodyHydrationVersion, + sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson, + }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.documents, + eq(schema.documents.id, schema.contentDatabaseItems.documentId), + ) + .innerJoin( + schema.contentDatabaseSourceRows, + eq( + schema.contentDatabaseSourceRows.databaseItemId, + schema.contentDatabaseItems.id, + ), + ) + .where(eq(schema.contentDatabaseItems.databaseId, databaseId)); + const remainingQueue = await db + .select({ id: schema.contentDatabaseBodyHydrationQueue.id }) + .from(schema.contentDatabaseBodyHydrationQueue) + .where(eq(schema.contentDatabaseBodyHydrationQueue.sourceId, sourceId)); + + expect(result).toMatchObject({ + processed: count, + succeeded: count, + failed: 0, + remaining: 0, + }); + expect(hydrated).toHaveLength(count); + expect(hydrated.every((row) => row.status === "hydrated")).toBe(true); + expect(hydrated.every((row) => Boolean(row.version))).toBe(true); + expect(hydrated.every((row) => row.content.includes("Bulk body"))).toBe(true); + expect( + hydrated.every((row) => + JSON.parse(row.sourceValuesJson)[BUILDER_CMS_BODY_CONTENT_KEY]?.includes( + "Bulk body", + ), + ), + ).toBe(true); + expect(remainingQueue).toEqual([]); + expect(builderReadMock.calls).toHaveLength(1); + expect(builderReadMock.singleEntryCalls).toEqual([]); +}); + it("repairs a stale remote body hash without overwriting a locally diverged document", async () => { const db = getDb(); const now = new Date().toISOString(); @@ -4482,12 +4978,38 @@ it("terminates an unbuildable empty Builder body job at the hydration cap", asyn }, }), priority: 0, - attempts: 4, + attempts: 0, createdAt: now, updatedAt: now, }); - await hydrateQueuedBodies({ sourceId, limit: 1 }); + await hydrateQueuedBodies({ sourceId, limit: 1, preloadBodies: true }); + + const [retryable] = await db + .select({ + status: schema.contentDatabaseItems.bodyHydrationStatus, + attempts: schema.contentDatabaseBodyHydrationQueue.attempts, + lastAttemptedAt: schema.contentDatabaseBodyHydrationQueue.lastAttemptedAt, + }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.contentDatabaseBodyHydrationQueue, + eq( + schema.contentDatabaseBodyHydrationQueue.databaseItemId, + schema.contentDatabaseItems.id, + ), + ) + .where(eq(schema.contentDatabaseItems.documentId, documentId)); + + expect(retryable).toMatchObject({ + status: "pending", + attempts: 1, + lastAttemptedAt: null, + }); + + for (let attempt = 1; attempt < 5; attempt += 1) { + await hydrateQueuedBodies({ sourceId, limit: 1, preloadBodies: true }); + } const [after] = await db .select({ diff --git a/templates/content/actions/review-content-database-source-change-set.ts b/templates/content/actions/review-content-database-source-change-set.ts index 6d78c01e2a..9ea5187750 100644 --- a/templates/content/actions/review-content-database-source-change-set.ts +++ b/templates/content/actions/review-content-database-source-change-set.ts @@ -116,6 +116,6 @@ export default defineAction({ .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/set-content-database-source-write-mode.ts b/templates/content/actions/set-content-database-source-write-mode.ts index 2473f1b0c5..25d71f5b37 100644 --- a/templates/content/actions/set-content-database-source-write-mode.ts +++ b/templates/content/actions/set-content-database-source-write-mode.ts @@ -121,6 +121,6 @@ export default defineAction({ "Builder source settings changed repeatedly while saving write mode.", }); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/stage-builder-revision.ts b/templates/content/actions/stage-builder-revision.ts index ab4a07f113..96902c4979 100644 --- a/templates/content/actions/stage-builder-revision.ts +++ b/templates/content/actions/stage-builder-revision.ts @@ -114,6 +114,6 @@ export default defineAction({ .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/update-content-database-personal-view.test.ts b/templates/content/actions/update-content-database-personal-view.test.ts index 8804cdaa60..8b46e77136 100644 --- a/templates/content/actions/update-content-database-personal-view.test.ts +++ b/templates/content/actions/update-content-database-personal-view.test.ts @@ -8,7 +8,9 @@ import { normalizePersonalDatabaseViewOverrides, PERSONAL_DATABASE_VIEW_OVERRIDES_VERSION, } from "./_content-database-personal-view"; -import action from "./update-content-database-personal-view"; +import action, { + personalSidebarOrderItemIds, +} from "./update-content-database-personal-view"; describe("update content database personal view", () => { it("accepts grouped filter overrides for the current user", () => { @@ -52,6 +54,32 @@ describe("update content database personal view", () => { ).toBeNull(); }); + it("only validates item ids that a sidebar order actually references", () => { + expect( + personalSidebarOrderItemIds({ + version: PERSONAL_DATABASE_VIEW_OVERRIDES_VERSION, + views: [ + { + id: "table", + sorts: [{ key: "date", label: "Date", direction: "asc" }], + filters: [], + filterMode: "and", + }, + { + id: "files", + sorts: [], + filters: [], + filterMode: "and", + sidebarOrder: { + mode: "custom", + itemIds: ["item-b", "item-a", "item-b"], + }, + }, + ], + }), + ).toEqual(["item-b", "item-a"]); + }); + it("preserves a personal sidebar order and normalizes legacy v2 views", () => { const parsed = action.schema.parse({ databaseId: "database", diff --git a/templates/content/actions/update-content-database-personal-view.ts b/templates/content/actions/update-content-database-personal-view.ts index ff2c6db418..3e84393f50 100644 --- a/templates/content/actions/update-content-database-personal-view.ts +++ b/templates/content/actions/update-content-database-personal-view.ts @@ -1,14 +1,26 @@ import { defineAction } from "@agent-native/core"; import { deleteUserSetting, putUserSetting } from "@agent-native/core/settings"; +import { and, eq, inArray } from "drizzle-orm"; import { z } from "zod"; +import { getDb, schema } from "../server/db/index.js"; +import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js"; import { assertContentDatabaseViewerAccess, normalizePersonalDatabaseViewOverrides, personalDatabaseViewSettingKey, personalViewOverridesSchema, } from "./_content-database-personal-view.js"; -import { getContentDatabaseResponse } from "./_database-utils.js"; + +export function personalSidebarOrderItemIds( + overrides: z.infer, +) { + return [ + ...new Set( + overrides.views.flatMap((view) => view.sidebarOrder?.itemIds ?? []), + ), + ]; +} export default defineAction({ description: @@ -23,10 +35,24 @@ export default defineAction({ const key = personalDatabaseViewSettingKey(databaseId); if (overrides) { - const visibleDatabase = await getContentDatabaseResponse(databaseId); + const requestedItemIds = personalSidebarOrderItemIds(overrides); + const validItemIds = new Set(); + const itemIdChunkSize = Math.max(1, bulkChunkSizeForColumnCount(1) - 1); + for (const itemIds of chunks(requestedItemIds, itemIdChunkSize)) { + const rows = await getDb() + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where( + and( + eq(schema.contentDatabaseItems.databaseId, databaseId), + inArray(schema.contentDatabaseItems.id, itemIds), + ), + ); + for (const row of rows) validItemIds.add(row.id); + } overrides = normalizePersonalDatabaseViewOverrides( overrides, - new Set(visibleDatabase.items.map((item) => item.id)), + validItemIds, ); await putUserSetting(ctx.userEmail, key, overrides); } else { diff --git a/templates/content/actions/update-content-database-view.ts b/templates/content/actions/update-content-database-view.ts index 8dbe954e88..edbf7ac923 100644 --- a/templates/content/actions/update-content-database-view.ts +++ b/templates/content/actions/update-content-database-view.ts @@ -136,6 +136,6 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); - return getContentDatabaseResponse(databaseId); + return getContentDatabaseResponse(databaseId, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/validate-builder-source-execution.ts b/templates/content/actions/validate-builder-source-execution.ts index 3cae203b8d..26574db458 100644 --- a/templates/content/actions/validate-builder-source-execution.ts +++ b/templates/content/actions/validate-builder-source-execution.ts @@ -153,6 +153,6 @@ export default defineAction({ .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/app/components/editor/DocumentProperties.layout.test.ts b/templates/content/app/components/editor/DocumentProperties.layout.test.ts index 21406c2f5e..36228346b9 100644 --- a/templates/content/app/components/editor/DocumentProperties.layout.test.ts +++ b/templates/content/app/components/editor/DocumentProperties.layout.test.ts @@ -107,6 +107,17 @@ describe("document property layout", () => { expect(source).toContain('role="alert"'); }); + it("uses the optimistic source-field mutation for Add Property", () => { + const source = readPropertiesSource(); + + expect(source).toContain( + "useAddContentDatabaseSourceFieldProperty(documentId)", + ); + expect(source).not.toContain( + '>("add-content-database-source-field-property", {', + ); + }); + it("makes editable property value triggers fill the database cell", () => { const source = readPropertiesSource(); diff --git a/templates/content/app/components/editor/DocumentProperties.tsx b/templates/content/app/components/editor/DocumentProperties.tsx index a94354bb2c..642693a730 100644 --- a/templates/content/app/components/editor/DocumentProperties.tsx +++ b/templates/content/app/components/editor/DocumentProperties.tsx @@ -19,10 +19,8 @@ import { } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; import type { - AddContentDatabaseSourceFieldPropertyRequest, BindContentDatabaseSourceFieldRequest, ContentDatabaseResponse, - ContentDatabaseSourceFieldPropertyResponse, ContentDatabaseSource, DocumentProperty, } from "@shared/api"; @@ -130,7 +128,7 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; -import { applySourceFieldPropertyToDatabaseResponse } from "@/hooks/use-content-database"; +import { useAddContentDatabaseSourceFieldProperty } from "@/hooks/use-content-database"; import { documentPropertiesResponseMatchesScope, useConfigureDocumentProperty, @@ -3148,25 +3146,8 @@ export function AddProperty({ }) { const t = useT(); const configure = useConfigureDocumentProperty(documentId, databaseId); - const queryClient = useQueryClient(); - const addSourceFieldProperty = useActionMutation< - ContentDatabaseSourceFieldPropertyResponse, - AddContentDatabaseSourceFieldPropertyRequest - >("add-content-database-source-field-property", { - onSuccess: (data) => { - queryClient.setQueriesData( - { queryKey: ["action", "get-content-database"] }, - (current) => applySourceFieldPropertyToDatabaseResponse(current, data), - ); - queryClient.invalidateQueries({ - queryKey: [ - "action", - "list-document-properties", - { documentId, databaseId }, - ], - }); - }, - }); + const addSourceFieldProperty = + useAddContentDatabaseSourceFieldProperty(documentId); const [open, setOpen] = useState(false); const [typeQuery, setTypeQuery] = useState(""); const filteredPropertyTypes = filterDocumentPropertyTypes(typeQuery); diff --git a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx index d013c5966b..8ac3d43332 100644 --- a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx @@ -1,6 +1,7 @@ import type { BuilderCmsModelSummary, ContentDatabaseResponse, + ContentDatabaseTableQuery, } from "@shared/api"; // @vitest-environment happy-dom // @@ -111,16 +112,28 @@ vi.mock("@agent-native/core/client/settings", () => ({ vi.mock("@/hooks/use-content-database", () => ({ isContentDatabaseUnavailable: () => false, - useContentDatabase: (documentId: string, limit: number) => { - contentDatabaseQueryMock(documentId, limit); + useContentDatabase: ( + documentId: string, + limit: number, + tableQuery?: ContentDatabaseTableQuery, + ) => { + contentDatabaseQueryMock(documentId, limit, tableQuery); return { data: databaseResponse, isLoading: false, - isFetching: limit !== databasePagination.limit, + isFetching: limit !== databasePagination.limit || Boolean(tableQuery), }; }, useAddDatabaseItem: () => addItemMutation, + useAddContentDatabaseSourceFieldProperty: () => benignMutation, useAttachContentDatabaseSource: () => attachSourceMutation, + useBuilderCmsAttachPreview: () => ({ + data: undefined, + isLoading: false, + isFetching: false, + error: null, + }), + writeBuilderAttachPreviewToCache: vi.fn(), useChangeContentDatabaseSourceRole: () => benignMutation, useRefreshContentDatabaseSource: () => benignMutation, useDisconnectContentDatabaseSource: () => benignMutation, @@ -353,7 +366,7 @@ describe("DatabaseView UI regressions", () => { ); }); - it("requests the whole bounded search window and hides the partial no-match state", async () => { + it("keeps search page-bounded and hides the partial no-match state", async () => { databasePagination.totalItems = 571; databasePagination.hasMore = true; await renderDatabaseView(); @@ -379,7 +392,11 @@ describe("DatabaseView UI regressions", () => { await Promise.resolve(); }); - expect(contentDatabaseQueryMock).toHaveBeenCalledWith("document-1", 571); + expect(contentDatabaseQueryMock).toHaveBeenCalledWith( + "document-1", + 100, + expect.objectContaining({ search: "Quiet Comet" }), + ); expect(container.textContent).toContain( messagesByLocale["en-US"].database.loadingDatabase, ); diff --git a/templates/content/app/components/editor/database/DatabaseView.test.ts b/templates/content/app/components/editor/database/DatabaseView.test.ts index 6962d78a4f..f79c17f8b6 100644 --- a/templates/content/app/components/editor/database/DatabaseView.test.ts +++ b/templates/content/app/components/editor/database/DatabaseView.test.ts @@ -48,6 +48,7 @@ import { databaseSearchExpandedItemLimit, databaseSearchExpansionIsPending, databaseAttachedBuilderSources, + databaseActiveBuilderReview, databaseNextBuilderContinuationSource, databaseNextBuilderContinuationWatchdogSource, databaseNextBuilderHydrationSource, @@ -55,6 +56,9 @@ import { databaseItemPagePath, databaseRecordBuilderContinuationAttempt, databaseSourceOperationIsPending, + databaseSourceChangeSetsAreComplete, + databaseTableShouldShowBlockingLoader, + databaseViewRequiresCompleteClientDataset, pendingMutationSourceId, previewDocumentSaveResult, releaseDatabaseSourceOperation, @@ -63,6 +67,81 @@ import { preparedBuilderReviewMatches, } from "./DatabaseView"; +describe("database source page projections", () => { + it("does not present page-scoped review counts as complete", () => { + expect( + databaseSourceChangeSetsAreComplete({ + projection: { rows: "page", changeSets: "page" }, + } as ContentDatabaseSource), + ).toBe(false); + expect( + databaseSourceChangeSetsAreComplete({ + projection: { rows: "complete", changeSets: "complete" }, + } as ContentDatabaseSource), + ).toBe(true); + expect(databaseSourceChangeSetsAreComplete(null)).toBe(true); + }); + + it("shows page review changes while the complete review loads", () => { + const page = { + sourceId: "source-1", + } as unknown as ContentDatabaseSourceReviewPayload; + const complete = { + sourceId: "source-1", + preparedRowLimit: 100, + } as unknown as ContentDatabaseSourceReviewPayload; + expect( + databaseActiveBuilderReview({ + prepared: null, + complete: null, + page, + open: true, + }), + ).toBe(page); + expect( + databaseActiveBuilderReview({ + prepared: null, + complete, + page, + open: true, + }), + ).toBe(complete); + }); +}); + +describe("database table loading", () => { + it("keeps useful rows visible while a constrained result refetches", () => { + expect(databaseTableShouldShowBlockingLoader(true, 100)).toBe(false); + expect(databaseTableShouldShowBlockingLoader(true, 0)).toBe(true); + expect(databaseTableShouldShowBlockingLoader(false, 0)).toBe(false); + }); + + it("keeps constrained tables bounded while preserving complete calendar data", () => { + expect( + databaseViewRequiresCompleteClientDataset({ + viewType: "table", + activeConstraintCount: 1, + grouped: false, + }), + ).toBe(false); + expect( + databaseViewRequiresCompleteClientDataset({ + viewType: "calendar", + activeConstraintCount: 0, + grouped: false, + }), + ).toBe(true); + expect( + databaseViewRequiresCompleteClientDataset({ + viewType: "table", + activeConstraintCount: 1, + grouped: false, + tableQueryMode: "client-required", + }), + ).toBe(true); + }); +}); + describe("database preview property saves", () => { it("carries the database and its page through full-page navigation", () => { expect( diff --git a/templates/content/app/components/editor/database/DatabaseView.tsx b/templates/content/app/components/editor/database/DatabaseView.tsx index fb344a60f1..edab812af2 100644 --- a/templates/content/app/components/editor/database/DatabaseView.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.tsx @@ -15,6 +15,7 @@ import { type BuilderCmsModelSummary, type ContentDatabaseItem, type ContentDatabaseResponse, + type ContentDatabaseSourceAttachmentResult, type ContentDatabaseSource, type ContentDatabaseSourceChangeSet, type ContentDatabaseSourceJoinRequest, @@ -34,6 +35,7 @@ import { type ContentDatabaseRowDensity, type ContentDatabaseSort, type ContentDatabaseSortDirection, + type ContentDatabaseTableQuery, type ContentDatabaseViewType, type Document, type DocumentProperty, @@ -42,6 +44,7 @@ import { type DocumentPropertyValue, } from "@shared/api"; import { contentDatabaseFormQuestions } from "@shared/database-form"; +import { applyContentDatabaseTableQuery } from "@shared/database-query"; import { type DocumentPropertyOptionColor, countWords, @@ -165,6 +168,7 @@ import { useAddDatabaseItem, useAddContentDatabaseSourceFieldProperty, useAttachContentDatabaseSource, + useBuilderCmsAttachPreview, useBuilderCmsModels, useCancelPreparedBuilderSourceUpdate, useChangeContentDatabaseSourceRole, @@ -188,6 +192,7 @@ import { useSuggestSourceJoinKey, useUpdateContentDatabasePersonalView, useUpdateContentDatabaseView, + writeBuilderAttachPreviewToCache, } from "@/hooks/use-content-database"; import { useContentSpaces, @@ -441,6 +446,7 @@ export function databaseCreatedItemForImmediatePreview( now?: string; }, ): ContentDatabaseItem | null { + if (response.createdItem) return response.createdItem; const returnedItem = response.items.find( (item) => item.id === response.createdItemId, ); @@ -769,15 +775,51 @@ function DatabaseTable({ SELECTED_CONTENT_SPACE_STORAGE_KEY, null, ); + const [searchQuery, setSearchQuery] = useState(""); + const [viewConfig, setViewConfig] = useState( + defaultDatabaseViewConfig(), + ); + const activeView = useMemo( + () => activeDatabaseView(viewConfig), + [viewConfig], + ); + const tableQuery = useMemo(() => { + if ( + activeView.type !== "table" || + activeDatabaseConstraintCount( + searchQuery, + activeView.sorts, + activeView.filters, + ) === 0 + ) { + return undefined; + } + return { + search: searchQuery, + filters: activeView.filters, + sorts: activeView.sorts, + filterMode: activeView.filterMode ?? "and", + }; + }, [activeView, searchQuery]); const [manualDatabaseItemLimit, setManualDatabaseItemLimit] = useState( CONTENT_DATABASE_PAGE_SIZE, ); const [databaseRequestItemLimit, setDatabaseRequestItemLimit] = useState( CONTENT_DATABASE_PAGE_SIZE, ); - const database = useContentDatabase(document.id, databaseRequestItemLimit); + const database = useContentDatabase( + document.id, + databaseRequestItemLimit, + tableQuery, + ); + // A deleted/missing database resolves to the unavailable union (no + // `database` field) — treat it as no data; the inline-block wrapper owns + // the user-facing "Database unavailable" state. + const data = isContentDatabaseUnavailable(database.data) + ? undefined + : database.data; const addItem = useAddDatabaseItem(document.id); - const attachSource = useAttachContentDatabaseSource(document.id); + const attachSource = useAttachContentDatabaseSource(document.id, data); const changeSourceRole = useChangeContentDatabaseSourceRole(document.id); const refreshSource = useRefreshContentDatabaseSource(document.id); const disconnectSource = useDisconnectContentDatabaseSource(document.id); @@ -794,12 +836,8 @@ function DatabaseTable({ document.id, ); const updateView = useUpdateContentDatabaseView(document.id); - // A deleted/missing database resolves to the unavailable union (no - // `database` field) — treat it as no data; the inline-block wrapper owns - // the user-facing "Database unavailable" state. - const data = isContentDatabaseUnavailable(database.data) - ? undefined - : database.data; + const attachPreviewActive = Boolean(data?.attachPreview); + const effectiveCanEdit = canEdit && !attachPreviewActive; const isWorkspaceCatalog = data?.database.systemRole === "workspaces"; const isCreatingDatabaseItem = addItem.isPending; const isDatabaseInitialLoading = database.isLoading && !data; @@ -841,7 +879,6 @@ function DatabaseTable({ string | null >(null); const [searchOpen, setSearchOpen] = useState(false); - const [searchQuery, setSearchQuery] = useState(""); const [settingsOpen, setSettingsOpen] = useState(false); const [inlineFilterControlsOpen, setInlineFilterControlsOpen] = useState(false); @@ -876,19 +913,12 @@ function DatabaseTable({ useState | null>(null); const [settingsPanel, setSettingsPanel] = useState("main"); - const [viewConfig, setViewConfig] = useState( - defaultDatabaseViewConfig(), - ); const [savedViewConfig, setSavedViewConfig] = useState(defaultDatabaseViewConfig()); const [personalQueryDirty, setPersonalQueryDirty] = useState(false); const [dateViewMonth, setDateViewMonth] = useState(() => startOfMonth(new Date()), ); - const activeView = useMemo( - () => activeDatabaseView(viewConfig), - [viewConfig], - ); const orderedProperties = useMemo( () => orderDatabasePropertiesForView(properties, activeView), [properties, activeView], @@ -996,10 +1026,12 @@ function DatabaseTable({ visibleFilters, ); const requiresCompleteClientDataset = - activeConstraintCount > 0 || - !!databaseGroupProperty || - activeView.type === "calendar" || - activeView.type === "timeline"; + databaseViewRequiresCompleteClientDataset({ + viewType: activeView.type, + activeConstraintCount, + grouped: !!databaseGroupProperty, + tableQueryMode: data?.tableQueryMode, + }); const clientQueryExpandedItemLimit = databaseClientQueryExpandedItemLimit( requiresCompleteClientDataset, manualDatabaseItemLimit, @@ -1011,7 +1043,19 @@ function DatabaseTable({ data?.pagination?.limit ?? items.length, ); const isDatabaseViewLoading = - isDatabaseInitialLoading || isClientQueryExpansionPending; + isDatabaseInitialLoading || + isClientQueryExpansionPending || + (database.isFetching && Boolean(tableQuery)); + useEffect(() => { + if (isDatabaseViewLoading) return; + window.document.documentElement.dataset.contentDatabaseRowsVisibleDocumentId = + document.id; + window.dispatchEvent( + new CustomEvent("content-database-rows-visible", { + detail: { documentId: document.id }, + }), + ); + }, [document.id, isDatabaseViewLoading]); useEffect(() => { if (clientQueryExpandedItemLimit === databaseRequestItemLimit) return; setDatabaseRequestItemLimit(clientQueryExpandedItemLimit); @@ -1072,6 +1116,8 @@ function DatabaseTable({ () => builderReviewableChangeSets(builderReviewSource), [builderReviewSource], ); + const builderReviewCountIsComplete = + databaseSourceChangeSetsAreComplete(builderReviewSource); const builderReviewPreview = useMemo( () => builderReviewSource && builderReviewChangeSets.length > 0 @@ -1082,11 +1128,12 @@ function DatabaseTable({ : null, [builderReviewChangeSets, builderReviewSource], ); - const activeBuilderReview = - builderReviewResult ?? - (builderReviewOpen - ? (completeBuilderReviewPreview?.review ?? null) - : builderReviewPreview); + const activeBuilderReview = databaseActiveBuilderReview({ + prepared: builderReviewResult, + complete: completeBuilderReviewPreview?.review ?? null, + page: builderReviewPreview, + open: builderReviewOpen, + }); const sourcePendingOperations: DatabaseSourcePendingOperations = { attach: attachSource.isPending, changeRole: changeSourceRole.isPending, @@ -1135,6 +1182,7 @@ function DatabaseTable({ onError?: () => void, expectedBuilderContinuationOffset?: number, onSuccess?: () => void, + fullRefresh = false, ) => { if (!acquireDatabaseSourceOperation(refreshSourceInFlightRef, sourceId)) { return false; @@ -1144,6 +1192,8 @@ function DatabaseTable({ documentId: document.id, sourceId, expectedBuilderContinuationOffset, + fullRefresh: fullRefresh || undefined, + finishBuilderPagination: fullRefresh || undefined, }, { onError, @@ -1190,19 +1240,35 @@ function DatabaseTable({ ) { return false; } - processBuilderBodies.mutate( - { sourceId, ...request }, - { - onSuccess: options.onSuccess, - onError: options.onError, - onSettled: () => { - releaseDatabaseSourceOperation( - hydrationSourceInFlightRef, - sourceId, - ); + const pump = () => { + let result: ProcessBuilderBodyHydrationResponse | null = null; + processBuilderBodies.mutate( + { sourceId, ...request }, + { + onSuccess: (nextResult) => { + result = nextResult; + options.onSuccess?.(nextResult); + }, + onError: options.onError, + onSettled: () => { + if ( + !request.documentId && + result && + builderBodyHydrationMutationMadeProgress(result) && + result.remaining > 0 + ) { + pump(); + return; + } + releaseDatabaseSourceOperation( + hydrationSourceInFlightRef, + sourceId, + ); + }, }, - }, - ); + ); + }; + pump(); return true; }, [processBuilderBodies.mutate], @@ -1246,6 +1312,7 @@ function DatabaseTable({ () => handleBuilderContinuationError(continuationKey), candidate.metadata.lastReadNextOffset, () => handleBuilderContinuationSuccess(continuationKey), + true, ) ) { autoContinueBuilderSourceRef.current.delete(continuationKey); @@ -1325,6 +1392,7 @@ function DatabaseTable({ () => handleBuilderContinuationError(continuationKey), candidate.metadata.lastReadNextOffset, () => handleBuilderContinuationSuccess(continuationKey), + true, ) ) { builderContinuationWatchdogRef.current.set( @@ -2264,7 +2332,7 @@ function DatabaseTable({ : viewConfig; const nextKey = databaseViewStateKey(databaseId, sharedViewConfig); if (databaseViewStateKey(databaseId, savedViewConfig) === nextKey) return; - if (!canEdit) return; + if (!effectiveCanEdit) return; if (saveViewTimerRef.current) { clearTimeout(saveViewTimerRef.current); } @@ -2344,7 +2412,7 @@ function DatabaseTable({
@@ -2460,12 +2528,16 @@ function DatabaseTable({ size="sm" aria-label={ builderReviewChangeSets.length > 0 - ? `Database settings, ${builderReviewChangeSets.length} Builder update pending` + ? builderReviewCountIsComplete + ? `Database settings, ${builderReviewChangeSets.length} Builder update pending` + : "Database settings, Builder updates pending" : "Database settings" } title={ builderReviewChangeSets.length > 0 - ? `${builderReviewChangeSets.length} Builder update pending` + ? builderReviewCountIsComplete + ? `${builderReviewChangeSets.length} Builder update pending` + : "Builder updates pending" : "Database settings" } className={cn( @@ -2485,12 +2557,24 @@ function DatabaseTable({ > {builderReviewChangeSets.length > 0 ? ( - - {formatCompactCountBadge(builderReviewChangeSets.length)} - + builderReviewCountIsComplete ? ( + + {formatCompactCountBadge(builderReviewChangeSets.length)} + + ) : ( + + ) ) : null} - {canEdit && isWorkspaceCatalog ? ( + {data?.attachPreview ? ( + + {data.attachPreview.complete && + typeof data.attachPreview.importedItemCount === "number" + ? `${data.attachPreview.importedItemCount} rows fetched` + : dbText("attachingReadOnly")} + + ) : null} + {effectiveCanEdit && isWorkspaceCatalog ? ( - ) : canEdit ? ( + ) : effectiveCanEdit ? (
- {isLoading ? ( + {databaseTableShouldShowBlockingLoader(isLoading, items.length) ? (
{dbText("loadingDatabase")} @@ -5863,6 +5954,29 @@ function DatabaseTableView({ ); } +export function databaseTableShouldShowBlockingLoader( + isLoading: boolean, + itemCount: number, +) { + return isLoading && itemCount === 0; +} + +export function databaseViewRequiresCompleteClientDataset(args: { + viewType: ContentDatabaseViewType; + activeConstraintCount: number; + grouped: boolean; + tableQueryMode?: ContentDatabaseResponse["tableQueryMode"]; +}) { + return ( + (args.viewType === "table" && args.tableQueryMode === "client-required") || + (args.viewType !== "table" && + (args.activeConstraintCount > 0 || + args.grouped || + args.viewType === "calendar" || + args.viewType === "timeline")) + ); +} + function DatabaseActiveConstraintsBar({ documentId, items, @@ -7305,11 +7419,11 @@ function DatabaseSettingsPanelSheet({ onAttachBuilderSource: ( model: BuilderCmsModelSummary, relationshipMode?: "items" | "details", - ) => Promise; + ) => Promise; onFederateSource: ( candidate: PendingSourceCandidate, join: ContentDatabaseSourceJoinRequest, - ) => Promise; + ) => Promise; onChangeSourceRole: ( sourceId: string, relationshipMode: "items" | "details", @@ -7333,10 +7447,44 @@ function DatabaseSettingsPanelSheet({ onHideEmptyGroupsChange: (hideEmptyGroups: boolean) => void; onGroupsCollapsedChange: (groupIds: string[], collapsed: boolean) => void; }) { + const queryClient = useQueryClient(); // Local drill-down path *within* the Source(s) panel. Kept here (not in the // flat panel enum) because the levels are dynamic — space/model names aren't // known at compile time. The sheet's back button pops this stack first. const [sourceNavStack, setSourceNavStack] = useState([]); + const sourceNavTop = sourceNavStack[sourceNavStack.length - 1]; + const previewModel = + sourceNavTop?.kind === "model" ? (sourceNavTop.model ?? null) : null; + const previewFieldPaths = useMemo( + () => previewModel?.fields.map((field) => field.name), + [previewModel], + ); + const previewAlreadyAttached = previewModel + ? Boolean( + databaseAttachedBuilderSource(sources, source, { + modelName: previewModel.name, + }), + ) + : false; + const builderAttachPreview = useBuilderCmsAttachPreview({ + documentId, + sourceTable: previewModel?.name ?? null, + fieldPaths: previewFieldPaths, + enabled: + open && + panel === "source" && + canEdit && + Boolean(previewModel) && + !previewAlreadyAttached, + }); + useEffect(() => { + if (!sourceActionPending || !builderAttachPreview.data) return; + writeBuilderAttachPreviewToCache( + queryClient, + documentId, + builderAttachPreview.data, + ); + }, [builderAttachPreview.data, documentId, queryClient, sourceActionPending]); useEffect(() => { // Always re-enter the Sources panel at its root, and don't retain a path // across close/reopen. @@ -7425,6 +7573,7 @@ function DatabaseSettingsPanelSheet({ onReviewBuilderUpdate={onReviewBuilderUpdate} onSetBuilderLiveWrites={onSetBuilderLiveWrites} sourceActionPending={sourceActionPending} + builderAttachPreviewPending={builderAttachPreview.isFetching} sourcePendingOperations={sourcePendingOperations} /> ) : panel === "layout" ? ( @@ -7488,7 +7637,9 @@ function DatabaseSettingsMainPanel({ onPanelChange: (panel: DatabaseSettingsPanel) => void; }) { const groupLabel = activeView.groupByPropertyId ? "On" : ""; - const sourceBadgeCount = builderReviewableChangeSets(source).length; + const sourceBadgeCount = databaseSourceChangeSetsAreComplete(source) + ? builderReviewableChangeSets(source).length + : undefined; return (
@@ -7550,6 +7701,23 @@ export function builderReviewableChangeSets( ); } +export function databaseActiveBuilderReview(args: { + prepared: ContentDatabaseSourceReviewPayload | null; + complete: ContentDatabaseSourceReviewPayload | null; + page: ContentDatabaseSourceReviewPayload | null; + open: boolean; +}) { + return ( + args.prepared ?? (args.open ? (args.complete ?? args.page) : args.page) + ); +} + +export function databaseSourceChangeSetsAreComplete( + source: ContentDatabaseSource | null, +) { + return source?.projection?.changeSets !== "page"; +} + function sourceReviewRiskRank( risk: ContentDatabaseSourceReviewPayload["riskLevel"], ) { @@ -7807,6 +7975,7 @@ function DatabaseSettingsSourcePanel({ onReviewBuilderUpdate, onSetBuilderLiveWrites, sourceActionPending, + builderAttachPreviewPending, sourcePendingOperations, }: { source: ContentDatabaseSource | null; @@ -7823,11 +7992,11 @@ function DatabaseSettingsSourcePanel({ onAttachBuilderSource: ( model: BuilderCmsModelSummary, relationshipMode?: "items" | "details", - ) => Promise; + ) => Promise; onFederateSource: ( candidate: PendingSourceCandidate, join: ContentDatabaseSourceJoinRequest, - ) => Promise; + ) => Promise; onChangeSourceRole: ( sourceId: string, relationshipMode: "items" | "details", @@ -7840,13 +8009,17 @@ function DatabaseSettingsSourcePanel({ onReviewBuilderUpdate: (sourceId: string) => void; onSetBuilderLiveWrites: (settings: BuilderSourceWriteSettingsInput) => void; sourceActionPending: boolean; + builderAttachPreviewPending: boolean; sourcePendingOperations: DatabaseSourcePendingOperations; }) { const { isCodeMode } = useCodeMode(); const navigate = useNavigate(); const builderSources = databaseAttachedBuilderSources(sources, source); const builderStatus = useBuilderStatus(); + const builderAttachPending = + sourceActionPending || builderAttachPreviewPending; const builderConfigured = builderStatus.status?.configured === true; + const builderModelsQuery = useBuilderCmsModels(builderConfigured); const builderOrgName = builderStatus.status?.orgName ?? null; // Real space name(s) from the Admin API, falling back to the generic org // name (then a constant) so the drill-down never renders a blank label. @@ -7873,11 +8046,15 @@ function DatabaseSettingsSourcePanel({ sources={sources} builderConfigured={builderConfigured} builderSpaceLabel={builderSpaceLabel} - reviewableCount={builderSources.reduce( - (total, candidate) => - total + builderReviewableChangeSets(candidate).length, - 0, - )} + reviewableCount={ + builderSources.every(databaseSourceChangeSetsAreComplete) + ? builderSources.reduce( + (total, candidate) => + total + builderReviewableChangeSets(candidate).length, + 0, + ) + : undefined + } onOpenBuilder={(builderSource) => onNavPush( builderSource @@ -8003,6 +8180,11 @@ function DatabaseSettingsSourcePanel({ join, ) : await onFederateSource(top.candidate, join); + if ("responseProjection" in result) { + throw new Error( + "Detail-source attachment returned an item-source acknowledgement.", + ); + } const detailsSource = findDetailsSource(result, top.candidate); onNavReplace( detailsSource @@ -8095,6 +8277,7 @@ function DatabaseSettingsSourcePanel({ return ( { const attached = databaseAttachedBuilderSource(sources, source, { modelName: model.name, @@ -8144,7 +8327,7 @@ function DatabaseSettingsSourcePanel({ displayName: model.displayName, }} canEdit={canEdit} - pending={sourceActionPending} + pending={builderAttachPending} onAddDetails={() => onNavPush({ kind: "keyConfirm", @@ -8166,7 +8349,7 @@ function DatabaseSettingsSourcePanel({