From d8e886f8bad880092421680579989433fc9f0f6c Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:10:57 -0400 Subject: [PATCH 01/21] Improve large Content database performance --- ...content-blog-table-performance-research.md | 808 ++++++++++++++++++ templates/content/actions/_batch-utils.ts | 12 + templates/content/actions/_content-files.ts | 52 +- .../actions/_database-source-utils.test.ts | 47 + .../content/actions/_database-source-utils.ts | 197 +++-- templates/content/actions/_database-utils.ts | 142 ++- .../content/actions/add-database-item.ts | 14 +- .../actions/attach-content-database-source.ts | 30 +- .../bind-content-database-source-field.ts | 4 +- .../builder-source-review-gates.db.test.ts | 2 +- .../cancel-prepared-builder-source-update.ts | 5 +- .../content/actions/content-spaces.db.test.ts | 46 +- .../content/actions/delete-database-items.ts | 10 +- .../disconnect-content-database-source.ts | 4 +- .../actions/duplicate-database-item.ts | 16 +- .../actions/duplicate-database-items.ts | 19 +- .../execute-builder-source-execution.ts | 3 +- .../content/actions/get-content-database.ts | 53 +- .../content/actions/move-database-item.ts | 5 +- .../prepare-builder-source-execution.ts | 2 +- .../actions/prepare-builder-source-review.ts | 12 +- .../actions/preview-builder-source-review.ts | 131 +-- ...view-content-database-source-change-set.ts | 2 +- .../set-content-database-source-write-mode.ts | 2 +- .../content/actions/stage-builder-revision.ts | 2 +- ...ate-content-database-personal-view.test.ts | 30 +- .../update-content-database-personal-view.ts | 32 +- .../actions/update-content-database-view.ts | 2 +- .../validate-builder-source-execution.ts | 2 +- .../DatabaseView.error-toasts.test.tsx | 19 +- .../editor/database/DatabaseView.test.ts | 79 ++ .../editor/database/DatabaseView.tsx | 211 +++-- .../sidebar/DocumentSidebar.layout.test.ts | 6 +- .../components/sidebar/DocumentSidebar.tsx | 83 +- .../app/hooks/use-content-database.test.ts | 48 +- .../content/app/hooks/use-content-database.ts | 104 ++- .../app/hooks/use-document-properties.ts | 13 +- ...arge-databases-keep-useful-rows-visible.md | 6 + templates/content/shared/api.ts | 19 +- .../content/shared/database-query.test.ts | 99 +++ templates/content/shared/database-query.ts | 359 ++++++++ 41 files changed, 2402 insertions(+), 330 deletions(-) create mode 100644 plans/content-blog-table-performance-research.md create mode 100644 templates/content/changelog/2026-07-29-large-databases-keep-useful-rows-visible.md create mode 100644 templates/content/shared/database-query.test.ts create mode 100644 templates/content/shared/database-query.ts diff --git a/plans/content-blog-table-performance-research.md b/plans/content-blog-table-performance-research.md new file mode 100644 index 0000000000..56521eb38a --- /dev/null +++ b/plans/content-blog-table-performance-research.md @@ -0,0 +1,808 @@ +# Why did the Content blog table feel unusably slow? + +Date: 2026-07-28 + +## Answer + +The 16:01 Clips recording separates four different kinds of delay that Rewind had blurred together: + +- creating the replacement Blog database left a titled but empty surface on screen for **at least 25 seconds**, and the usable empty table only appeared after Alice navigated away and back; +- attaching the Builder source produced the first useful imported rows in roughly **5-10 seconds**, but truthful background source/body hydration continued for roughly **3 minutes 15 seconds**; +- sorting by Date replaced the populated table with `Loading database` for roughly **5-10 seconds**; +- opening Builder's review surface spent roughly **1 minute 40 seconds** on `Loading the complete Builder diff` before showing the review. + +Opening an imported row was materially better but still staged: useful article content appeared in roughly **5 seconds**, with its property metadata settling roughly **10-15 seconds later**. Property-picker dwell was long, but the clip shows Alice browsing and choosing fields during much of it, so it is not valid to label the whole picker-open interval as application latency. + +The current code makes those delays plausible. Opening a database is a composed read with more than a dozen visible SQL phases. Even though the UI initially asks for only 100 rows, it still loads complete attached-source snapshots and filters them to the visible page afterward. Several mutations also return a complete, sometimes unpaginated database snapshot and then invalidate queries that read the database again. Search, filter, sort, grouping, calendar, and timeline modes may expand the request from 100 rows to as many as 5,000, which the table renders without row virtualization. + +The implementation shape is now frozen. Work should first capture exact browser and server spans for the representative workflow, then deliver four bounded changes: a page-bounded table/source projection, server-bounded table constraints, delta-shaped mutations with exact cache updates, and progressive Builder review/body hydration. Each slice must be accepted against the same visual boundaries and approved budgets without weakening source-review correctness, sharing, cross-tab sync, or Yjs row-body behavior. + +## Rewind Evidence + +### Coverage and limits + +- Screen Memory was enabled and not paused, but status reported `segmentCount: 0`. +- The chapter search returned no matching current chapter and carried a stale `generatedAt` value from 2026-07-21. +- A 32-minute local OCR search covered 10:33:25-10:52:04 EDT and reported 66 Blog-matching observations; its broad result returned the newest 50 and omitted 16. Targeted searches recovered the important earlier transitions. +- Four five-minute contact-sheet requests covering 10:34-10:54 and an exact-frame request at 10:50:44 all failed with `No clean retained Rewind frames`. +- Therefore the observations below are OCR screen-state samples, normally ten seconds apart. They measure visible-state dwell, not exact click-to-response latency. Deliberate user pauses cannot be distinguished from an unresponsive UI unless a start and completion state are both visible. + +### Observed timeline + +| Local time | Visible state | Measurement | Confidence | +| ----------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ------------------------- | +| 10:37:05 | Blog table with `Sort Descending` visible | Operation boundary unknown | Low | +| 10:37:15 | Blog table with `Duplicate property` visible | Picker/menu changed within about 10 seconds | Medium | +| 10:37:25 | Only `Blog (ALPHA TESTING)` remained visible | Later coverage gap prevents a latency claim | Low | +| 10:42:44-10:43:14 | `Questions to Ask Beth...` visible under Blog | State persisted for at least 30 seconds; no click boundary | Low | +| 10:43:24-10:43:34 | `Agent-Native Plan` title/shell only | Body not yet visible | Medium | +| 10:43:44 | Agent-Native Plan body visible | About 20 seconds title-to-body | Medium; not click-to-body | +| 10:43:44-10:44:04 | Delete-page, then delete-database UI, then body | Each visible transition happened within one 10-second sample | Medium | +| 10:44:14 | `Blog` shell/title only | Start of clearest table-load interval | High | +| 10:44:24-10:44:34 | Blog table frame/title, no useful rows detected | Still waiting | High | +| 10:44:54 | First useful row detected | Roughly 30-40 seconds shell-to-first-row | Medium-high | +| 10:45:04 | `100 changes / ready` and one row detected | Intermediate state | Medium | +| 10:45:34 | Multiple rows detected | Roughly 50-80 seconds shell-to-populated-table because 10:45:14/24 samples did not match the query | Medium | +| 10:45:34-10:47:44 | Same populated table state | Stable view; not operation latency | High | +| 10:49:14-10:49:44 | Property search/picker visible | 30-second dwell; user choice versus UI delay unknown | Low | +| 10:49:54 | Picker closed and table visible | Completion occurred within the next sample | Medium | +| 10:50:24-10:51:04 | Property picker visible again with changing choices | At least 40 seconds of interaction; latency cannot be isolated | Low | +| 10:51:14 | KPMG row body visible | Start click unknown | Low | +| 10:51:24 | Back at table | Visible round trip completed within about 10 seconds | Medium | +| 10:51:34-10:52:04 | Property picker visible | At least 30 seconds; no completion captured | Low | + +## Clips Evidence + +### Coverage and method + +- Public recording: `https://clips.agent-native.com/share/BfEnyRiC4Pu7?ref=clip_share`. +- Recording duration: 16:00.905, 1280x830, captured in Safari on 2026-07-28. +- The Clips transcript was still `pending`, so timings come from the video clock and visible UI states. +- The full recording was sampled every five seconds, then the operation boundaries below were checked against the neighboring frames. Durations are therefore reported as ranges, normally with a +/- 2.5-second boundary tolerance, rather than invented millisecond precision. +- `First useful` means Alice can see or act on real rows/content. `Settled` means the visible loading/sync state for that operation has completed. A persistent background sync is reported separately rather than pretending the earlier usable state was still blank. + +### Measured operation ledger + +| Clip time | User operation | First useful / completion | Measured wait | What was visibly happening | Confidence | +| ----------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------ | ------------------------------------------------------------------------------ | ------------------------------------------------------ | +| 00:17-00:22 | Permanently delete the old `Blog (ALPHA TESTING)` database | Dialog disappears and deletion toast appears | <=5s | Destructive mutation | Medium-high | +| 00:27-00:42 | Create a new page/database and name it `Blog (ALPHA TEST)` | Titled database shell appears | ~15s to shell | Page/database creation | Medium | +| 00:42-01:07 | Wait on the new database shell | No usable table appears before Alice leaves for Notion | >=25s unresolved | Empty/titled surface with no table controls or rows | High for visible dwell; root cause unknown | +| 01:42-02:06 | Return to Content, navigate away from the stuck Blog page, then back | Empty Blog table with Name and Content columns appears | ~24s end-to-end; includes navigation | Recovery/navigation plus database read | Medium; not a single request | +| 02:32-02:52 | Find the Builder Blog model and choose Attach | Attach command issued | ~20s user interaction | Search/browse/selection, not system latency | High; exclude from performance budget | +| 02:52-03:01 | Attach Builder source | First imported rows and source-progress UI appear | ~5-10s to useful rows | Source attach and initial projection | Medium-high | +| 03:01-06:17 | Let attached source finish fetching/hydrating | Progress advances from 24/100 through 584 rows and body sync; normal table remains available | ~3m15s to settled | Background row fetch/body hydration | High; table was usable during most of it | +| 06:37-06:42 | Open `You Optimized the Wrong 0.6%` | Article title/body visible in side panel | ~5s to useful body | Imported-row/document read | Medium-high | +| 06:52-07:16 | Add/reveal Author property | Author column appears populated | ~24s total picker-to-column | Mostly property browsing plus a final mutation/read | Low as latency; exact choice click needs request trace | +| 07:37-07:57 | Add/reveal Date property | Date column appears populated | ~20s total picker-to-column | Mostly property browsing plus a final mutation/read | Low as latency; exact choice click needs request trace | +| 08:07-08:16 | Sort Date descending | Sorted populated table returns | ~5-10s | Whole table replaced by `Loading database` | High | +| 08:22-08:27 | Reorder Date and Author columns | New column order visible | <=5s | Local/view configuration | Medium-high | +| 08:52-09:02 | Rename database to `Blog Test` | New title stable | ~5-10s including typing | Title mutation | Medium | +| 09:47-09:57 | Open `The Future of SaaS Is Cloneable` | Useful article body visible | ~5-10s | Row/document read | Medium-high | +| 09:57-10:12 | Wait for row metadata to settle | Kind, Parent, and Source fields become visible | ~10-15s after body | Property/metadata read after useful content | Medium-high | +| 10:52-10:57 | Open database settings | Settings panel visible | <=5s | Local settings surface | Medium-high | +| 11:37-13:17 | Open Review Builder update | Complete diff finally appears | ~1m40s | Modal remains on `Loading the complete Builder diff` / `Preparing full review` | High | +| 14:42-14:52 | Prepare/review one selected Builder update | Validation error becomes visible | ~5-10s | `Preparing` ends in typed author-model validation failure | Medium-high | +| 15:27-15:32 | Close the review modal | Table visible again | <=5s | Local modal close | High | + +### Longest confirmed waits + +1. Attached-source fetch/body hydration: approximately **3m15s** to settle, although useful rows appeared after 5-10 seconds. +2. Complete Builder review diff: approximately **1m40s** before review content appeared. +3. New database shell with no usable table: **at least 25s**, unresolved until Alice navigated away. +4. Return/recovery to the empty table: approximately **24s**, including Alice's navigation rather than one isolated request. +5. Date sort: approximately **5-10s** with a blocking whole-table loading state. + +The clip therefore disproves a single "table takes a minute to load" story. The worst elapsed operation is background hydration; the worst blocking routine table operation captured is sort; and the longest blocking source-review operation is diff preparation. + +## Current-Code Evidence + +### Database open carries too much serial work + +- The editor deliberately requires an authoritative `get-document` read; a seeded database snapshot does not satisfy it (`templates/content/app/hooks/use-documents.ts`, `templates/content/app/components/editor/DocumentEditor.tsx`). +- The database page mounts another document read and independently starts `get-content-database`, personal-view, and content-space reads (`templates/content/app/components/editor/DocumentDatabase.tsx`, `templates/content/app/components/editor/database/DatabaseView.tsx`). +- `get-content-database` resolves the database, checks access, builds the composed response, reloads the backing document, and computes context (`templates/content/actions/get-content-database.ts`). +- `getContentDatabaseResponse` serially reads the database, backing document, count, items, documents, favorites, shares, row properties, schema properties, hydration queue, attached sources, federation, and context (`templates/content/actions/_database-utils.ts`). +- Property values are batched, but batch chunks and rollups can still run serially. Rollups can grow with rows times rollup properties (`templates/content/actions/_property-utils.ts`). + +### Pagination does not bound attached-source work + +- The UI initially requests 100 rows (`DatabaseView.tsx`). +- Every attached source snapshot is nevertheless loaded serially (`templates/content/actions/_database-source-utils.ts`). +- Each snapshot reads complete source rows and can repeat a consistency sequence of marker -> rows -> marker up to three times. +- Only after complete source snapshots exist does `getContentDatabaseResponse` filter their rows to the visible page (`_database-utils.ts`). +- This is the strongest code-backed explanation for a Builder-backed Blog table taking tens of seconds while an ordinary page is much lighter. It remains an inference until exact server spans identify the dominant phase. + +### Several writes perform avoidable full reads + +- `create-content-database` and `add-database-item` return a complete database response after the write (`templates/content/actions/create-content-database.ts`, `add-database-item.ts`). +- Add, move, duplicate, bulk delete, and bulk duplicate omit the page limit when rebuilding the response, so the returned snapshot can be unpaginated. +- Successful mutations broadly invalidate active action queries unless a hook opts out. Several database hooks then issue their own targeted invalidations (`packages/core/src/client/use-action.ts`, `templates/content/app/hooks/use-content-database.ts`). +- The result can be: perform write -> rebuild full snapshot -> return it -> invalidate -> fetch again. +- Property configuration is narrower, but it still reconstructs row properties and then invalidates properties, document, and database reads (`templates/content/actions/configure-document-property.ts`, `templates/content/app/hooks/use-document-properties.ts`). + +### Large constrained views move the problem into the browser too + +- Search, filter, sort, grouping, calendar, and timeline constraints can expand from the initial 100 rows to the complete dataset, capped at 5,000 (`DatabaseView.tsx`). +- Table rows are rendered with an unvirtualized `items.map`. +- This can combine a larger server response, more source/federation work, more transferred bytes, and a large React commit. + +### Existing protections that should not be blamed or removed + +- Ordinary client navigation no longer revalidates the root locale loader (`templates/content/app/root.tsx`); prior independent slow-network QA measured a 65.3 ms route commit under an artificial eight-second delay. Route commitment is not the same as useful table content. +- Core `useDbSync` ignores the current tab's own action echo, coalesces invalidations, does not cancel and restart matching reads already in flight, and suppresses broad invalidation for source refresh/body hydration events (`packages/core/src/client/use-db-sync.ts`, `templates/content/app/hooks/use-db-sync.ts`). +- Property values and several document metadata edits already use optimistic cache patches with rollback. Preserve that direction. + +## Existing Observability + +The Core action client already emits `action.response` for every action taking at least one second and for a sample of faster successes. The event includes action name, request ID, total duration, time to first byte, body-read duration, server duration, network overhead, framework startup wait, database wall time/connect time/slowest operation, response bytes, outcome, and status (`packages/core/src/client/use-action.ts`). + +This is useful but not sufficient. It does not identify which phase inside `get-content-database` consumed the server time, how many SQL round trips ran, how many source rows/change sets were scanned, how many duplicate client requests followed one gesture, or when the interface first became useful. + +## Inferences + +Ranked hypotheses to test: + +1. **Complete source snapshots are blocking basic table reads.** This best matches a Builder-backed Blog table and the 30-80 second observed load. +2. **Writes rebuild an unbounded snapshot and then cause another read.** This best explains why ordinary row/property operations can feel as slow as opening the table. +3. **A constrained view expands to thousands of rows and renders them all.** This would amplify both server and browser cost after sort/filter/search/group changes. +4. **Authoritative row reads and Builder/Yjs hydration delay useful/editable row content.** This matches the observed title-before-body interval. +5. **Rollup evaluation can produce row-by-property query growth.** Relevant only if this Blog schema uses rollups; not established from Rewind. + +## Uncertainties + +- Clip boundaries are accurate to roughly one five-second sampling interval, not milliseconds; matching request IDs are still needed for exact action latency. +- The video distinguishes picker deliberation from blocking states, but the exact property-choice click falls between sampled frames. +- No matching production analytics query capability is available in this task, so the existing `action.response` events for Alice's exact session were not retrieved. +- The clip proves one attached Builder source and a 584-row imported table. Payload sizes, rollups, database latency, source-snapshot retry count, and React commit duration remain unmeasured. +- It is not yet known which server phase dominated the stuck new-database shell, Date sort, complete-diff preparation, or the delayed row metadata. + +## Architecture Constraints + +### Demonstrated caller and request + +- Alice, using the production Agent Native Content interface, opened and edited the Builder-backed `Blog (ALPHA TESTING)` database and experienced routine operations as unusably slow. +- The required workflow is ordinary table use: open the database, reveal/add a property, edit a value, open a row, and return. + +### Existing primitives and ownership boundaries + +- UI operations already use Content actions and React Query hooks; they should continue to do so. +- Core owns action transport, Server-Timing parsing, `action.response` telemetry, request-source identity, and database-sync invalidation behavior. +- Content owns database read composition, source snapshot serialization, pagination, source review state, optimistic cache updates, and row/document hydration. +- Builder is a source provider. Basic table visibility should consume persisted SQL projections; live provider work must not become an implicit prerequisite for every interaction. + +### Legacy contracts that must remain unchanged + +- Access checks and ownable-data scoping. +- Multi-source federation and exact source-review/change-set semantics. +- Agent/script/other-tab changes still refresh the current UI. +- Current-tab optimistic edits roll back on failure. +- Open documents still use an authoritative body path and preserve Yjs collaboration. +- Initial table pagination, source status, body hydration status, and review counts remain truthful rather than being replaced by plausible empty values. + +### Smallest compatible delta + +Do not begin with a broad cache rewrite. First capture one exact representative trace and split `get-content-database` into named timing phases. Then make the basic 100-row table projection independent of complete source-review snapshots, returning only page-bounded source overlay/status data needed to render those rows. Load full review/change-set/audit state only for the review surface that consumes it. In parallel, replace full post-write snapshots with bounded deltas or page-bounded responses and remove invalidations made redundant by an exact optimistic patch. + +### Deferred capabilities + +- A general query planner for every Content database shape. +- Infinite scrolling or a redesigned pagination model. +- A complete rollup engine rewrite. +- A new sync transport. +- Provider live reads during ordinary table render. + +### Direct evidence versus inference + +- Direct: OCR state timestamps, frame-retention failures, current action/hook/read paths, existing telemetry fields, current 100/5,000 limits, serial source-snapshot loading, post-read page filtering, full-snapshot mutation responses, broad-plus-targeted invalidations, unvirtualized row rendering. +- Inference: which phase dominated Alice's exact session and the relative contribution of cold start, database network latency, source snapshots, client refetches, and React rendering. + +## Recommendation + +### First Work slice: exact trace and critical-path separation + +1. Reproduce the now-defined blocking cases separately: create/open the new database, sort Date, open an imported row, and open the complete Builder review diff. The supplied 16-minute clip is the UX baseline; no replacement clip is required. +2. Capture each reproduction's browser network waterfall and Core `action.response` request IDs. +3. Add server spans inside `get-content-database` for access, count/items, documents, properties/rollups, per-source snapshots and consistency retries, federation, context, and serialization. Attach query count, sequential critical-path round trips, rows scanned/returned, source/change-set/execution counts, response bytes, and cold-start/connect timing. +4. Record browser marks for intent -> route/shell -> first useful rows -> settled table, plus intent -> optimistic paint -> action acknowledgement -> cache settled. +5. Compare an unsourced table with the same row count against the Blog Builder source, then 0/100/1,000 rows, no rollups versus rollups, and unconstrained versus filtered/sorted/grouped views. + +### Likely second Work slice, if the trace confirms hypothesis 1 + +- Keep the initial 100-row database projection page-bounded end to end. +- Read only the source rows/overlays needed for visible documents. +- Return lightweight source status/counts with the table; fetch full change sets, reviews, executions, and consistency material when the review surface opens. +- Preserve a typed unavailable/error state; never turn a failed source snapshot into an empty successful table. + +### Likely third Work slice, if the trace confirms hypothesis 2 or 3 + +- Make row/property mutations optimistically useful immediately and return bounded deltas instead of complete unpaginated snapshots. +- Suppress default broad invalidation where the hook performs an exact targeted update; keep other-tab and agent refresh semantics. +- Keep constrained reads server-side and paginated rather than expanding silently to 5,000 client-rendered rows; virtualize only if a legitimately large client result remains part of the accepted product contract. + +## Proposed Acceptance Story + +On the production-like Builder-backed Blog fixture, Alice can open the table, reveal/add a property, edit one value, open a row, and return without losing context or waiting through a blank/stale interface. + +Required assertions: + +- Route/shell responds to every gesture within 100 ms. +- Warm table open shows first useful rows within 1 second; cold open within 2 seconds, excluding an explicitly measured and separately reported platform cold start. +- Property picker opens locally within 100 ms. +- Property/value changes paint optimistically within 100 ms, acknowledge within 1 second warm / 2 seconds cold, roll back visibly on failure, and do not blank the table. +- Row shell opens within 100 ms; authoritative body appears within 1 second warm / 2 seconds cold; editability reports Yjs or Builder hydration separately instead of holding the entire page in an undifferentiated wait. +- One gesture produces a bounded, asserted action/request count with no full unpaginated snapshot or redundant table refetch. +- Initial table read work is proportional to the requested page and attached sources needed for that page, not the complete provider corpus or review history. +- 100-, 1,000-, and 5,000-row fixtures meet explicit server, payload, and React-commit budgets; constrained views do not silently force an unvirtualized 5,000-row client render. +- Sharing, multi-source federation, source-review truth, other-tab/agent refresh, optimistic rollback, Yjs collaboration, and typed failure states remain correct. + +### Approved operation budgets + +Alice approved these values on 2026-07-29 as the acceptance target for this lane: + +| Operation | Accepted target | +| --------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| Delete database | Immediate removal; server acknowledgement within 1 second | +| Create and name database | Shell within 100 ms; usable empty table within 1 second warm / 2 seconds cold | +| Recover or return to table | Route shell within 100 ms; useful table within 1 second warm / 2 seconds cold | +| Open Builder model picker | Picker within 100 ms; search results within 300 ms | +| Attach Builder source | Feedback within 100 ms; first useful rows within 2 seconds | +| Hydrate a 584-row attached source | Never block table interaction; metadata within 10 seconds and bodies within 30-60 seconds in the background | +| Open imported row | Shell within 100 ms; useful body within 1 second warm / 2 seconds cold | +| Reveal/add a property | Picker within 100 ms; selected column paints within 100 ms; acknowledgement within 1 second | +| Sort table | Existing rows remain visible; sorted result within 500 ms warm / 1 second cold | +| Reorder columns | Within 100 ms | +| Rename database | Optimistic title within 100 ms; acknowledgement within 1 second | +| Load row metadata | Within 1 second warm / 2 seconds cold, without delaying the body | +| Open database settings | Within 100 ms | +| Open Builder review | Shell within 100 ms; first changes within 2 seconds; full diff within 10 seconds or progressively loaded | +| Prepare selected Builder update | Success or typed validation error within 2 seconds | +| Close review surface | Within 100 ms | + +The acceptance run must measure visible intent-to-paint and action completion separately. Background completion cannot be reported as blocking latency, and an optimistic paint cannot be reported as success before acknowledgement or rollback is observed. + +## Local Performance Environment + +### Current preflight, 2026-07-29 + +The repository contains the intended local database-mode path, but this worktree is not currently test-ready: + +- repo root pins `pnpm@10.29.1`; +- `templates/content/package.json` still routes `dev:database` through `scripts/check-native-deps.mjs` and `scripts/dev-database.mjs`; +- Node is v22.21.1, ABI 127, arm64; +- repository `node_modules` is absent; +- `templates/content/.env.local` and repo-root `.env` are absent; +- no Content dev server is running; +- the native preflight fails loudly with `better-sqlite3 is not installed`. + +The canonical checkout at `/Users/alicemoore/Developer/agent-native` does have repository dependencies plus both relevant env files. Their values were not read or printed. Work can use that existing local setup as the bootstrap source after verifying key presence and checkout compatibility, so no Alice-owned setup step is currently known. + +This is environment absence, not evidence about Content runtime performance. Shape does not repair it. Work should restore dependencies with the repo-pinned package manager, provision the local env without printing secrets, cold-start `dev:database`, and verify the root, database backend, and auth shape before taking a baseline. + +### Work proving ground + +Use a task-owned local database and deterministic fixtures through Content's normal action surface. The primary fixture should reproduce the captured table shape without production/customer data: + +- 584 imported-looking rows; +- Name, Content, Author, and Date properties; +- one attached Builder-shaped source using fixture/read-only adapter data; +- queued body hydration with independently observable progress; +- representative review changes large enough to exercise complete-diff preparation; +- imported rows with useful body content plus later metadata; +- stable ownership marker and an explicit local cleanup/absence check. + +Run the real local UI for create/open, attach, reveal property, sort, reorder, rename, row open, settings, and review. Measure browser intent -> first paint -> useful state -> action acknowledgement -> settled background state. Pair those marks with `action.response` request IDs and named server phases. Repeat cold and warm cases after every material optimization so the lane works the measured waits downward rather than relying on one favorable run. + +Local acceptance proves application behavior under a controlled database/source fixture. It does not by itself prove production database latency, cold platform startup, or live Builder provider behavior; those remain separate destination evidence for Land. + +### Correlated instrumentation protocol + +Every performance run must produce one joined record from the user's gesture through the action/server path to the visible UI result. A fast server log without a fast screen is not a pass; a fast optimistic paint without acknowledgement or truthful settlement is not a pass either. + +For each operation, capture the same correlation identity across: + +1. **Intent:** pointer/keyboard gesture and browser monotonic timestamp. +2. **Immediate visual response:** shell, optimistic paint, retained rows, or explicit progress state. +3. **Client transport:** action name, request ID, request start, time to first byte, response completion, response bytes, and invalidations/refetches caused by that gesture. +4. **Server phases:** access, database connection, count/items, documents, properties/rollups, each attached-source snapshot and consistency retry, federation, review-diff preparation, serialization, and total duration. +5. **Useful visual state:** the semantic UI condition Alice can act on, such as real rows visible, sorted order visible, article body present, or first review changes present. +6. **Acknowledgement:** mutation success or typed failure, including rollback when applicable. +7. **Settled background state:** source/body hydration or complete review work, reported separately from the earlier useful state. + +Use the existing Core `action.response` telemetry and Server-Timing/request IDs before adding new machinery. Temporary probes may add named phase durations and query/row/retry counts where the existing event is too coarse. Browser marks must be paired with semantic UI assertions and timestamped screenshots or video checkpoints from the same run. The visual clock and log clock must be joined through the interaction/request identity and a recorded run-start offset; do not correlate by eyeballing two unrelated timestamps. + +The measurement report for one gesture should therefore read as a single waterfall, for example: + +`gesture -> optimistic/shell paint -> request -> server phases -> response -> useful UI -> acknowledgement -> background settled` + +The acceptance budget is judged on the relevant visual boundary. Logs diagnose which phase consumed that time; they do not redefine success. + +### Temporary-probe hygiene + +- Gate verbose phase logging to the local performance environment and task-owned fixture. +- Record identifiers, durations, counts, status, and coarse error classes; do not log document bodies, source payloads, credentials, or customer data. +- Keep an explicit inventory of every temporary probe and its purpose. +- Before the Work artifact is handed to Land, either remove each probe or deliberately promote a low-overhead, content-free measurement into the existing observability surface with tests and documentation. +- Verify the final diff contains no forgotten debug logging, local-only flags, fixture IDs, or accidental timing behavior. +- Re-run the clean production-like workflow after temporary probes are removed or disabled, because instrumentation overhead can otherwise become the last performance bug. The stopwatch has occasionally eaten the runner. + +## Frozen Implementation Plan + +### Exact acceptance fixture + +The live provider fixture is the isolated Builder collection Alice used in the clip: + +- visible Builder model label: **`Agent Native Blog Article Test`**; +- canonical Builder model name: **`agent-native-blog-article-test`**; +- expected scale: the clip showed **584 rows**; +- direct code corroboration: Builder model discovery deliberately prioritizes the canonical name `agent-native-blog-article-test` in `actions/_builder-cms-read-client.ts`; +- authority: Alice explicitly identified this as the disconnected test collection and authorized writes because it does not feed another product surface. + +The display label or model name alone is not mutation authority. At Work activation, call the read-only `list-builder-cms-models` action and require exactly one result whose `name` and `displayName` match the pair above. Persist its returned Builder model `id`, row count, and a marker-absence baseline in the run ledger. A missing, duplicate, renamed, or differently identified model fails closed before any Builder write. + +Existing collection rows are a read-only scale corpus. Provider mutations are limited to task-created rows whose title contains a run-unique marker such as `__an_content_perf___`. Work must never bulk-edit or delete the existing 584-row corpus, never delete the model, and never use an unmarked row for mutation acceptance. + +Use two additional task-owned local fixtures through Content actions: + +- a **1,000-row** SQL database with the same Name, Content, Author, and Date shape; +- a **5,000-row** SQL database with deterministic values and enough matching/non-matching rows to exercise search, filter, and sort boundaries. + +These local fixtures establish scaling independently of live Builder/provider variance. They contain synthetic text only and are deleted with an independent absence check before Work completes. + +### Delivery slices + +#### Slice 0 — restore the proving ground and capture the old path + +1. Use the current worktree and repo-pinned `pnpm@10.29.1`; restore dependencies without changing branches or printing secrets. +2. Provision the Content local database environment from the compatible canonical checkout configuration without copying values into logs or source. +3. Start the real `dev:database` UI, verify the local database/auth/provider shape, resolve the exact Builder model identity read-only, and create the task-owned local database plus one marked Builder row only after the Work resource gate is active. +4. Run the complete operation ledger once on the unchanged path. Save a joined browser/server waterfall, semantic screenshot checkpoints, action request IDs, response sizes, SQL/query counts, and the old visual durations. This is the comparison baseline, not an acceptance pass. + +#### Slice 1 — correlated instrumentation + +1. Add a Content-local timing collector for `get-content-database` and reuse the existing Builder timing collector for review/execute paths. +2. Instrument `actions/_database-utils.ts` with named spans for database/access resolution, count/page items, list-document projection, shares/favorites, properties/rollups, hydration status, source summaries/page overlays, federation, context, and serialization. +3. Instrument `actions/_database-source-utils.ts` with source ID/type, consistency attempt count, source rows scanned/returned, change-set/execution counts, body-field inclusion, and per-source duration. +4. Extend `actions/prepare-builder-source-review.ts` / `preview-builder-source-review.ts` timing around candidate discovery, authoritative target-row loading, body conversion/diff construction, validation, reconciliation, and response construction. +5. Add browser marks in `DatabaseView.tsx` at the actual intent handlers and semantic UI boundaries: shell/optimistic paint, first useful rows, stable sorted order, article body, first review changes, acknowledgement/rollback, and background settled. +6. Join browser marks to the existing Core action request ID and `action.response`/Server-Timing event. Change Core transport only if the existing request identity cannot be surfaced to the Content mark; do not invent a second unrelated trace ID. + +The Work ledger must inventory each temporary probe. Logging remains local/preview gated and content-free. Remove it before the clean final replay, except for low-overhead counts/timings deliberately promoted into existing telemetry with tests. + +#### Slice 2 — make the basic table read page-bounded end to end + +1. Split the ordinary table projection from the complete source snapshot in `getContentDatabaseResponse`. +2. Query source metadata/status plus only the source rows needed for the returned document IDs. Do not call `getAllContentDatabaseSourceSnapshots` and then filter a complete corpus after the fact. +3. Keep full change sets, execution history, heavy Builder body values, consistency/reconciliation material, and full review payloads off the table-open critical path. Load them only through the source/review action that consumes them. +4. Parallelize independent list reads after access and visible item IDs are known, while retaining typed failures. An unavailable source projection must remain distinguishable from a valid source with zero rows. +5. Return and assert page diagnostics in local timing only: requested limit, database rows scanned/returned, source rows scanned/returned, attached source count, query count, response bytes, and sequential critical-path database round trips. + +Primary files: `actions/_database-utils.ts`, `actions/_database-source-utils.ts`, `actions/get-content-database.ts`, `shared/api.ts`, and focused database/source tests. + +#### Slice 3 — stop sort/filter/search from becoming a 5,000-row client fetch + +1. Extend the existing `get-content-database` action contract with typed table-query inputs for search, active filters, sort order, and page cursor/limit. Keep this on the action surface; do not add a REST twin. +2. Apply table constraints in portable Drizzle queries and return one bounded result page plus truthful total/more state. Add only additive indexes demonstrated by query plans on hot membership/property/sort columns; no destructive schema changes or dialect-specific app calls. +3. Remove the `DatabaseView.tsx` behavior that expands a constrained table request to as many as 5,000 rows and maps the entire result into React. +4. Retain the current rows while a changed query is in flight. A sort gesture immediately updates the control and either reorders cached-complete data or keeps the prior rows visible until the bounded server result commits; it never swaps the table for `Loading database`. +5. Preserve the legacy behavior for board/calendar/timeline semantics unless the same portable query contract can represent them without changing product meaning. They must not regress, but the frozen latency target is the large **table** workflow shown in the clip. + +Primary files: `actions/get-content-database.ts`, `actions/_database-utils.ts`, schema/query helpers and additive migrations only if traces require them, `app/hooks/use-content-database.ts`, `DatabaseView.tsx`, and view/query unit plus database tests. + +#### Slice 4 — make routine writes bounded and optimistic + +1. Inventory every operation in the approved ledger and record its current mutation response shape and invalidation set. +2. For create/name, reveal property, edit value, reorder columns, rename database, and row operations, paint the exact optimistic delta within 100 ms with rollback state retained. +3. Replace complete or unpaginated post-write database reconstruction with the smallest typed delta or the caller's current bounded page. Do not return a plausible empty/success state when reconstruction fails. +4. Set `skipActionQueryInvalidation` only where the hook applies an exact cache update. Keep Core other-tab, agent/script, and external-source invalidation behavior intact. +5. Assert one gesture's action count, cache writes, invalidations, and follow-up reads so a write cannot regress into `write -> full response -> broad invalidate -> full refetch`. + +Primary files: the relevant database mutation actions, `app/hooks/use-content-database.ts`, `app/hooks/use-document-properties.ts`, and Core `use-action` only if a framework-level invalidation defect is directly demonstrated. + +#### Slice 5 — make attach, hydration, row open, and Builder review progressive + +1. Attach returns the source binding and first page of lightweight metadata quickly; continuation fetches remaining metadata in bounded pages without blocking table interaction. +2. Keep heavy Builder body fields in the existing durable hydration queue. Expose queued/progress/error counts truthfully and process bounded batches until all 584 bodies settle or a typed partial failure is visible. +3. Opening an imported row renders the shell immediately and loads its one authoritative body independently from later metadata/Yjs readiness. +4. Opening Builder review renders a local shell immediately, requests only the first review page/candidate summaries, and progressively loads full body diffs for visible or selected changes. `prepare-builder-source-review` must not rebuild an unpaginated database response after validation. +5. Preserve exact Builder target identity, change-set state, publication transition checks, dry-run/live-write gates, and reconciliation behavior. Performance is not permission to make review truth approximate. + +Primary files: `actions/_database-source-utils.ts`, refresh/hydration actions, `preview-builder-source-review.ts`, `prepare-builder-source-review.ts`, `app/hooks/use-content-database.ts`, and the Builder review/hydration portions of `DatabaseView.tsx`. + +### Measurement and acceptance loop + +Each slice follows the same loop: clean baseline -> one bounded change -> focused tests -> real browser replay -> joined waterfall -> compare against the prior run. Work continues until every operation in the approved budget table passes or a material architecture/acceptance change returns the lane to Shape. + +For warm operations, run ten repetitions after one discarded warm-up and require the approved target at p95. For cold table/row opens, run three fresh browser-context repetitions and require each to meet the 2-second target, with framework startup recorded separately rather than silently subtracted. Attach/hydration/review are repeated three times because they mutate durable local state; each run gets a fresh task-owned local database and clean source binding. + +Every measurement row contains: + +- fixture identity and row count; +- cold/warm classification; +- gesture and browser monotonic time; +- request ID and action name; +- shell/optimistic, first-useful, acknowledgement, and settled timestamps; +- server phase durations, query/row/retry counts, response bytes, and action/refetch count; +- semantic assertion and same-run screenshot/video checkpoint; +- pass/fail against the approved operation budget. + +Acceptance must cover the exact 584-row Builder fixture and synthetic 1,000/5,000-row SQL fixtures. A fast synthetic fixture cannot substitute for the Builder run; provider time cannot excuse a table path that reads the entire provider corpus. Conversely, the live collection is not used to manufacture 5,000 remote records. + +### Verification gates + +Work is complete only when all of these are current on the exact artifact: + +- focused unit/database tests prove page-bounded source reads, portable table constraints, typed failure states, bounded mutation responses, exact invalidation counts, hydration continuation, and progressive review scoping; +- `pnpm test` for Content passes; +- `pnpm typecheck` and `pnpm build` pass; +- modified source is formatted with oxfmt; +- the real local UI passes the operation ledger at the approved numbers; +- a production-like branch preview repeats the large-table acceptance with the new path and no temporary instrumentation overhead; +- an off-path/regression pass covers sharing/access, multi-source federation, other-tab/agent refresh, optimistic rollback, source-review identity/state, Builder validation failures, and Yjs row editing; +- the final diff contains no bodies/secrets, hard-coded returned Builder IDs, fixture markers, forgotten debug logging, or local-only performance switches; +- every task-created local and Builder resource is independently proven absent. + +No production performance claim may be made from unit tests, server logs alone, or a visually fast optimistic paint without acknowledgement. Local and preview evidence are reported separately. + +### Risk and rollback strategy + +The frozen strategy is **system-ready**, not a permanent production feature flag. This change preserves an existing public action/product contract and must be fully accepted before merge; merging an unaccepted dormant alternative would only move the uncertainty downstream. During Work, an explicitly local, temporary benchmark switch may compare old and new read implementations against the same fixture, but it is not a rollout mechanism and must be removed before Work completion. + +Keep each slice reversible in review: timing only, table projection, constrained queries, mutation deltas, then progressive source/review. If a slice fails correctness or its acceptance budget, revert that slice's implementation while preserving its measurements and continue from the last passing boundary. No migration may require rollback by dropping or renaming data. + +Stop and return to Shape if meeting the target requires changing the action architecture, source-review truth model, public table semantics, named shipping surface, accepted budgets, or system-ready risk strategy. Fail closed before provider writes if the Builder model identity or task-owned marker boundary cannot be proven. + +## Sources + +- Local Clips Screen Memory status, chapter search, recent-context OCR, contact-sheet attempts, and exact-frame attempt on 2026-07-28. +- Public 16:01 Clips recording `BfEnyRiC4Pu7`, inspected at five-second intervals with key visible state boundaries recorded above. +- Current checkout `9d16c580b` under `templates/content` and `packages/core`. +- `templates/content/actions/get-content-database.ts` +- `templates/content/actions/_database-utils.ts` +- `templates/content/actions/_database-source-utils.ts` +- `templates/content/actions/_property-utils.ts` +- `templates/content/actions/create-content-database.ts` +- `templates/content/actions/add-database-item.ts` +- `templates/content/actions/configure-document-property.ts` +- `templates/content/app/components/editor/database/DatabaseView.tsx` +- `templates/content/app/hooks/use-content-database.ts` +- `templates/content/app/hooks/use-document-properties.ts` +- `packages/core/src/client/use-action.ts` +- `packages/core/src/client/use-db-sync.ts` +- Prior Content slow-network route and property-save isolation research, revalidated against the current checkout before use. + +## Lifecycle + +```yaml +stage: work +ledger-revision: work-blog-table-latency-r7 +authority-source: >- + Alice invoked $work on 2026-07-29 against the frozen + shape-blog-table-latency-r5 artifact after approving the operation budgets + and authorizing the disconnected Builder test collection used in clip + BfEnyRiC4Pu7 as the proving surface +authorized-scope: + repositories: + - /Users/alicemoore/.codex/worktrees/8c63/agent-native + product-surfaces: + - Agent Native Content large database table UI + - Builder-backed Content source attach, hydration, row, and review surfaces + outcome: >- + Make ordinary Content table operations on Builder-backed and SQL databases + over 500 rows meet Alice's approved visual and acknowledgement budgets +allowed-mutations: + - artifact-write + - ephemeral-test-resource + - branch + - commit + - push + - pull-request + - deploy + - schema +write-targets: + artifacts: + - /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md +execution-lane: + worktree: /Users/alicemoore/.codex/worktrees/8c63/agent-native + branch: codex/content-large-database-performance + refreshed-base: origin/main@ce426feef1ebeb370fca581291c9391339a757ed + remote-default-branch: origin/main + calling-task-id: unavailable from current host surface + parent-task-id: none + descendants: + - /root/hot_path_inventory + - /root/performance_technical_review +work-progress: + completed-slices: + - page-scoped primary Builder rows, change sets, reviews, and executions + - bounded table search, filter, and sort response pages with retained rows + - bounded routine mutation responses with explicit created and duplicated rows + - immediate page-scoped Builder review while authoritative review loads + - exact federated-field fallback without expanding ordinary large tables + - source-free setup projections and a 100-row fast path for fresh Builder attachment + - personal sort persistence that preserves existing visibility rules without source snapshots + - correlated loopback-only server and visible-row timing probe + verification: + - shared table-query TypeScript compilation passed + - shared table-query focused Vitest passed + - oxfmt and git diff whitespace checks passed + - independent technical review found no remaining P1 or P2 after three correction passes + pending: + - dependency-compatible full typecheck and focused database/UI suites + - real dev-database fixture creation and joined browser/server measurements + - exact read-only Builder model identity and marker-absence baseline + - 584, 1000, and 5000 row acceptance runs + - temporary-probe removal and clean replay + - independent human QA, PR, and CI +temporary-probes: + - id: content-database-response-trace + gate: >- + CONTENT_DATABASE_PERFORMANCE_TRACE=1 on a verified loopback action request + records: >- + trace ID, phase durations, requested and returned row counts, source row + and change-set counts, per-source duration, consistency attempts, review + count, and execution count; no titles, bodies, provider payloads, or secrets + removal-trigger: after the final joined acceptance replay + - id: content-database-visible-row-trace + gate: response carries content-database-response-trace + records: >- + response trace ID, local interaction ID, intent-to-row-commit duration, + server duration, and returned item count + removal-trigger: after the final joined acceptance replay +test-resources: + - id: builder-perf-entry-agent-native-blog-article-test + kind: record + surface: >- + Builder model name agent-native-blog-article-test, expected display name + Agent Native Blog Article Test, exact returned model ID recorded read-only + before activation + ownership-marker: __an_content_perf___ in every task-created title + baseline: >- + list-builder-cms-models returns exactly one matching model and a provider + list query returns zero entries carrying the run marker before creation + allowed-actions: + - create + - update + - exercise + - delete + cleanup-trigger: after each live-provider acceptance run and before Work completion + cleanup-method: >- + delete every exact returned Builder entry ID created with the run marker; + never delete the model or pre-existing corpus + cleanup-proof: >- + independent reads show every returned entry ID absent and zero entries + matching the run marker + shared-impact: none + isolation: isolated-test-surface + ownership: task-created + production-data: false + customer-data: false + cost: none + boundary-evidence: + - Alice identified this disconnected collection as freely writable test data + - code recognizes canonical model name agent-native-blog-article-test + - trusted read-only model discovery must bind the exact returned provider ID + max-lifetime-minutes: 1440 + declared-at: 2026-07-29T19:00:06Z + expires-at: 2026-07-30T19:00:06Z + status: declared + phase: work + - id: content-large-database-local-fixtures + kind: database + surface: >- + task-owned local Content databases marked __an_content_perf___ + with deterministic 584, 1000, and 5000 row fixtures + ownership-marker: __an_content_perf___ database and row titles + baseline: no local database or document carries the run marker + allowed-actions: + - create + - update + - exercise + - delete + cleanup-trigger: after each destructive acceptance run and before Work completion + cleanup-method: delete task-created databases through Content actions + cleanup-proof: >- + list/search actions and scoped database reads independently report no + database, document, or row carrying the run marker + shared-impact: none + isolation: local-runtime + ownership: task-created + production-data: false + customer-data: false + cost: none + boundary-evidence: + - local database runtime and unique task marker + - creation result IDs retained in the resource ledger + max-lifetime-minutes: 1440 + declared-at: 2026-07-29T19:00:06Z + expires-at: 2026-07-30T19:00:06Z + status: active + active-run-id: 20260729_1715 + active-resources: + - kind: sqlite-runtime + path: /tmp/agent-native-content-perf.GnFBS0/app.db + state: acquired + - kind: content-database + database-id: TYhMNt9YVgjW + document-id: YXN4WiFhHCo0 + title: __an_content_perf_20260729_1715__ 1000 rows + intended-row-count: 1000 + state: created + - kind: content-database + database-id: pyehc2Pq3FT1 + document-id: 4h6N49c1GNkj + title: __an_content_perf_20260729_1715__ 5000 rows + intended-row-count: 5000 + state: created + - kind: content-database + database-id: FLeIYkBrKXeC + document-id: 4UiIEVdifuyF + title: __an_content_perf_20260729_1715__ 584 rows + intended-row-count: 584 + state: created + phase: work +governing-artifact: + path: /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md + revision: shape-blog-table-latency-r5 +architecture-fingerprint: + outcome: >- + Large Content tables remain immediately usable and meet the approved + operation budgets at 584, 1000, and 5000 rows + shipping-surfaces: + - id: agent-native-content-large-table-performance + repository: /Users/alicemoore/.codex/worktrees/8c63/agent-native + product-surface: Agent Native Content database table and Builder source workflow + constituency: >- + source-blind developers and Content users working with databases over + 500 rows, including Builder-backed tables + durable-destination: merged Agent Native public repository implementation + integration-action: merge + governing-architecture: >- + Content actions remain the single UI/agent data surface; ordinary table reads + use persisted SQL page projections, provider/body/review work stays progressive, + and client caches apply bounded optimistic deltas with truthful rollback + acceptance-story: + id: content-large-table-approved-budgets-v1 + summary: >- + On the exact isolated 584-row Builder model and deterministic 1000/5000-row + SQL fixtures, Alice can create/open a table, attach the source, reveal/edit + properties, sort, open a row, return, and review Builder changes without a + blank interface and within the approved budget table in this artifact + required-assertions: + - gesture-to-shell or optimistic response is at most 100 ms where specified + - warm reads and acknowledgements meet 1 second and cold paths meet 2 seconds where specified + - sort retains rows and settles within 500 ms warm or 1 second cold + - attach shows first useful rows within 2 seconds and hydration never blocks interaction + - 584-row metadata settles within 10 seconds and bodies within 30-60 seconds in the background + - Builder review shows first changes within 2 seconds and full or progressive diff within 10 seconds + - every gesture has one correlated visual/client/server waterfall and bounded action/refetch count + - work scales with requested page or selected review scope rather than complete corpus size + - typed failure, access, sharing, federation, sync, rollback, Yjs, and Builder review truth remain correct + - all temporary probes and task-created resources are removed with current independent proof + risk-strategy: + kind: system-ready + production-validation-after-merge: false +architecture-grounding: + applicability: required + reason: >- + The work changes shared Content action response composition, source projection, + query pagination, cache invalidation, and Builder review boundaries + status: grounded + demonstrated-callers: + - Alice's 584-row Builder-backed Blog table workflow in clip BfEnyRiC4Pu7 + - get-content-database called by DatabaseView with a 100-row initial limit + existing-primitives: + - defineAction plus useActionQuery/useActionMutation + - persisted Content SQL source rows and body hydration queue + - Core action.response request IDs and Server-Timing + - Builder source timing collector and review actions + - Core useDbSync current-tab echo suppression and targeted invalidation hooks + ownership-boundaries: + - Core owns action transport, request identity, timing headers, and DB-sync mechanics + - Content owns database projection, pagination, source/review state, hydration, and optimistic cache policy + - Builder owns provider content; only the exact isolated test model is writable in acceptance + legacy-contracts: + - action surface parity for UI and agent callers + - ownable-data access and sharing scope + - truthful multi-source and review/change-set state + - other-tab and agent/script refresh + - optimistic rollback and authoritative Yjs document behavior + shared-vocabulary: + - first useful + - acknowledgement + - settled background state + - page-bounded table projection + - progressive Builder review + smallest-compatible-delta: >- + Page-bound get-content-database source overlays and table constraints before + changing mutation/review response shapes; no broad cache or sync rewrite + deferred-capabilities: + - general query planner for every database view type + - redesigned infinite scrolling + - rollup engine rewrite + - new real-time transport + - provider live reads on ordinary table render + reversibility: >- + Six ordered slices isolate environment/baseline, timing, read projection, + constraints, mutations, and progressive provider work; all schema changes are additive and each slice + can be removed without deleting user data + direct-evidence: + - clip operation ledger and approved budgets in this artifact + - getContentDatabaseResponse loads complete source snapshots before page filtering + - DatabaseView expands constrained reads to 5000 and renders items with map + - Builder review timing already names snapshot/diff and reconciliation phases + - model discovery prioritizes canonical name agent-native-blog-article-test + inferences: + - relative contribution of source snapshots, query waterfalls, payload, and React commit before joined baseline + unresolved-owner-questions: [] +delegation-ceiling: + - read-only + - artifact-write +product-boundary-gates: + agent-native-public-constituency: >- + The implementation and synthetic fixtures work without Alice's vault or + credentials; her isolated Builder model supplies provider acceptance only, + not a runtime dependency or hard-coded identity + bowerbird-product-boundary: not-applicable +acceptance-state: + status: pending + summary: >- + Local implementation and synthetic browser acceptance are complete. The + 5000-row table returns and paints only 100 rows, retained-row sorting meets + the warm target, and reconciliation now remains portable at 5000 rows. + Exact Builder-provider acceptance and independent human QA remain pending; + task-local fixture cleanup is complete. + local-evidence: + - >- + 5000-row warm open, ten settled repetitions: first useful React commit + 146-189 ms; correlated server read 5-12 ms; 100 of 5000 rows returned + - >- + 5000-row Date sort, ten repetitions: existing rows remained visible and + the sorted paint completed in 272-285 ms + - >- + representative 1000-row and 584-row opens committed 100 useful rows in + 217 ms and 210 ms respectively, with 12 ms and 6 ms server reads + - >- + 584-row row preview showed shell, metadata, and synthetic authoritative + body within 526 ms including browser-control overhead + - >- + Final local replay retained 100 of 5000 rows through Date sort reversal + and opened the first row preview with Author, Date, and authoritative body + - >- + final ABI-matched verification: Content typecheck and 228 focused action, + batching, table, sidebar, personal-view, and query tests passed + blockers: + - >- + The isolated local runtime has no Builder vault credential, so the exact + Agent Native Test Blog model identity, attach/hydration, and progressive + review budgets cannot yet be accepted against the provider fixture. + - >- + Independent human QA preflight was blocked before H1-H5 because the tester + host did not expose the in-app browser; no frozen-test action was executed. + PR review and CI have not yet run. + cleanup-evidence: + - >- + Stopped the traced local Content server and moved the exact task database + directory plus task-local ignored environment copies to macOS Trash; all + three original paths were verified absent. + - >- + Removed the environment-gated server phase trace, source snapshot trace, + response trace fields, and correlated browser console timing after the + final joined replay; no temporary Content performance probe remains. + last-land-packet: null +status: active +``` + +Frozen-five status: **frozen at `shape-blog-table-latency-r5`**. + +Output: `/Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md` + +Authority: Work on the current branch. Product code, tests, changelog, this +artifact, and recoverable task-local fixtures changed; no Builder record, branch +operation, deployment, or merge is authorized by this phase. + +Invalidated by: a change to the approved outcome/budgets, named Agent Native Content shipping surface or constituency, action/SQL/progressive-provider architecture, acceptance story, or system-ready risk strategy. + +Task attention: `shape-complete`; title unchanged because no calling-task title adapter is available in this surface. + +Next: `/work /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md` diff --git a/templates/content/actions/_batch-utils.ts b/templates/content/actions/_batch-utils.ts index f4ff5ded55..36e816ed6f 100644 --- a/templates/content/actions/_batch-utils.ts +++ b/templates/content/actions/_batch-utils.ts @@ -1,3 +1,5 @@ +import { getDialect, type Dialect } from "@agent-native/core/db"; + export function chunks(items: T[], size: number): T[][] { if (items.length === 0) return []; const out: T[][] = []; @@ -6,3 +8,13 @@ export function chunks(items: T[], size: number): T[][] { } return out; } + +export function bulkChunkSizeForColumnCount( + columnCount: number, + dialect: Dialect = getDialect(), +) { + // D1 rejects statements with more than 100 bound params, so derive every + // bulk chunk from the statement's column count instead of a fixed row count. + const budget = dialect === "d1" ? 90 : 900; + return Math.max(1, Math.floor(budget / Math.max(1, columnCount))); +} diff --git a/templates/content/actions/_content-files.ts b/templates/content/actions/_content-files.ts index a82e0fbb42..68d43fbd1d 100644 --- a/templates/content/actions/_content-files.ts +++ b/templates/content/actions/_content-files.ts @@ -4,6 +4,7 @@ import { accessFilter } from "@agent-native/core/sharing"; import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { schema } from "../server/db/index.js"; +import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js"; import { listContentOrganizationMemberships, normalizeContentSpaceEmail, @@ -32,23 +33,29 @@ export function contentFilesItemId(databaseId: string, documentId: string) { async function remapItemReferences(db: Db, replacements: Map) { if (!replacements.size) return; - const duplicateIds = [...replacements.keys()]; - const remap = (column: any) => { - let expression = sql`CASE ${column}`; - for (const [duplicateId, canonicalId] of replacements) - expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`; - return sql`${expression} ELSE ${column} END`; - }; - for (const table of [ - schema.contentSpaceCatalogItems, - schema.contentDatabaseBodyHydrationQueue, - schema.contentDatabaseSourceRows, - schema.contentDatabaseSourceChangeSets, - ]) { - await db - .update(table) - .set({ databaseItemId: remap(table.databaseItemId) }) - .where(inArray(table.databaseItemId, duplicateIds)); + for (const replacementBatch of chunks( + [...replacements.entries()], + bulkChunkSizeForColumnCount(3), + )) { + const batchReplacements = new Map(replacementBatch); + const duplicateIds = [...batchReplacements.keys()]; + const remap = (column: any) => { + let expression = sql`CASE ${column}`; + for (const [duplicateId, canonicalId] of batchReplacements) + expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`; + return sql`${expression} ELSE ${column} END`; + }; + for (const table of [ + schema.contentSpaceCatalogItems, + schema.contentDatabaseBodyHydrationQueue, + schema.contentDatabaseSourceRows, + schema.contentDatabaseSourceChangeSets, + ]) { + await db + .update(table) + .set({ databaseItemId: remap(table.databaseItemId) }) + .where(inArray(table.databaseItemId, duplicateIds)); + } } } @@ -170,15 +177,18 @@ async function reconcileDocuments(args: { } } await remapItemReferences(args.db, replacements); - if (deleteIds.size) { + for (const deleteBatch of chunks( + [...deleteIds], + bulkChunkSizeForColumnCount(1), + )) { await args.db .delete(schema.contentDatabaseItems) - .where(inArray(schema.contentDatabaseItems.id, [...deleteIds])); + .where(inArray(schema.contentDatabaseItems.id, deleteBatch)); } - if (inserts.length) { + for (const insertBatch of chunks(inserts, bulkChunkSizeForColumnCount(8))) { await args.db .insert(schema.contentDatabaseItems) - .values(inserts) + .values(insertBatch) .onConflictDoNothing(); } return { inserted: inserts.length, removed: deleteIds.size }; diff --git a/templates/content/actions/_database-source-utils.test.ts b/templates/content/actions/_database-source-utils.test.ts index 738397bcbe..bb16cb6884 100644 --- a/templates/content/actions/_database-source-utils.test.ts +++ b/templates/content/actions/_database-source-utils.test.ts @@ -51,6 +51,7 @@ import { serializeSourceMetadataRecord, sourceSnapshotValuesJsonProjectionSql, sourceSnapshotDocumentSelection, + sourceSnapshotPageDocumentIds, sourceValuesForSnapshot, sourceValuesForSeededSourceRow, sourceChangeSetKey, @@ -103,6 +104,52 @@ function item(id: string, title: string): ContentDatabaseItem { } describe("database source helpers", () => { + it("page-scopes primary Builder rows without truncating secondary federation", () => { + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "builder-cms", + metadataJson: "{}", + documentIds: ["doc-1", "doc-2"], + }), + ).toEqual(["doc-1", "doc-2"]); + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "builder-cms", + metadataJson: "{}", + documentIds: [], + }), + ).toEqual([]); + + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "builder-cms", + metadataJson: JSON.stringify({ + federation: { + role: "secondary", + keyField: "slug", + normalizationFormula: "lower(trim(value))", + join: { + kind: "identity", + collection: null, + localExpr: "{Slug}", + remoteKeyField: "slug", + normalizationFormula: "lower(trim(value))", + }, + }, + }), + documentIds: ["doc-1"], + }), + ).toBeUndefined(); + + expect( + sourceSnapshotPageDocumentIds({ + sourceType: "local-table", + metadataJson: "{}", + documentIds: ["doc-1"], + }), + ).toBeUndefined(); + }); + it("keeps the provider-bound property when duplicate local labels collide", () => { const existingFields = [ { diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 08b3bdd4d3..17f6c52dbb 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -55,7 +55,8 @@ import { type DocumentPropertyOptionColor, } from "../shared/properties.js"; import { sanitizeNormalizationFormula } from "../shared/properties.js"; -import { chunks } from "./_batch-utils.js"; +import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js"; +export { bulkChunkSizeForColumnCount } from "./_batch-utils.js"; import { readBuilderCmsContentEntry, readBuilderCmsContentEntries, @@ -966,16 +967,6 @@ const BUILDER_BODY_HYDRATION_CODEC_VERSION = const BUILDER_CMS_REFRESH_INITIAL_PAGES = 1; const BUILDER_BODY_NOT_AVAILABLE_ERROR = "body not yet available from Builder"; -export function bulkChunkSizeForColumnCount( - columnCount: number, - dialect: Dialect = getDialect(), -) { - // D1 rejects statements with more than 100 bound params, so derive every - // bulk chunk from the statement's column count instead of a fixed row count. - const budget = dialect === "d1" ? 90 : 900; - return Math.max(1, Math.floor(budget / Math.max(1, columnCount))); -} - function idChunkSize() { return bulkChunkSizeForColumnCount(1); } @@ -3514,6 +3505,7 @@ export async function getContentDatabaseSourceSnapshotForReview( */ export async function getAllContentDatabaseSourceSnapshots( database: ContentDatabaseRow | ContentDatabase, + options: { documentIds?: string[] } = {}, ): Promise { if ("deletedAt" in database && database.deletedAt) { throw new Error(`Database "${database.id}" not found`); @@ -3527,15 +3519,31 @@ export async function getAllContentDatabaseSourceSnapshots( asc(schema.contentDatabaseSources.createdAt), asc(schema.contentDatabaseSources.id), ); - const snapshots: ContentDatabaseSource[] = []; - for (const source of sources) { - snapshots.push( - await loadSourceSnapshot(source, database, { + return Promise.all( + sources.map((source) => { + return loadSourceSnapshot(source, database, { includeHeavyBuilderBodyValues: false, - }), - ); - } - return snapshots; + documentIds: sourceSnapshotPageDocumentIds({ + sourceType: source.sourceType, + metadataJson: source.metadataJson, + documentIds: options.documentIds, + }), + }); + }), + ); +} + +export function sourceSnapshotPageDocumentIds(args: { + sourceType: string; + metadataJson: string | null; + documentIds?: string[]; +}) { + const metadata = parseObject(args.metadataJson); + const federation = normalizeSourceFederation(metadata?.federation); + return normalizeSourceType(args.sourceType) === "builder-cms" && + federation?.role !== "secondary" + ? args.documentIds + : undefined; } async function readSourceSnapshotRowsOnce(args: { @@ -3546,9 +3554,7 @@ async function readSourceSnapshotRowsOnce(args: { documentIds?: string[]; }) { const db = getDb(); - const documentScope = args.documentIds?.length - ? new Set(args.documentIds) - : null; + const documentScope = args.documentIds ? new Set(args.documentIds) : null; const rowRows = await db .select( sourceSnapshotRowSelection({ @@ -3660,9 +3666,7 @@ async function sourceSnapshotConsistencyMarker(args: { documentIds?: string[]; }) { const db = getDb(); - const documentScope = args.documentIds?.length - ? new Set(args.documentIds) - : null; + const documentScope = args.documentIds ? new Set(args.documentIds) : null; const [rows] = await db .select({ count: sql`COUNT(*)`, @@ -3737,9 +3741,14 @@ async function loadSourceSnapshotRowsOptimistically(args: { const before = await sourceSnapshotConsistencyMarker(args); latest = await readSourceSnapshotRowsOnce(args); const after = await sourceSnapshotConsistencyMarker(args); - if (sourceSnapshotConsistencyMarkersEqual(before, after)) return latest; + if (sourceSnapshotConsistencyMarkersEqual(before, after)) { + return { ...latest, consistencyAttempts: attempt + 1 }; + } } - return latest ?? (await readSourceSnapshotRowsOnce(args)); + return { + ...(latest ?? (await readSourceSnapshotRowsOnce(args))), + consistencyAttempts: 3, + }; } async function loadSourceSnapshot( @@ -3748,50 +3757,76 @@ async function loadSourceSnapshot( options: { includeHeavyBuilderBodyValues: boolean; documentIds?: string[] }, ): Promise { const db = getDb(); - const [fieldRows, changeRows, reviewRows, executionRows, propertyDefs] = - await Promise.all([ - db - .select() - .from(schema.contentDatabaseSourceFields) - .where(eq(schema.contentDatabaseSourceFields.sourceId, source.id)) - .orderBy(asc(schema.contentDatabaseSourceFields.createdAt)), - db - .select() - .from(schema.contentDatabaseSourceChangeSets) - .where( - options.documentIds?.length - ? and( - eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), - inArray( - schema.contentDatabaseSourceChangeSets.documentId, - options.documentIds, + const [fieldRows, changeRows, propertyDefs] = await Promise.all([ + db + .select() + .from(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.sourceId, source.id)) + .orderBy(asc(schema.contentDatabaseSourceFields.createdAt)), + db + .select() + .from(schema.contentDatabaseSourceChangeSets) + .where( + options.documentIds !== undefined + ? and( + eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), + inArray( + schema.contentDatabaseSourceChangeSets.documentId, + options.documentIds, + ), + ) + : eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), + ) + .orderBy(asc(schema.contentDatabaseSourceChangeSets.createdAt)), + db + .select({ + id: schema.documentPropertyDefinitions.id, + name: schema.documentPropertyDefinitions.name, + type: schema.documentPropertyDefinitions.type, + optionsJson: schema.documentPropertyDefinitions.optionsJson, + }) + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)), + ]); + const scopedChangeSetIds = changeRows.map((changeSet) => changeSet.id); + const [reviewRows, executionRows] = + options.documentIds !== undefined && scopedChangeSetIds.length === 0 + ? [[], []] + : await Promise.all([ + db + .select() + .from(schema.contentDatabaseSourceChangeReviews) + .where( + and( + eq( + schema.contentDatabaseSourceChangeReviews.sourceId, + source.id, ), - ) - : eq(schema.contentDatabaseSourceChangeSets.sourceId, source.id), - ) - .orderBy(asc(schema.contentDatabaseSourceChangeSets.createdAt)), - db - .select() - .from(schema.contentDatabaseSourceChangeReviews) - .where( - eq(schema.contentDatabaseSourceChangeReviews.sourceId, source.id), - ) - .orderBy(asc(schema.contentDatabaseSourceChangeReviews.createdAt)), - db - .select() - .from(schema.contentDatabaseSourceExecutions) - .where(eq(schema.contentDatabaseSourceExecutions.sourceId, source.id)) - .orderBy(asc(schema.contentDatabaseSourceExecutions.createdAt)), - db - .select({ - id: schema.documentPropertyDefinitions.id, - name: schema.documentPropertyDefinitions.name, - type: schema.documentPropertyDefinitions.type, - optionsJson: schema.documentPropertyDefinitions.optionsJson, - }) - .from(schema.documentPropertyDefinitions) - .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)), - ]); + options.documentIds !== undefined + ? inArray( + schema.contentDatabaseSourceChangeReviews.changeSetId, + scopedChangeSetIds, + ) + : undefined, + ), + ) + .orderBy(asc(schema.contentDatabaseSourceChangeReviews.createdAt)), + db + .select() + .from(schema.contentDatabaseSourceExecutions) + .where( + and( + eq(schema.contentDatabaseSourceExecutions.sourceId, source.id), + options.documentIds !== undefined + ? inArray( + schema.contentDatabaseSourceExecutions.changeSetId, + scopedChangeSetIds, + ) + : undefined, + ), + ) + .orderBy(asc(schema.contentDatabaseSourceExecutions.createdAt)), + ]); const propertyNameById = new Map( propertyDefs.map((row) => [row.id, row.name]), @@ -3844,6 +3879,7 @@ async function loadSourceSnapshot( allDocumentIds, rowDocuments, propertyValueRows, + consistencyAttempts, } = await loadSourceSnapshotRowsOptimistically({ source, database, @@ -4224,6 +4260,10 @@ async function loadSourceSnapshot( fields, rows, changeSets, + projection: + options.documentIds !== undefined + ? { rows: "page", changeSets: "page" } + : { rows: "complete", changeSets: "complete" }, bodyHydration, }; } @@ -6971,15 +7011,24 @@ export async function getSourceRows(sourceId: string) { .where(eq(schema.contentDatabaseSourceRows.sourceId, sourceId)); } -export async function listDatabasePropertiesAndItems(databaseId: string) { +export async function listDatabasePropertiesAndItems( + databaseId: string, + options: { limit?: number; offset?: number; documentIds?: string[] } = {}, +) { const { getContentDatabaseResponse } = await import("./_database-utils.js"); - return getContentDatabaseResponse(databaseId); + return getContentDatabaseResponse(databaseId, { + ...options, + includeSources: false, + }); } -export async function sourceSetupPayload(databaseId: string) { +export async function sourceSetupPayload( + databaseId: string, + options: { limit?: number; offset?: number; documentIds?: string[] } = {}, +) { const [properties, response] = await Promise.all([ listPropertiesForDatabase(databaseId), - listDatabasePropertiesAndItems(databaseId), + listDatabasePropertiesAndItems(databaseId, options), ]); return { properties, response }; } diff --git a/templates/content/actions/_database-utils.ts b/templates/content/actions/_database-utils.ts index 6ec1470fdf..829d32f805 100644 --- a/templates/content/actions/_database-utils.ts +++ b/templates/content/actions/_database-utils.ts @@ -20,7 +20,12 @@ import type { ContentDatabaseBodyHydration, ContentDatabaseMembership, ContentDatabaseResponse, + ContentDatabaseTableQuery, } from "../shared/api.js"; +import { + applyContentDatabaseTableQuery, + contentDatabaseTableQueryUsesProperties, +} from "../shared/database-query.js"; import { favoriteDocumentIds } from "./_content-favorites.js"; import { listContentOrganizationMemberships, @@ -44,6 +49,44 @@ export { getDocumentContextPath }; export const CONTENT_DATABASE_MAX_READ_LIMIT = 5_000; +async function contentDatabaseTableQueryMode( + databaseId: string, + query: ContentDatabaseTableQuery | undefined, +) { + if (!query) return undefined; + const sourceFields = await getDb() + .select({ + metadataJson: schema.contentDatabaseSources.metadataJson, + propertyId: schema.contentDatabaseSourceFields.propertyId, + }) + .from(schema.contentDatabaseSourceFields) + .innerJoin( + schema.contentDatabaseSources, + eq( + schema.contentDatabaseSources.id, + schema.contentDatabaseSourceFields.sourceId, + ), + ) + .where(eq(schema.contentDatabaseSources.databaseId, databaseId)); + const secondaryPropertyIds = new Set(); + for (const field of sourceFields) { + let role: unknown = null; + try { + role = JSON.parse(field.metadataJson || "{}").federation?.role; + } catch { + role = null; + } + if (role === "secondary" && field.propertyId) { + secondaryPropertyIds.add(field.propertyId); + } + } + const usesSecondaryField = contentDatabaseTableQueryUsesProperties( + query, + secondaryPropertyIds, + ); + return usesSecondaryField ? "client-required" : "server"; +} + function canManageRole(role: string) { return role === "owner" || role === "admin"; } @@ -279,7 +322,13 @@ function serializeDocument( export async function getContentDatabaseResponse( databaseId: string, - options: { limit?: number; offset?: number } = {}, + options: { + limit?: number; + offset?: number; + tableQuery?: ContentDatabaseTableQuery; + includeSources?: boolean; + documentIds?: string[]; + } = {}, ): Promise { const db = getDb(); const [database] = await db @@ -304,6 +353,12 @@ export async function getContentDatabaseResponse( // shared one a viewer is opening) must not mutate schema. const { limit, offset } = normalizeContentDatabasePageOptions(options); + const tableQuery = options.tableQuery; + const tableQueryMode = await contentDatabaseTableQueryMode( + databaseId, + tableQuery, + ); + const serverTableQuery = tableQueryMode === "server" ? tableQuery : undefined; const userEmail = getRequestUserEmail(); const normalizedUserEmail = userEmail ? normalizeContentSpaceEmail(userEmail) @@ -401,6 +456,11 @@ export async function getContentDatabaseResponse( : undefined; const visibleItemFilter = and( eq(schema.contentDatabaseItems.databaseId, databaseId), + options.documentIds !== undefined + ? options.documentIds.length > 0 + ? inArray(schema.contentDatabaseItems.documentId, options.documentIds) + : sql`1 = 0` + : undefined, sql`exists ( select 1 from ${schema.documents} where ${schema.documents.id} = ${schema.contentDatabaseItems.documentId} @@ -428,6 +488,12 @@ export async function getContentDatabaseResponse( .select({ count: sql`COUNT(*)` }) .from(schema.contentDatabaseItems) .where(visibleItemFilter); + const totalVisibleItems = Number(itemCount?.count ?? 0); + if (tableQuery && totalVisibleItems > CONTENT_DATABASE_MAX_READ_LIMIT) { + throw new Error( + `Table constraints support up to ${CONTENT_DATABASE_MAX_READ_LIMIT} rows; this database has ${totalVisibleItems}.`, + ); + } let itemsQuery = db .select() @@ -435,7 +501,9 @@ export async function getContentDatabaseResponse( .where(visibleItemFilter) .orderBy(asc(schema.contentDatabaseItems.position)) .$dynamic(); - if (limit !== null) { + if (serverTableQuery) { + itemsQuery = itemsQuery.limit(CONTENT_DATABASE_MAX_READ_LIMIT); + } else if (limit !== null) { itemsQuery = itemsQuery.limit(limit).offset(offset); } const items = await itemsQuery; @@ -586,12 +654,12 @@ export async function getContentDatabaseResponse( ) : new Set(); - const serializedItems = []; + const serializedCandidateItems = []; for (const item of items) { const document = documentById.get(item.documentId); if (!document) continue; const bodyHydrationQueued = queuedBodyHydrationItemIds.has(item.id); - serializedItems.push({ + serializedCandidateItems.push({ id: item.id, databaseId: item.databaseId, document: serializeDocument( @@ -612,7 +680,27 @@ export async function getContentDatabaseResponse( }); } - const sourceSnapshots = await getAllContentDatabaseSourceSnapshots(database); + const constrainedItems = serverTableQuery + ? applyContentDatabaseTableQuery( + serializedCandidateItems, + responseProperties, + serverTableQuery, + ) + : serializedCandidateItems; + const serializedItems = + serverTableQuery && limit !== null + ? constrainedItems.slice(offset, offset + limit) + : constrainedItems; + + const serializedDocumentIds = new Set( + serializedItems.map((item) => item.document.id), + ); + const sourceSnapshots = + options.includeSources === false + ? [] + : await getAllContentDatabaseSourceSnapshots(database, { + documentIds: limit !== null ? [...serializedDocumentIds] : undefined, + }); const organizationVisibleDocumentIds = organizationFilesItemFilter ? new Set( ( @@ -631,25 +719,26 @@ export async function getContentDatabaseResponse( ), ) : sourceSnapshots; - const serializedDocumentIds = new Set( - serializedItems.map((item) => item.document.id), - ); - // When paginating, scope every DOCUMENT-BACKED source's rows to the visible - // page, plus the small set referenced by actionable reviews. The dialog gets - // change sets independently of the item page and needs those rows to retain - // the linked provider target instead of misclassifying an off-page update as - // a create. Federated join rows carry no document (empty documentId), so - // they're kept intact — only matched ones overlay anyway. + // Keep the returned source overlay aligned to the visible item page. + // Secondary federation sources stay complete until their join-key lookup can + // be bounded independently; only matched rows overlay the returned items. const pagedSources = limit !== null - ? sources.map((source) => ({ - ...source, - rows: filterContentDatabaseSourceRowsForPage({ + ? sources.map((source) => { + const rows = filterContentDatabaseSourceRowsForPage({ rows: source.rows, changeSets: source.changeSets, visibleDocumentIds: serializedDocumentIds, - }), - })) + }); + return { + ...source, + rows, + projection: { + rows: "page" as const, + changeSets: source.projection?.changeSets ?? "complete", + }, + }; + }) : sources; const pagedPrimary = pagedSources[0] ?? null; @@ -660,12 +749,13 @@ export async function getContentDatabaseResponse( // Opt-in federated columns (a secondary field the user added via the picker) // get their per-row values from the matched overlay at read time. const itemsWithOverlay = applyFederatedOverlayValues(federatedItems); + const contextPath = databaseDocument + ? await getDocumentContextPath(databaseDocument) + : []; return { database: serializeDatabase(database, databaseDocument?.description ?? ""), - contextPath: databaseDocument - ? await getDocumentContextPath(databaseDocument) - : [], + contextPath, properties: responseProperties, items: itemsWithOverlay, source: pagedPrimary, @@ -675,12 +765,16 @@ export async function getContentDatabaseResponse( ? { offset, limit, - totalItems: Number(itemCount?.count ?? 0), + totalItems: serverTableQuery + ? constrainedItems.length + : totalVisibleItems, returnedItems: serializedItems.length, hasMore: - offset + serializedItems.length < Number(itemCount?.count ?? 0), + offset + serializedItems.length < + (serverTableQuery ? constrainedItems.length : totalVisibleItems), } : undefined, + tableQueryMode, }; } diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 1ef8642638..5732f0f2f1 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -213,8 +213,20 @@ export default defineAction({ // SQLite writer briefly blocks this best-effort refresh hint. }); + const response = await getContentDatabaseResponse(databaseId, { + limit: 100, + offset: 0, + }); + const createdPage = response.items.some((item) => item.id === itemId) + ? null + : await getContentDatabaseResponse(databaseId, { + limit: 1, + offset: Math.max(0, (response.pagination?.totalItems ?? 1) - 1), + }); + const createdItem = createdPage?.items.find((item) => item.id === itemId); return { - ...(await getContentDatabaseResponse(databaseId)), + ...response, + createdItem, createdItemId: itemId, createdDocumentId: documentId, createdDocumentUpdatedAt: now, diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index e038791a83..8ce889e4ba 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -1,7 +1,9 @@ import { defineAction } from "@agent-native/core"; import { assertAccess } from "@agent-native/core/sharing"; +import { eq } from "drizzle-orm"; import { z } from "zod"; +import { getDb, schema } from "../server/db/index.js"; import type { BuilderCmsModelFieldSummary, ContentDatabaseResponse, @@ -445,11 +447,14 @@ export default defineAction({ sourceTable, now, }); - // Snapshot existing items BEFORE importing so we can bind the new source - // to ONLY the rows it imports — never the primary's existing rows. - const beforeSetup = await sourceSetupPayload(database.id); + // Snapshot membership IDs before importing so the new source binds only + // its own rows without serializing the existing database. + const priorItems = await getDb() + .select({ documentId: schema.contentDatabaseItems.documentId }) + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, database.id)); const priorDocumentIds = new Set( - beforeSetup.response.items.map((item) => item.document.id), + priorItems.map((item) => item.documentId), ); let importedEntriesByDocumentId = new Map< string, @@ -466,7 +471,12 @@ export default defineAction({ }); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; } - const additionalSetup = await sourceSetupPayload(database.id); + const importedDocumentIds = [...importedEntriesByDocumentId.keys()]; + const additionalSetup = await sourceSetupPayload(database.id, { + documentIds: importedDocumentIds, + limit: Math.max(1, importedDocumentIds.length), + offset: 0, + }); // Only the items this collection just created — exclude the primary's. const importedItems = additionalSetup.response.items.filter( (item) => !priorDocumentIds.has(item.document.id), @@ -576,7 +586,15 @@ export default defineAction({ importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; } - const refreshedSetup = await sourceSetupPayload(database.id); + const everyInitialEntryWasImported = + builderRead?.state === "live" && + importedEntriesByDocumentId.size === builderEntries.length; + const refreshedSetup = await sourceSetupPayload( + database.id, + everyInitialEntryWasImported + ? { limit: args.limit, offset: args.offset } + : undefined, + ); const builderEntriesByDocumentId = builderRead?.state === "live" ? mapBuilderCmsEntriesToLocalItems({ diff --git a/templates/content/actions/bind-content-database-source-field.ts b/templates/content/actions/bind-content-database-source-field.ts index 305e474e44..e706be76af 100644 --- a/templates/content/actions/bind-content-database-source-field.ts +++ b/templates/content/actions/bind-content-database-source-field.ts @@ -105,7 +105,7 @@ export default defineAction({ .update(schema.contentDatabaseSources) .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); } // ── Bind to an existing column ───────────────────────────────────────── @@ -261,6 +261,6 @@ export default defineAction({ } } - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/builder-source-review-gates.db.test.ts b/templates/content/actions/builder-source-review-gates.db.test.ts index 9e05c8e1e2..7e4c85b3f9 100644 --- a/templates/content/actions/builder-source-review-gates.db.test.ts +++ b/templates/content/actions/builder-source-review-gates.db.test.ts @@ -689,7 +689,7 @@ describe("Builder source review execution gates", () => { expect(heavySnapshotReads).toMatchObject({ review: 1, target: 1, - allSources: 1, + allSources: 0, }); expect(persistedChangeSet.state).toBe("approved"); expect(response.review.rows[0]?.fieldChanges).toEqual(persistedFields); diff --git a/templates/content/actions/cancel-prepared-builder-source-update.ts b/templates/content/actions/cancel-prepared-builder-source-update.ts index e993f0b362..b6ab409134 100644 --- a/templates/content/actions/cancel-prepared-builder-source-update.ts +++ b/templates/content/actions/cancel-prepared-builder-source-update.ts @@ -295,7 +295,10 @@ export default defineAction({ }); return { - ...(await getContentDatabaseResponse(database.id)), + ...(await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + })), cancellation: { sourceId: args.sourceId, changeSetId: args.changeSetId, diff --git a/templates/content/actions/content-spaces.db.test.ts b/templates/content/actions/content-spaces.db.test.ts index 2f77151788..296c4e827d 100644 --- a/templates/content/actions/content-spaces.db.test.ts +++ b/templates/content/actions/content-spaces.db.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { getDbExec } from "@agent-native/core/db"; import { runWithRequestContext } from "@agent-native/core/server"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; const TEST_DB_PATH = join( @@ -40,6 +40,7 @@ const OWNER = "owner@example.com"; const MEMBER = "member@example.com"; const OUTSIDER = "outsider@example.com"; const WORKSPACE_OWNER = "workspace-owner@example.com"; +const LARGE_DATABASE_OWNER = "large-database-owner@example.com"; beforeAll(async () => { process.env.DATABASE_URL = `file:${TEST_DB_PATH}`; @@ -112,6 +113,49 @@ async function addMember( } describe("Content space provisioning", () => { + it("batches Files reconciliation for databases with 5,000 row documents", async () => { + const provisioned = await runWithRequestContext( + { userEmail: LARGE_DATABASE_OWNER }, + () => provisionContentSpaces(getDb(), LARGE_DATABASE_OWNER), + ); + const now = new Date().toISOString(); + const documents = Array.from({ length: 5_000 }, (_, index) => ({ + id: `large-database-document-${index}`, + ownerEmail: LARGE_DATABASE_OWNER, + orgId: null, + spaceId: provisioned.personalSpaceId, + parentId: null, + title: `Large database row ${index}`, + content: "Synthetic row", + position: index, + visibility: "private" as const, + createdAt: now, + updatedAt: now, + })); + for (let start = 0; start < documents.length; start += 250) { + await getDb() + .insert(schema.documents) + .values(documents.slice(start, start + 250)); + } + + await expect( + runWithRequestContext({ userEmail: LARGE_DATABASE_OWNER }, () => + ensureContentSpacesAction.run({}), + ), + ).resolves.toBeDefined(); + + const [membershipCount] = await getDb() + .select({ count: sql`count(*)` }) + .from(schema.contentDatabaseItems) + .where( + eq( + schema.contentDatabaseItems.databaseId, + provisioned.personalFilesDatabaseId, + ), + ); + expect(Number(membershipCount.count)).toBe(5_000); + }); + it("creates an idempotent private named workspace with canonical Files", async () => { const provisioned = await runWithRequestContext( { userEmail: WORKSPACE_OWNER }, diff --git a/templates/content/actions/delete-database-items.ts b/templates/content/actions/delete-database-items.ts index a059abdd63..f8e65f45dd 100644 --- a/templates/content/actions/delete-database-items.ts +++ b/templates/content/actions/delete-database-items.ts @@ -38,7 +38,10 @@ export default defineAction({ }); await writeAppState("refresh-signal", { ts: Date.now() }); return { - ...(await getContentDatabaseResponse(database.id)), + ...(await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + })), deletedItemIds: removedItemIds, deletedDocumentIds: [], deletedCount: 0, @@ -74,7 +77,10 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); return { - ...(await getContentDatabaseResponse(database.id)), + ...(await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + })), deletedItemIds, deletedDocumentIds, deletedCount: deletedItemIds.length, diff --git a/templates/content/actions/disconnect-content-database-source.ts b/templates/content/actions/disconnect-content-database-source.ts index a552b22448..092bdd5abd 100644 --- a/templates/content/actions/disconnect-content-database-source.ts +++ b/templates/content/actions/disconnect-content-database-source.ts @@ -71,7 +71,7 @@ export default defineAction({ ), ); if (target) await deleteSourceRecords(target.id); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); } const source = await getExistingSource(database.id); @@ -79,6 +79,6 @@ export default defineAction({ await deleteSourceRecords(source.id); } - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/duplicate-database-item.ts b/templates/content/actions/duplicate-database-item.ts index 5f28ecb52d..7094b10761 100644 --- a/templates/content/actions/duplicate-database-item.ts +++ b/templates/content/actions/duplicate-database-item.ts @@ -176,8 +176,22 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); + const response = await getContentDatabaseResponse(row.item.databaseId, { + limit: 100, + offset: 0, + }); + const duplicatedPage = response.items.some((item) => item.id === nextItemId) + ? null + : await getContentDatabaseResponse(row.item.databaseId, { + limit: 1, + offset: nextPosition, + }); + const duplicatedItem = duplicatedPage?.items.find( + (item) => item.id === nextItemId, + ); return { - ...(await getContentDatabaseResponse(row.item.databaseId)), + ...response, + duplicatedItems: duplicatedItem ? [duplicatedItem] : [], duplicatedItemId: nextItemId, duplicatedDocumentId: nextDocumentId, }; diff --git a/templates/content/actions/duplicate-database-items.ts b/templates/content/actions/duplicate-database-items.ts index 40d4cda9a8..a0e1f929ea 100644 --- a/templates/content/actions/duplicate-database-items.ts +++ b/templates/content/actions/duplicate-database-items.ts @@ -186,8 +186,25 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); + const response = await getContentDatabaseResponse(database.id, { + limit: 100, + offset: 0, + }); + const duplicatedPage = await getContentDatabaseResponse(database.id, { + limit: duplicates.length, + offset: insertionPosition, + }); + const duplicatedItemIds = new Set( + duplicates.map((duplicate) => duplicate.duplicatedItemId), + ); + const appendedDuplicatedItems = duplicatedPage.items.filter( + (item) => + duplicatedItemIds.has(item.id) && + !response.items.some((candidate) => candidate.id === item.id), + ); return { - ...(await getContentDatabaseResponse(database.id)), + ...response, + duplicatedItems: appendedDuplicatedItems, duplicatedItemId: duplicates[0]?.duplicatedItemId, duplicatedDocumentId: duplicates[0]?.duplicatedDocumentId, duplicatedItemIds: duplicates.map( diff --git a/templates/content/actions/execute-builder-source-execution.ts b/templates/content/actions/execute-builder-source-execution.ts index 26125d15de..f90d57b7fd 100644 --- a/templates/content/actions/execute-builder-source-execution.ts +++ b/templates/content/actions/execute-builder-source-execution.ts @@ -854,7 +854,8 @@ export function realExecutionDeps( executeWrite: (args) => executeBuilderCmsWrite(args), readLiveEntry: (args) => readBuilderCmsEntryLiveState(args), reconcileWrite: reconcileBuilderCmsWrite, - getResponse: (databaseId) => getContentDatabaseResponse(databaseId), + getResponse: (databaseId) => + getContentDatabaseResponse(databaseId, { limit: 100, offset: 0 }), lookupSafeModelIntent: lookupBuilderCmsSafeModelIntent, }; } diff --git a/templates/content/actions/get-content-database.ts b/templates/content/actions/get-content-database.ts index 3f183d7a71..37d470dfa0 100644 --- a/templates/content/actions/get-content-database.ts +++ b/templates/content/actions/get-content-database.ts @@ -12,7 +12,6 @@ import { resolveContentSpaceAccess } from "./_content-space-access.js"; import { CONTENT_DATABASE_MAX_READ_LIMIT, getContentDatabaseResponse, - getDocumentContextPath, } from "./_database-utils.js"; export default defineAction({ @@ -28,6 +27,46 @@ export default defineAction({ .max(CONTENT_DATABASE_MAX_READ_LIMIT) .optional(), offset: z.coerce.number().int().min(0).optional(), + tableQuery: z + .object({ + search: z.string().max(500), + filters: z + .array( + z.object({ + key: z.string(), + label: z.string(), + operator: z.enum([ + "contains", + "equals", + "does_not_equal", + "greater_than", + "less_than", + "before", + "after", + "between", + "is_checked", + "is_unchecked", + "is_empty", + "is_not_empty", + ]), + value: z.string(), + filterGroupId: z.string().optional(), + parentFilterGroupId: z.string().optional(), + }), + ) + .max(50), + sorts: z + .array( + z.object({ + key: z.string(), + label: z.string(), + direction: z.enum(["asc", "desc"]), + }), + ) + .max(20), + filterMode: z.enum(["and", "or"]), + }) + .optional(), }), http: { method: "GET" }, readOnly: true, @@ -36,6 +75,7 @@ export default defineAction({ documentId, limit, offset, + tableQuery, }): Promise => { const db = getDb(); let resolvedDatabaseId = databaseId; @@ -88,17 +128,10 @@ export default defineAction({ }; } - const response = await getContentDatabaseResponse(resolvedDatabaseId, { + return getContentDatabaseResponse(resolvedDatabaseId, { limit, offset, + tableQuery, }); - const [document] = await db - .select() - .from(schema.documents) - .where(eq(schema.documents.id, database.documentId)); - return { - ...response, - contextPath: document ? await getDocumentContextPath(document) : [], - }; }, }); diff --git a/templates/content/actions/move-database-item.ts b/templates/content/actions/move-database-item.ts index 90052bea6e..751c8de55a 100644 --- a/templates/content/actions/move-database-item.ts +++ b/templates/content/actions/move-database-item.ts @@ -135,6 +135,9 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); - return getContentDatabaseResponse(row.item.databaseId); + return getContentDatabaseResponse(row.item.databaseId, { + limit: 100, + offset: 0, + }); }, }); diff --git a/templates/content/actions/prepare-builder-source-execution.ts b/templates/content/actions/prepare-builder-source-execution.ts index d1359e7430..6606dbea45 100644 --- a/templates/content/actions/prepare-builder-source-execution.ts +++ b/templates/content/actions/prepare-builder-source-execution.ts @@ -155,7 +155,7 @@ export default defineAction({ timing.record("gate_preparation_and_dry_run_validation", gateStartedAt); const response = await timing.measure("response_load", () => - getContentDatabaseResponse(database.id), + getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }), ); const result = { ...response, timings: timing.finish() }; timing.log("succeeded"); diff --git a/templates/content/actions/prepare-builder-source-review.ts b/templates/content/actions/prepare-builder-source-review.ts index 03909f0d30..687ccc9539 100644 --- a/templates/content/actions/prepare-builder-source-review.ts +++ b/templates/content/actions/prepare-builder-source-review.ts @@ -34,7 +34,6 @@ import { serializeSourceRowRecord, sourceChangeSetKey, } from "./_database-source-utils.js"; -import { getContentDatabaseResponse } from "./_database-utils.js"; export const BUILDER_SOURCE_REVIEW_PREPARE_LIMIT = 100; @@ -763,18 +762,16 @@ export default defineAction({ approvalStartedAt, ); - const { reviewedSnapshot, response } = await timing.measure( - "reconciliation_and_response_load", - async () => ({ - reviewedSnapshot: await getContentDatabaseSourceSnapshotForWrite( + const reviewedSnapshot = await timing.measure( + "reconciliation_snapshot_load", + () => + getContentDatabaseSourceSnapshotForWrite( database, args.sourceId, reviewableChanges.every((changeSet) => changeSet.documentId) ? reviewableChanges.map((changeSet) => changeSet.documentId!) : args.documentIds, ), - response: await getContentDatabaseResponse(database.id), - }), ); if (!reviewedSnapshot) throw new Error("Builder source disappeared."); // Build the review payload from the TARGET source snapshot, not @@ -797,7 +794,6 @@ export default defineAction({ review.preparedRowLimit = reviewableChanges.length; const result = { - ...response, review, preparedChangeSetMappings, timings: timing.finish(), diff --git a/templates/content/actions/preview-builder-source-review.ts b/templates/content/actions/preview-builder-source-review.ts index bc620deea7..96353446a3 100644 --- a/templates/content/actions/preview-builder-source-review.ts +++ b/templates/content/actions/preview-builder-source-review.ts @@ -6,6 +6,7 @@ import type { PreviewBuilderSourceReviewRequest, PreviewBuilderSourceReviewResponse, } from "../shared/api.js"; +import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { getContentDatabaseSourceSnapshotForReview, resolveDatabaseForSourceMutation, @@ -36,67 +37,83 @@ export default defineAction({ run: async ( args: PreviewBuilderSourceReviewRequest, ): Promise => { - const database = await resolveDatabaseForSourceMutation(args); - if (!database) throw new Error("Database not found."); - await assertAccess("document", database.documentId, "editor"); + const timing = createBuilderSourceTiming("preview_builder_source_review"); + try { + const database = await timing.measure("access_and_database", async () => { + const database = await resolveDatabaseForSourceMutation(args); + if (!database) throw new Error("Database not found."); + await assertAccess("document", database.documentId, "editor"); + return database; + }); - const selectedDocumentIds = new Set(args.documentIds ?? []); - if (args.scope === "selected" && selectedDocumentIds.size === 0) { - throw new Error("Select at least one Builder row before reviewing it."); - } - - const source = await getContentDatabaseSourceSnapshotForReview( - database, - args.sourceId, - args.scope === "selected" ? args.documentIds : undefined, - ); - if (!source || source.sourceType !== "builder-cms") { - throw new Error("Attach a Builder CMS source before reviewing updates."); - } + const selectedDocumentIds = new Set(args.documentIds ?? []); + if (args.scope === "selected" && selectedDocumentIds.size === 0) { + throw new Error("Select at least one Builder row before reviewing it."); + } - const allReviewableChanges = source.changeSets.filter((changeSet) => { - if ( - changeSet.direction !== "outbound" || - (changeSet.state !== "pending_push" && - changeSet.state !== "staged_revision" && - changeSet.state !== "approved") - ) { - return false; + const source = await timing.measure("snapshot_and_candidate_diff", () => + getContentDatabaseSourceSnapshotForReview( + database, + args.sourceId, + args.scope === "selected" ? args.documentIds : undefined, + ), + ); + if (!source || source.sourceType !== "builder-cms") { + throw new Error( + "Attach a Builder CMS source before reviewing updates.", + ); } - return ( - args.scope === "all" || - (!!changeSet.documentId && - selectedDocumentIds.has(changeSet.documentId)) + + const allReviewableChanges = source.changeSets.filter((changeSet) => { + if ( + changeSet.direction !== "outbound" || + (changeSet.state !== "pending_push" && + changeSet.state !== "staged_revision" && + changeSet.state !== "approved") + ) { + return false; + } + return ( + args.scope === "all" || + (!!changeSet.documentId && + selectedDocumentIds.has(changeSet.documentId)) + ); + }); + const changeSets = [...allReviewableChanges] + .sort( + (left, right) => + reviewPreparePriority(left, source) - + reviewPreparePriority(right, source), + ) + .slice(0, BUILDER_SOURCE_REVIEW_PREPARE_LIMIT); + const authoritativeSource = await timing.measure( + "authoritative_target_rows", + () => withAuthoritativeBuilderTargetRows({ source, changeSets }), ); - }); - const changeSets = [...allReviewableChanges] - .sort( - (left, right) => - reviewPreparePriority(left, source) - - reviewPreparePriority(right, source), - ) - .slice(0, BUILDER_SOURCE_REVIEW_PREPARE_LIMIT); - const authoritativeSource = await withAuthoritativeBuilderTargetRows({ - source, - changeSets, - }); - const review = - changeSets.length > 0 - ? buildBuilderSourceReviewPayload({ - source: authoritativeSource, - changeSets, - }) - : null; - if (review) { - review.totalRowCount = allReviewableChanges.length; - review.preparedRowLimit = changeSets.length; - } + const review = + changeSets.length > 0 + ? buildBuilderSourceReviewPayload({ + source: authoritativeSource, + changeSets, + }) + : null; + if (review) { + review.totalRowCount = allReviewableChanges.length; + review.preparedRowLimit = changeSets.length; + } - return { - sourceId: source.id, - sourceTable: source.sourceTable, - changeSetIds: changeSets.map((changeSet) => changeSet.id), - review, - }; + const response = { + sourceId: source.id, + sourceTable: source.sourceTable, + changeSetIds: changeSets.map((changeSet) => changeSet.id), + review, + timings: timing.finish(), + }; + timing.log("succeeded"); + return response; + } catch (error) { + timing.log("failed"); + throw error; + } }, }); diff --git a/templates/content/actions/review-content-database-source-change-set.ts b/templates/content/actions/review-content-database-source-change-set.ts index 6d78c01e2a..9ea5187750 100644 --- a/templates/content/actions/review-content-database-source-change-set.ts +++ b/templates/content/actions/review-content-database-source-change-set.ts @@ -116,6 +116,6 @@ export default defineAction({ .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/set-content-database-source-write-mode.ts b/templates/content/actions/set-content-database-source-write-mode.ts index 2473f1b0c5..25d71f5b37 100644 --- a/templates/content/actions/set-content-database-source-write-mode.ts +++ b/templates/content/actions/set-content-database-source-write-mode.ts @@ -121,6 +121,6 @@ export default defineAction({ "Builder source settings changed repeatedly while saving write mode.", }); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/stage-builder-revision.ts b/templates/content/actions/stage-builder-revision.ts index ab4a07f113..96902c4979 100644 --- a/templates/content/actions/stage-builder-revision.ts +++ b/templates/content/actions/stage-builder-revision.ts @@ -114,6 +114,6 @@ export default defineAction({ .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/update-content-database-personal-view.test.ts b/templates/content/actions/update-content-database-personal-view.test.ts index 8804cdaa60..8b46e77136 100644 --- a/templates/content/actions/update-content-database-personal-view.test.ts +++ b/templates/content/actions/update-content-database-personal-view.test.ts @@ -8,7 +8,9 @@ import { normalizePersonalDatabaseViewOverrides, PERSONAL_DATABASE_VIEW_OVERRIDES_VERSION, } from "./_content-database-personal-view"; -import action from "./update-content-database-personal-view"; +import action, { + personalSidebarOrderItemIds, +} from "./update-content-database-personal-view"; describe("update content database personal view", () => { it("accepts grouped filter overrides for the current user", () => { @@ -52,6 +54,32 @@ describe("update content database personal view", () => { ).toBeNull(); }); + it("only validates item ids that a sidebar order actually references", () => { + expect( + personalSidebarOrderItemIds({ + version: PERSONAL_DATABASE_VIEW_OVERRIDES_VERSION, + views: [ + { + id: "table", + sorts: [{ key: "date", label: "Date", direction: "asc" }], + filters: [], + filterMode: "and", + }, + { + id: "files", + sorts: [], + filters: [], + filterMode: "and", + sidebarOrder: { + mode: "custom", + itemIds: ["item-b", "item-a", "item-b"], + }, + }, + ], + }), + ).toEqual(["item-b", "item-a"]); + }); + it("preserves a personal sidebar order and normalizes legacy v2 views", () => { const parsed = action.schema.parse({ databaseId: "database", diff --git a/templates/content/actions/update-content-database-personal-view.ts b/templates/content/actions/update-content-database-personal-view.ts index ff2c6db418..3e84393f50 100644 --- a/templates/content/actions/update-content-database-personal-view.ts +++ b/templates/content/actions/update-content-database-personal-view.ts @@ -1,14 +1,26 @@ import { defineAction } from "@agent-native/core"; import { deleteUserSetting, putUserSetting } from "@agent-native/core/settings"; +import { and, eq, inArray } from "drizzle-orm"; import { z } from "zod"; +import { getDb, schema } from "../server/db/index.js"; +import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js"; import { assertContentDatabaseViewerAccess, normalizePersonalDatabaseViewOverrides, personalDatabaseViewSettingKey, personalViewOverridesSchema, } from "./_content-database-personal-view.js"; -import { getContentDatabaseResponse } from "./_database-utils.js"; + +export function personalSidebarOrderItemIds( + overrides: z.infer, +) { + return [ + ...new Set( + overrides.views.flatMap((view) => view.sidebarOrder?.itemIds ?? []), + ), + ]; +} export default defineAction({ description: @@ -23,10 +35,24 @@ export default defineAction({ const key = personalDatabaseViewSettingKey(databaseId); if (overrides) { - const visibleDatabase = await getContentDatabaseResponse(databaseId); + const requestedItemIds = personalSidebarOrderItemIds(overrides); + const validItemIds = new Set(); + const itemIdChunkSize = Math.max(1, bulkChunkSizeForColumnCount(1) - 1); + for (const itemIds of chunks(requestedItemIds, itemIdChunkSize)) { + const rows = await getDb() + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where( + and( + eq(schema.contentDatabaseItems.databaseId, databaseId), + inArray(schema.contentDatabaseItems.id, itemIds), + ), + ); + for (const row of rows) validItemIds.add(row.id); + } overrides = normalizePersonalDatabaseViewOverrides( overrides, - new Set(visibleDatabase.items.map((item) => item.id)), + validItemIds, ); await putUserSetting(ctx.userEmail, key, overrides); } else { diff --git a/templates/content/actions/update-content-database-view.ts b/templates/content/actions/update-content-database-view.ts index 8dbe954e88..edbf7ac923 100644 --- a/templates/content/actions/update-content-database-view.ts +++ b/templates/content/actions/update-content-database-view.ts @@ -136,6 +136,6 @@ export default defineAction({ await writeAppState("refresh-signal", { ts: Date.now() }); - return getContentDatabaseResponse(databaseId); + return getContentDatabaseResponse(databaseId, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/actions/validate-builder-source-execution.ts b/templates/content/actions/validate-builder-source-execution.ts index 3cae203b8d..26574db458 100644 --- a/templates/content/actions/validate-builder-source-execution.ts +++ b/templates/content/actions/validate-builder-source-execution.ts @@ -153,6 +153,6 @@ export default defineAction({ .set({ updatedAt: now }) .where(eq(schema.contentDatabaseSources.id, source.id)); - return getContentDatabaseResponse(database.id); + return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); }, }); diff --git a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx index d013c5966b..1278f37ec8 100644 --- a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx @@ -1,6 +1,7 @@ import type { BuilderCmsModelSummary, ContentDatabaseResponse, + ContentDatabaseTableQuery, } from "@shared/api"; // @vitest-environment happy-dom // @@ -111,12 +112,16 @@ vi.mock("@agent-native/core/client/settings", () => ({ vi.mock("@/hooks/use-content-database", () => ({ isContentDatabaseUnavailable: () => false, - useContentDatabase: (documentId: string, limit: number) => { - contentDatabaseQueryMock(documentId, limit); + useContentDatabase: ( + documentId: string, + limit: number, + tableQuery?: ContentDatabaseTableQuery, + ) => { + contentDatabaseQueryMock(documentId, limit, tableQuery); return { data: databaseResponse, isLoading: false, - isFetching: limit !== databasePagination.limit, + isFetching: limit !== databasePagination.limit || Boolean(tableQuery), }; }, useAddDatabaseItem: () => addItemMutation, @@ -353,7 +358,7 @@ describe("DatabaseView UI regressions", () => { ); }); - it("requests the whole bounded search window and hides the partial no-match state", async () => { + it("keeps search page-bounded and hides the partial no-match state", async () => { databasePagination.totalItems = 571; databasePagination.hasMore = true; await renderDatabaseView(); @@ -379,7 +384,11 @@ describe("DatabaseView UI regressions", () => { await Promise.resolve(); }); - expect(contentDatabaseQueryMock).toHaveBeenCalledWith("document-1", 571); + expect(contentDatabaseQueryMock).toHaveBeenCalledWith( + "document-1", + 100, + expect.objectContaining({ search: "Quiet Comet" }), + ); expect(container.textContent).toContain( messagesByLocale["en-US"].database.loadingDatabase, ); diff --git a/templates/content/app/components/editor/database/DatabaseView.test.ts b/templates/content/app/components/editor/database/DatabaseView.test.ts index 6962d78a4f..f79c17f8b6 100644 --- a/templates/content/app/components/editor/database/DatabaseView.test.ts +++ b/templates/content/app/components/editor/database/DatabaseView.test.ts @@ -48,6 +48,7 @@ import { databaseSearchExpandedItemLimit, databaseSearchExpansionIsPending, databaseAttachedBuilderSources, + databaseActiveBuilderReview, databaseNextBuilderContinuationSource, databaseNextBuilderContinuationWatchdogSource, databaseNextBuilderHydrationSource, @@ -55,6 +56,9 @@ import { databaseItemPagePath, databaseRecordBuilderContinuationAttempt, databaseSourceOperationIsPending, + databaseSourceChangeSetsAreComplete, + databaseTableShouldShowBlockingLoader, + databaseViewRequiresCompleteClientDataset, pendingMutationSourceId, previewDocumentSaveResult, releaseDatabaseSourceOperation, @@ -63,6 +67,81 @@ import { preparedBuilderReviewMatches, } from "./DatabaseView"; +describe("database source page projections", () => { + it("does not present page-scoped review counts as complete", () => { + expect( + databaseSourceChangeSetsAreComplete({ + projection: { rows: "page", changeSets: "page" }, + } as ContentDatabaseSource), + ).toBe(false); + expect( + databaseSourceChangeSetsAreComplete({ + projection: { rows: "complete", changeSets: "complete" }, + } as ContentDatabaseSource), + ).toBe(true); + expect(databaseSourceChangeSetsAreComplete(null)).toBe(true); + }); + + it("shows page review changes while the complete review loads", () => { + const page = { + sourceId: "source-1", + } as unknown as ContentDatabaseSourceReviewPayload; + const complete = { + sourceId: "source-1", + preparedRowLimit: 100, + } as unknown as ContentDatabaseSourceReviewPayload; + expect( + databaseActiveBuilderReview({ + prepared: null, + complete: null, + page, + open: true, + }), + ).toBe(page); + expect( + databaseActiveBuilderReview({ + prepared: null, + complete, + page, + open: true, + }), + ).toBe(complete); + }); +}); + +describe("database table loading", () => { + it("keeps useful rows visible while a constrained result refetches", () => { + expect(databaseTableShouldShowBlockingLoader(true, 100)).toBe(false); + expect(databaseTableShouldShowBlockingLoader(true, 0)).toBe(true); + expect(databaseTableShouldShowBlockingLoader(false, 0)).toBe(false); + }); + + it("keeps constrained tables bounded while preserving complete calendar data", () => { + expect( + databaseViewRequiresCompleteClientDataset({ + viewType: "table", + activeConstraintCount: 1, + grouped: false, + }), + ).toBe(false); + expect( + databaseViewRequiresCompleteClientDataset({ + viewType: "calendar", + activeConstraintCount: 0, + grouped: false, + }), + ).toBe(true); + expect( + databaseViewRequiresCompleteClientDataset({ + viewType: "table", + activeConstraintCount: 1, + grouped: false, + tableQueryMode: "client-required", + }), + ).toBe(true); + }); +}); + describe("database preview property saves", () => { it("carries the database and its page through full-page navigation", () => { expect( diff --git a/templates/content/app/components/editor/database/DatabaseView.tsx b/templates/content/app/components/editor/database/DatabaseView.tsx index fb344a60f1..bba6ba9873 100644 --- a/templates/content/app/components/editor/database/DatabaseView.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.tsx @@ -34,6 +34,7 @@ import { type ContentDatabaseRowDensity, type ContentDatabaseSort, type ContentDatabaseSortDirection, + type ContentDatabaseTableQuery, type ContentDatabaseViewType, type Document, type DocumentProperty, @@ -42,6 +43,7 @@ import { type DocumentPropertyValue, } from "@shared/api"; import { contentDatabaseFormQuestions } from "@shared/database-form"; +import { applyContentDatabaseTableQuery } from "@shared/database-query"; import { type DocumentPropertyOptionColor, countWords, @@ -441,6 +443,7 @@ export function databaseCreatedItemForImmediatePreview( now?: string; }, ): ContentDatabaseItem | null { + if (response.createdItem) return response.createdItem; const returnedItem = response.items.find( (item) => item.id === response.createdItemId, ); @@ -769,13 +772,43 @@ function DatabaseTable({ SELECTED_CONTENT_SPACE_STORAGE_KEY, null, ); + const [searchQuery, setSearchQuery] = useState(""); + const [viewConfig, setViewConfig] = useState( + defaultDatabaseViewConfig(), + ); + const activeView = useMemo( + () => activeDatabaseView(viewConfig), + [viewConfig], + ); + const tableQuery = useMemo(() => { + if ( + activeView.type !== "table" || + activeDatabaseConstraintCount( + searchQuery, + activeView.sorts, + activeView.filters, + ) === 0 + ) { + return undefined; + } + return { + search: searchQuery, + filters: activeView.filters, + sorts: activeView.sorts, + filterMode: activeView.filterMode ?? "and", + }; + }, [activeView, searchQuery]); const [manualDatabaseItemLimit, setManualDatabaseItemLimit] = useState( CONTENT_DATABASE_PAGE_SIZE, ); const [databaseRequestItemLimit, setDatabaseRequestItemLimit] = useState( CONTENT_DATABASE_PAGE_SIZE, ); - const database = useContentDatabase(document.id, databaseRequestItemLimit); + const database = useContentDatabase( + document.id, + databaseRequestItemLimit, + tableQuery, + ); const addItem = useAddDatabaseItem(document.id); const attachSource = useAttachContentDatabaseSource(document.id); const changeSourceRole = useChangeContentDatabaseSourceRole(document.id); @@ -841,7 +874,6 @@ function DatabaseTable({ string | null >(null); const [searchOpen, setSearchOpen] = useState(false); - const [searchQuery, setSearchQuery] = useState(""); const [settingsOpen, setSettingsOpen] = useState(false); const [inlineFilterControlsOpen, setInlineFilterControlsOpen] = useState(false); @@ -876,19 +908,12 @@ function DatabaseTable({ useState | null>(null); const [settingsPanel, setSettingsPanel] = useState("main"); - const [viewConfig, setViewConfig] = useState( - defaultDatabaseViewConfig(), - ); const [savedViewConfig, setSavedViewConfig] = useState(defaultDatabaseViewConfig()); const [personalQueryDirty, setPersonalQueryDirty] = useState(false); const [dateViewMonth, setDateViewMonth] = useState(() => startOfMonth(new Date()), ); - const activeView = useMemo( - () => activeDatabaseView(viewConfig), - [viewConfig], - ); const orderedProperties = useMemo( () => orderDatabasePropertiesForView(properties, activeView), [properties, activeView], @@ -996,10 +1021,12 @@ function DatabaseTable({ visibleFilters, ); const requiresCompleteClientDataset = - activeConstraintCount > 0 || - !!databaseGroupProperty || - activeView.type === "calendar" || - activeView.type === "timeline"; + databaseViewRequiresCompleteClientDataset({ + viewType: activeView.type, + activeConstraintCount, + grouped: !!databaseGroupProperty, + tableQueryMode: data?.tableQueryMode, + }); const clientQueryExpandedItemLimit = databaseClientQueryExpandedItemLimit( requiresCompleteClientDataset, manualDatabaseItemLimit, @@ -1011,7 +1038,19 @@ function DatabaseTable({ data?.pagination?.limit ?? items.length, ); const isDatabaseViewLoading = - isDatabaseInitialLoading || isClientQueryExpansionPending; + isDatabaseInitialLoading || + isClientQueryExpansionPending || + (database.isFetching && Boolean(tableQuery)); + useEffect(() => { + if (isDatabaseViewLoading) return; + window.document.documentElement.dataset.contentDatabaseRowsVisibleDocumentId = + document.id; + window.dispatchEvent( + new CustomEvent("content-database-rows-visible", { + detail: { documentId: document.id }, + }), + ); + }, [document.id, isDatabaseViewLoading]); useEffect(() => { if (clientQueryExpandedItemLimit === databaseRequestItemLimit) return; setDatabaseRequestItemLimit(clientQueryExpandedItemLimit); @@ -1072,6 +1111,8 @@ function DatabaseTable({ () => builderReviewableChangeSets(builderReviewSource), [builderReviewSource], ); + const builderReviewCountIsComplete = + databaseSourceChangeSetsAreComplete(builderReviewSource); const builderReviewPreview = useMemo( () => builderReviewSource && builderReviewChangeSets.length > 0 @@ -1082,11 +1123,12 @@ function DatabaseTable({ : null, [builderReviewChangeSets, builderReviewSource], ); - const activeBuilderReview = - builderReviewResult ?? - (builderReviewOpen - ? (completeBuilderReviewPreview?.review ?? null) - : builderReviewPreview); + const activeBuilderReview = databaseActiveBuilderReview({ + prepared: builderReviewResult, + complete: completeBuilderReviewPreview?.review ?? null, + page: builderReviewPreview, + open: builderReviewOpen, + }); const sourcePendingOperations: DatabaseSourcePendingOperations = { attach: attachSource.isPending, changeRole: changeSourceRole.isPending, @@ -2460,12 +2502,16 @@ function DatabaseTable({ size="sm" aria-label={ builderReviewChangeSets.length > 0 - ? `Database settings, ${builderReviewChangeSets.length} Builder update pending` + ? builderReviewCountIsComplete + ? `Database settings, ${builderReviewChangeSets.length} Builder update pending` + : "Database settings, Builder updates pending" : "Database settings" } title={ builderReviewChangeSets.length > 0 - ? `${builderReviewChangeSets.length} Builder update pending` + ? builderReviewCountIsComplete + ? `${builderReviewChangeSets.length} Builder update pending` + : "Builder updates pending" : "Database settings" } className={cn( @@ -2485,9 +2531,13 @@ function DatabaseTable({ > {builderReviewChangeSets.length > 0 ? ( - - {formatCompactCountBadge(builderReviewChangeSets.length)} - + builderReviewCountIsComplete ? ( + + {formatCompactCountBadge(builderReviewChangeSets.length)} + + ) : ( + + ) ) : null} {canEdit && isWorkspaceCatalog ? ( @@ -3896,13 +3946,18 @@ export function databaseBulkScalarInputState( type DuplicatedDatabaseItemResponse = { items: ContentDatabaseResponse["items"]; duplicatedItemId?: ContentDatabaseResponse["duplicatedItemId"]; + duplicatedItems?: ContentDatabaseResponse["duplicatedItems"]; }; export function databaseDuplicatedItemFromResponse( response: DuplicatedDatabaseItemResponse, ) { return ( - response.items.find((item) => item.id === response.duplicatedItemId) ?? null + response.duplicatedItems?.find( + (item) => item.id === response.duplicatedItemId, + ) ?? + response.items.find((item) => item.id === response.duplicatedItemId) ?? + null ); } @@ -5697,7 +5752,7 @@ function DatabaseTableView({ ) : null} - {isLoading ? ( + {databaseTableShouldShowBlockingLoader(isLoading, items.length) ? (
{dbText("loadingDatabase")} @@ -5863,6 +5918,29 @@ function DatabaseTableView({ ); } +export function databaseTableShouldShowBlockingLoader( + isLoading: boolean, + itemCount: number, +) { + return isLoading && itemCount === 0; +} + +export function databaseViewRequiresCompleteClientDataset(args: { + viewType: ContentDatabaseViewType; + activeConstraintCount: number; + grouped: boolean; + tableQueryMode?: ContentDatabaseResponse["tableQueryMode"]; +}) { + return ( + (args.viewType === "table" && args.tableQueryMode === "client-required") || + (args.viewType !== "table" && + (args.activeConstraintCount > 0 || + args.grouped || + args.viewType === "calendar" || + args.viewType === "timeline")) + ); +} + function DatabaseActiveConstraintsBar({ documentId, items, @@ -7488,7 +7566,9 @@ function DatabaseSettingsMainPanel({ onPanelChange: (panel: DatabaseSettingsPanel) => void; }) { const groupLabel = activeView.groupByPropertyId ? "On" : ""; - const sourceBadgeCount = builderReviewableChangeSets(source).length; + const sourceBadgeCount = databaseSourceChangeSetsAreComplete(source) + ? builderReviewableChangeSets(source).length + : undefined; return (
@@ -7550,6 +7630,23 @@ export function builderReviewableChangeSets( ); } +export function databaseActiveBuilderReview(args: { + prepared: ContentDatabaseSourceReviewPayload | null; + complete: ContentDatabaseSourceReviewPayload | null; + page: ContentDatabaseSourceReviewPayload | null; + open: boolean; +}) { + return ( + args.prepared ?? (args.open ? (args.complete ?? args.page) : args.page) + ); +} + +export function databaseSourceChangeSetsAreComplete( + source: ContentDatabaseSource | null, +) { + return source?.projection?.changeSets !== "page"; +} + function sourceReviewRiskRank( risk: ContentDatabaseSourceReviewPayload["riskLevel"], ) { @@ -7873,11 +7970,15 @@ function DatabaseSettingsSourcePanel({ sources={sources} builderConfigured={builderConfigured} builderSpaceLabel={builderSpaceLabel} - reviewableCount={builderSources.reduce( - (total, candidate) => - total + builderReviewableChangeSets(candidate).length, - 0, - )} + reviewableCount={ + builderSources.every(databaseSourceChangeSetsAreComplete) + ? builderSources.reduce( + (total, candidate) => + total + builderReviewableChangeSets(candidate).length, + 0, + ) + : undefined + } onOpenBuilder={(builderSource) => onNavPush( builderSource @@ -8195,6 +8296,8 @@ function DatabaseSettingsSourcePanel({ } const selectedSource = selectedBuilderSource; + const selectedSourceChangeSetsComplete = + databaseSourceChangeSetsAreComplete(selectedSource); const reviewableBuilderChangeSets = builderReviewableChangeSets(selectedSource); const outboundChangeSets = reviewableBuilderChangeSets; @@ -8408,11 +8511,13 @@ function DatabaseSettingsSourcePanel({
- {conflictChangeSets.length > 0 + {selectedSourceChangeSetsComplete && + conflictChangeSets.length > 0 ? `${conflictChangeSets.length} change${ conflictChangeSets.length === 1 ? "" : "s" } need review` - : reviewableBuilderChangeSets.length > 0 + : selectedSourceChangeSetsComplete && + reviewableBuilderChangeSets.length > 0 ? `${reviewableBuilderChangeSets.length} change${ reviewableBuilderChangeSets.length === 1 ? "" : "s" } ready to push` @@ -8562,7 +8667,7 @@ function SourcesListView({ sources: ContentDatabaseSource[]; builderConfigured: boolean; builderSpaceLabel: string | null; - reviewableCount: number; + reviewableCount?: number; onOpenBuilder: (source?: ContentDatabaseSource) => void; onOpenNotion: () => void; onOpenLocalFolder: () => void; @@ -16362,37 +16467,11 @@ export function applyDatabaseView( sorts: DatabaseSort[], filterMode: DatabaseFilterMode = "and", ) { - const query = searchQuery.trim().toLowerCase(); - const searched = query - ? items.filter((item) => - databaseItemSearchText(item, properties).toLowerCase().includes(query), - ) - : items; - const activeFilters = filters.filter(isActiveFilter); - const filtered = activeFilters.length - ? searched.filter((item) => - databaseItemMatchesFilterTree( - item, - properties, - activeFilters, - filterMode, - ), - ) - : searched; - - if (sorts.length === 0) return filtered; - - return [...filtered].sort((a, b) => { - for (const sort of sorts) { - const comparison = compareDatabaseSortValues( - databaseItemSortValue(a, properties, sort.key), - databaseItemSortValue(b, properties, sort.key), - ); - if (comparison !== 0) { - return sort.direction === "asc" ? comparison : -comparison; - } - } - return 0; + return applyContentDatabaseTableQuery(items, properties, { + search: searchQuery, + filters, + sorts, + filterMode, }); } diff --git a/templates/content/app/components/sidebar/DocumentSidebar.layout.test.ts b/templates/content/app/components/sidebar/DocumentSidebar.layout.test.ts index 1538010ebd..4075ed6967 100644 --- a/templates/content/app/components/sidebar/DocumentSidebar.layout.test.ts +++ b/templates/content/app/components/sidebar/DocumentSidebar.layout.test.ts @@ -224,9 +224,9 @@ describe("document sidebar layout", () => { expect(sidebar).toContain(""); expect(sidebar).toContain(" { + const wasExpanded = previouslyExpanded.current; + previouslyExpanded.current = expanded; + if (!expanded) { + setReady(false); + return; + } + if (!wasExpanded) { + setReady(true); + return; + } + + if (deferUntilDocumentId) { + if ( + window.document.documentElement.dataset + .contentDatabaseRowsVisibleDocumentId === deferUntilDocumentId + ) { + setReady(true); + return; + } + setReady(false); + const handleRowsVisible = (event: Event) => { + if ( + (event as CustomEvent<{ documentId?: string }>).detail?.documentId === + deferUntilDocumentId + ) { + setReady(true); + } + }; + window.addEventListener(DATABASE_ROWS_VISIBLE_EVENT, handleRowsVisible); + const fallback = window.setTimeout( + () => setReady(true), + DATABASE_PAGE_READY_FALLBACK_MS, + ); + return () => { + window.removeEventListener( + DATABASE_ROWS_VISIBLE_EVENT, + handleRowsVisible, + ); + window.clearTimeout(fallback); + }; + } + + // An already-expanded workspace can contain thousands of files. Give the + // selected page's critical read one turn before starting that inventory; + // direct expansion remains immediate. + setReady(false); + const timeout = window.setTimeout( + () => setReady(true), + INITIAL_EXPANDED_WORKSPACE_READ_DELAY_MS, + ); + return () => window.clearTimeout(timeout); + }, [databaseId, deferUntilDocumentId, expanded]); + + return expanded && ready ? databaseId : null; +} + function WorkspaceSidebarItem({ space, selected, expanded, + deferInitialReadUntilDocumentId, reorder, createDocumentPending, activeDocumentId, @@ -329,6 +399,7 @@ function WorkspaceSidebarItem({ space: ContentSpaceSummary; selected: boolean; expanded: boolean; + deferInitialReadUntilDocumentId: string | null; reorder?: ContentFilesSidebarRenderReorder; createDocumentPending: boolean; activeDocumentId: string | null; @@ -349,7 +420,11 @@ function WorkspaceSidebarItem({ onToggleFavorite: (item: ContentDatabaseItem) => void; }) { const t = useT(); - const activeFilesDatabaseId = expanded ? space.filesDatabaseId : null; + const activeFilesDatabaseId = useDeferredFilesDatabaseId( + space.filesDatabaseId, + expanded, + deferInitialReadUntilDocumentId, + ); const filesDatabase = useContentDatabaseById(activeFilesDatabaseId); const filesDatabaseData = isContentDatabaseUnavailable(filesDatabase.data) ? undefined @@ -1776,6 +1851,12 @@ export function DocumentSidebar({ space={space} selected={selectedSpace?.id === space.id} expanded={expandedWorkspaceIds.includes(space.id)} + deferInitialReadUntilDocumentId={ + activeDocumentId && + databaseDocuments.some((document) => document.id === activeDocumentId) + ? activeDocumentId + : null + } reorder={reorder} createDocumentPending={createDocument.isPending} activeDocumentId={activeDocumentId} diff --git a/templates/content/app/hooks/use-content-database.test.ts b/templates/content/app/hooks/use-content-database.test.ts index c8362947f2..7d016049a3 100644 --- a/templates/content/app/hooks/use-content-database.test.ts +++ b/templates/content/app/hooks/use-content-database.test.ts @@ -13,6 +13,7 @@ import { applyOptimisticSourceFieldPropertyToDatabaseResponse, applySourceFieldPropertyToDatabaseResponse, clearDeletedContentDatabaseFromCache, + contentDatabaseResponseCanSeedQuery, contentDatabaseQueryKey, invalidateBuilderBodyHydrationQueries, invalidateContentDatabaseSourceRefreshQueries, @@ -644,7 +645,17 @@ describe("writeContentDatabaseResponseToCache", () => { items: [], source: null, }; - const attached = databaseResponse(); + const attached = { + ...databaseResponse(), + items: databaseResponse().items.slice(0, 100), + pagination: { + offset: 0, + limit: 100, + totalItems: 500, + returnedItems: 100, + hasMore: true, + }, + }; const queryClient = new QueryClient(); const visibleQueryKey = [ "action", @@ -661,7 +672,40 @@ describe("writeContentDatabaseResponseToCache", () => { const visibleCache = queryClient.getQueryData(visibleQueryKey); expect(visibleCache?.source?.sourceTable).toBe("blog-article"); - expect(visibleCache?.items).toHaveLength(500); + expect(visibleCache?.items).toHaveLength(100); + }); + + it("does not overwrite a constrained cache with an unconstrained response", () => { + const response = { + ...databaseResponse(), + pagination: { + offset: 0, + limit: 100, + totalItems: 500, + returnedItems: 100, + hasMore: true, + }, + }; + expect( + contentDatabaseResponseCanSeedQuery( + [ + "action", + "get-content-database", + { + documentId: "database-page", + limit: 100, + tableQuery: { + search: "alpha", + filters: [], + sorts: [], + filterMode: "and", + }, + }, + ], + "database-page", + response, + ), + ).toBe(false); }); }); diff --git a/templates/content/app/hooks/use-content-database.ts b/templates/content/app/hooks/use-content-database.ts index 1a6a512785..fa1643e8d8 100644 --- a/templates/content/app/hooks/use-content-database.ts +++ b/templates/content/app/hooks/use-content-database.ts @@ -11,6 +11,7 @@ import type { CancelPreparedBuilderSourceUpdateResponse, ChangeContentDatabaseSourceRoleRequest, ContentDatabaseResponse, + ContentDatabaseTableQuery, ContentDatabaseItem, ContentDatabasePersonalViewResponse, ContentDatabaseSourceFieldMapping, @@ -174,21 +175,59 @@ export function contentDatabaseQueryFilter(documentId: string) { }; } +export function contentDatabaseConstrainedQueryFilter(documentId: string) { + return { + queryKey: ["action", "get-content-database"], + predicate: (query: Query) => { + if (!isContentDatabaseQueryForDocument(query.queryKey, documentId)) { + return false; + } + const params = query.queryKey[2] as { tableQuery?: unknown }; + return params.tableQuery !== undefined; + }, + }; +} + export function writeContentDatabaseResponseToCache( queryClient: Pick, documentId: string, data: ContentDatabaseResponse, ) { - queryClient.setQueryData( - contentDatabaseQueryKey(documentId), - data, - ); + if (!data.pagination) { + queryClient.setQueryData( + contentDatabaseQueryKey(documentId), + data, + ); + } queryClient.setQueriesData( - contentDatabaseQueryFilter(documentId), + { + queryKey: ["action", "get-content-database"], + predicate: (query) => + contentDatabaseResponseCanSeedQuery(query.queryKey, documentId, data), + }, data, ); } +export function contentDatabaseResponseCanSeedQuery( + queryKey: readonly unknown[], + documentId: string, + data: ContentDatabaseResponse, +) { + if (!isContentDatabaseQueryForDocument(queryKey, documentId)) return false; + const params = queryKey[2] as { + limit?: unknown; + offset?: unknown; + tableQuery?: unknown; + }; + if (params.tableQuery !== undefined) return false; + if (!data.pagination) return params.limit === undefined; + return ( + params.limit === data.pagination.limit && + (params.offset ?? 0) === data.pagination.offset + ); +} + export function applyOptimisticBuilderWriteMode( current: ContentDatabaseResponse | undefined, request: SetContentDatabaseSourceWriteModeRequest, @@ -583,11 +622,15 @@ function removeOptimisticSourceFieldProperty( }; } -export function useContentDatabase(documentId: string | null, limit?: number) { +export function useContentDatabase( + documentId: string | null, + limit?: number, + tableQuery?: ContentDatabaseTableQuery, +) { const queryClient = useQueryClient(); return useActionQuery( "get-content-database", - documentId ? { documentId, limit } : undefined, + documentId ? { documentId, limit, tableQuery } : undefined, { enabled: !!documentId, retry: false, @@ -749,12 +792,28 @@ export function useAddDatabaseItem(documentId: string) { return useActionMutation( "add-database-item", { + skipActionQueryInvalidation: true, onSuccess: (data) => { - // The action returns the committed row and full database snapshot. - // Seed every active pagination key before invalidating so navigating - // away from the creation side-peek cannot briefly lose an appended row - // behind an older 100/200-row response. - writeContentDatabaseResponseToCache(queryClient, documentId, data); + if (data.createdItem) { + queryClient.setQueriesData( + { + queryKey: ["action", "get-content-database"], + predicate: (query) => { + if ( + !isContentDatabaseQueryForDocument(query.queryKey, documentId) + ) { + return false; + } + const params = query.queryKey[2] as { + tableQuery?: unknown; + }; + return params.tableQuery === undefined; + }, + }, + (current) => + applyOptimisticItemToContentDatabase(current, data.createdItem!), + ); + } queryClient.invalidateQueries({ queryKey: contentDatabaseQueryKey(documentId), }); @@ -912,13 +971,18 @@ export function useUpdateContentDatabaseView(documentId: string) { ContentDatabaseResponse, UpdateContentDatabaseViewRequest >("update-content-database-view", { - onSuccess: () => { - queryClient.invalidateQueries({ - queryKey: contentDatabaseQueryKey(documentId), - }); - queryClient.invalidateQueries({ - queryKey: ["action", "get-content-database-source", { documentId }], - }); + skipActionQueryInvalidation: true, + onSuccess: (data) => { + queryClient.setQueriesData( + contentDatabaseQueryFilter(documentId), + (current) => + current + ? { + ...current, + database: data.database, + } + : current, + ); }, }); } @@ -997,6 +1061,7 @@ export function useAttachContentDatabaseSource(documentId: string) { ContentDatabaseResponse, AttachContentDatabaseSourceRequest >("attach-content-database-source", { + skipActionQueryInvalidation: true, onSuccess: (data) => { writeContentDatabaseResponseToCache(queryClient, documentId, data); queryClient.invalidateQueries({ @@ -1032,6 +1097,7 @@ export function useAddContentDatabaseSourceFieldProperty(documentId: string) { ContentDatabaseSourceFieldPropertyResponse, AddContentDatabaseSourceFieldPropertyRequest >("add-content-database-source-field-property", { + skipActionQueryInvalidation: true, onMutate: async (variables) => { await queryClient.cancelQueries({ queryKey: contentDatabaseQueryKey(documentId), diff --git a/templates/content/app/hooks/use-document-properties.ts b/templates/content/app/hooks/use-document-properties.ts index bf3007b571..7d31bf19cd 100644 --- a/templates/content/app/hooks/use-document-properties.ts +++ b/templates/content/app/hooks/use-document-properties.ts @@ -18,6 +18,7 @@ import { applyDocumentPropertiesToDatabaseResponse, applyDocumentPropertyValueToDatabaseResponse, contentDatabaseQueryFilter, + contentDatabaseConstrainedQueryFilter, contentDatabaseQueryKey, removeDocumentPropertyFromDatabaseResponse, } from "./use-content-database"; @@ -93,9 +94,9 @@ export function useConfigureDocumentProperty( queryClient.invalidateQueries({ queryKey: ["action", "get-document", { id: documentId }], }); - queryClient.invalidateQueries({ - ...contentDatabaseQueryFilter(databaseDocumentId), - }); + queryClient.invalidateQueries( + contentDatabaseConstrainedQueryFilter(databaseDocumentId), + ); }, }); return withDatabaseScope(mutation, databaseId); @@ -160,9 +161,9 @@ export function useSetDocumentProperty( queryClient.invalidateQueries({ queryKey: ["action", "get-document", { id: variables.documentId }], }); - queryClient.invalidateQueries({ - ...contentDatabaseQueryFilter(databaseDocumentId), - }); + queryClient.invalidateQueries( + contentDatabaseConstrainedQueryFilter(databaseDocumentId), + ); queryClient.invalidateQueries({ queryKey: [ "action", diff --git a/templates/content/changelog/2026-07-29-large-databases-keep-useful-rows-visible.md b/templates/content/changelog/2026-07-29-large-databases-keep-useful-rows-visible.md new file mode 100644 index 0000000000..f7eccaa901 --- /dev/null +++ b/templates/content/changelog/2026-07-29-large-databases-keep-useful-rows-visible.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-29 +--- + +Large databases keep useful rows visible while table sorting and Builder review details load. diff --git a/templates/content/shared/api.ts b/templates/content/shared/api.ts index 0b47e8893a..5eb52f4a95 100644 --- a/templates/content/shared/api.ts +++ b/templates/content/shared/api.ts @@ -276,6 +276,13 @@ export interface ContentDatabaseFilter { parentFilterGroupId?: string; } +export interface ContentDatabaseTableQuery { + search: string; + filters: ContentDatabaseFilter[]; + sorts: ContentDatabaseSort[]; + filterMode: ContentDatabaseFilterMode; +} + export type ContentDatabaseColumnCalculation = | "count_all" | "count_values" @@ -707,6 +714,10 @@ export interface ContentDatabaseSource { fields: ContentDatabaseSourceFieldMapping[]; rows: ContentDatabaseSourceRow[]; changeSets: ContentDatabaseSourceChangeSet[]; + projection?: { + rows: "complete" | "page"; + changeSets: "complete" | "page"; + }; bodyHydration?: ContentDatabaseBodyHydrationSummary; } @@ -774,15 +785,18 @@ export interface ContentDatabaseResponse { hasMore: boolean; }; createdItemId?: string; + createdItem?: ContentDatabaseItem; createdDocumentId?: string; createdDocumentUpdatedAt?: string; duplicatedItemId?: string; duplicatedDocumentId?: string; duplicatedItemIds?: string[]; + duplicatedItems?: ContentDatabaseItem[]; duplicatedDocumentIds?: string[]; deletedItemIds?: string[]; deletedDocumentIds?: string[]; timings?: BuilderActionTiming[]; + tableQueryMode?: "server" | "client-required"; } export interface BuilderActionTiming { @@ -1256,13 +1270,10 @@ export interface PreviewBuilderSourceReviewResponse { sourceTable: string; changeSetIds: string[]; review: ContentDatabaseSourceReviewPayload | null; + timings?: BuilderActionTiming[]; } export interface PrepareBuilderSourceReviewResponse { - database: ContentDatabase; - properties: DocumentProperty[]; - items: ContentDatabaseItem[]; - source: ContentDatabaseSource | null; review: ContentDatabaseSourceReviewPayload; /** * Maps the operator-selected diff identities to the immutable change-set diff --git a/templates/content/shared/database-query.test.ts b/templates/content/shared/database-query.test.ts new file mode 100644 index 0000000000..1a1c3e6a5d --- /dev/null +++ b/templates/content/shared/database-query.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; + +import type { ContentDatabaseItem, DocumentProperty } from "./api"; +import { + applyContentDatabaseTableQuery, + contentDatabaseTableQueryUsesProperties, +} from "./database-query"; + +function item(id: string, title: string, date: string): ContentDatabaseItem { + return { + id: `item-${id}`, + databaseId: "database-1", + position: 0, + document: { + id, + parentId: "database-page", + title, + content: "", + icon: null, + position: 0, + isFavorite: false, + hideFromSearch: false, + visibility: "private", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + properties: [ + { + ...dateProperty, + value: { start: date, includeTime: false }, + }, + ], + }; +} + +const dateProperty = { + definition: { + id: "date", + databaseId: "database-1", + name: "Date", + type: "date", + visibility: "always_show", + options: {}, + position: 0, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + value: null, + editable: true, +} satisfies DocumentProperty; + +describe("content database table query", () => { + it("applies search, filters, and Date sorting before a caller pages rows", () => { + const result = applyContentDatabaseTableQuery( + [ + item("one", "Alpha launch", "2026-01-01"), + item("two", "Alpha follow-up", "2026-03-01"), + item("three", "Beta launch", "2026-02-01"), + ], + [dateProperty], + { + search: "alpha", + filters: [ + { + key: "date", + label: "Date", + operator: "after", + value: "2026-01-15", + }, + ], + sorts: [{ key: "date", label: "Date", direction: "desc" }], + filterMode: "and", + }, + ); + + expect(result.map((row) => row.document.id)).toEqual(["two"]); + }); + + it("requires complete client data only when a query can read federated properties", () => { + const base = { + search: "", + filters: [], + sorts: [{ key: "date", label: "Date", direction: "desc" as const }], + filterMode: "and" as const, + }; + expect( + contentDatabaseTableQueryUsesProperties(base, new Set(["date"])), + ).toBe(true); + expect( + contentDatabaseTableQueryUsesProperties(base, new Set(["author"])), + ).toBe(false); + expect( + contentDatabaseTableQueryUsesProperties( + { ...base, search: "Alice", sorts: [] }, + new Set(["author"]), + ), + ).toBe(true); + }); +}); diff --git a/templates/content/shared/database-query.ts b/templates/content/shared/database-query.ts new file mode 100644 index 0000000000..1e03f447d2 --- /dev/null +++ b/templates/content/shared/database-query.ts @@ -0,0 +1,359 @@ +import type { + ContentDatabaseFilter, + ContentDatabaseFilterMode, + ContentDatabaseItem, + ContentDatabaseSort, + ContentDatabaseTableQuery, + DocumentProperty, +} from "./api.js"; +import { documentPropertyDatePart, formulaValueText } from "./properties.js"; + +export function applyContentDatabaseTableQuery( + items: ContentDatabaseItem[], + properties: DocumentProperty[], + query: ContentDatabaseTableQuery, +) { + const search = query.search.trim().toLowerCase(); + const searched = search + ? items.filter((item) => + itemSearchText(item, properties).toLowerCase().includes(search), + ) + : items; + const activeFilters = query.filters.filter(isActiveFilter); + const filtered = activeFilters.length + ? searched.filter((item) => + itemMatchesFilterTree( + item, + properties, + activeFilters, + query.filterMode, + ), + ) + : searched; + + if (query.sorts.length === 0) return filtered; + return [...filtered].sort((left, right) => { + for (const sort of query.sorts) { + const comparison = compareSortValues( + itemSortValue(left, properties, sort), + itemSortValue(right, properties, sort), + ); + if (comparison !== 0) { + return sort.direction === "asc" ? comparison : -comparison; + } + } + return 0; + }); +} + +export function contentDatabaseTableQueryUsesProperties( + query: ContentDatabaseTableQuery, + propertyIds: ReadonlySet, +) { + if (propertyIds.size === 0) return false; + if (query.search.trim()) return true; + return [...query.filters, ...query.sorts].some( + (constraint) => + constraint.key !== "name" && propertyIds.has(constraint.key), + ); +} + +function itemMatchesFilterTree( + item: ContentDatabaseItem, + properties: DocumentProperty[], + filters: ContentDatabaseFilter[], + filterMode: ContentDatabaseFilterMode, +) { + const rootFilters = filters.filter((filter) => !filter.parentFilterGroupId); + const nestedGroups = new Map(); + for (const filter of filters) { + if (!filter.parentFilterGroupId || !filter.filterGroupId) continue; + nestedGroups.set(filter.filterGroupId, [ + ...(nestedGroups.get(filter.filterGroupId) ?? []), + filter, + ]); + } + const matches = [ + ...rootFilters.map((filter) => itemMatchesFilter(item, properties, filter)), + ...[...nestedGroups.values()].map((group) => + combineMatches( + group.map((filter) => itemMatchesFilter(item, properties, filter)), + filterMode, + ), + ), + ]; + return combineMatches(matches, filterMode); +} + +function combineMatches( + matches: boolean[], + filterMode: ContentDatabaseFilterMode, +) { + if (matches.length === 0) return true; + return filterMode === "or" ? matches.some(Boolean) : matches.every(Boolean); +} + +function isActiveFilter(filter: ContentDatabaseFilter) { + if ( + !["is_empty", "is_not_empty", "is_checked", "is_unchecked"].includes( + filter.operator, + ) + ) { + return selectedFilterValues(filter.value).length > 0; + } + return true; +} + +function itemMatchesFilter( + item: ContentDatabaseItem, + properties: DocumentProperty[], + filter: ContentDatabaseFilter, +) { + const value = itemFilterValue(item, properties, filter.key); + const property = itemFilterProperty(item, properties, filter.key); + if (filter.operator === "is_empty") return !value.trim(); + if (filter.operator === "is_not_empty") return !!value.trim(); + if (filter.operator === "is_checked") return property?.value === true; + if (filter.operator === "is_unchecked") return property?.value !== true; + + if (filter.operator === "greater_than" || filter.operator === "less_than") { + const current = propertyNumberValue(property); + const target = Number(filter.value.trim()); + if (!Number.isFinite(current) || !Number.isFinite(target)) return false; + return filter.operator === "greater_than" + ? current > target + : current < target; + } + + if ( + filter.operator === "before" || + filter.operator === "after" || + filter.operator === "between" + ) { + const current = propertyDateValue(property); + if (!Number.isFinite(current)) return false; + if (filter.operator === "between") { + const range = filterDateRange(filter.value); + return !!range && current >= range[0] && current <= range[1]; + } + const target = new Date(filter.value.trim()).getTime(); + if (!Number.isFinite(target)) return false; + return filter.operator === "before" ? current < target : current > target; + } + + const candidateValues = itemFilterCandidateValues( + item, + properties, + filter.key, + ).map((candidate) => candidate.trim().toLowerCase()); + const selectedValues = selectedFilterValues(filter.value).map((candidate) => + candidate.trim().toLowerCase(), + ); + const normalizedValue = value.trim().toLowerCase(); + const normalizedFilter = selectedValues[0] ?? ""; + const usesDiscreteValues = + property?.definition.type === "select" || + property?.definition.type === "status" || + property?.definition.type === "multi_select" || + property?.definition.type === "person"; + if ( + usesDiscreteValues && + (filter.operator === "equals" || filter.operator === "contains") + ) { + return selectedValues.some((filterValue) => + candidateValues.includes(filterValue), + ); + } + if (usesDiscreteValues && filter.operator === "does_not_equal") { + return selectedValues.every( + (filterValue) => !candidateValues.includes(filterValue), + ); + } + if (filter.operator === "equals") { + return candidateValues.includes(normalizedFilter); + } + if (filter.operator === "does_not_equal") { + return !candidateValues.includes(normalizedFilter); + } + return normalizedValue.includes(normalizedFilter); +} + +function selectedFilterValues(value: string) { + const trimmed = value.trim(); + if (!trimmed) return []; + try { + const parsed = JSON.parse(trimmed) as unknown; + if (!Array.isArray(parsed)) return [trimmed]; + return [ + ...new Set( + parsed + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter(Boolean), + ), + ]; + } catch { + return [trimmed]; + } +} + +function filterDateRange(value: string): [number, number] | null { + const values = selectedFilterValues(value); + const start = new Date(values[0] ?? "").getTime(); + const end = new Date(values[1] ?? "").getTime(); + if (!Number.isFinite(start) || !Number.isFinite(end)) return null; + return start <= end ? [start, end] : [end, start]; +} + +function itemSearchText( + item: ContentDatabaseItem, + properties: DocumentProperty[], +) { + return [ + item.document.title || "Untitled", + ...properties.map((property) => + propertyValueText( + item.properties.find( + (candidate) => candidate.definition.id === property.definition.id, + ) ?? property, + ), + ), + ].join(" "); +} + +function itemSortValue( + item: ContentDatabaseItem, + properties: DocumentProperty[], + sort: ContentDatabaseSort, +) { + if (sort.key === "name") return item.document.title || ""; + return propertyValueText( + item.properties.find((candidate) => candidate.definition.id === sort.key) ?? + properties.find((candidate) => candidate.definition.id === sort.key) ?? + null, + ); +} + +function itemFilterValue( + item: ContentDatabaseItem, + properties: DocumentProperty[], + key: string, +) { + if (key === "name") return item.document.title || ""; + return propertyValueText(itemFilterProperty(item, properties, key)); +} + +function itemFilterProperty( + item: ContentDatabaseItem, + properties: DocumentProperty[], + key: string, +) { + if (key === "name") return null; + return ( + item.properties.find((candidate) => candidate.definition.id === key) ?? + properties.find((candidate) => candidate.definition.id === key) ?? + null + ); +} + +function itemFilterCandidateValues( + item: ContentDatabaseItem, + properties: DocumentProperty[], + key: string, +) { + if (key === "name") return [item.document.title || ""]; + const property = itemFilterProperty(item, properties, key); + if (!property || property.value === null || property.value === undefined) { + return [""]; + } + if (Array.isArray(property.value)) { + return property.value.flatMap((id) => { + const optionName = + property.definition.options.options?.find((option) => option.id === id) + ?.name ?? id; + return [id, optionName]; + }); + } + if ( + property.definition.type === "select" || + property.definition.type === "status" + ) { + const id = String(property.value); + const optionName = + property.definition.options.options?.find((option) => option.id === id) + ?.name ?? id; + return [id, optionName]; + } + return [propertyValueText(property)]; +} + +function propertyValueText(property: DocumentProperty | null | undefined) { + if (!property || property.value === null || property.value === undefined) { + return ""; + } + if (Array.isArray(property.value)) { + return property.value + .map( + (id) => + property.definition.options.options?.find( + (option) => option.id === id, + )?.name ?? id, + ) + .join(" "); + } + if ( + property.definition.type === "select" || + property.definition.type === "status" + ) { + return ( + property.definition.options.options?.find( + (option) => option.id === String(property.value), + )?.name ?? String(property.value) + ); + } + if (property.definition.type === "checkbox") { + return property.value ? "Checked" : "Unchecked"; + } + return formulaValueText(property.value); +} + +function compareSortValues(left: string, right: string) { + const leftNumber = Number(left); + const rightNumber = Number(right); + if ( + left.trim() && + right.trim() && + Number.isFinite(leftNumber) && + Number.isFinite(rightNumber) + ) { + return leftNumber - rightNumber; + } + return left.localeCompare(right, undefined, { + numeric: true, + sensitivity: "base", + }); +} + +function propertyNumberValue(property: DocumentProperty | null | undefined) { + if ( + !property || + property.value === null || + property.value === undefined || + property.value === "" + ) { + return Number.NaN; + } + const value = + typeof property.value === "number" + ? property.value + : Number(String(property.value).trim()); + return Number.isFinite(value) ? value : Number.NaN; +} + +function propertyDateValue(property: DocumentProperty | null | undefined) { + if (!property || !property.value) return Number.NaN; + const value = new Date( + documentPropertyDatePart(property.value, "start") || String(property.value), + ).getTime(); + return Number.isFinite(value) ? value : Number.NaN; +} From d07d402f679c4fee57f6bd2db68673315c0a38a1 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:01:53 -0400 Subject: [PATCH 02/21] Fix exact Content mutation projections --- ...content-blog-table-performance-research.md | 66 ++++++++++++++----- .../content/actions/add-database-item.ts | 14 ++-- .../actions/attach-content-database-source.ts | 28 ++++++-- .../content-database-source-actions.test.ts | 38 +++++++++++ .../database-row-batch-actions.db.test.ts | 50 ++++++++++++++ .../actions/duplicate-database-item.ts | 16 ++--- .../actions/duplicate-database-items.ts | 13 ++-- 7 files changed, 183 insertions(+), 42 deletions(-) diff --git a/plans/content-blog-table-performance-research.md b/plans/content-blog-table-performance-research.md index 56521eb38a..00e93f9f4d 100644 --- a/plans/content-blog-table-performance-research.md +++ b/plans/content-blog-table-performance-research.md @@ -521,21 +521,26 @@ work-progress: - exact federated-field fallback without expanding ordinary large tables - source-free setup projections and a 100-row fast path for fresh Builder attachment - personal sort persistence that preserves existing visibility rules without source snapshots - - correlated loopback-only server and visible-row timing probe + - correlated loopback-only server and visible-row timing replay, with probes removed afterward + - exact imported-document reload for Builder source binding and hydration + - exact identity-based created and duplicated mutation response projections verification: - shared table-query TypeScript compilation passed - shared table-query focused Vitest passed - oxfmt and git diff whitespace checks passed - - independent technical review found no remaining P1 or P2 after three correction passes + - live PR review found four correctness defects; exact Builder binding and + identity-based mutation response repairs were implemented + - Content typecheck and 273 focused action, database, batching, table, + sidebar, personal-view, source, and query tests passed on Node 26 + - independent repair review found no remaining P1 or P2 in the four paths pending: - - dependency-compatible full typecheck and focused database/UI suites - - real dev-database fixture creation and joined browser/server measurements - exact read-only Builder model identity and marker-absence baseline - - 584, 1000, and 5000 row acceptance runs - - temporary-probe removal and clean replay - - independent human QA, PR, and CI + - exact Builder-backed attach, hydration, property, and progressive-review acceptance + - independent human QA on a host that exposes the in-app browser + - final repair verification, exact-head PR prose receipt, review, and CI temporary-probes: - id: content-database-response-trace + status: removed gate: >- CONTENT_DATABASE_PERFORMANCE_TRACE=1 on a verified loopback action request records: >- @@ -544,6 +549,7 @@ temporary-probes: count, and execution count; no titles, bodies, provider payloads, or secrets removal-trigger: after the final joined acceptance replay - id: content-database-visible-row-trace + status: removed gate: response carries content-database-response-trace records: >- response trace ID, local interaction ID, intent-to-row-commit duration, @@ -616,30 +622,47 @@ test-resources: max-lifetime-minutes: 1440 declared-at: 2026-07-29T19:00:06Z expires-at: 2026-07-30T19:00:06Z - status: active + status: cleaned active-run-id: 20260729_1715 active-resources: - kind: sqlite-runtime path: /tmp/agent-native-content-perf.GnFBS0/app.db - state: acquired + state: removed-after-marker-free-proof - kind: content-database database-id: TYhMNt9YVgjW document-id: YXN4WiFhHCo0 title: __an_content_perf_20260729_1715__ 1000 rows intended-row-count: 1000 - state: created + state: permanently-deleted - kind: content-database database-id: pyehc2Pq3FT1 document-id: 4h6N49c1GNkj title: __an_content_perf_20260729_1715__ 5000 rows intended-row-count: 5000 - state: created + state: permanently-deleted - kind: content-database database-id: FLeIYkBrKXeC document-id: 4UiIEVdifuyF title: __an_content_perf_20260729_1715__ 584 rows intended-row-count: 584 - state: created + state: permanently-deleted + cleanup-completed-at: 2026-07-30T03:58:00Z + cleanup-observation: + - >- + Content action-backed UI soft-deleted database IDs TYhMNt9YVgjW, + pyehc2Pq3FT1, and FLeIYkBrKXeC with their exact root documents. + - >- + permanently-delete-document removed root IDs YXN4WiFhHCo0, + 4h6N49c1GNkj, and 4UiIEVdifuyF with 1001, 5001, and 585 subtree + documents respectively. + - >- + list-content-databases and search-documents returned zero marker + matches; get-content-database returned typed not_found for all three + exact database IDs; read-only SQL corroborated zero marker documents, + fixture databases, and fixture roots. + - >- + The marker-free SQLite runtime and generated task-local Content env + file were moved to macOS Trash and their original paths verified absent. phase: work governing-artifact: path: /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md @@ -771,6 +794,10 @@ acceptance-state: - >- final ABI-matched verification: Content typecheck and 228 focused action, batching, table, sidebar, personal-view, and query tests passed + - >- + post-review repair verification: Content typecheck and 273 focused tests + passed, including exact imported Builder binding, sparse-position + duplicate projections, and concurrent exact created-item responses blockers: - >- The isolated local runtime has no Builder vault credential, so the exact @@ -779,12 +806,19 @@ acceptance-state: - >- Independent human QA preflight was blocked before H1-H5 because the tester host did not expose the in-app browser; no frozen-test action was executed. - PR review and CI have not yet run. + Exact-head PR prose, review, and CI must refresh after the repair commit. cleanup-evidence: - >- - Stopped the traced local Content server and moved the exact task database - directory plus task-local ignored environment copies to macOS Trash; all - three original paths were verified absent. + The first file-only cleanup was audited and rejected because the restored + database still contained every marker row. The exact fixture was then + recovered and cleaned through Content actions: all three database roots + were soft-deleted, permanently deleted by exact document ID, and proved + absent through list/search/not-found action reads plus corroborating SQL. + - >- + After action-level marker-absence proof, the stopped marker-free runtime + and regenerated task-local Content env file were moved to macOS Trash and + their original paths were verified absent; ambiguous older Trash env files + were preserved untouched. - >- Removed the environment-gated server phase trace, source snapshot trace, response trace fields, and correlated browser console timing after the diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 5732f0f2f1..0cc29cb55e 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -217,13 +217,15 @@ export default defineAction({ limit: 100, offset: 0, }); - const createdPage = response.items.some((item) => item.id === itemId) - ? null - : await getContentDatabaseResponse(databaseId, { + const createdItem = + response.items.find((item) => item.id === itemId) ?? + ( + await getContentDatabaseResponse(databaseId, { limit: 1, - offset: Math.max(0, (response.pagination?.totalItems ?? 1) - 1), - }); - const createdItem = createdPage?.items.find((item) => item.id === itemId); + offset: 0, + documentIds: [documentId], + }) + ).items.find((item) => item.id === itemId); return { ...response, createdItem, diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index 8ce889e4ba..826192de26 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -124,6 +124,24 @@ export function builderCmsAttachReadMetadata(read: BuilderCmsReadResult) { }; } +export function initialBuilderAttachmentSetupOptions(args: { + builderRead: BuilderCmsReadResult | null; + importedEntriesByDocumentId: ReadonlyMap; +}) { + if ( + args.builderRead?.state !== "live" || + args.importedEntriesByDocumentId.size !== args.builderRead.entries.length + ) { + return undefined; + } + const documentIds = [...args.importedEntriesByDocumentId.keys()]; + return { + documentIds, + limit: Math.max(1, documentIds.length), + offset: 0, + }; +} + // Per-source key mapping the UI commits after the canonical-key confirm step. const normalizationFormulaSchema = z .string() @@ -586,14 +604,12 @@ export default defineAction({ importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; } - const everyInitialEntryWasImported = - builderRead?.state === "live" && - importedEntriesByDocumentId.size === builderEntries.length; const refreshedSetup = await sourceSetupPayload( database.id, - everyInitialEntryWasImported - ? { limit: args.limit, offset: args.offset } - : undefined, + initialBuilderAttachmentSetupOptions({ + builderRead, + importedEntriesByDocumentId, + }), ); const builderEntriesByDocumentId = builderRead?.state === "live" diff --git a/templates/content/actions/content-database-source-actions.test.ts b/templates/content/actions/content-database-source-actions.test.ts index 3539a5ecbf..8849967aa6 100644 --- a/templates/content/actions/content-database-source-actions.test.ts +++ b/templates/content/actions/content-database-source-actions.test.ts @@ -10,6 +10,7 @@ import addSourceFieldProperty, { } from "./add-content-database-source-field-property"; import attachSource, { builderCmsAttachReadMetadata, + initialBuilderAttachmentSetupOptions, readInitialBuilderCmsAttachEntries, readInitialBuilderCmsAttachSource, } from "./attach-content-database-source"; @@ -376,6 +377,43 @@ describe("content database source actions", () => { }); }); + it("binds a fully imported initial Builder page by exact document identity", () => { + const read = { + state: "live", + entries: [{ id: "entry-1" }, { id: "entry-2" }], + } as BuilderCmsReadResult; + + expect( + initialBuilderAttachmentSetupOptions({ + builderRead: read, + importedEntriesByDocumentId: new Map([ + ["document-2", read.entries[1]!], + ["document-1", read.entries[0]!], + ]), + }), + ).toEqual({ + documentIds: ["document-2", "document-1"], + limit: 2, + offset: 0, + }); + }); + + it("keeps complete setup when an initial Builder entry was not imported", () => { + const read = { + state: "live", + entries: [{ id: "entry-1" }, { id: "entry-2" }], + } as BuilderCmsReadResult; + + expect( + initialBuilderAttachmentSetupOptions({ + builderRead: read, + importedEntriesByDocumentId: new Map([ + ["document-1", read.entries[0]!], + ]), + }), + ).toBeUndefined(); + }); + it("rejects unsafe source federation normalization formulas", () => { expect(() => attachSource.schema.parse({ diff --git a/templates/content/actions/database-row-batch-actions.db.test.ts b/templates/content/actions/database-row-batch-actions.db.test.ts index 37d9cd976a..c5f3491289 100644 --- a/templates/content/actions/database-row-batch-actions.db.test.ts +++ b/templates/content/actions/database-row-batch-actions.db.test.ts @@ -239,6 +239,9 @@ describe("database row batch actions", () => { ).resolves.toEqual([{ spaceId }, { spaceId }]); expect(result.duplicatedItemId).toBe(result.duplicatedItemIds?.[0]); expect(result.duplicatedDocumentId).toBe(result.duplicatedDocumentIds?.[0]); + expect(result.duplicatedItems?.map((item) => item.id)).toEqual( + result.duplicatedItemIds, + ); expect(result.sourceItemIds).toEqual([rows[1].itemId, rows[2].itemId]); expect(result.sourceDocumentIds).toEqual([ rows[1].documentId, @@ -315,6 +318,49 @@ describe("database row batch actions", () => { expect(await orderedRows(second.databaseId)).toHaveLength(1); }); + it("returns exact duplicate projections when stored positions are sparse", async () => { + const db = getDb(); + const { databaseId, rows } = await createDatabaseWithRows(3); + await Promise.all( + rows.map((row, index) => + db + .update(schema.contentDatabaseItems) + .set({ position: [10, 30, 50][index] }) + .where(eq(schema.contentDatabaseItems.id, row.itemId)), + ), + ); + await Promise.all( + rows.map((row, index) => + db + .update(schema.documents) + .set({ position: [10, 30, 50][index] }) + .where(eq(schema.documents.id, row.documentId)), + ), + ); + + const single = await runWithRequestContext({ userEmail: OWNER }, () => + duplicateDatabaseItemAction.run({ itemId: rows[0].itemId }), + ); + expect(single.duplicatedItems).toHaveLength(1); + expect(single.duplicatedItems?.[0]).toMatchObject({ + id: single.duplicatedItemId, + document: { id: single.duplicatedDocumentId }, + }); + + const batch = await runWithRequestContext({ userEmail: OWNER }, () => + duplicateDatabaseItemsAction.run({ + databaseId, + itemIds: [rows[1].itemId, rows[2].itemId], + }), + ); + expect(batch.duplicatedItems?.map((item) => item.id)).toEqual( + batch.duplicatedItemIds, + ); + expect(batch.duplicatedItems?.map((item) => item.document.id)).toEqual( + batch.duplicatedDocumentIds, + ); + }); + it("deletes selected rows recursively in one batch and renumbers survivors", async () => { const db = getDb(); const { databaseId, databaseDocumentId, rows } = @@ -479,6 +525,10 @@ describe("database row batch actions", () => { const createdItemIds = results.map((result) => result.createdItemId); expect(new Set(createdItemIds).size).toBe(concurrentAdds); for (const result of results) { + expect(result.createdItem).toMatchObject({ + id: result.createdItemId, + document: { id: result.createdDocumentId }, + }); const [createdDocument] = await getDb() .select({ updatedAt: schema.documents.updatedAt }) .from(schema.documents) diff --git a/templates/content/actions/duplicate-database-item.ts b/templates/content/actions/duplicate-database-item.ts index 7094b10761..9bcb548120 100644 --- a/templates/content/actions/duplicate-database-item.ts +++ b/templates/content/actions/duplicate-database-item.ts @@ -180,15 +180,15 @@ export default defineAction({ limit: 100, offset: 0, }); - const duplicatedPage = response.items.some((item) => item.id === nextItemId) - ? null - : await getContentDatabaseResponse(row.item.databaseId, { + const duplicatedItem = + response.items.find((item) => item.id === nextItemId) ?? + ( + await getContentDatabaseResponse(row.item.databaseId, { limit: 1, - offset: nextPosition, - }); - const duplicatedItem = duplicatedPage?.items.find( - (item) => item.id === nextItemId, - ); + offset: 0, + documentIds: [nextDocumentId], + }) + ).items.find((item) => item.id === nextItemId); return { ...response, duplicatedItems: duplicatedItem ? [duplicatedItem] : [], diff --git a/templates/content/actions/duplicate-database-items.ts b/templates/content/actions/duplicate-database-items.ts index a0e1f929ea..ae6f00d713 100644 --- a/templates/content/actions/duplicate-database-items.ts +++ b/templates/content/actions/duplicate-database-items.ts @@ -192,19 +192,20 @@ export default defineAction({ }); const duplicatedPage = await getContentDatabaseResponse(database.id, { limit: duplicates.length, - offset: insertionPosition, + offset: 0, + documentIds: duplicates.map( + (duplicate) => duplicate.duplicatedDocumentId, + ), }); const duplicatedItemIds = new Set( duplicates.map((duplicate) => duplicate.duplicatedItemId), ); - const appendedDuplicatedItems = duplicatedPage.items.filter( - (item) => - duplicatedItemIds.has(item.id) && - !response.items.some((candidate) => candidate.id === item.id), + const duplicatedItems = duplicatedPage.items.filter((item) => + duplicatedItemIds.has(item.id), ); return { ...response, - duplicatedItems: appendedDuplicatedItems, + duplicatedItems, duplicatedItemId: duplicates[0]?.duplicatedItemId, duplicatedDocumentId: duplicates[0]?.duplicatedDocumentId, duplicatedItemIds: duplicates.map( From 03c1a6f406637d3895ccb986dcb5b9f48af5285a Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:14:01 -0400 Subject: [PATCH 03/21] Record blocked Content performance landing --- ...content-blog-table-performance-research.md | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/plans/content-blog-table-performance-research.md b/plans/content-blog-table-performance-research.md index 00e93f9f4d..349ff0015f 100644 --- a/plans/content-blog-table-performance-research.md +++ b/plans/content-blog-table-performance-research.md @@ -474,8 +474,8 @@ Stop and return to Shape if meeting the target requires changing the action arch ## Lifecycle ```yaml -stage: work -ledger-revision: work-blog-table-latency-r7 +stage: land +ledger-revision: land-blog-table-latency-r8 authority-source: >- Alice invoked $work on 2026-07-29 against the frozen shape-blog-table-latency-r5 artifact after approving the operation budgets @@ -499,6 +499,7 @@ allowed-mutations: - pull-request - deploy - schema + - merge write-targets: artifacts: - /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md @@ -768,13 +769,12 @@ product-boundary-gates: not a runtime dependency or hard-coded identity bowerbird-product-boundary: not-applicable acceptance-state: - status: pending + status: blocked summary: >- - Local implementation and synthetic browser acceptance are complete. The - 5000-row table returns and paints only 100 rows, retained-row sorting meets - the warm target, and reconciliation now remains portable at 5000 rows. - Exact Builder-provider acceptance and independent human QA remain pending; - task-local fixture cleanup is complete. + Local implementation, deterministic large-database acceptance, repair + verification, independent technical review, and resource cleanup are + complete. Exact Builder-provider acceptance and independent human QA remain + unproved, so the system-ready risk strategy prohibits merge and closure. local-evidence: - >- 5000-row warm open, ten settled repetitions: first useful React commit @@ -823,7 +823,34 @@ acceptance-state: Removed the environment-gated server phase trace, source snapshot trace, response trace fields, and correlated browser console timing after the final joined replay; no temporary Content performance probe remains. - last-land-packet: null + last-land-packet: + observed-at: 2026-07-30T04:13:17Z + artifact-revision: land-audit-blocked-r8 + passed: >- + deterministic 584/1000/5000-row browser budgets; Content typecheck and + 273 focused tests; exact imported-row and identity-projection repairs; + independent no-P1/P2 repair review; 28 exact-head CI checks with zero + failures at observation time; action-level fixture marker-absence proof + missing-acceptance-evidence: + - >- + exact isolated Builder model identity, attach, hydration, property, + row, and progressive-review budgets on a credentialed runtime + - >- + frozen H1-H5 independent human QA on a host exposing the in-app browser + - >- + two CI jobs were still pending at the observation time, and the + external Review Agent skipped the repair rerun + feature-flags: >- + none; the frozen system-ready strategy has no code-ready-only or off-state + merge fallback + repository-governance: not-satisfied + independent-technical-review: evidenced + acceptance-story: not-satisfied + merge-permitted: false + enablement-permitted: false + may-call-shipped: false + task-may-close: false + readiness: blocked status: active ``` @@ -831,12 +858,16 @@ Frozen-five status: **frozen at `shape-blog-table-latency-r5`**. Output: `/Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md` -Authority: Work on the current branch. Product code, tests, changelog, this -artifact, and recoverable task-local fixtures changed; no Builder record, branch -operation, deployment, or merge is authorized by this phase. +Authority: Land under `land-blog-table-latency-r8`. Bare Land carries the frozen +PR merge action, but the persisted no-go packet prohibits exercising it. No +Builder record, branch operation, deployment, merge, cleanup, or archival was +performed by Land. Invalidated by: a change to the approved outcome/budgets, named Agent Native Content shipping surface or constituency, action/SQL/progressive-provider architecture, acceptance story, or system-ready risk strategy. -Task attention: `shape-complete`; title unchanged because no calling-task title adapter is available in this surface. +Task attention: `manual-custody`; exact Builder credentials and an independent +browser-capable QA host are required before Land can recompute system-ready. -Next: `/work /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md` +Next: Work resumes internally when the credentialed Builder runtime and +independent browser-capable QA host are available; then rerun exact-provider and +H1-H5 acceptance against the unchanged frozen story. From 59409da471efe8936c5742cdf8254dcc1edc5e74 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:08:36 -0400 Subject: [PATCH 04/21] Fix nested GET action parameters --- .changeset/tidy-action-query-objects.md | 5 ++ packages/core/src/client/use-action.spec.ts | 23 ++++++++++ packages/core/src/client/use-action.ts | 6 +++ .../core/src/server/action-routes.spec.ts | 46 ++++++++++++++++++- 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 .changeset/tidy-action-query-objects.md diff --git a/.changeset/tidy-action-query-objects.md b/.changeset/tidy-action-query-objects.md new file mode 100644 index 0000000000..223a5801e2 --- /dev/null +++ b/.changeset/tidy-action-query-objects.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Preserve nested object parameters when browser clients call GET actions. diff --git a/packages/core/src/client/use-action.spec.ts b/packages/core/src/client/use-action.spec.ts index 8e4e32aeea..03b7fa186d 100644 --- a/packages/core/src/client/use-action.spec.ts +++ b/packages/core/src/client/use-action.spec.ts @@ -54,6 +54,29 @@ describe("serializeActionQueryParams", () => { expect(params.has("empty")).toBe(false); expect(params.has("none")).toBe(false); }); + + it("serializes nested object GET params as JSON", () => { + const tableQuery = { + filters: [ + { + propertyId: "status", + operator: "equals", + value: "published", + }, + ], + sorts: [{ propertyId: "date", direction: "desc" }], + }; + + const query = serializeActionQueryParams({ + documentId: "doc-1", + tableQuery, + }); + + const params = new URLSearchParams(query); + expect(params.get("documentId")).toBe("doc-1"); + expect(params.get("tableQuery")).toBe(JSON.stringify(tableQuery)); + expect(query).not.toContain("%5Bobject+Object%5D"); + }); }); describe("callAction", () => { diff --git a/packages/core/src/client/use-action.ts b/packages/core/src/client/use-action.ts index f1cd36c4e2..af99ec43be 100644 --- a/packages/core/src/client/use-action.ts +++ b/packages/core/src/client/use-action.ts @@ -207,6 +207,12 @@ function appendActionQueryParam( } return; } + if (typeof value === "object") { + // defineAction restores JSON strings when the schema expects an object. + // Preserve nested GET params instead of collapsing them to "[object Object]". + qs.append(key, JSON.stringify(value)); + return; + } qs.append(key, String(value)); } diff --git a/packages/core/src/server/action-routes.spec.ts b/packages/core/src/server/action-routes.spec.ts index f3268d141f..0a5b437343 100644 --- a/packages/core/src/server/action-routes.spec.ts +++ b/packages/core/src/server/action-routes.spec.ts @@ -823,6 +823,23 @@ describe("mountActionRoutes", () => { baseCurrency: z.string().optional(), includeSeries: z.boolean().optional(), limit: z.number().optional(), + tableQuery: z + .object({ + filters: z.array( + z.object({ + propertyId: z.string(), + operator: z.string(), + value: z.string(), + }), + ), + sorts: z.array( + z.object({ + propertyId: z.string(), + direction: z.enum(["asc", "desc"]), + }), + ), + }) + .optional(), }), run: async (params: any) => ({ params }), }); @@ -836,7 +853,24 @@ describe("mountActionRoutes", () => { const result = await mounted[0].handler({ _method: "GET", req: { - url: "http://app.test/_agent-native/actions/instrument-overview?portfolioId=p1&isin=US67066G1040&includeSeries=true&limit=5", + url: `http://app.test/_agent-native/actions/instrument-overview?${new URLSearchParams( + { + portfolioId: "p1", + isin: "US67066G1040", + includeSeries: "true", + limit: "5", + tableQuery: JSON.stringify({ + filters: [ + { + propertyId: "status", + operator: "equals", + value: "published", + }, + ], + sorts: [{ propertyId: "date", direction: "desc" }], + }), + }, + )}`, }, }); @@ -846,6 +880,16 @@ describe("mountActionRoutes", () => { isin: "US67066G1040", includeSeries: true, limit: 5, + tableQuery: { + filters: [ + { + propertyId: "status", + operator: "equals", + value: "published", + }, + ], + sorts: [{ propertyId: "date", direction: "desc" }], + }, }, }); }); From b95ceb056fab9e978bc76765df4cd9fcc3e0889d Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:36:17 -0400 Subject: [PATCH 05/21] Avoid redundant Builder field backfills --- .../actions/_builder-cms-read-client.test.ts | 2 + .../actions/_builder-cms-read-client.ts | 62 +++++++++++++++---- ...d-content-database-source-field.db.test.ts | 41 ++++++++++++ ...s-now-appear-immediately-when-connected.md | 6 ++ 4 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index f6bb60f401..214f5bdee9 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -428,6 +428,7 @@ describe("Builder CMS read client", () => { "data.customModelField", "data.Status", "data.status", + "data.optionalField", "data.blocks", ], fetchImpl: fetchImpl as unknown as typeof fetch, @@ -448,6 +449,7 @@ describe("Builder CMS read client", () => { "data.customModelField": "Preserved", "data.Status": "Editorial", "data.status": "published", + "data.optionalField": null, }, }, ], diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index 04287d0c08..9c24cd886a 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -218,6 +218,36 @@ function normalizeBuilderCmsListFieldPath(fieldPath: string) { return normalized; } +function preserveProjectedBuilderFieldAbsence( + read: BuilderCmsReadResult, + fieldPaths: readonly string[] | undefined, + rawData: boolean | undefined, +): BuilderCmsReadResult { + if (read.state !== "live" || rawData === true || !fieldPaths?.length) { + return read; + } + const projectedFieldPaths = [ + ...new Set( + fieldPaths + .map(normalizeBuilderCmsListFieldPath) + .filter((fieldPath): fieldPath is string => fieldPath !== null), + ), + ]; + if (projectedFieldPaths.length === 0) return read; + return { + ...read, + entries: read.entries.map((entry) => { + const sourceValues = { ...entry.sourceValues }; + for (const fieldPath of projectedFieldPaths) { + if (!Object.prototype.hasOwnProperty.call(sourceValues, fieldPath)) { + sourceValues[fieldPath] = null; + } + } + return { ...entry, sourceValues }; + }), + }; +} + export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) { const fields = new Map(); for (const fieldPath of [ @@ -1273,22 +1303,32 @@ export async function readBuilderCmsContentEntries(args: { publicKey, privateKey: privateKey ?? undefined, }); - if (contentApiRead.state === "live") return contentApiRead; + if (contentApiRead.state === "live") { + return preserveProjectedBuilderFieldAbsence( + contentApiRead, + args.fieldPaths, + args.rawData, + ); + } if (!privateKey) return contentApiRead; } if (privateKey) { try { - return await readBuilderCmsContentEntriesViaMcp({ - model: args.model, - fieldPaths: args.fieldPaths, - rawData: args.rawData, - limit: args.limit, - maxPages: args.maxPages, - offset: args.offset, - fetchImpl, - privateKey, - }); + return preserveProjectedBuilderFieldAbsence( + await readBuilderCmsContentEntriesViaMcp({ + model: args.model, + fieldPaths: args.fieldPaths, + rawData: args.rawData, + limit: args.limit, + maxPages: args.maxPages, + offset: args.offset, + fetchImpl, + privateKey, + }), + args.fieldPaths, + args.rawData, + ); } catch (error) { return { state: "error", diff --git a/templates/content/actions/bind-content-database-source-field.db.test.ts b/templates/content/actions/bind-content-database-source-field.db.test.ts index d9d67212be..0581576838 100644 --- a/templates/content/actions/bind-content-database-source-field.db.test.ts +++ b/templates/content/actions/bind-content-database-source-field.db.test.ts @@ -469,6 +469,47 @@ describe("bind-content-database-source-field (row-union)", () => { }); describe("add-content-database-source-field-property Builder refresh", () => { + it("creates an explicitly empty projected field without rereading Builder", async () => { + const f = await seedStaleBuilderTopicsSnapshot(); + const db = getDb(); + const rows = await db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, f.sourceId)); + await Promise.all( + rows.map((row) => + db + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: JSON.stringify({ + ...JSON.parse(row.sourceValuesJson), + "data.topics": null, + }), + }) + .where(eq(schema.contentDatabaseSourceRows.id, row.id)), + ), + ); + const readBuilderEntries = vi.spyOn( + await import("./_builder-cms-read-client.js"), + "readBuilderCmsContentEntries", + ); + + const result = await asOwner(() => + addSourceFieldPropertyAction.run({ + documentId: f.databaseDocId, + sourceFieldId: f.fieldId, + }), + ); + + expect(readBuilderEntries).not.toHaveBeenCalled(); + expect(result.itemValues).toEqual([]); + const properties = await db + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, f.databaseId)); + expect(properties).toHaveLength(1); + }); + it("refreshes a stale Builder snapshot before creating and populating a Topics property", async () => { const f = await seedStaleBuilderTopicsSnapshot(); const readBuilderEntries = vi diff --git a/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md b/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md new file mode 100644 index 0000000000..729b7ec9db --- /dev/null +++ b/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md @@ -0,0 +1,6 @@ +--- +type: improved +date: 2026-07-30 +--- + +Builder source columns now appear immediately when connected entries leave the field empty. From 1d2dca31d1d13c3e0c65632374e9af593704e767 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:06:05 -0400 Subject: [PATCH 06/21] Project Builder source fields on reveal --- ...-content-database-source-field-property.ts | 96 +++++++++++++++---- ...d-content-database-source-field.db.test.ts | 52 ++++++++++ ...s-now-appear-immediately-when-connected.md | 2 +- 3 files changed, 129 insertions(+), 21 deletions(-) diff --git a/templates/content/actions/add-content-database-source-field-property.ts b/templates/content/actions/add-content-database-source-field-property.ts index 3b56927751..a97c0272f0 100644 --- a/templates/content/actions/add-content-database-source-field-property.ts +++ b/templates/content/actions/add-content-database-source-field-property.ts @@ -1,4 +1,5 @@ import { defineAction } from "@agent-native/core"; +import { getDialect } from "@agent-native/core/db"; import { assertAccess } from "@agent-native/core/sharing"; import { and, eq, isNull, sql } from "drizzle-orm"; import { z } from "zod"; @@ -61,6 +62,32 @@ function hasSourceFieldValue( ); } +function sourceFieldValueJsonProjection(sourceFieldKey: string) { + const sourceValuesJson = schema.contentDatabaseSourceRows.sourceValuesJson; + if (getDialect() === "postgres") { + return sql< + string | null + >`(${sourceValuesJson}::jsonb -> ${sourceFieldKey})::text`; + } + + const path = `$."${sourceFieldKey}"`; + const type = sql`json_type(${sourceValuesJson}, ${path})`; + return sql`CASE + WHEN ${type} IS NULL THEN NULL + WHEN ${type} = 'true' THEN 'true' + WHEN ${type} = 'false' THEN 'false' + ELSE json_quote(json_extract(${sourceValuesJson}, ${path})) + END`; +} + +function compactSourceValuesJson( + sourceFieldKey: string, + sourceFieldValueJson: string, +) { + const escapedKey = sourceFieldKey.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + return `{"${escapedKey}":${sourceFieldValueJson}}`; +} + function mergeBuilderFieldIntoSourceRows(args: { rows: SourceRow[]; entries: BuilderCmsSourceEntry[]; @@ -386,15 +413,17 @@ export default defineAction({ const initialSourceRows = isSecondary ? [] : await db - .select() + .select({ + sourceFieldValueJson: sourceFieldValueJsonProjection( + field.sourceFieldKey, + ), + }) .from(schema.contentDatabaseSourceRows) .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); const shouldRefreshBuilderField = !isSecondary && source.sourceType === "builder-cms" && - initialSourceRows.some( - (row) => !hasSourceFieldValue(row, field.sourceFieldKey), - ); + initialSourceRows.some((row) => row.sourceFieldValueJson === null); let builderEntries: BuilderCmsSourceEntry[] | null = null; if (shouldRefreshBuilderField) { // Read before writing so a Builder outage cannot leave behind a cleanly @@ -458,15 +487,18 @@ export default defineAction({ throw new Error("Source field does not belong to this database."); } - let sourceRows = isSecondary - ? [] - : await tx - .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); + let sourceRows: Array<{ + databaseItemId: string; + documentId: string; + sourceValuesJson: string | null; + }> = []; if (builderEntries) { + const currentSourceRows = await tx + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); const merged = mergeBuilderFieldIntoSourceRows({ - rows: sourceRows, + rows: currentSourceRows, entries: builderEntries, sourceTable: source.sourceTable, sourceFieldKey: currentField.sourceFieldKey, @@ -478,7 +510,7 @@ export default defineAction({ ); } for (let index = 0; index < merged.rows.length; index += 1) { - const currentRow = sourceRows[index]; + const currentRow = currentSourceRows[index]; const mergedRow = merged.rows[index]; if ( !currentRow || @@ -510,14 +542,38 @@ export default defineAction({ } } sourceRows = merged.rows; - } else if ( - source.sourceType === "builder-cms" && - sourceRows.some( - (row) => !hasSourceFieldValue(row, currentField.sourceFieldKey), - ) - ) { - throw new Error( - "The Builder source changed while adding this property. No property was created; try again.", + } else if (!isSecondary) { + const projectedSourceRows = await tx + .select({ + databaseItemId: schema.contentDatabaseSourceRows.databaseItemId, + documentId: schema.contentDatabaseSourceRows.documentId, + sourceFieldValueJson: sourceFieldValueJsonProjection( + currentField.sourceFieldKey, + ), + }) + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); + if ( + source.sourceType === "builder-cms" && + projectedSourceRows.some((row) => row.sourceFieldValueJson === null) + ) { + throw new Error( + "The Builder source changed while adding this property. No property was created; try again.", + ); + } + sourceRows = projectedSourceRows.flatMap((row) => + row.sourceFieldValueJson === null + ? [] + : [ + { + databaseItemId: row.databaseItemId, + documentId: row.documentId, + sourceValuesJson: compactSourceValuesJson( + currentField.sourceFieldKey, + row.sourceFieldValueJson, + ), + }, + ], ); } diff --git a/templates/content/actions/bind-content-database-source-field.db.test.ts b/templates/content/actions/bind-content-database-source-field.db.test.ts index 0581576838..b7500aba3d 100644 --- a/templates/content/actions/bind-content-database-source-field.db.test.ts +++ b/templates/content/actions/bind-content-database-source-field.db.test.ts @@ -510,6 +510,58 @@ describe("add-content-database-source-field-property Builder refresh", () => { expect(properties).toHaveLength(1); }); + it("materializes a projected field without returning unrelated source content", async () => { + const f = await seedStaleBuilderTopicsSnapshot(); + const db = getDb(); + const rows = await db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, f.sourceId)); + await Promise.all( + rows.map((row, index) => + db + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: JSON.stringify({ + ...JSON.parse(row.sourceValuesJson), + "data.topics": [ + index === 0 ? "Agent Native" : "Developer Experience", + ], + "_builder.bodyContent": "unrelated".repeat(5_000), + }), + }) + .where(eq(schema.contentDatabaseSourceRows.id, row.id)), + ), + ); + const readBuilderEntries = vi.spyOn( + await import("./_builder-cms-read-client.js"), + "readBuilderCmsContentEntries", + ); + + const result = await asOwner(() => + addSourceFieldPropertyAction.run({ + documentId: f.databaseDocId, + sourceFieldId: f.fieldId, + }), + ); + + expect(readBuilderEntries).not.toHaveBeenCalled(); + expect(result.itemValues).toEqual([ + { + itemId: f.rows[0].itemId, + documentId: f.rows[0].documentId, + value: ["agent-native"], + }, + { + itemId: f.rows[1].itemId, + documentId: f.rows[1].documentId, + value: ["developer-experience"], + }, + ]); + expect(JSON.stringify(result)).not.toContain("_builder.bodyContent"); + expect(JSON.stringify(result)).not.toContain("unrelated"); + }); + it("refreshes a stale Builder snapshot before creating and populating a Topics property", async () => { const f = await seedStaleBuilderTopicsSnapshot(); const readBuilderEntries = vi diff --git a/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md b/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md index 729b7ec9db..00d243d958 100644 --- a/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md +++ b/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md @@ -3,4 +3,4 @@ type: improved date: 2026-07-30 --- -Builder source columns now appear immediately when connected entries leave the field empty. +Builder source columns now appear immediately when connected entries already contain the selected field, including empty values. From b0c28b1d0af1e723b2b9c454dbcf36205ac72760 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:12:05 -0400 Subject: [PATCH 07/21] Paint Builder source columns optimistically --- .../editor/DocumentProperties.layout.test.ts | 11 ++++++++ .../components/editor/DocumentProperties.tsx | 25 +++---------------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/templates/content/app/components/editor/DocumentProperties.layout.test.ts b/templates/content/app/components/editor/DocumentProperties.layout.test.ts index 21406c2f5e..36228346b9 100644 --- a/templates/content/app/components/editor/DocumentProperties.layout.test.ts +++ b/templates/content/app/components/editor/DocumentProperties.layout.test.ts @@ -107,6 +107,17 @@ describe("document property layout", () => { expect(source).toContain('role="alert"'); }); + it("uses the optimistic source-field mutation for Add Property", () => { + const source = readPropertiesSource(); + + expect(source).toContain( + "useAddContentDatabaseSourceFieldProperty(documentId)", + ); + expect(source).not.toContain( + '>("add-content-database-source-field-property", {', + ); + }); + it("makes editable property value triggers fill the database cell", () => { const source = readPropertiesSource(); diff --git a/templates/content/app/components/editor/DocumentProperties.tsx b/templates/content/app/components/editor/DocumentProperties.tsx index a94354bb2c..642693a730 100644 --- a/templates/content/app/components/editor/DocumentProperties.tsx +++ b/templates/content/app/components/editor/DocumentProperties.tsx @@ -19,10 +19,8 @@ import { } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; import type { - AddContentDatabaseSourceFieldPropertyRequest, BindContentDatabaseSourceFieldRequest, ContentDatabaseResponse, - ContentDatabaseSourceFieldPropertyResponse, ContentDatabaseSource, DocumentProperty, } from "@shared/api"; @@ -130,7 +128,7 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; -import { applySourceFieldPropertyToDatabaseResponse } from "@/hooks/use-content-database"; +import { useAddContentDatabaseSourceFieldProperty } from "@/hooks/use-content-database"; import { documentPropertiesResponseMatchesScope, useConfigureDocumentProperty, @@ -3148,25 +3146,8 @@ export function AddProperty({ }) { const t = useT(); const configure = useConfigureDocumentProperty(documentId, databaseId); - const queryClient = useQueryClient(); - const addSourceFieldProperty = useActionMutation< - ContentDatabaseSourceFieldPropertyResponse, - AddContentDatabaseSourceFieldPropertyRequest - >("add-content-database-source-field-property", { - onSuccess: (data) => { - queryClient.setQueriesData( - { queryKey: ["action", "get-content-database"] }, - (current) => applySourceFieldPropertyToDatabaseResponse(current, data), - ); - queryClient.invalidateQueries({ - queryKey: [ - "action", - "list-document-properties", - { documentId, databaseId }, - ], - }); - }, - }); + const addSourceFieldProperty = + useAddContentDatabaseSourceFieldProperty(documentId); const [open, setOpen] = useState(false); const [typeQuery, setTypeQuery] = useState(""); const filteredPropertyTypes = filterDocumentPropertyTypes(typeQuery); From 57be73a1f5b371f79ef8099b7f5428985c9cc3a1 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:18:42 -0400 Subject: [PATCH 08/21] Avoid duplicate Builder field scans --- ...-content-database-source-field-property.ts | 531 +++++++++--------- 1 file changed, 263 insertions(+), 268 deletions(-) diff --git a/templates/content/actions/add-content-database-source-field-property.ts b/templates/content/actions/add-content-database-source-field-property.ts index a97c0272f0..89edc80300 100644 --- a/templates/content/actions/add-content-database-source-field-property.ts +++ b/templates/content/actions/add-content-database-source-field-property.ts @@ -38,6 +38,12 @@ const BUILDER_FIELD_REFRESH_MINIMUM_LIMIT = 500; type SourceRow = typeof schema.contentDatabaseSourceRows.$inferSelect; +class MissingBuilderFieldValuesError extends Error { + constructor(readonly rowCount: number) { + super("Builder source field values are missing from the local snapshot."); + } +} + function parseSourceValues( value: string | null | undefined, ): Record { @@ -410,295 +416,284 @@ export default defineAction({ } const isSecondary = federationRole === "secondary"; - const initialSourceRows = isSecondary - ? [] - : await db - .select({ - sourceFieldValueJson: sourceFieldValueJsonProjection( - field.sourceFieldKey, - ), - }) - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); - const shouldRefreshBuilderField = - !isSecondary && - source.sourceType === "builder-cms" && - initialSourceRows.some((row) => row.sourceFieldValueJson === null); - let builderEntries: BuilderCmsSourceEntry[] | null = null; - if (shouldRefreshBuilderField) { - // Read before writing so a Builder outage cannot leave behind a cleanly - // reported but empty property. Start at zero even when the source's - // broader refresh is incremental: the stored rows we are backfilling - // begin with that first page. - const builderRead = await readBuilderCmsContentEntries({ - model: source.sourceTable, - fieldPaths: [field.sourceFieldKey], - limit: Math.max( - BUILDER_FIELD_REFRESH_MINIMUM_LIMIT, - initialSourceRows.length, - ), - offset: 0, - }); - if (builderRead.state !== "live") { - throw new Error( - builderRead.message ?? - "Builder CMS could not refresh this field. No property was created.", - ); - } - if (initialSourceRows.length > 0 && builderRead.entries.length === 0) { - throw new Error( - "Builder CMS returned no entries for this source. No property was created; refresh the source and try again.", - ); - } - builderEntries = builderRead.entries; - } - // Keep the local snapshot, property definition, mapping, and materialized - // values atomic. Re-read rows after the provider request so a concurrent - // source refresh cannot be overwritten by the older pre-request snapshot. - return await db.transaction(async (tx) => { - const now = new Date().toISOString(); - const visibility = normalizePropertyVisibility(undefined); - const propertyId = nanoid(); - const [currentField] = await tx - .select() - .from(schema.contentDatabaseSourceFields) - .where(eq(schema.contentDatabaseSourceFields.id, field.id)); - if (!currentField) throw new Error("Source field not found."); - if (currentField.propertyId) { - throw new Error("Source field is already mapped to a property."); - } - if (currentField.mappingType === "title") { - throw new Error("The title source field is already mapped to Name."); - } - if (currentField.sourceId !== source.id) { - throw new Error("Source field does not belong to this database."); - } - const [currentSource] = await tx - .select() - .from(schema.contentDatabaseSources) - .where( - and( - eq(schema.contentDatabaseSources.id, source.id), - eq(schema.contentDatabaseSources.databaseId, database.id), - ), - ); - if (!currentSource) { - throw new Error("Source field does not belong to this database."); - } - - let sourceRows: Array<{ - databaseItemId: string; - documentId: string; - sourceValuesJson: string | null; - }> = []; - if (builderEntries) { - const currentSourceRows = await tx + // values atomic. The common complete-snapshot path needs one projected row + // read. If values are missing, that transaction exits before writes, the + // provider read happens outside it, and a second transaction re-reads the + // rows so concurrent source refreshes cannot be overwritten. + const materializeProperty = ( + builderEntries: BuilderCmsSourceEntry[] | null, + ) => + db.transaction(async (tx) => { + const now = new Date().toISOString(); + const visibility = normalizePropertyVisibility(undefined); + const propertyId = nanoid(); + const [currentField] = await tx .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); - const merged = mergeBuilderFieldIntoSourceRows({ - rows: currentSourceRows, - entries: builderEntries, - sourceTable: source.sourceTable, - sourceFieldKey: currentField.sourceFieldKey, - now, - }); - if (merged.matchedMissingRows < merged.missingRows) { - throw new Error( - "Builder CMS did not return entries for every stored source row. No property was created; refresh the source and try again.", + .from(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.id, field.id)); + if (!currentField) throw new Error("Source field not found."); + if (currentField.propertyId) { + throw new Error("Source field is already mapped to a property."); + } + if (currentField.mappingType === "title") { + throw new Error("The title source field is already mapped to Name."); + } + if (currentField.sourceId !== source.id) { + throw new Error("Source field does not belong to this database."); + } + const [currentSource] = await tx + .select() + .from(schema.contentDatabaseSources) + .where( + and( + eq(schema.contentDatabaseSources.id, source.id), + eq(schema.contentDatabaseSources.databaseId, database.id), + ), ); + if (!currentSource) { + throw new Error("Source field does not belong to this database."); } - for (let index = 0; index < merged.rows.length; index += 1) { - const currentRow = currentSourceRows[index]; - const mergedRow = merged.rows[index]; - if ( - !currentRow || - !mergedRow || - currentRow.sourceValuesJson === mergedRow.sourceValuesJson - ) { - continue; + + let sourceRows: Array<{ + databaseItemId: string; + documentId: string; + sourceValuesJson: string | null; + }> = []; + if (builderEntries) { + const currentSourceRows = await tx + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); + const merged = mergeBuilderFieldIntoSourceRows({ + rows: currentSourceRows, + entries: builderEntries, + sourceTable: source.sourceTable, + sourceFieldKey: currentField.sourceFieldKey, + now, + }); + if (merged.matchedMissingRows < merged.missingRows) { + throw new Error( + "Builder CMS did not return entries for every stored source row. No property was created; refresh the source and try again.", + ); } - const [updatedRow] = await tx - .update(schema.contentDatabaseSourceRows) - .set({ - sourceValuesJson: mergedRow.sourceValuesJson, - updatedAt: now, - }) - .where( - and( - eq(schema.contentDatabaseSourceRows.id, currentRow.id), - eq( - schema.contentDatabaseSourceRows.sourceValuesJson, - currentRow.sourceValuesJson, + for (let index = 0; index < merged.rows.length; index += 1) { + const currentRow = currentSourceRows[index]; + const mergedRow = merged.rows[index]; + if ( + !currentRow || + !mergedRow || + currentRow.sourceValuesJson === mergedRow.sourceValuesJson + ) { + continue; + } + const [updatedRow] = await tx + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: mergedRow.sourceValuesJson, + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseSourceRows.id, currentRow.id), + eq( + schema.contentDatabaseSourceRows.sourceValuesJson, + currentRow.sourceValuesJson, + ), ), + ) + .returning({ id: schema.contentDatabaseSourceRows.id }); + if (!updatedRow) { + throw new Error( + "The Builder source changed while adding this property. No property was created; try again.", + ); + } + } + sourceRows = merged.rows; + } else if (!isSecondary) { + const projectedSourceRows = await tx + .select({ + databaseItemId: schema.contentDatabaseSourceRows.databaseItemId, + documentId: schema.contentDatabaseSourceRows.documentId, + sourceFieldValueJson: sourceFieldValueJsonProjection( + currentField.sourceFieldKey, ), - ) - .returning({ id: schema.contentDatabaseSourceRows.id }); - if (!updatedRow) { - throw new Error( - "The Builder source changed while adding this property. No property was created; try again.", + }) + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); + if ( + source.sourceType === "builder-cms" && + projectedSourceRows.some((row) => row.sourceFieldValueJson === null) + ) { + throw new MissingBuilderFieldValuesError( + projectedSourceRows.length, ); } + sourceRows = projectedSourceRows.flatMap((row) => + row.sourceFieldValueJson === null + ? [] + : [ + { + databaseItemId: row.databaseItemId, + documentId: row.documentId, + sourceValuesJson: compactSourceValuesJson( + currentField.sourceFieldKey, + row.sourceFieldValueJson, + ), + }, + ], + ); } - sourceRows = merged.rows; - } else if (!isSecondary) { - const projectedSourceRows = await tx + + const builderMetadata = builderMetadataForSourceField({ + sourceFieldKey: currentField.sourceFieldKey, + sourceMetadataJson: currentSource.metadataJson, + }); + const type = propertyTypeForSourceField( + currentField.sourceFieldType, + builderMetadata, + ); + const options = sourceFieldPropertyOptions({ + type, + metadata: builderMetadata, + rows: sourceRows, + sourceFieldKey: currentField.sourceFieldKey, + }); + const [maxPos] = await tx .select({ - databaseItemId: schema.contentDatabaseSourceRows.databaseItemId, - documentId: schema.contentDatabaseSourceRows.documentId, - sourceFieldValueJson: sourceFieldValueJsonProjection( - currentField.sourceFieldKey, - ), + max: sql`COALESCE(MAX(position), -1)`, }) - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); - if ( - source.sourceType === "builder-cms" && - projectedSourceRows.some((row) => row.sourceFieldValueJson === null) - ) { - throw new Error( - "The Builder source changed while adding this property. No property was created; try again.", + .from(schema.documentPropertyDefinitions) + .where( + and( + eq( + schema.documentPropertyDefinitions.ownerEmail, + database.ownerEmail, + ), + eq(schema.documentPropertyDefinitions.databaseId, database.id), + ), ); - } - sourceRows = projectedSourceRows.flatMap((row) => - row.sourceFieldValueJson === null - ? [] - : [ - { - databaseItemId: row.databaseItemId, - documentId: row.documentId, - sourceValuesJson: compactSourceValuesJson( - currentField.sourceFieldKey, - row.sourceFieldValueJson, - ), - }, - ], - ); - } + const position = (maxPos?.max ?? -1) + 1; + + await tx.insert(schema.documentPropertyDefinitions).values({ + id: propertyId, + ownerEmail: database.ownerEmail, + orgId: database.orgId ?? null, + databaseId: database.id, + name: currentField.sourceFieldLabel, + type, + visibility, + optionsJson: serializePropertyOptions(options), + position, + createdAt: now, + updatedAt: now, + }); - const builderMetadata = builderMetadataForSourceField({ - sourceFieldKey: currentField.sourceFieldKey, - sourceMetadataJson: currentSource.metadataJson, - }); - const type = propertyTypeForSourceField( - currentField.sourceFieldType, - builderMetadata, - ); - const options = sourceFieldPropertyOptions({ - type, - metadata: builderMetadata, - rows: sourceRows, - sourceFieldKey: currentField.sourceFieldKey, - }); - const [maxPos] = await tx - .select({ - max: sql`COALESCE(MAX(position), -1)`, - }) - .from(schema.documentPropertyDefinitions) - .where( - and( - eq( - schema.documentPropertyDefinitions.ownerEmail, - database.ownerEmail, + const [mappedField] = await tx + .update(schema.contentDatabaseSourceFields) + .set({ + propertyId, + localFieldKey: propertyId, + mappingType: "property", + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseSourceFields.id, currentField.id), + isNull(schema.contentDatabaseSourceFields.propertyId), ), - eq(schema.documentPropertyDefinitions.databaseId, database.id), - ), - ); - const position = (maxPos?.max ?? -1) + 1; - - await tx.insert(schema.documentPropertyDefinitions).values({ - id: propertyId, - ownerEmail: database.ownerEmail, - orgId: database.orgId ?? null, - databaseId: database.id, - name: currentField.sourceFieldLabel, - type, - visibility, - optionsJson: serializePropertyOptions(options), - position, - createdAt: now, - updatedAt: now, - }); + ) + .returning({ id: schema.contentDatabaseSourceFields.id }); + if (!mappedField) { + throw new Error("Source field is already mapped to a property."); + } - const [mappedField] = await tx - .update(schema.contentDatabaseSourceFields) - .set({ - propertyId, - localFieldKey: propertyId, - mappingType: "property", - updatedAt: now, - }) - .where( - and( - eq(schema.contentDatabaseSourceFields.id, currentField.id), - isNull(schema.contentDatabaseSourceFields.propertyId), - ), - ) - .returning({ id: schema.contentDatabaseSourceFields.id }); - if (!mappedField) { - throw new Error("Source field is already mapped to a property."); - } + await tx + .update(schema.contentDatabaseSources) + .set({ updatedAt: now }) + .where(eq(schema.contentDatabaseSources.id, currentSource.id)); - await tx - .update(schema.contentDatabaseSources) - .set({ updatedAt: now }) - .where(eq(schema.contentDatabaseSources.id, currentSource.id)); + const itemValues = sourceFieldPropertyValuesFromRows( + sourceRows, + currentField.sourceFieldKey, + type, + options, + ); + if (itemValues.length > 0) { + for (const chunk of chunks(itemValues, 200)) { + await tx.insert(schema.documentPropertyValues).values( + chunk.map((row) => ({ + id: nanoid(), + ownerEmail: database.ownerEmail, + documentId: row.documentId, + propertyId, + valueJson: serializePropertyValue(row.value), + createdAt: now, + updatedAt: now, + })), + ); + } + } - const itemValues = sourceFieldPropertyValuesFromRows( - sourceRows, - currentField.sourceFieldKey, - type, - options, - ); - if (itemValues.length > 0) { - for (const chunk of chunks(itemValues, 200)) { - await tx.insert(schema.documentPropertyValues).values( - chunk.map((row) => ({ - id: nanoid(), - ownerEmail: database.ownerEmail, - documentId: row.documentId, - propertyId, - valueJson: serializePropertyValue(row.value), + const sourceField = serializeSourceField( + { + ...currentField, + propertyId, + localFieldKey: propertyId, + mappingType: "property", + updatedAt: now, + }, + currentField.sourceFieldLabel, + ); + + return { + databaseId: database.id, + documentId: database.documentId, + property: { + definition: { + id: propertyId, + databaseId: database.id, + name: currentField.sourceFieldLabel, + type, + visibility, + options, + position, createdAt: now, updatedAt: now, - })), - ); - } - } + }, + value: null, + editable: true, + }, + sourceField, + itemValues, + }; + }); - const sourceField = serializeSourceField( - { - ...currentField, - propertyId, - localFieldKey: propertyId, - mappingType: "property", - updatedAt: now, - }, - currentField.sourceFieldLabel, - ); + try { + return await materializeProperty(null); + } catch (error) { + if (!(error instanceof MissingBuilderFieldValuesError)) throw error; - return { - databaseId: database.id, - documentId: database.documentId, - property: { - definition: { - id: propertyId, - databaseId: database.id, - name: currentField.sourceFieldLabel, - type, - visibility, - options, - position, - createdAt: now, - updatedAt: now, - }, - value: null, - editable: true, - }, - sourceField, - itemValues, - }; - }); + // Read before retrying the transaction so a Builder outage cannot leave + // behind a cleanly reported but empty property. + const builderRead = await readBuilderCmsContentEntries({ + model: source.sourceTable, + fieldPaths: [field.sourceFieldKey], + limit: Math.max(BUILDER_FIELD_REFRESH_MINIMUM_LIMIT, error.rowCount), + offset: 0, + }); + if (builderRead.state !== "live") { + throw new Error( + builderRead.message ?? + "Builder CMS could not refresh this field. No property was created.", + ); + } + if (error.rowCount > 0 && builderRead.entries.length === 0) { + throw new Error( + "Builder CMS returned no entries for this source. No property was created; refresh the source and try again.", + ); + } + return await materializeProperty(builderRead.entries); + } }, }); From e717fdb3b8895b1433d822ce801d4804ea0c8a51 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:25:33 -0400 Subject: [PATCH 09/21] Cover large Builder field reveals --- ...d-content-database-source-field.db.test.ts | 40 +++++++++++++++++++ .../DatabaseView.error-toasts.test.tsx | 1 + 2 files changed, 41 insertions(+) diff --git a/templates/content/actions/bind-content-database-source-field.db.test.ts b/templates/content/actions/bind-content-database-source-field.db.test.ts index b7500aba3d..ee26a0696f 100644 --- a/templates/content/actions/bind-content-database-source-field.db.test.ts +++ b/templates/content/actions/bind-content-database-source-field.db.test.ts @@ -562,6 +562,46 @@ describe("add-content-database-source-field-property Builder refresh", () => { expect(JSON.stringify(result)).not.toContain("unrelated"); }); + it("materializes 584 already-projected rows without rereading Builder", async () => { + const f = await seedStaleBuilderTopicsSnapshot(584); + const db = getDb(); + await db + .update(schema.contentDatabaseSourceRows) + .set({ + sourceValuesJson: JSON.stringify({ + "data.topics": ["Agent Native"], + "_builder.bodyContent": "unrelated".repeat(500), + }), + }) + .where(eq(schema.contentDatabaseSourceRows.sourceId, f.sourceId)); + const readBuilderEntries = vi.spyOn( + await import("./_builder-cms-read-client.js"), + "readBuilderCmsContentEntries", + ); + + const result = await asOwner(() => + addSourceFieldPropertyAction.run({ + documentId: f.databaseDocId, + sourceFieldId: f.fieldId, + }), + ); + + expect(readBuilderEntries).not.toHaveBeenCalled(); + expect(result.itemValues).toHaveLength(584); + expect(result.itemValues[0]?.value).toEqual(["agent-native"]); + expect(JSON.stringify(result)).not.toContain("_builder.bodyContent"); + const propertyValues = await db + .select() + .from(schema.documentPropertyValues) + .where( + eq( + schema.documentPropertyValues.propertyId, + result.property.definition.id, + ), + ); + expect(propertyValues).toHaveLength(584); + }); + it("refreshes a stale Builder snapshot before creating and populating a Topics property", async () => { const f = await seedStaleBuilderTopicsSnapshot(); const readBuilderEntries = vi diff --git a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx index 1278f37ec8..e1de35dbd3 100644 --- a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx @@ -125,6 +125,7 @@ vi.mock("@/hooks/use-content-database", () => ({ }; }, useAddDatabaseItem: () => addItemMutation, + useAddContentDatabaseSourceFieldProperty: () => benignMutation, useAttachContentDatabaseSource: () => attachSourceMutation, useChangeContentDatabaseSourceRole: () => benignMutation, useRefreshContentDatabaseSource: () => benignMutation, From 83c45829f2db33c590c0c5298a6836dd5edfe35e Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:40:46 -0400 Subject: [PATCH 10/21] Finish Builder imports in one continuation --- .../app/components/editor/database/DatabaseView.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/templates/content/app/components/editor/database/DatabaseView.tsx b/templates/content/app/components/editor/database/DatabaseView.tsx index bba6ba9873..e8636a1f65 100644 --- a/templates/content/app/components/editor/database/DatabaseView.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.tsx @@ -1177,6 +1177,7 @@ function DatabaseTable({ onError?: () => void, expectedBuilderContinuationOffset?: number, onSuccess?: () => void, + fullRefresh = false, ) => { if (!acquireDatabaseSourceOperation(refreshSourceInFlightRef, sourceId)) { return false; @@ -1186,6 +1187,7 @@ function DatabaseTable({ documentId: document.id, sourceId, expectedBuilderContinuationOffset, + fullRefresh: fullRefresh || undefined, }, { onError, @@ -1288,6 +1290,7 @@ function DatabaseTable({ () => handleBuilderContinuationError(continuationKey), candidate.metadata.lastReadNextOffset, () => handleBuilderContinuationSuccess(continuationKey), + true, ) ) { autoContinueBuilderSourceRef.current.delete(continuationKey); @@ -1367,6 +1370,7 @@ function DatabaseTable({ () => handleBuilderContinuationError(continuationKey), candidate.metadata.lastReadNextOffset, () => handleBuilderContinuationSuccess(continuationKey), + true, ) ) { builderContinuationWatchdogRef.current.set( @@ -2657,6 +2661,7 @@ function DatabaseTable({ () => handleBuilderContinuationError(continuationKey), builderSource.metadata.lastReadNextOffset, () => handleBuilderContinuationSuccess(continuationKey), + true, ); }} /> @@ -7944,6 +7949,7 @@ function DatabaseSettingsSourcePanel({ const builderSources = databaseAttachedBuilderSources(sources, source); const builderStatus = useBuilderStatus(); const builderConfigured = builderStatus.status?.configured === true; + const builderModelsQuery = useBuilderCmsModels(builderConfigured); const builderOrgName = builderStatus.status?.orgName ?? null; // Real space name(s) from the Admin API, falling back to the generic org // name (then a constant) so the drill-down never renders a blank label. @@ -8196,6 +8202,7 @@ function DatabaseSettingsSourcePanel({ return ( { const attached = databaseAttachedBuilderSource(sources, source, { modelName: model.name, @@ -9894,12 +9901,13 @@ function SourceDetailsFieldPicker({ // is marked; selecting a row opens that model's leaf. function BuilderSpaceModelsView({ attachedModelNames, + modelsQuery, onOpenModel, }: { attachedModelNames: string[]; + modelsQuery: ReturnType; onOpenModel: (model: BuilderCmsModelSummary) => void; }) { - const modelsQuery = useBuilderCmsModels(true); const models = modelsQuery.data?.models ?? []; const [query, setQuery] = useState(""); From 46ffed6a3aca377242fc20a02c9bacdb0364e3e3 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:51:07 -0400 Subject: [PATCH 11/21] Resume full Builder syncs from their cursor --- .../content/actions/_database-source-utils.ts | 35 +++++++++++-------- .../refresh-content-database-source.ts | 7 ++++ .../editor/database/DatabaseView.tsx | 1 + templates/content/shared/api.ts | 1 + 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 17f6c52dbb..29d922c90d 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -5905,6 +5905,7 @@ export async function resyncBuilderCmsSourceSnapshot(args: { source: ContentDatabaseSourceRowDb; now: string; runFullRefresh?: boolean; + finishPagination?: boolean; refreshClaimId?: string; }) { let { properties, response } = await sourceSetupPayload(args.database.id); @@ -5917,7 +5918,7 @@ export async function resyncBuilderCmsSourceSnapshot(args: { ? sourceMetadata.activeReadSourceRowIds : []; const continueOffset = - !args.runFullRefresh && + (!args.runFullRefresh || args.finishPagination) && sourceMetadata.sourceFetchState === "fetching" && sourceMetadata.lastReadHasMore === true && activeReadSourceRowIds.length > 0 && @@ -5931,20 +5932,24 @@ export async function resyncBuilderCmsSourceSnapshot(args: { .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id)); let builderModelFields: BuilderCmsModelFieldSummary[] | undefined; let builderModelFieldsReadFailed = false; - try { - builderModelFields = await readBuilderCmsModelFields({ - model: args.source.sourceTable, - }); - builderModelFields = mergeBuilderCmsModelFieldsPreservingReferenceModels({ - existing: sourceMetadata.builderModelFields, - refreshed: builderModelFields, - }); - } catch (error) { - builderModelFieldsReadFailed = true; - const message = error instanceof Error ? error.message : String(error); - console.warn( - `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`, - ); + if (continueOffset > 0 && sourceMetadata.builderModelFields?.length) { + builderModelFields = sourceMetadata.builderModelFields; + } else { + try { + builderModelFields = await readBuilderCmsModelFields({ + model: args.source.sourceTable, + }); + builderModelFields = mergeBuilderCmsModelFieldsPreservingReferenceModels({ + existing: sourceMetadata.builderModelFields, + refreshed: builderModelFields, + }); + } catch (error) { + builderModelFieldsReadFailed = true; + const message = error instanceof Error ? error.message : String(error); + console.warn( + `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`, + ); + } } const projectionModelFields = builderModelFields && builderModelFields.length > 0 diff --git a/templates/content/actions/refresh-content-database-source.ts b/templates/content/actions/refresh-content-database-source.ts index 5d576815be..1c9f2f4ea9 100644 --- a/templates/content/actions/refresh-content-database-source.ts +++ b/templates/content/actions/refresh-content-database-source.ts @@ -35,6 +35,12 @@ export default defineAction({ .describe( "For paginated Builder CMS or Notion sources, read a bounded multi-page snapshot in this refresh.", ), + finishBuilderPagination: z + .boolean() + .optional() + .describe( + "Internal Builder continuation mode. Read every remaining page from the persisted continuation offset.", + ), expectedBuilderContinuationOffset: z .number() .int() @@ -75,6 +81,7 @@ export default defineAction({ source: claimedSource.source, now, runFullRefresh: args.fullRefresh === true, + finishPagination: args.finishBuilderPagination === true, refreshClaimId: claimedSource.claimId, }).catch(async (error: unknown) => { await releaseBuilderCmsSourceRefreshClaim({ diff --git a/templates/content/app/components/editor/database/DatabaseView.tsx b/templates/content/app/components/editor/database/DatabaseView.tsx index e8636a1f65..4b1b165b69 100644 --- a/templates/content/app/components/editor/database/DatabaseView.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.tsx @@ -1188,6 +1188,7 @@ function DatabaseTable({ sourceId, expectedBuilderContinuationOffset, fullRefresh: fullRefresh || undefined, + finishBuilderPagination: fullRefresh || undefined, }, { onError, diff --git a/templates/content/shared/api.ts b/templates/content/shared/api.ts index 5eb52f4a95..e824504c28 100644 --- a/templates/content/shared/api.ts +++ b/templates/content/shared/api.ts @@ -999,6 +999,7 @@ export interface RefreshContentDatabaseSourceRequest { documentId?: string; sourceId?: string; fullRefresh?: boolean; + finishBuilderPagination?: boolean; expectedBuilderContinuationOffset?: number; } From 212c4e84f7b3f0456a925443d5a9faf48194d7a0 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:01:27 -0400 Subject: [PATCH 12/21] Record hosted large-table acceptance --- ...content-blog-table-performance-research.md | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/plans/content-blog-table-performance-research.md b/plans/content-blog-table-performance-research.md index 349ff0015f..70c774e4e2 100644 --- a/plans/content-blog-table-performance-research.md +++ b/plans/content-blog-table-performance-research.md @@ -259,6 +259,79 @@ Alice approved these values on 2026-07-29 as the acceptance target for this lane | Prepare selected Builder update | Success or typed validation error within 2 seconds | | Close review surface | Within 100 ms | +## Hosted acceptance ledger — 2026-07-30 + +The Builder connection was restored and the production-like 584-row source was +tested on PR #2522's exact Netlify deploy previews. The writable fixture was a +task-owned Content database; the Builder model remained read-only. + +### Runtime improvements verified + +- Before the source-field projection fix, revealing Author took **15.49s**. + The bounded SQL projection reduced that to **969ms**; the later single-scan + path produced three hosted acknowledgements of **1,087ms**, **977ms**, and + **745ms** (median **977ms**) while the column painted optimistically in + **34.2ms** and the existing 100 rows never disappeared. +- Sort retained the existing rows immediately. The hosted background request + was **1.49-1.59s** cold, so visual continuity passes while the cold + acknowledgement remains above the 1s target. +- Opening the Builder review showed progressive changes within **465ms**. The + complete review request remained slow at **29.97s** (`app=26.14s`, + `db=3.81s`), so progressive acceptance passes but eager full completion does + not. +- Prefetching Builder models when Sources opens reduced the measured + space-to-model paint from **1,263ms** to **43ms**. + +### Fresh 584-row attach measurements + +Exact deploy `6a6b7e943e99e3000899a44a` at commit `83c45829f`: + +- feedback: **93ms**; +- first 100 useful rows: **5.56s**; +- attach request: **5.40s** (`app=5.20s`, `db=2.46s`); +- one complete continuation replaced five client-driven page continuations, + but still took **22.61s** (`app=22.35s`, `db=11.15s`); +- complete metadata was therefore about **28.1s**, improved from the earlier + run that was still at 300 rows after **59.5s** and only known complete by + **182s**. + +Exact deploy `6a6b8100dac828000830b1de` at commit `46ffed6a3` reused the +persisted row cursor and field schema: + +- feedback: **78ms**; +- first 100 useful rows: **6.18s**; +- attach request: **5.94s** (`app=5.64s`, `db=2.97s`); +- remaining-row continuation: **20.30s** (`app=20.07s`, `db=10.29s`); +- complete metadata: **29.38s** from pointer intent. + +The model-picker and immediate-feedback budgets now pass. First useful rows, +complete metadata, and background bodies do not: the prior completed run +reached 584 metadata rows between **59.5s and 182s**, then reported 582/584 +usable bodies ready by **280.5s**. The single-continuation implementation makes +metadata substantially faster, but the accepted **2s / 10s / 30-60s** attach +ladder is not yet met. + +### Verification and cleanup + +- Focused UI tests: **64 passed**. +- Focused Builder resync tests: **4 passed**, including explicit full refresh, + continuation convergence, a 597-row snapshot, and typed model-field failure. +- Content typecheck passed (with the existing Node 22.21/react-router version + warning). +- PR fast lanes, Content DB tests, typecheck, build, security, and parity were + green at `e717fdb3b`; the unrelated standalone Chat template E2E was red. +- All five task-owned hosted pages/databases were moved to Trash and permanently + deleted. The final Personal and Trash snapshots contained no + `__an_content_perf_2522_` marker. No Builder provider rows were written or + deleted. + +### Land decision + +Do not merge yet. The remaining ingestion work needs a separate bounded slice: +parallelize the Builder list-page reads, reduce hosted SQL round trips while +importing the remaining 484 rows, and bulk/lazily schedule body conversion +without making metadata acknowledgement await the full hydration queue. + The acceptance run must measure visible intent-to-paint and action completion separately. Background completion cannot be reported as blocking latency, and an optimistic paint cannot be reported as success before acknowledgement or rollback is observed. ## Local Performance Environment From f006d6a9709cacbd851202a251b9fcb3a51f026d Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:08:08 -0400 Subject: [PATCH 13/21] Improve large Builder database loading --- ...content-blog-table-performance-research.md | 262 +++++++++++++++++- .../actions/_builder-cms-read-client.test.ts | 161 ++++++++++- .../actions/_builder-cms-read-client.ts | 161 ++++++----- .../content/actions/_database-source-utils.ts | 124 +++++++-- templates/content/actions/_database-utils.ts | 197 +++++++++++-- .../actions/attach-content-database-source.ts | 171 +++++++----- .../content-database-lifecycle.db.test.ts | 78 ++++++ .../content-database-source-actions.test.ts | 65 +++++ .../content/actions/get-content-database.ts | 105 +------ .../preview-content-database-source-attach.ts | 100 +++++++ .../actions/process-builder-body-hydration.ts | 51 ++-- .../actions/query-content-database-items.ts | 73 +++++ .../refresh-content-database-source.ts | 57 ++-- .../resync-content-database-source.db.test.ts | 16 ++ .../DatabaseView.error-toasts.test.tsx | 7 + .../editor/database/DatabaseView.tsx | 108 ++++++-- .../app/hooks/use-content-database.test.ts | 52 +++- .../content/app/hooks/use-content-database.ts | 155 ++++++++++- ...tables-show-useful-rows-sooner-and-fini.md | 6 + templates/content/parity/matrix.md | 3 +- templates/content/parity/matrix.ts | 25 ++ templates/content/shared/api.ts | 24 ++ 22 files changed, 1635 insertions(+), 366 deletions(-) create mode 100644 templates/content/actions/preview-content-database-source-attach.ts create mode 100644 templates/content/actions/query-content-database-items.ts create mode 100644 templates/content/changelog/2026-07-30-large-builder-backed-tables-show-useful-rows-sooner-and-fini.md diff --git a/plans/content-blog-table-performance-research.md b/plans/content-blog-table-performance-research.md index 70c774e4e2..a538499e50 100644 --- a/plans/content-blog-table-performance-research.md +++ b/plans/content-blog-table-performance-research.md @@ -332,6 +332,228 @@ parallelize the Builder list-page reads, reduce hosted SQL round trips while importing the remaining 484 rows, and bulk/lazily schedule body conversion without making metadata acknowledgement await the full hydration queue. +## Shaped implementation plan for the four remaining failures + +Shaped on 2026-07-30 after hosted acceptance at `46ffed6a3`. This section is +the governing repair plan for PR #2522. It does not change the approved +operation budgets or permit a code-ready-only merge. + +### Refresher and current truth + +Routine interaction is no longer the large-table problem: Builder model +results paint in 43ms, attach feedback in 78-93ms, source columns in 34.2ms, +and progressive review in 465ms. Four assertions still fail: + +| Boundary | Current | Required | +| --- | ---: | ---: | +| First useful Builder rows after Attach | 6.18s | <=2s | +| Complete metadata for 584 rows | 29.38s | <=10s | +| Usable bodies for the 584-row source | <=280.5s in the last completed run | 30-60s | +| Cold Date-sort acknowledgement | 1.49-1.59s | <=1s | + +The first three failures share one ingestion path. The initial attach waits on +provider discovery, a projected first page, SQL import, source-row seeding, and +hydration-queue creation. The continuation then reads Builder pages serially +and repeats database setup reads before scheduling bodies. Body hydration +claims 25 jobs at a time but persists each converted row through its own +multi-statement transaction. Sort is separate: it retains rows visually, but a +constraint change still recomposes database/schema/source/context data when it +only needs a newly ordered item page. + +### Frozen implementation boundary + +- **Outcome:** every approved operation budget passes on the read-only + `Agent Native Blog Article Test` model with more than 500 rows. +- **Shipping surface:** `BuilderIO/agent-native`, Content database UI and + actions, for any Content user attaching and operating a large provider-backed + database; durable destination is the existing PR #2522 merged to `main`. +- **Governing architecture:** actions remain the only data-operation surface; + SQL remains authoritative; provider reads are projected and bounded; useful + UI is optimistic/progressive; unfinished body work remains durable and typed. +- **Acceptance story:** Alice attaches the 584-row Builder model, sees real + rows within 2s, receives complete metadata within 10s and bodies within 60s, + then sorts within 1s without blanking or losing the table. +- **Risk strategy:** system-ready. No feature flag and no merge until all four + assertions and the already-passing interaction assertions pass on the exact + PR head. + +Architecture grounding is required because the repair touches the shared +Content action contract, provider pagination, SQL import transactions, and the +durable hydration queue. Existing primitives to preserve are +`defineAction`, `useActionQuery`/`useActionMutation`, the Builder read client, +stable Builder import identities, refresh claims/cursors, the hydration queue, +targeted cache invalidation, and the current row-body conflict/CAS rules. No +custom REST route, provider write, direct UI fetch, new scheduler, large SQL +blob, or client-trusted provider snapshot belongs in this repair. + +### Slice 1 — make sort a page read, not a database re-open + +Add a page-shaped action, `query-content-database-items`, that reuses the +existing access filter, server-side constraint evaluator, item serializer, and +page-bounded source overlay but returns only: + +- the ordered/filtered 100-row item page; +- pagination/count and `tableQueryMode`; +- property values and source overlays needed by those rows. + +It must not reload database metadata, schema properties, full source status, +context path, review state, or hydration state unrelated to the returned page. +`DatabaseView` keeps the last good page visible with placeholder data and swaps +only the page cache when a sort changes. Other-tab/agent invalidation must still +refresh this action key. + +Instrumentation for this slice separates constraint planning, item IDs, +documents, property values, overlay, and serialization, aligned to +pointer-to-paint and request acknowledgement. Remove or gate temporary phase +logging before completion. + +Acceptance: + +- existing rows never disappear; +- sorted paint and acknowledgement are <=500ms warm / <=1s cold across five + hosted samples; +- exactly one page action follows one sort gesture; +- the response contains at most 100 rows and no document bodies/full source + snapshots; +- access, filters, multi-sort, null ordering, pagination, federation, and + other-tab refresh tests remain green. + +Stop and reshape if the existing server constraint evaluator cannot produce a +page without first materializing all documents/properties in application +memory; that would require a broader SQL query-planner decision. + +### Slice 2 — show truthful rows before attachment persistence settles + +Add a read-only `preview-content-database-source-attach` action and query it +when the user opens a Builder model leaf. It returns the same projected first +100 provider rows the attach path consumes, without bodies or writes. On +Attach, the hook paints those real provider rows immediately in a typed +`attaching` state. They can be opened as read-only previews but are not +represented as persisted/editable until the attach action succeeds. Failure +rolls the preview back and leaves the original database intact. + +In parallel, make the server attach read model fields and the first projected +content page concurrently, then batch the minimal document/item/source-identity +writes. The server must independently read and validate Builder data; it must +not trust provider rows echoed back by the browser. The successful response +reconciles the optimistic preview to stable document/item IDs without reflow. + +Acceptance: + +- feedback remains <=100ms; +- real first rows paint <=2s even when Attach is clicked before prefetch + completes; +- rows are visibly typed as attaching/read-only until persistence succeeds; +- success preserves selection/order and failure removes only the preview; +- no provider body or credential enters the client cache or SQL list rows; +- attach idempotency, duplicate-title identity, access, and rollback tests pass. + +Stop and reshape if a useful row must be editable before stable SQL identity +exists. The approved plan treats early rows as truthful read-only provider +previews, not fictional local rows. + +### Slice 3 — finish all metadata in one bounded provider/SQL pass + +Keep the current persisted continuation cursor, then change the complete read +to a bounded parallel window: + +- fetch offsets 100-500 with at most four concurrent Builder requests; +- preserve offset order and stable-ID deduplication; +- stop at the first short page; +- retry transient page failures through the existing retry policy; +- report a typed incomplete/error result if any required page fails rather + than treating a hole as completion. + +On the SQL side, make `importBuilderCmsEntriesAsDatabaseItems` return the +minimal serialized imported rows it just created. Use that result to avoid the +second `sourceSetupPayload` and full existing-row reread. In one bounded +transaction, bulk-write documents, memberships, database items, source rows, +materialized values for already-bound properties, the compact hydration queue +identity/version envelope, and final source progress. Do not reseed unchanged +source fields. Body conversion is explicitly outside the metadata +acknowledgement; only durable queue creation belongs inside it. + +Temporary spans must separately report provider-page wall time, import SQL +round trips, source-row/value writes, queue insertion, and final metadata +publication. The intended hosted budget allocation is <=3s provider, +<=5s SQL, and <=2s framework/network margin. + +Acceptance: + +- 584 metadata rows are complete <=10s from Attach across five hosted runs; +- the initial 100 rows remain visible throughout; +- the final active-source identity set is exactly 584 and stale rows prune only + after complete coverage; +- 584/597/1,000-row deterministic fixtures prove ordering, cursor completion, + duplicate IDs, short final pages, retries, suspicious-empty preservation, + concurrent refresh claims, and provider failure; +- SQL/query-count assertions prove no second full setup read and no per-row + write loop. + +Stop and reduce provider concurrency if Builder returns 429s or the existing +retry envelope cannot bound the window safely. Do not trade correctness for an +optimistic completed count. + +### Slice 4 — bulk body conversion without per-row database chatter + +Retain the durable hydration queue and open-row priority, but refactor one pump +into three explicit phases: + +1. claim and preload a bounded batch in bulk; +2. convert bodies with measured bounded concurrency; +3. persist successful, unavailable, superseded, and failed results with + chunked CASE/CAS updates rather than one multi-query transaction per row. + +The compare-and-swap protections remain mandatory: a local/Yjs edit, a newer +queue payload, or a changed source baseline must win over stale hydration. Use +portable Drizzle/shared SQL helpers and reviewed `getDbExec().execute()` only if +the shared query builder cannot express the chunked CAS. Queue payloads remain +compact identity/version/source references; raw bodies, screenshots, or large +Builder entries do not move into SQL. + +Start with 50 jobs per pump and conversion concurrency 8, then tune from hosted +phase spans rather than raising both blindly. The client immediately requests +the next batch from the returned queue count without interleaving a complete +database/source refetch. Existing `useDbSync` suppression and targeted +invalidation remain in force. Two genuinely bodyless rows may end in typed +`unavailable`; they count as settled, not hydrated. + +Acceptance: + +- the table remains interactive for the entire run; +- 582 hydrated plus two typed unavailable rows settle <=60s in five hosted + runs, with first-open row priority still <=2s; +- queue depth is monotonic except for explicit newer-version replacement; +- closing/reopening the page resumes durable work without duplication; +- supersession, local-edit conflict, retry cap, empty-body, bodyless-row, + source-change, and partial-batch failure tests pass; +- query-count evidence shows O(chunks) writes, not O(rows) transactions. + +Stop and reshape toward the existing framework job substrate only if the +durable client-driven pump cannot meet 60s while the table remains open. A new +background scheduler is deliberately deferred from the first repair. + +### Cross-slice verification and Land gate + +Use one deterministic local 584-row fixture for regression speed and one +task-owned hosted Content database attached read-only to +`Agent Native Blog Article Test`. Every hosted run records visual marks, +action request IDs, Server-Timing phases, request counts, row/body counts, and +cleanup proof. Instrumentation must align each server request with the visible +state transition it caused. + +Before Land, rerun all previously passing budgets as non-regression gates: +43ms model results, <=100ms attach feedback, <=100ms source-column paint, +<=1s median property acknowledgement with no >2s cold sample, progressive +review <=2s, row shell/body targets, and no table blanking. Run Content DB, +source/resync/hydration, hook/cache, parity, typecheck, format, build, security, +and independent H1-H5 browser QA against the exact PR head. Permanently delete +every task-owned hosted fixture and independently prove its marker absent. + +Land remains blocked until all four numbers and every preserved assertion pass. +The next authorized stage, after Alice approves this shaped plan, is `/work` +against this section of the existing brief. + The acceptance run must measure visible intent-to-paint and action completion separately. Background completion cannot be reported as blocking latency, and an optimistic paint cannot be reported as success before acknowledgement or rollback is observed. ## Local Performance Environment @@ -547,13 +769,12 @@ Stop and return to Shape if meeting the target requires changing the action arch ## Lifecycle ```yaml -stage: land -ledger-revision: land-blog-table-latency-r8 +stage: work +ledger-revision: work-2026-07-30-four-failures-v1 authority-source: >- - Alice invoked $work on 2026-07-29 against the frozen - shape-blog-table-latency-r5 artifact after approving the operation budgets - and authorizing the disconnected Builder test collection used in clip - BfEnyRiC4Pu7 as the proving surface + Alice invoked $work on 2026-07-30 after approving the four remaining + operation budgets and reconnecting the PR #2522 deployment to the isolated + Builder test collection used in clip BfEnyRiC4Pu7 authorized-scope: repositories: - /Users/alicemoore/.codex/worktrees/8c63/agent-native @@ -572,14 +793,13 @@ allowed-mutations: - pull-request - deploy - schema - - merge write-targets: artifacts: - /Users/alicemoore/.codex/worktrees/8c63/agent-native/plans/content-blog-table-performance-research.md execution-lane: worktree: /Users/alicemoore/.codex/worktrees/8c63/agent-native branch: codex/content-large-database-performance - refreshed-base: origin/main@ce426feef1ebeb370fca581291c9391339a757ed + refreshed-base: origin/main@58bdd10a7f60ba08c01fa425f836732c73c5f5b6 remote-default-branch: origin/main calling-task-id: unavailable from current host surface parent-task-id: none @@ -613,6 +833,23 @@ work-progress: - independent human QA on a host that exposes the in-app browser - final repair verification, exact-head PR prose receipt, review, and CI temporary-probes: + - id: content-four-failure-action-phases + status: active + gate: PR #2522 preview and isolated local performance runtime only + records: >- + content-free phase names and durations for Builder attach preview, + primary attach, metadata continuation, body hydration batches, and + constrained table pages; action responses carry the same timing array + so browser network evidence can be joined to the visible boundary + removal-trigger: after the final joined preview acceptance replay + - id: content-four-failure-browser-network-waterfall + status: active + gate: browser developer timing capture during the frozen H1-H5 replay + records: >- + pointer intent, semantic row/progress state, action URL, request and + response timestamps, x-agent-native-request-id, status, and response + timing phases; no bodies, credentials, or provider payloads + removal-trigger: after the final joined preview acceptance replay - id: content-database-response-trace status: removed gate: >- @@ -938,9 +1175,8 @@ performed by Land. Invalidated by: a change to the approved outcome/budgets, named Agent Native Content shipping surface or constituency, action/SQL/progressive-provider architecture, acceptance story, or system-ready risk strategy. -Task attention: `manual-custody`; exact Builder credentials and an independent -browser-capable QA host are required before Land can recompute system-ready. +Task attention: `autonomous`; the connected PR preview and in-app browser are +available for the four-failure Work loop. -Next: Work resumes internally when the credentialed Builder runtime and -independent browser-capable QA host are available; then rerun exact-provider and -H1-H5 acceptance against the unchanged frozen story. +Next: deploy the exact Work artifact to PR #2522, run correlated exact-provider +acceptance, remove temporary probes, and replay the clean artifact before Land. diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index 214f5bdee9..cb8826e64e 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -946,7 +946,7 @@ describe("Builder CMS read client", () => { }); expect(requests.map((request) => request.offset)).toEqual([ - 0, 100, 200, 300, 400, 500, + 0, 100, 200, 300, 400, 500, 600, 700, 800, ]); expect(requests.every((request) => request.limit > 0)).toBe(true); expect(result.entries).toHaveLength(597); @@ -958,6 +958,165 @@ describe("Builder CMS read client", () => { }); }); + it("reads projected Content API pages in bounded parallel windows and preserves offset order", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + let activeRequests = 0; + let maxActiveRequests = 0; + const resolvers = new Map void>(); + const fetchImpl = vi.fn(async (input: URL) => { + const offset = Number(input.searchParams.get("offset")); + activeRequests += 1; + maxActiveRequests = Math.max(maxActiveRequests, activeRequests); + return await new Promise((resolve) => { + resolvers.set(offset, () => { + resolvers.delete(offset); + activeRequests -= 1; + resolve( + new Response( + JSON.stringify({ + results: Array.from({ length: 100 }, (_, index) => ({ + id: `builder-entry-${offset + index + 1}`, + data: { title: `Builder title ${offset + index + 1}` }, + })), + }), + { status: 200 }, + ), + ); + }); + }); + }); + + const read = readBuilderCmsContentEntries({ + model: "blog_article", + limit: 500, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await vi.waitFor(() => expect(resolvers.size).toBe(1)); + resolvers.get(0)?.(); + await vi.waitFor(() => expect(resolvers.size).toBe(4)); + expect(maxActiveRequests).toBe(4); + for (const offset of [400, 300, 200, 100]) resolvers.get(offset)?.(); + + const result = await read; + + expect(result.entries.map((entry) => entry.id)).toEqual( + Array.from({ length: 500 }, (_, index) => `builder-entry-${index + 1}`), + ); + expect(maxActiveRequests).toBeLessThanOrEqual(4); + }); + + it("stops at the first short projected page and ignores later in-flight pages", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const requestedOffsets: number[] = []; + const fetchImpl = vi.fn(async (input: URL) => { + const offset = Number(input.searchParams.get("offset")); + requestedOffsets.push(offset); + const resultCount = offset === 100 ? 10 : 100; + return new Response( + JSON.stringify({ + results: Array.from({ length: resultCount }, (_, index) => ({ + id: `builder-entry-${offset + index + 1}`, + data: { title: `Builder title ${offset + index + 1}` }, + })), + }), + { status: 200 }, + ); + }); + + const result = await readBuilderCmsContentEntries({ + model: "blog_article", + limit: 400, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(requestedOffsets).toEqual([0, 100, 200, 300]); + expect(result.entries.map((entry) => entry.id)).toEqual( + Array.from({ length: 110 }, (_, index) => `builder-entry-${index + 1}`), + ); + expect(result.progress).toMatchObject({ + nextOffset: 110, + fetchedEntryCount: 110, + hasMore: false, + partial: false, + }); + }); + + it("keeps stable-ID deduplication across projected Content API windows", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + const offset = Number(input.searchParams.get("offset")); + const ids = + offset === 0 + ? Array.from({ length: 100 }, (_, index) => index + 1) + : offset === 100 + ? Array.from({ length: 100 }, (_, index) => index + 51) + : Array.from({ length: 50 }, (_, index) => index + 151); + return new Response( + JSON.stringify({ + results: ids.map((id) => ({ + id: `builder-entry-${id}`, + data: { title: `Builder title ${id}` }, + })), + }), + { status: 200 }, + ); + }); + + const result = await readBuilderCmsContentEntries({ + model: "blog_article", + limit: 200, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(result.entries.map((entry) => entry.id)).toEqual( + Array.from({ length: 200 }, (_, index) => `builder-entry-${index + 1}`), + ); + }); + + it("returns a typed error when a required projected page fails", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + if (input.searchParams.get("offset") === "100") { + return new Response("bad request", { status: 400 }); + } + return new Response( + JSON.stringify({ + results: Array.from({ length: 100 }, (_, index) => ({ + id: `builder-entry-${index + 1}`, + data: { title: `Builder title ${index + 1}` }, + })), + }), + { status: 200 }, + ); + }); + + await expect( + readBuilderCmsContentEntries({ + model: "blog_article", + limit: 400, + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toMatchObject({ + state: "error", + entries: [], + message: "Builder CMS read failed with HTTP 400.", + progress: { readMode: "builder-api" }, + }); + }); + it("continues a 597-entry MCP source from offset 500", async () => { resolveBuilderCredentialMock.mockImplementation(async (key) => key === "BUILDER_PRIVATE_KEY" ? "private-key" : null, diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index 9c24cd886a..923caa75da 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -163,6 +163,7 @@ type BuilderMcpToolResult = { const BUILDER_CMS_DEFAULT_READ_LIMIT = 500; const BUILDER_CMS_MAX_READ_LIMIT = 10_000; const BUILDER_CMS_PAGE_SIZE = 100; +const BUILDER_CMS_PROJECTED_READ_CONCURRENCY = 4; const BUILDER_CMS_READ_RETRIES = 2; const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [ "id", @@ -192,6 +193,10 @@ const BUILDER_CMS_HEAVY_BODY_FIELD_PATHS = [ const BUILDER_CMS_FIELD_PATH_PATTERN = /^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/; +type BuilderCmsContentPageResult = + | { pageLimit: number; pageEntries: BuilderCmsSourceEntry[] } + | { pageLimit: number; error: string }; + function normalizeBuilderCmsListFieldPath(fieldPath: string) { const trimmed = fieldPath.trim(); if (!trimmed || !BUILDER_CMS_FIELD_PATH_PATTERN.test(trimmed)) return null; @@ -1008,73 +1013,103 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { const seenIds = new Set(); let pagesRead = 0; let hasMore = false; - for ( - let offset = startOffset; - entries.length < limit; - offset += BUILDER_CMS_PAGE_SIZE - ) { - const pageUrl = new URL(url); - const pageLimit = readPageLimit(limit - entries.length); - pageUrl.searchParams.set("limit", String(pageLimit)); - pageUrl.searchParams.set("offset", String(offset)); + const concurrency = + args.rawData === true ? 1 : BUILDER_CMS_PROJECTED_READ_CONCURRENCY; + const errorResult = (message: string): BuilderCmsReadResult => ({ + state: "error", + entries: [], + fetchedAt, + message, + progress: { + requestedLimit: limit, + pageSize: BUILDER_CMS_PAGE_SIZE, + startOffset, + nextOffset: startOffset + entries.length, + fetchedEntryCount: startOffset + entries.length, + hasMore, + partial: Boolean(args.maxPages) && hasMore, + readMode: "builder-api", + }, + }); - let response: Response; - try { - response = await fetchBuilderContentPage({ - fetchImpl: args.fetchImpl, - url: pageUrl, - privateKey: args.privateKey, - }); - } catch (error) { - return { - state: "error", - entries: [], - fetchedAt, - message: - error instanceof Error - ? `Builder CMS read failed: ${error.message}` - : "Builder CMS read failed.", - progress: { - requestedLimit: limit, - pageSize: BUILDER_CMS_PAGE_SIZE, - startOffset, - nextOffset: startOffset + entries.length, - fetchedEntryCount: startOffset + entries.length, - hasMore, - partial: Boolean(args.maxPages) && hasMore, - readMode: "builder-api", + while (entries.length < limit) { + const pagesRemaining = args.maxPages + ? args.maxPages - pagesRead + : Number.POSITIVE_INFINITY; + if (pagesRemaining <= 0) break; + const pageCount = Math.min( + pagesRead === 0 ? 1 : concurrency, + pagesRemaining, + Math.ceil((limit - entries.length) / BUILDER_CMS_PAGE_SIZE), + ); + const pageRequests = Array.from({ length: pageCount }, (_, index) => { + const pageUrl = new URL(url); + const pageLimit = readPageLimit( + limit - entries.length - index * BUILDER_CMS_PAGE_SIZE, + ); + pageUrl.searchParams.set("limit", String(pageLimit)); + pageUrl.searchParams.set( + "offset", + String(startOffset + (pagesRead + index) * BUILDER_CMS_PAGE_SIZE), + ); + return { pageLimit, pageUrl }; + }); + const pageResults = await Promise.all( + pageRequests.map( + async ({ + pageLimit, + pageUrl, + }): Promise => { + try { + const response = await fetchBuilderContentPage({ + fetchImpl: args.fetchImpl, + url: pageUrl, + privateKey: args.privateKey, + }); + if (!response.ok) { + return { + pageLimit, + error: `Builder CMS read failed with HTTP ${response.status}.`, + }; + } + const json = (await response.json()) as unknown; + const pageEntries = entryArrayFromResponse(json) + .map((entry) => normalizeBuilderCmsApiEntry(entry, args.model)) + .filter((entry): entry is BuilderCmsSourceEntry => + Boolean(entry), + ); + return { pageLimit, pageEntries }; + } catch (error) { + return { + pageLimit, + error: + error instanceof Error + ? `Builder CMS read failed: ${error.message}` + : "Builder CMS read failed.", + }; + } }, - }; - } + ), + ); - if (!response.ok) { - return { - state: "error", - entries: [], - fetchedAt, - message: `Builder CMS read failed with HTTP ${response.status}.`, - progress: { - requestedLimit: limit, - pageSize: BUILDER_CMS_PAGE_SIZE, - startOffset, - nextOffset: startOffset + entries.length, - fetchedEntryCount: startOffset + entries.length, - hasMore, - partial: Boolean(args.maxPages) && hasMore, - readMode: "builder-api", - }, - }; + let stoppedOnShortPage = false; + for (const pageResult of pageResults) { + if ("error" in pageResult) return errorResult(pageResult.error); + const appended = appendUniqueBuilderEntries( + entries, + seenIds, + pageResult.pageEntries, + ); + pagesRead += 1; + hasMore = + pageResult.pageEntries.length >= pageResult.pageLimit && appended > 0; + if (args.maxPages && pagesRead >= args.maxPages) break; + if (!hasMore) { + stoppedOnShortPage = true; + break; + } } - - const json = (await response.json()) as unknown; - const pageEntries = entryArrayFromResponse(json) - .map((entry) => normalizeBuilderCmsApiEntry(entry, args.model)) - .filter((entry): entry is BuilderCmsSourceEntry => Boolean(entry)); - const appended = appendUniqueBuilderEntries(entries, seenIds, pageEntries); - pagesRead += 1; - hasMore = pageEntries.length >= pageLimit && appended > 0; - if (args.maxPages && pagesRead >= args.maxPages) break; - if (!hasMore) break; + if (stoppedOnShortPage || !hasMore) break; } return { diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 29d922c90d..043c12f445 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -124,6 +124,24 @@ function stableBuilderImportId( .slice(0, 24)}`; } +export function builderCmsImportIds(args: { + ownerEmail: string; + databaseId: string; + sourceTable: string; + entryId: string; +}) { + const identity = [ + args.ownerEmail, + args.databaseId, + args.sourceTable, + args.entryId, + ]; + return { + documentId: stableBuilderImportId("builder-doc", identity), + itemId: stableBuilderImportId("builder-item", identity), + }; +} + const DEFAULT_SOURCE_CAPABILITIES: ContentDatabaseSourceCapabilities = { canRefresh: true, canCreateChangeSets: true, @@ -959,8 +977,8 @@ function builderBodyUsesCurrentMediaConverter( const BUILDER_BODY_HYDRATION_BACKGROUND_PRIORITY = 10; const BUILDER_BODY_HYDRATION_OPEN_PRIORITY = 0; -const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 25; -const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 6; +const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 50; +const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 24; const BUILDER_BODY_HYDRATION_MAX_ATTEMPTS = 5; const BUILDER_BODY_HYDRATION_CODEC_VERSION = "readable-native-images-authoritative-raw-baseline-v9"; @@ -5680,10 +5698,11 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { }): Promise<{ imported: number; importedEntriesByDocumentId: Map; + importedItems: ContentDatabaseItem[]; }> { const importedEntriesByDocumentId = new Map(); if (args.entries.length === 0) { - return { imported: 0, importedEntriesByDocumentId }; + return { imported: 0, importedEntriesByDocumentId, importedItems: [] }; } const db = getDb(); const [databaseDocument] = await db @@ -5772,6 +5791,9 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { .filter((row) => !representedDocumentIds.has(row.document.id)) .map((row) => row.document.title.trim().toLowerCase()), ); + const currentItemByDocumentId = new Map( + currentItems.map((row) => [row.document.id, row.item]), + ); // Reads MAX(position) for both `documents` and `content_database_items` // then batch-inserts at MAX+1.. — serialize the whole read-through-write @@ -5817,26 +5839,29 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { continue; } + const { documentId, itemId } = builderCmsImportIds({ + ownerEmail: args.database.ownerEmail, + databaseId: args.database.id, + sourceTable: args.sourceTable, + entryId: entry.id, + }); + const existingDeterministicItem = + currentItemByDocumentId.get(documentId); + if (existingDeterministicItem?.id === itemId) { + // A prior attach can commit the deterministic document/item and + // fail before linking its source row. Treat that pair as the + // same Builder identity so refresh repairs the missing link + // without synthesizing a duplicate response item. + importedEntriesByDocumentId.set(documentId, entry); + continue; + } + const title = entry.title.trim() || entry.id; const titleKey = title.toLowerCase(); if (!args.skipTitleDedup && existingUnlinkedTitles.has(titleKey)) { continue; } - const importIdentity = [ - args.database.ownerEmail, - args.database.id, - args.sourceTable, - entry.id, - ]; - const documentId = stableBuilderImportId( - "builder-doc", - importIdentity, - ); - const itemId = stableBuilderImportId( - "builder-item", - importIdentity, - ); documentRows.push({ id: documentId, spaceId: databaseSpaceId, @@ -5894,7 +5919,44 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { ); }); - return { imported: itemRows.length, importedEntriesByDocumentId }; + const importedItems: ContentDatabaseItem[] = itemRows.map( + (item, index) => { + const document = documentRows[index]; + return { + id: item.id!, + databaseId: args.database.id, + document: { + id: document.id!, + parentId: document.parentId ?? null, + title: document.title ?? "Untitled", + content: "", + icon: document.icon ?? null, + position: document.position ?? 0, + isFavorite: false, + hideFromSearch: Boolean(document.hideFromSearch), + visibility: document.visibility ?? "private", + accessRole: "owner", + canEdit: true, + canManage: true, + createdAt: document.createdAt!, + updatedAt: document.updatedAt!, + }, + position: item.position ?? 0, + properties: [], + bodyHydration: { + status: "pending", + attemptedAt: null, + error: null, + version: null, + }, + }; + }, + ); + return { + imported: itemRows.length, + importedEntriesByDocumentId, + importedItems, + }; }, ), ); @@ -5908,7 +5970,7 @@ export async function resyncBuilderCmsSourceSnapshot(args: { finishPagination?: boolean; refreshClaimId?: string; }) { - let { properties, response } = await sourceSetupPayload(args.database.id); + const setupPromise = sourceSetupPayload(args.database.id); const db = getDb(); const sourceMetadata = parseObject(args.source.metadataJson) ?? {}; @@ -5955,6 +6017,10 @@ export async function resyncBuilderCmsSourceSnapshot(args: { builderModelFields && builderModelFields.length > 0 ? builderModelFields : (sourceMetadata.builderModelFields ?? []); + const existingRowsPromise = db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); const builderRead = await readBuilderCmsContentEntries({ model: args.source.sourceTable, fieldPaths: [ @@ -5990,10 +6056,9 @@ export async function resyncBuilderCmsSourceSnapshot(args: { "Builder source refresh claim was lost before snapshot mutation.", ); } - let existingRows = await db - .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); + const { properties, response: initialResponse } = await setupPromise; + let response = initialResponse; + const existingRows = await existingRowsPromise; const readStartOffset = builderRead.progress?.startOffset ?? 0; const activeReadSourceRowIdSet = new Set(activeReadSourceRowIds); const suspiciousEmptyRead = @@ -6040,11 +6105,14 @@ export async function resyncBuilderCmsSourceSnapshot(args: { }); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; if (importResult.imported > 0) { - ({ properties, response } = await sourceSetupPayload(args.database.id)); - existingRows = await db - .select() - .from(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); + const existingItemIds = new Set(response.items.map((item) => item.id)); + const importedItems = importResult.importedItems.filter( + (item) => !existingItemIds.has(item.id), + ); + response = { + ...response, + items: [...response.items, ...importedItems], + }; } } const builderEntriesByDocumentId = diff --git a/templates/content/actions/_database-utils.ts b/templates/content/actions/_database-utils.ts index 829d32f805..2b1127d59d 100644 --- a/templates/content/actions/_database-utils.ts +++ b/templates/content/actions/_database-utils.ts @@ -5,9 +5,11 @@ import { import { accessFilter, ROLE_RANK, + resolveAccess, type ShareRole, } from "@agent-native/core/sharing"; import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; import { getDocumentContextPath } from "../server/lib/document-context.js"; @@ -30,6 +32,7 @@ import { favoriteDocumentIds } from "./_content-favorites.js"; import { listContentOrganizationMemberships, normalizeContentSpaceEmail, + resolveContentSpaceAccess, } from "./_content-space-access.js"; import { getAllContentDatabaseSourceSnapshots } from "./_database-source-utils.js"; import { @@ -49,6 +52,47 @@ export { getDocumentContextPath }; export const CONTENT_DATABASE_MAX_READ_LIMIT = 5_000; +export const contentDatabaseTableQuerySchema = z + .object({ + search: z.string().max(500), + filters: z + .array( + z.object({ + key: z.string(), + label: z.string(), + operator: z.enum([ + "contains", + "equals", + "does_not_equal", + "greater_than", + "less_than", + "before", + "after", + "between", + "is_checked", + "is_unchecked", + "is_empty", + "is_not_empty", + ]), + value: z.string(), + filterGroupId: z.string().optional(), + parentFilterGroupId: z.string().optional(), + }), + ) + .max(50), + sorts: z + .array( + z.object({ + key: z.string(), + label: z.string(), + direction: z.enum(["asc", "desc"]), + }), + ) + .max(20), + filterMode: z.enum(["and", "or"]), + }) + .optional(); + async function contentDatabaseTableQueryMode( databaseId: string, query: ContentDatabaseTableQuery | undefined, @@ -320,7 +364,87 @@ function serializeDocument( }; } -export async function getContentDatabaseResponse( +export type ContentDatabasePageResponse = Pick< + ContentDatabaseResponse, + "items" | "source" | "sources" | "pagination" | "tableQueryMode" +>; + +export type ContentDatabaseReadResolution = + | { + available: true; + database: typeof schema.contentDatabases.$inferSelect; + } + | { + available: false; + reason: "not_found" | "deleted"; + databaseId: string; + documentId: string | null; + deletedAt?: string | null; + message: string; + }; + +export async function resolveContentDatabaseRead(args: { + databaseId?: string; + documentId?: string; +}): Promise { + const db = getDb(); + let databaseId = args.databaseId; + if (!databaseId && args.documentId) { + const [database] = await db + .select({ id: schema.contentDatabases.id }) + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.documentId, args.documentId)); + databaseId = database?.id; + } + if (!databaseId) { + throw new Error("Either databaseId or documentId is required."); + } + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, databaseId)); + if (!database) { + return { + available: false, + reason: "not_found", + databaseId, + documentId: args.documentId ?? null, + message: `Database "${databaseId}" not found`, + }; + } + + let canRead = Boolean(await resolveAccess("document", database.documentId)); + if (!canRead && database.systemRole === "files" && database.spaceId) { + try { + await resolveContentSpaceAccess(database.spaceId); + canRead = true; + } catch { + canRead = false; + } + } + if (!canRead) throw new Error(`Database "${databaseId}" not found`); + + if (database.deletedAt) { + return { + available: false, + reason: "deleted", + databaseId: database.id, + documentId: database.documentId, + deletedAt: database.deletedAt, + message: `Database "${database.id}" has been deleted`, + }; + } + + return { available: true, database }; +} + +type ContentDatabasePageBuild = ContentDatabasePageResponse & { + databaseRecord: typeof schema.contentDatabases.$inferSelect; + properties: ContentDatabaseResponse["properties"]; +}; + +export async function getContentDatabasePageResponse( databaseId: string, options: { limit?: number; @@ -328,26 +452,22 @@ export async function getContentDatabaseResponse( tableQuery?: ContentDatabaseTableQuery; includeSources?: boolean; documentIds?: string[]; + database?: typeof schema.contentDatabases.$inferSelect; } = {}, -): Promise { +): Promise { const db = getDb(); - const [database] = await db - .select() - .from(schema.contentDatabases) - .where(eq(schema.contentDatabases.id, databaseId)); + const database = + options.database ?? + ( + await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, databaseId)) + )[0]; if (!database || database.deletedAt) { throw new Error(`Database "${databaseId}" not found`); } - const [databaseDocument] = await db - .select({ - id: schema.documents.id, - parentId: schema.documents.parentId, - description: schema.documents.description, - }) - .from(schema.documents) - .where(eq(schema.documents.id, database.documentId)); - // PURE read: the primary "Content" Blocks field is seeded at create time and // by the one-time startup repair — never here. Reading a database (including a // shared one a viewer is opening) must not mutate schema. @@ -749,13 +869,8 @@ export async function getContentDatabaseResponse( // Opt-in federated columns (a secondary field the user added via the picker) // get their per-row values from the matched overlay at read time. const itemsWithOverlay = applyFederatedOverlayValues(federatedItems); - const contextPath = databaseDocument - ? await getDocumentContextPath(databaseDocument) - : []; - return { - database: serializeDatabase(database, databaseDocument?.description ?? ""), - contextPath, + databaseRecord: database, properties: responseProperties, items: itemsWithOverlay, source: pagedPrimary, @@ -778,6 +893,46 @@ export async function getContentDatabaseResponse( }; } +export async function getContentDatabaseResponse( + databaseId: string, + options: { + limit?: number; + offset?: number; + tableQuery?: ContentDatabaseTableQuery; + includeSources?: boolean; + documentIds?: string[]; + database?: typeof schema.contentDatabases.$inferSelect; + } = {}, +): Promise { + const page = await getContentDatabasePageResponse(databaseId, options); + const db = getDb(); + const [databaseDocument] = await db + .select({ + id: schema.documents.id, + parentId: schema.documents.parentId, + description: schema.documents.description, + }) + .from(schema.documents) + .where(eq(schema.documents.id, page.databaseRecord.documentId)); + const contextPath = databaseDocument + ? await getDocumentContextPath(databaseDocument) + : []; + + return { + database: serializeDatabase( + page.databaseRecord, + databaseDocument?.description ?? "", + ), + contextPath, + properties: page.properties, + items: page.items, + source: page.source, + sources: page.sources, + pagination: page.pagination, + tableQueryMode: page.tableQueryMode, + }; +} + export async function isSoftDeletedDatabaseDocument(documentId: string) { const db = getDb(); const [ownedDatabase] = await db diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index 826192de26..d49fd1530e 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -17,6 +17,7 @@ import { type BuilderCmsReadResult, } from "./_builder-cms-read-client.js"; import type { BuilderCmsSourceEntry } from "./_builder-cms-source-adapter.js"; +import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { getContentDatabaseSourceAdapter } from "./_content-database-source-adapters.js"; import { databaseSourceExistsForTable, @@ -66,24 +67,20 @@ export async function readInitialBuilderCmsAttachSource( dependencies: { readModelFields?: typeof readBuilderCmsModelFields; readEntries?: typeof readBuilderCmsContentEntries; + fieldPaths?: readonly string[]; } = {}, ) { const readModelFields = dependencies.readModelFields ?? readBuilderCmsModelFields; const readEntries = dependencies.readEntries ?? readBuilderCmsContentEntries; - let modelFields: BuilderCmsModelFieldSummary[] = []; - let modelFieldsError: unknown = null; - try { - modelFields = await readModelFields({ model: sourceTable }); - } catch (error) { - modelFieldsError = error; - } - const read = await readInitialBuilderCmsAttachEntries( - sourceTable, - readEntries, - modelFields.map((field) => `data.${field.name}`), - ); - if (modelFieldsError) throw modelFieldsError; + const [modelFields, read] = await Promise.all([ + readModelFields({ model: sourceTable }), + readInitialBuilderCmsAttachEntries( + sourceTable, + readEntries, + dependencies.fieldPaths, + ), + ]); return { read, modelFields }; } @@ -231,6 +228,7 @@ export default defineAction({ .string() .optional() .describe("Source table/model name, for example content_items."), + builderFieldPaths: z.array(z.string().max(500)).max(200).optional(), relationshipMode: z .enum(["items", "details"]) .optional() @@ -259,6 +257,12 @@ export default defineAction({ const now = new Date().toISOString(); const sourceType = (args.sourceType ?? "mock-local") as ContentDatabaseSourceType; + const timing = + sourceType === "builder-cms" + ? createBuilderSourceTiming("attach-content-database-source") + : null; + const measure = (name: string, operation: () => Promise) => + timing ? timing.measure(name, operation) : operation(); if (sourceType === "notion-database" && !args.sourceTable?.trim()) { throw new Error( "sourceTable must be a Notion data-source ID returned by list-notion-database-sources.", @@ -578,38 +582,48 @@ export default defineAction({ : []; const builderInitial = sourceType === "builder-cms" - ? await readInitialBuilderCmsAttachSource(sourceTable) + ? await measure("provider-first-page", () => + readInitialBuilderCmsAttachSource(sourceTable, { + fieldPaths: args.builderFieldPaths, + }), + ) : null; - const sourceId = await replaceSourceMetadata({ - database, - source: existingSource, - sourceType, - sourceName, - sourceTable, - now, - }); + const sourceId = await measure("persist-source", () => + replaceSourceMetadata({ + database, + source: existingSource, + sourceType, + sourceName, + sourceTable, + now, + }), + ); const builderModelFields = builderInitial?.modelFields ?? []; const builderRead = builderInitial?.read ?? null; const builderEntries = builderRead?.state === "live" ? builderRead.entries : []; let importedEntriesByDocumentId = new Map(); if (builderRead?.state === "live") { - const importResult = await importBuilderCmsEntriesAsDatabaseItems({ - database, - entries: builderEntries, - now, - sourceTable, - existingSourceRows, - }); + const importResult = await measure("import-first-page", () => + importBuilderCmsEntriesAsDatabaseItems({ + database, + entries: builderEntries, + now, + sourceTable, + existingSourceRows, + }), + ); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; } - const refreshedSetup = await sourceSetupPayload( - database.id, - initialBuilderAttachmentSetupOptions({ - builderRead, - importedEntriesByDocumentId, - }), + const refreshedSetup = await measure("read-imported-page", () => + sourceSetupPayload( + database.id, + initialBuilderAttachmentSetupOptions({ + builderRead, + importedEntriesByDocumentId, + }), + ), ); const builderEntriesByDocumentId = builderRead?.state === "live" @@ -625,53 +639,62 @@ export default defineAction({ builderEntriesByDocumentId?.set(documentId, entry); } - await seedMockSourceFields({ - sourceId, - ownerEmail: database.ownerEmail, - sourceType, - properties: refreshedSetup.properties, - builderModelFields, - builderSampleEntries: builderEntries, - now, - }); - await seedMockSourceRows({ - sourceId, - ownerEmail: database.ownerEmail, - sourceType, - sourceTable, - items: refreshedSetup.response.items, - now, - builderEntriesByDocumentId, - }); - if (sourceType === "builder-cms" && builderRead?.state === "live") { - await enqueueBuilderBodyHydrationForItems({ + await measure("persist-first-page", async () => { + await seedMockSourceFields({ sourceId, ownerEmail: database.ownerEmail, - orgId: database.orgId, - sourceTable, - items: refreshedSetup.response.items, - builderEntriesByDocumentId, + sourceType, + properties: refreshedSetup.properties, + builderModelFields, + builderSampleEntries: builderEntries, now, }); - } - if (sourceType === "builder-cms" && builderRead) { - await updateBuilderCmsSourceReadMetadata({ + await seedMockSourceRows({ sourceId, + ownerEmail: database.ownerEmail, + sourceType, sourceTable, - readState: builderRead.state, - entryCount: builderRead.entries.length, - matchedRowCount: builderEntriesByDocumentId?.size ?? 0, - fetchedAt: builderRead.fetchedAt, + items: refreshedSetup.response.items, now, - message: builderRead.message, - builderModelFields, - ...builderCmsAttachReadMetadata(builderRead), + builderEntriesByDocumentId, }); - } - - return getContentDatabaseResponse(database.id, { - limit: args.limit, - offset: args.offset, + if (sourceType === "builder-cms" && builderRead?.state === "live") { + await enqueueBuilderBodyHydrationForItems({ + sourceId, + ownerEmail: database.ownerEmail, + orgId: database.orgId, + sourceTable, + items: refreshedSetup.response.items, + builderEntriesByDocumentId, + now, + }); + } + if (sourceType === "builder-cms" && builderRead) { + await updateBuilderCmsSourceReadMetadata({ + sourceId, + sourceTable, + readState: builderRead.state, + entryCount: builderRead.entries.length, + matchedRowCount: builderEntriesByDocumentId?.size ?? 0, + fetchedAt: builderRead.fetchedAt, + now, + message: builderRead.message, + builderModelFields, + ...builderCmsAttachReadMetadata(builderRead), + }); + } }); + + const response = await measure("response", () => + getContentDatabaseResponse(database.id, { + limit: args.limit, + offset: args.offset, + }), + ); + if (timing) { + response.timings = timing.finish(); + timing.log("succeeded"); + } + return response; }, }); diff --git a/templates/content/actions/content-database-lifecycle.db.test.ts b/templates/content/actions/content-database-lifecycle.db.test.ts index f18914fe17..c64d5b14fc 100644 --- a/templates/content/actions/content-database-lifecycle.db.test.ts +++ b/templates/content/actions/content-database-lifecycle.db.test.ts @@ -21,6 +21,7 @@ let moveDocumentAction: typeof import("./move-document.js").default; let deleteContentDatabaseAction: typeof import("./delete-content-database.js").default; let restoreContentDatabaseAction: typeof import("./restore-content-database.js").default; let getContentDatabaseAction: typeof import("./get-content-database.js").default; +let queryContentDatabaseItemsAction: typeof import("./query-content-database-items.js").default; let listDocumentsAction: typeof import("./list-documents.js").default; let listTrashedContentDatabasesAction: typeof import("./list-trashed-content-databases.js").default; let getDocumentAction: typeof import("./get-document.js").default; @@ -53,6 +54,9 @@ beforeAll(async () => { .default; getContentDatabaseAction = (await import("./get-content-database.js")) .default; + queryContentDatabaseItemsAction = ( + await import("./query-content-database-items.js") + ).default; listDocumentsAction = (await import("./list-documents.js")).default; listTrashedContentDatabasesAction = ( await import("./list-trashed-content-databases.js") @@ -864,6 +868,14 @@ describe("content database soft-delete actions and reads", () => { reason: "deleted", databaseId, }); + const pageResponse = await runWithRequestContext({ userEmail: OWNER }, () => + queryContentDatabaseItemsAction.run({ databaseId }), + ); + expect(pageResponse).toMatchObject({ + available: false, + reason: "deleted", + databaseId, + }); const listResponse = await runWithRequestContext({ userEmail: OWNER }, () => listDocumentsAction.run({}), @@ -874,6 +886,72 @@ describe("content database soft-delete actions and reads", () => { expect(listedIds.has(rowDocumentId)).toBe(false); }); + it("returns only the ordered, filtered database page and preserves read access", async () => { + const { databaseId, databaseDocumentId } = await createDatabase({}); + const db = getDb(); + const now = new Date().toISOString(); + const rows = await Promise.all( + [ + { title: "Zebra", position: 0 }, + { title: "Apricot", position: 1 }, + { title: "Banana", position: 2 }, + ].map(async ({ title, position }) => { + const documentId = await createDocument({ + parentId: databaseDocumentId, + title, + }); + const id = nextId("item"); + await db.insert(schema.contentDatabaseItems).values({ + id, + ownerEmail: OWNER, + databaseId, + documentId, + position, + createdAt: now, + updatedAt: now, + }); + return { id, documentId }; + }), + ); + + const response = await runWithRequestContext({ userEmail: OWNER }, () => + queryContentDatabaseItemsAction.run({ + databaseId, + limit: 1, + offset: 1, + tableQuery: { + search: "a", + filters: [], + sorts: [{ key: "name", label: "Name", direction: "asc" }], + filterMode: "and", + }, + }), + ); + expect(response).toMatchObject({ + tableQueryMode: "server", + pagination: { + offset: 1, + limit: 1, + totalItems: 3, + returnedItems: 1, + hasMore: true, + }, + }); + expect(response.items.map((item) => item.document.title)).toEqual([ + "Banana", + ]); + expect(response).not.toHaveProperty("database"); + expect(response).not.toHaveProperty("contextPath"); + expect(response).not.toHaveProperty("properties"); + expect(response.items[0].id).toBe(rows[2].id); + + await expect( + runWithRequestContext({ userEmail: COLLABORATOR }, () => + queryContentDatabaseItemsAction.run({ databaseId }), + ), + ).rejects.toThrow(`Database "${databaseId}" not found`); + }); + it("blocks direct document and property reads for soft-deleted database pages", async () => { const deletedAt = new Date().toISOString(); const { databaseId, databaseDocumentId } = await createDatabase({ diff --git a/templates/content/actions/content-database-source-actions.test.ts b/templates/content/actions/content-database-source-actions.test.ts index 8849967aa6..30c8c46c1c 100644 --- a/templates/content/actions/content-database-source-actions.test.ts +++ b/templates/content/actions/content-database-source-actions.test.ts @@ -29,6 +29,7 @@ import prepareReview, { reviewPreparePriority, } from "./prepare-builder-source-review"; import previewReview from "./preview-builder-source-review"; +import previewSourceAttach from "./preview-content-database-source-attach"; import refreshSource from "./refresh-content-database-source"; import reviewChangeSet from "./review-content-database-source-change-set"; import setWriteMode from "./set-content-database-source-write-mode"; @@ -37,6 +38,20 @@ import stageBulkUpdate from "./stage-builder-source-bulk-update"; import validateExecution from "./validate-builder-source-execution"; describe("content database source actions", () => { + it("bounds Builder attachment previews to one database and safe field paths", () => { + expect( + previewSourceAttach.schema.parse({ + documentId: "database-page", + sourceTable: "agent-native-blog-article-test", + fieldPaths: ["topics", "data.author"], + }), + ).toEqual({ + documentId: "database-page", + sourceTable: "agent-native-blog-article-test", + fieldPaths: ["topics", "data.author"], + }); + }); + it("accepts database or document IDs for source status reads", () => { expect(getSource.schema.parse({ documentId: "database-page" })).toEqual({ documentId: "database-page", @@ -287,6 +302,56 @@ describe("content database source actions", () => { expect(calls).toEqual(["model-fields", "entries"]); }); + it("starts Builder field discovery and the projected first page together", async () => { + const calls: string[] = []; + let releaseFields!: () => void; + let releaseEntries!: () => void; + const fieldsReady = new Promise((resolve) => { + releaseFields = resolve; + }); + const entriesReady = new Promise((resolve) => { + releaseEntries = resolve; + }); + const pending = readInitialBuilderCmsAttachSource("blog-article", { + fieldPaths: ["topics", "data.author"], + readModelFields: async () => { + calls.push("model-fields"); + await fieldsReady; + return []; + }, + readEntries: async (args) => { + calls.push("entries"); + expect(args.fieldPaths).toEqual(["topics", "data.author"]); + await entriesReady; + return { + state: "live", + entries: [], + fetchedAt: "2026-01-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset: 0, + nextOffset: 0, + fetchedEntryCount: 0, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + }, + }); + + await Promise.resolve(); + expect(calls).toEqual(["model-fields", "entries"]); + releaseEntries(); + releaseFields(); + await expect(pending).resolves.toMatchObject({ + read: { state: "live" }, + modelFields: [], + }); + }); + it("fails role-change preparation before mappings can be rewritten when model discovery fails", async () => { const calls: string[] = []; const existingMappings = ["data.topics", "data.tags"]; diff --git a/templates/content/actions/get-content-database.ts b/templates/content/actions/get-content-database.ts index 37d470dfa0..d31f4b605e 100644 --- a/templates/content/actions/get-content-database.ts +++ b/templates/content/actions/get-content-database.ts @@ -1,17 +1,15 @@ import { defineAction } from "@agent-native/core"; -import { resolveAccess } from "@agent-native/core/sharing"; -import { eq } from "drizzle-orm"; import { z } from "zod"; -import { getDb, schema } from "../server/db/index.js"; import type { ContentDatabaseResponse, ContentDatabaseUnavailableResponse, } from "../shared/api.js"; -import { resolveContentSpaceAccess } from "./_content-space-access.js"; import { CONTENT_DATABASE_MAX_READ_LIMIT, + contentDatabaseTableQuerySchema, getContentDatabaseResponse, + resolveContentDatabaseRead, } from "./_database-utils.js"; export default defineAction({ @@ -27,46 +25,7 @@ export default defineAction({ .max(CONTENT_DATABASE_MAX_READ_LIMIT) .optional(), offset: z.coerce.number().int().min(0).optional(), - tableQuery: z - .object({ - search: z.string().max(500), - filters: z - .array( - z.object({ - key: z.string(), - label: z.string(), - operator: z.enum([ - "contains", - "equals", - "does_not_equal", - "greater_than", - "less_than", - "before", - "after", - "between", - "is_checked", - "is_unchecked", - "is_empty", - "is_not_empty", - ]), - value: z.string(), - filterGroupId: z.string().optional(), - parentFilterGroupId: z.string().optional(), - }), - ) - .max(50), - sorts: z - .array( - z.object({ - key: z.string(), - label: z.string(), - direction: z.enum(["asc", "desc"]), - }), - ) - .max(20), - filterMode: z.enum(["and", "or"]), - }) - .optional(), + tableQuery: contentDatabaseTableQuerySchema, }), http: { method: "GET" }, readOnly: true, @@ -77,61 +36,17 @@ export default defineAction({ offset, tableQuery, }): Promise => { - const db = getDb(); - let resolvedDatabaseId = databaseId; - - if (!resolvedDatabaseId && documentId) { - const [database] = await db - .select() - .from(schema.contentDatabases) - .where(eq(schema.contentDatabases.documentId, documentId)); - resolvedDatabaseId = database?.id; - } - - if (!resolvedDatabaseId) { - throw new Error("Either databaseId or documentId is required."); - } - - const [database] = await db - .select() - .from(schema.contentDatabases) - .where(eq(schema.contentDatabases.id, resolvedDatabaseId)); - if (!database) { - return { - available: false, - reason: "not_found", - databaseId: resolvedDatabaseId, - documentId: documentId ?? null, - message: `Database "${resolvedDatabaseId}" not found`, - }; - } - - let canRead = Boolean(await resolveAccess("document", database.documentId)); - if (!canRead && database.systemRole === "files" && database.spaceId) { - try { - await resolveContentSpaceAccess(database.spaceId); - canRead = true; - } catch { - canRead = false; - } - } - if (!canRead) throw new Error(`Database "${resolvedDatabaseId}" not found`); - - if (database.deletedAt) { - return { - available: false, - reason: "deleted", - databaseId: database.id, - documentId: database.documentId, - deletedAt: database.deletedAt, - message: `Database "${database.id}" has been deleted`, - }; - } + const resolved = await resolveContentDatabaseRead({ + databaseId, + documentId, + }); + if (!resolved.available) return resolved; - return getContentDatabaseResponse(resolvedDatabaseId, { + return getContentDatabaseResponse(resolved.database.id, { limit, offset, tableQuery, + database: resolved.database, }); }, }); diff --git a/templates/content/actions/preview-content-database-source-attach.ts b/templates/content/actions/preview-content-database-source-attach.ts new file mode 100644 index 0000000000..f0befa26ed --- /dev/null +++ b/templates/content/actions/preview-content-database-source-attach.ts @@ -0,0 +1,100 @@ +import { defineAction } from "@agent-native/core"; +import { assertAccess } from "@agent-native/core/sharing"; +import { z } from "zod"; + +import type { + BuilderCmsAttachPreviewResponse, + ContentDatabaseItem, +} from "../shared/api.js"; +import { readBuilderCmsContentEntries } from "./_builder-cms-read-client.js"; +import { createBuilderSourceTiming } from "./_builder-source-timings.js"; +import { + builderCmsImportIds, + resolveDatabaseForSourceMutation, +} from "./_database-source-utils.js"; + +export default defineAction({ + description: + "Preview the first projected Builder rows for a database source attachment without writing local or provider data.", + schema: z.object({ + databaseId: z.string().optional(), + documentId: z.string().optional(), + sourceTable: z.string().min(1).max(500), + fieldPaths: z.array(z.string().max(500)).max(200).optional(), + }), + http: { method: "GET" }, + readOnly: true, + run: async (args): Promise => { + const timing = createBuilderSourceTiming("preview-builder-attach"); + const database = await timing.measure("resolve-database", () => + resolveDatabaseForSourceMutation(args), + ); + if (!database) throw new Error("Database not found."); + await timing.measure("access", () => + assertAccess("document", database.documentId, "editor"), + ); + + const read = await timing.measure("provider-first-page", () => + readBuilderCmsContentEntries({ + model: args.sourceTable, + fieldPaths: args.fieldPaths, + maxPages: 1, + }), + ); + if (read.state !== "live") { + throw new Error( + read.message ?? "Builder rows are not available for preview.", + ); + } + + const mapStartedAt = timing.start(); + const items: ContentDatabaseItem[] = read.entries.map((entry, index) => { + const ids = builderCmsImportIds({ + ownerEmail: database.ownerEmail, + databaseId: database.id, + sourceTable: args.sourceTable, + entryId: entry.id, + }); + return { + id: ids.itemId, + databaseId: database.id, + document: { + id: ids.documentId, + parentId: database.documentId, + title: entry.title.trim() || entry.id, + content: "", + icon: null, + position: index, + isFavorite: false, + hideFromSearch: false, + accessRole: "viewer", + canEdit: false, + canManage: false, + createdAt: entry.updatedAt, + updatedAt: entry.updatedAt, + }, + position: index, + properties: [], + bodyHydration: { + status: "pending", + attemptedAt: null, + error: null, + version: null, + }, + }; + }); + timing.record("project-items", mapStartedAt); + + const response = { + databaseId: database.id, + documentId: database.documentId, + sourceTable: args.sourceTable, + items, + fetchedAt: read.fetchedAt, + hasMore: read.progress.hasMore, + timings: timing.finish(), + }; + timing.log("succeeded"); + return response; + }, +}); diff --git a/templates/content/actions/process-builder-body-hydration.ts b/templates/content/actions/process-builder-body-hydration.ts index 678f673654..f66f6e966a 100644 --- a/templates/content/actions/process-builder-body-hydration.ts +++ b/templates/content/actions/process-builder-body-hydration.ts @@ -8,6 +8,7 @@ import type { ProcessBuilderBodyHydrationRequest, ProcessBuilderBodyHydrationResponse, } from "../shared/api.js"; +import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { processBuilderBodyHydrationQueue } from "./_database-source-utils.js"; export default defineAction({ @@ -21,27 +22,37 @@ export default defineAction({ run: async ( args: ProcessBuilderBodyHydrationRequest, ): Promise => { + const timing = createBuilderSourceTiming("process-builder-body-hydration"); const db = getDb(); - const [source] = await db - .select({ - id: schema.contentDatabaseSources.id, - databaseDocumentId: schema.contentDatabases.documentId, - }) - .from(schema.contentDatabaseSources) - .innerJoin( - schema.contentDatabases, - eq( - schema.contentDatabases.id, - schema.contentDatabaseSources.databaseId, - ), - ) - .where(eq(schema.contentDatabaseSources.id, args.sourceId)); + const [source] = await timing.measure("resolve-source", () => + db + .select({ + id: schema.contentDatabaseSources.id, + databaseDocumentId: schema.contentDatabases.documentId, + }) + .from(schema.contentDatabaseSources) + .innerJoin( + schema.contentDatabases, + eq( + schema.contentDatabases.id, + schema.contentDatabaseSources.databaseId, + ), + ) + .where(eq(schema.contentDatabaseSources.id, args.sourceId)), + ); if (!source) throw new Error(`Source "${args.sourceId}" not found`); - await assertAccess("document", source.databaseDocumentId, "editor"); - return processBuilderBodyHydrationQueue({ - sourceId: args.sourceId, - documentId: args.documentId, - limit: args.limit, - }); + await timing.measure("access", () => + assertAccess("document", source.databaseDocumentId, "editor"), + ); + const result = await timing.measure("process-batch", () => + processBuilderBodyHydrationQueue({ + sourceId: args.sourceId, + documentId: args.documentId, + limit: args.limit, + }), + ); + const response = { ...result, timings: timing.finish() }; + timing.log("succeeded"); + return response; }, }); diff --git a/templates/content/actions/query-content-database-items.ts b/templates/content/actions/query-content-database-items.ts new file mode 100644 index 0000000000..fe466ffc3d --- /dev/null +++ b/templates/content/actions/query-content-database-items.ts @@ -0,0 +1,73 @@ +import { defineAction } from "@agent-native/core"; +import { z } from "zod"; + +import type { + ContentDatabaseItemsPageResponse, + ContentDatabaseUnavailableResponse, +} from "../shared/api.js"; +import { createBuilderSourceTiming } from "./_builder-source-timings.js"; +import { + CONTENT_DATABASE_MAX_READ_LIMIT, + contentDatabaseTableQuerySchema, + getContentDatabasePageResponse, + resolveContentDatabaseRead, +} from "./_database-utils.js"; + +export default defineAction({ + description: + "Query one ordered and filtered page of a content database without reopening its metadata.", + schema: z.object({ + databaseId: z.string().optional().describe("Database ID"), + documentId: z.string().optional().describe("Database document/page ID"), + limit: z.coerce + .number() + .int() + .min(1) + .max(CONTENT_DATABASE_MAX_READ_LIMIT) + .optional(), + offset: z.coerce.number().int().min(0).optional(), + tableQuery: contentDatabaseTableQuerySchema, + }), + http: { method: "GET" }, + readOnly: true, + agentTool: false, + run: async ({ + databaseId, + documentId, + limit, + offset, + tableQuery, + }): Promise< + ContentDatabaseItemsPageResponse | ContentDatabaseUnavailableResponse + > => { + const timing = createBuilderSourceTiming("query-content-database-items"); + const resolved = await timing.measure("resolve-access", () => + resolveContentDatabaseRead({ + databaseId, + documentId, + }), + ); + if (!resolved.available) return resolved; + + const page = await timing.measure("build-page", () => + getContentDatabasePageResponse(resolved.database.id, { + // This action is the bounded table replacement path; unlike the legacy + // database response, an omitted limit must not turn it into a full read. + limit: limit ?? 100, + offset, + tableQuery, + database: resolved.database, + }), + ); + const response = { + items: page.items, + source: page.source, + sources: page.sources, + pagination: page.pagination, + tableQueryMode: page.tableQueryMode, + timings: timing.finish(), + }; + timing.log("succeeded"); + return response; + }, +}); diff --git a/templates/content/actions/refresh-content-database-source.ts b/templates/content/actions/refresh-content-database-source.ts index 1c9f2f4ea9..64674cca52 100644 --- a/templates/content/actions/refresh-content-database-source.ts +++ b/templates/content/actions/refresh-content-database-source.ts @@ -3,6 +3,7 @@ import { assertAccess } from "@agent-native/core/sharing"; import { z } from "zod"; import type { ContentDatabaseSourceStatusResponse } from "../shared/api.js"; +import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { getContentDatabaseSourceAdapter } from "./_content-database-source-adapters.js"; import { claimBuilderCmsSourceRefresh, @@ -51,11 +52,18 @@ export default defineAction({ ), }), run: async (args): Promise => { - const database = await resolveDatabaseForSourceMutation(args); + const timing = createBuilderSourceTiming("refresh-content-database-source"); + const database = await timing.measure("resolve-database", () => + resolveDatabaseForSourceMutation(args), + ); if (!database) throw new Error("Database not found."); - await assertAccess("document", database.documentId, "editor"); + await timing.measure("access", () => + assertAccess("document", database.documentId, "editor"), + ); - const source = await getExistingSourceForWrite(database.id, args.sourceId); + const source = await timing.measure("resolve-source", () => + getExistingSourceForWrite(database.id, args.sourceId), + ); if (!source) { return { database: serializeDatabase(database), @@ -76,20 +84,24 @@ export default defineAction({ }); skippedOverlappingBuilderRefresh = !claimedSource; if (claimedSource) { - await resyncBuilderCmsSourceSnapshot({ - database, - source: claimedSource.source, - now, - runFullRefresh: args.fullRefresh === true, - finishPagination: args.finishBuilderPagination === true, - refreshClaimId: claimedSource.claimId, - }).catch(async (error: unknown) => { - await releaseBuilderCmsSourceRefreshClaim({ - sourceId: source.id, - claimId: claimedSource.claimId, + await timing + .measure("builder-resync", () => + resyncBuilderCmsSourceSnapshot({ + database, + source: claimedSource.source, + now, + runFullRefresh: args.fullRefresh === true, + finishPagination: args.finishBuilderPagination === true, + refreshClaimId: claimedSource.claimId, + }), + ) + .catch(async (error: unknown) => { + await releaseBuilderCmsSourceRefreshClaim({ + sourceId: source.id, + claimId: claimedSource.claimId, + }); + throw error; }); - throw error; - }); } } else if (source.sourceType === "local-table") { // Read-only federated secondary; its rows are re-read on demand, nothing @@ -130,9 +142,11 @@ export default defineAction({ } else { throw new Error(`Unsupported source type "${source.sourceType}".`); } - const snapshot = args.sourceId - ? await getContentDatabaseSourceSnapshotById(database, args.sourceId) - : await getContentDatabaseSourceSnapshot(database); + const snapshot = await timing.measure("status-snapshot", () => + args.sourceId + ? getContentDatabaseSourceSnapshotById(database, args.sourceId) + : getContentDatabaseSourceSnapshot(database), + ); const builderProgress = snapshot?.sourceType === "builder-cms" ? snapshot.metadata : null; @@ -144,7 +158,7 @@ export default defineAction({ ? builderProgress.lastReadFetchedEntryCount : undefined; - return { + const response: ContentDatabaseSourceStatusResponse = { database: serializeDatabase(database), mode: "source-backed", summary: snapshot @@ -157,6 +171,9 @@ export default defineAction({ : `${snapshot.sourceName} resynced locally; field mappings and row identity now reflect the current database snapshot.` : "Source metadata refreshed.", source: snapshot, + timings: timing.finish(), }; + timing.log("succeeded"); + return response; }, }); diff --git a/templates/content/actions/resync-content-database-source.db.test.ts b/templates/content/actions/resync-content-database-source.db.test.ts index a5cd253ea4..05877b430f 100644 --- a/templates/content/actions/resync-content-database-source.db.test.ts +++ b/templates/content/actions/resync-content-database-source.db.test.ts @@ -1951,6 +1951,22 @@ it("records freshly imported Builder row identities even when title and URL keys expect(concurrentRetryDocuments).toHaveLength(2); expect(concurrentRetryItems).toHaveLength(2); + const interruptedAttachRecovery = await importBuilderEntries({ + database, + entries, + now, + sourceTable: "collection-duplicates", + existingSourceRows: [], + skipTitleDedup: true, + }); + expect(interruptedAttachRecovery.imported).toBe(0); + expect(interruptedAttachRecovery.importedItems).toEqual([]); + expect( + Array.from(interruptedAttachRecovery.importedEntriesByDocumentId.values()) + .map((entry) => entry.id) + .sort(), + ).toEqual(["entry-dup-1", "entry-dup-2"]); + const importedItems = concurrentRetryItems.map( (item: { id: string; documentId: string }, index: number) => ({ id: item.id, diff --git a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx index e1de35dbd3..8ac3d43332 100644 --- a/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.error-toasts.test.tsx @@ -127,6 +127,13 @@ vi.mock("@/hooks/use-content-database", () => ({ useAddDatabaseItem: () => addItemMutation, useAddContentDatabaseSourceFieldProperty: () => benignMutation, useAttachContentDatabaseSource: () => attachSourceMutation, + useBuilderCmsAttachPreview: () => ({ + data: undefined, + isLoading: false, + isFetching: false, + error: null, + }), + writeBuilderAttachPreviewToCache: vi.fn(), useChangeContentDatabaseSourceRole: () => benignMutation, useRefreshContentDatabaseSource: () => benignMutation, useDisconnectContentDatabaseSource: () => benignMutation, diff --git a/templates/content/app/components/editor/database/DatabaseView.tsx b/templates/content/app/components/editor/database/DatabaseView.tsx index 4b1b165b69..155c029c04 100644 --- a/templates/content/app/components/editor/database/DatabaseView.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.tsx @@ -167,6 +167,7 @@ import { useAddDatabaseItem, useAddContentDatabaseSourceFieldProperty, useAttachContentDatabaseSource, + useBuilderCmsAttachPreview, useBuilderCmsModels, useCancelPreparedBuilderSourceUpdate, useChangeContentDatabaseSourceRole, @@ -190,6 +191,7 @@ import { useSuggestSourceJoinKey, useUpdateContentDatabasePersonalView, useUpdateContentDatabaseView, + writeBuilderAttachPreviewToCache, } from "@/hooks/use-content-database"; import { useContentSpaces, @@ -833,6 +835,8 @@ function DatabaseTable({ const data = isContentDatabaseUnavailable(database.data) ? undefined : database.data; + const attachPreviewActive = Boolean(data?.attachPreview); + const effectiveCanEdit = canEdit && !attachPreviewActive; const isWorkspaceCatalog = data?.database.systemRole === "workspaces"; const isCreatingDatabaseItem = addItem.isPending; const isDatabaseInitialLoading = database.isLoading && !data; @@ -1235,19 +1239,35 @@ function DatabaseTable({ ) { return false; } - processBuilderBodies.mutate( - { sourceId, ...request }, - { - onSuccess: options.onSuccess, - onError: options.onError, - onSettled: () => { - releaseDatabaseSourceOperation( - hydrationSourceInFlightRef, - sourceId, - ); + const pump = () => { + let result: ProcessBuilderBodyHydrationResponse | null = null; + processBuilderBodies.mutate( + { sourceId, ...request }, + { + onSuccess: (nextResult) => { + result = nextResult; + options.onSuccess?.(nextResult); + }, + onError: options.onError, + onSettled: () => { + if ( + !request.documentId && + result && + builderBodyHydrationMutationMadeProgress(result) && + result.remaining > 0 + ) { + pump(); + return; + } + releaseDatabaseSourceOperation( + hydrationSourceInFlightRef, + sourceId, + ); + }, }, - }, - ); + ); + }; + pump(); return true; }, [processBuilderBodies.mutate], @@ -2311,7 +2331,7 @@ function DatabaseTable({ : viewConfig; const nextKey = databaseViewStateKey(databaseId, sharedViewConfig); if (databaseViewStateKey(databaseId, savedViewConfig) === nextKey) return; - if (!canEdit) return; + if (!effectiveCanEdit) return; if (saveViewTimerRef.current) { clearTimeout(saveViewTimerRef.current); } @@ -2391,7 +2411,7 @@ function DatabaseTable({
@@ -2545,7 +2565,12 @@ function DatabaseTable({ ) ) : null} - {canEdit && isWorkspaceCatalog ? ( + {data?.attachPreview ? ( + + Attaching · read-only + + ) : null} + {effectiveCanEdit && isWorkspaceCatalog ? ( - ) : canEdit ? ( + ) : effectiveCanEdit ? ( {data?.attachPreview ? ( - Attaching · read-only + {dbText("attachingReadOnly")} ) : null} {effectiveCanEdit && isWorkspaceCatalog ? ( diff --git a/templates/content/app/i18n-data.ts b/templates/content/app/i18n-data.ts index 1508dc4ab0..c507f07790 100644 --- a/templates/content/app/i18n-data.ts +++ b/templates/content/app/i18n-data.ts @@ -26,6 +26,7 @@ const databaseMessages = { addingDetailsMatchedOn: "Adding details matched on {{field}}.", addingItems: "Adding items", addingItemsAsRows: "Adding items as their own rows in this database.", + attachingReadOnly: "Attaching · read-only", addMoreItemsToThisList: "Add more items to this list", addProperty: "Add property", allAvailableDetailFieldsAlreadyVisible: @@ -8962,6 +8963,7 @@ export const messagesByLocale = { builderRowsFetchedSoFar: "目前已获取 {{count}} 行。", builderRowsFinishingUp: "Builder 行即将完成加载。", builderRowsLoadingBackground: "Builder 仍在后台加载行。", + attachingReadOnly: "正在附加 · 只读", builderRowsLoadingHitSnag: "Builder 行加载遇到问题。", opening: "正在打开...", }, @@ -9147,6 +9149,7 @@ export const messagesByLocale = { "Las filas de Builder están terminando de cargarse.", builderRowsLoadingBackground: "Builder sigue cargando filas en segundo plano.", + attachingReadOnly: "Adjuntando · solo lectura", builderRowsLoadingHitSnag: "La carga de filas de Builder tuvo un problema.", opening: "Abriendo...", @@ -9344,6 +9347,7 @@ export const messagesByLocale = { builderRowsFinishingUp: "Les lignes Builder terminent leur chargement.", builderRowsLoadingBackground: "Builder charge encore des lignes en arrière-plan.", + attachingReadOnly: "Ajout en cours · lecture seule", builderRowsLoadingHitSnag: "Le chargement des lignes Builder a rencontré un problème.", opening: "Ouverture...", @@ -9540,6 +9544,7 @@ export const messagesByLocale = { builderRowsFinishingUp: "Builder-Zeilen werden fertig geladen.", builderRowsLoadingBackground: "Builder lädt weiterhin Zeilen im Hintergrund.", + attachingReadOnly: "Wird angehängt · schreibgeschützt", builderRowsLoadingHitSnag: "Beim Laden der Builder-Zeilen ist ein Problem aufgetreten.", opening: "Wird geöffnet...", @@ -9736,6 +9741,7 @@ export const messagesByLocale = { builderRowsFinishingUp: "Builder 行の読み込みを完了しています。", builderRowsLoadingBackground: "Builder はバックグラウンドで行を読み込み続けています。", + attachingReadOnly: "接続中 · 読み取り専用", builderRowsLoadingHitSnag: "Builder 行の読み込みで問題が発生しました。", opening: "開いています...", }, @@ -9928,6 +9934,7 @@ export const messagesByLocale = { builderRowsFinishingUp: "Builder 행 로드를 마무리하는 중입니다.", builderRowsLoadingBackground: "Builder가 백그라운드에서 행을 계속 로드하고 있습니다.", + attachingReadOnly: "연결 중 · 읽기 전용", builderRowsLoadingHitSnag: "Builder 행 로드 중 문제가 발생했습니다.", opening: "여는 중...", }, @@ -10115,6 +10122,7 @@ export const messagesByLocale = { "As linhas do Builder estão terminando de carregar.", builderRowsLoadingBackground: "O Builder ainda está carregando linhas em segundo plano.", + attachingReadOnly: "Anexando · somente leitura", builderRowsLoadingHitSnag: "O carregamento de linhas do Builder encontrou um problema.", opening: "Abrindo...", @@ -10307,6 +10315,7 @@ export const messagesByLocale = { builderRowsFinishingUp: "Builder पंक्तियां लोड होना पूरा कर रही हैं।", builderRowsLoadingBackground: "Builder अभी भी पृष्ठभूमि में पंक्तियां लोड कर रहा है।", + attachingReadOnly: "संलग्न किया जा रहा है · केवल पढ़ने के लिए", builderRowsLoadingHitSnag: "Builder पंक्तियां लोड करने में समस्या आई।", opening: "खोला जा रहा है...", }, @@ -10488,6 +10497,7 @@ export const messagesByLocale = { builderRowsFetchedSoFar: "تم جلب {{count}} صفًا حتى الآن.", builderRowsFinishingUp: "صفوف Builder توشك على إكمال التحميل.", builderRowsLoadingBackground: "لا يزال Builder يحمّل الصفوف في الخلفية.", + attachingReadOnly: "جارٍ الإرفاق · للقراءة فقط", builderRowsLoadingHitSnag: "واجه تحميل صفوف Builder مشكلة.", opening: "جارٍ الفتح...", }, diff --git a/templates/content/app/i18n/zh-TW.ts b/templates/content/app/i18n/zh-TW.ts index a64ff5c3c1..e302e6c842 100644 --- a/templates/content/app/i18n/zh-TW.ts +++ b/templates/content/app/i18n/zh-TW.ts @@ -722,6 +722,7 @@ const messages = { builderRowsFetchedSoFar: "目前已擷取 {{count}} 列。", builderRowsFinishingUp: "Builder 列即將完成載入。", builderRowsLoadingBackground: "Builder 仍在背景載入列。", + attachingReadOnly: "正在附加 · 唯讀", builderRowsLoadingHitSnag: "Builder 列載入遇到問題。", closeDatabaseSettings: "關閉資料庫設定", closeSearch: "關閉搜尋", From 06af4987585130dce93bfb3bc0f211f1b0867750 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:49:31 -0400 Subject: [PATCH 17/21] Batch Builder body hydration reads --- .../actions/_builder-cms-read-client.test.ts | 41 ++++++++++++++++ .../actions/_builder-cms-read-client.ts | 16 ++++++- .../content/actions/_database-source-utils.ts | 47 ++++++++++++++++++- .../actions/process-builder-body-hydration.ts | 3 +- 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index cb8826e64e..07c37c2a7b 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -456,6 +456,47 @@ describe("Builder CMS read client", () => { }); }); + it("can project Builder bodies in paged list reads for bulk hydration", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + expect(input.searchParams.get("enrich")).toBe("true"); + expect(input.searchParams.get("fields")?.split(",")).toEqual( + expect.arrayContaining([ + "data.title", + "data.blocks", + "data.blocksString", + ]), + ); + return new Response( + JSON.stringify({ + results: [ + { + id: "builder-entry-with-body", + data: { + title: "Builder body", + blocks: [{ id: "block-1" }], + }, + }, + ], + }), + { status: 200 }, + ); + }); + + const result = await readBuilderCmsContentEntries({ + model: "blog-article", + includeBodies: true, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(result.entries[0]?.rawEntry?.data).toMatchObject({ + blocks: [{ id: "block-1" }], + }); + }); + it("performs authenticated, cachebusted raw fidelity reads without projection or enrichment", async () => { process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; resolveBuilderCredentialMock.mockImplementation(async (key) => { diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index 923caa75da..dc740a1a7e 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -787,6 +787,7 @@ async function initializeBuilderMcp(args: { async function readBuilderCmsContentEntriesViaMcp(args: { model: string; fieldPaths?: readonly string[]; + includeBodies?: boolean; rawData?: boolean; limit?: number; maxPages?: number; @@ -810,7 +811,9 @@ async function readBuilderCmsContentEntriesViaMcp(args: { const contentEntries: BuilderCmsSourceEntry[] = []; const fields = args.rawData ? undefined - : builderCmsListEntryFields(args.fieldPaths); + : args.includeBodies + ? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}` + : builderCmsListEntryFields(args.fieldPaths); const seenContentIds = new Set(); let pagesRead = 0; let hasMore = false; @@ -976,6 +979,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: { async function readBuilderCmsContentEntriesViaContentApi(args: { model: string; fieldPaths?: readonly string[]; + includeBodies?: boolean; rawData?: boolean; limit?: number; maxPages?: number; @@ -1001,7 +1005,12 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { ); url.searchParams.set("includeUnpublished", "true"); if (args.rawData !== true) { - url.searchParams.set("fields", builderCmsListEntryFields(args.fieldPaths)); + url.searchParams.set( + "fields", + args.includeBodies + ? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}` + : builderCmsListEntryFields(args.fieldPaths), + ); } const limit = readLimit(args.limit); @@ -1296,6 +1305,7 @@ export async function readBuilderCmsModelFields(args: { export async function readBuilderCmsContentEntries(args: { model: string; fieldPaths?: readonly string[]; + includeBodies?: boolean; rawData?: boolean; requirePrivateKey?: boolean; limit?: number; @@ -1330,6 +1340,7 @@ export async function readBuilderCmsContentEntries(args: { const contentApiRead = await readBuilderCmsContentEntriesViaContentApi({ model: args.model, fieldPaths: args.fieldPaths, + includeBodies: args.includeBodies, rawData: args.rawData, limit: args.limit, maxPages: args.maxPages, @@ -1354,6 +1365,7 @@ export async function readBuilderCmsContentEntries(args: { await readBuilderCmsContentEntriesViaMcp({ model: args.model, fieldPaths: args.fieldPaths, + includeBodies: args.includeBodies, rawData: args.rawData, limit: args.limit, maxPages: args.maxPages, diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 8814048883..6d003bd5bb 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -977,7 +977,7 @@ function builderBodyUsesCurrentMediaConverter( const BUILDER_BODY_HYDRATION_BACKGROUND_PRIORITY = 10; const BUILDER_BODY_HYDRATION_OPEN_PRIORITY = 0; -const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 100; +const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 600; const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 72; const BUILDER_BODY_HYDRATION_MAX_ATTEMPTS = 5; const BUILDER_BODY_HYDRATION_CODEC_VERSION = @@ -1744,6 +1744,7 @@ async function processBuilderBodyHydrationJob( sourceRow?: ContentDatabaseSourceRecordRowDb | null; documentContent?: string | null; bodyHydrationVersion?: string | null; + bodyEntry?: BuilderCmsSourceEntry | null; }, ) { const db = getDb(); @@ -1753,9 +1754,19 @@ async function processBuilderBodyHydrationJob( entry.sourceValues, BUILDER_CMS_BODY_BLOCKS_HASH_KEY, ); + const bodyEntry = + preloaded?.bodyEntry?.id === entry.id + ? { + ...preloaded.bodyEntry, + sourceValues: { + ...entry.sourceValues, + ...preloaded.bodyEntry.sourceValues, + }, + } + : entry; let activeSourceEntryJson = row.sourceEntryJson; let entryWithBody = await refreshBuilderBodySourceValuesFromStoredLossless( - await withBuilderBodySourceValues(entry), + await withBuilderBodySourceValues(bodyEntry), ); const sourceRow = preloaded?.sourceRow != null @@ -2275,6 +2286,7 @@ export async function processBuilderBodyHydrationQueue(args: { documentId?: string | null; limit?: number | null; preloadedJobs?: ContentDatabaseBodyHydrationQueueRowDb[]; + preloadBodies?: boolean; }) { const db = getDb(); const limit = normalizeHydrationLimit(args.limit); @@ -2458,6 +2470,36 @@ export async function processBuilderBodyHydrationQueue(args: { const documentContentById = new Map( documents.map((document) => [document.id, document.content]), ); + const claimedSourceRowIds = new Set( + claimedJobs.map((job) => job.sourceRowId), + ); + const bodyEntryById = new Map(); + if ( + !args.documentId && + args.preloadBodies === true && + claimedJobs.length > 0 + ) { + const sourceTables = Array.from( + new Set(claimedJobs.map((job) => job.sourceTable)), + ); + const bodyReads = await Promise.all( + sourceTables.map((model) => + readBuilderCmsContentEntries({ + model, + includeBodies: true, + limit: 10_000, + }), + ), + ); + for (const read of bodyReads) { + if (read.state !== "live") continue; + for (const entry of read.entries) { + if (claimedSourceRowIds.has(entry.id)) { + bodyEntryById.set(entry.id, entry); + } + } + } + } await processInBatches( claimedJobs, BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY, @@ -2470,6 +2512,7 @@ export async function processBuilderBodyHydrationQueue(args: { sourceRow: sourceRowsByItemId.get(job.databaseItemId) ?? null, bodyHydrationVersion: bodyHydrationVersionByItemId.get(job.databaseItemId) ?? null, + bodyEntry: bodyEntryById.get(job.sourceRowId) ?? null, documentContent: documentContentById.has(job.documentId) ? (documentContentById.get(job.documentId) ?? null) : undefined, diff --git a/templates/content/actions/process-builder-body-hydration.ts b/templates/content/actions/process-builder-body-hydration.ts index 79d0fe6d80..4469369351 100644 --- a/templates/content/actions/process-builder-body-hydration.ts +++ b/templates/content/actions/process-builder-body-hydration.ts @@ -16,7 +16,7 @@ export default defineAction({ schema: z.object({ sourceId: z.string(), documentId: z.string().optional(), - limit: z.number().int().positive().max(100).optional(), + limit: z.number().int().positive().max(600).optional(), }), agentTool: false, run: async ( @@ -49,6 +49,7 @@ export default defineAction({ sourceId: args.sourceId, documentId: args.documentId, limit: args.limit, + preloadBodies: !args.documentId, }), ); const response = { ...result, timings: timing.finish() }; From a2fced09d9aabe0ef9848762846e607ccee2c493 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:21:49 -0400 Subject: [PATCH 18/21] Finish bounded large database reads --- .../content/actions/_batch-utils.test.ts | 24 ++ templates/content/actions/_batch-utils.ts | 21 ++ .../actions/_builder-cms-read-client.test.ts | 14 +- .../actions/_builder-cms-read-client.ts | 2 +- .../content/actions/_database-source-utils.ts | 61 ++-- templates/content/actions/_database-utils.ts | 274 ++++++++++++++++-- .../actions/attach-content-database-source.ts | 82 ++---- .../content-database-lifecycle.db.test.ts | 114 ++++++++ .../actions/preview-builder-source-review.ts | 131 ++++----- .../preview-content-database-source-attach.ts | 30 +- .../actions/process-builder-body-hydration.ts | 53 ++-- .../actions/query-content-database-items.ts | 35 +-- .../refresh-content-database-source.ts | 57 ++-- .../resync-content-database-source.db.test.ts | 98 +++++++ templates/content/shared/api.ts | 6 +- 15 files changed, 714 insertions(+), 288 deletions(-) create mode 100644 templates/content/actions/_batch-utils.test.ts diff --git a/templates/content/actions/_batch-utils.test.ts b/templates/content/actions/_batch-utils.test.ts new file mode 100644 index 0000000000..c46b20b412 --- /dev/null +++ b/templates/content/actions/_batch-utils.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; + +import { processWithConcurrency } from "./_batch-utils.js"; + +describe("processWithConcurrency", () => { + it("starts the next item as soon as one worker becomes available", async () => { + const releases = new Map void>(); + const started: number[] = []; + const run = processWithConcurrency([1, 2, 3, 4], 2, async (item) => { + started.push(item); + await new Promise((resolve) => releases.set(item, resolve)); + }); + + await expect.poll(() => started).toEqual([1, 2]); + releases.get(1)?.(); + await expect.poll(() => started).toEqual([1, 2, 3]); + releases.get(3)?.(); + await expect.poll(() => started).toEqual([1, 2, 3, 4]); + releases.get(2)?.(); + releases.get(4)?.(); + + await run; + }); +}); diff --git a/templates/content/actions/_batch-utils.ts b/templates/content/actions/_batch-utils.ts index 36e816ed6f..fde23f11ae 100644 --- a/templates/content/actions/_batch-utils.ts +++ b/templates/content/actions/_batch-utils.ts @@ -18,3 +18,24 @@ export function bulkChunkSizeForColumnCount( const budget = dialect === "d1" ? 90 : 900; return Math.max(1, Math.floor(budget / Math.max(1, columnCount))); } + +export async function processWithConcurrency( + items: readonly T[], + concurrency: number, + worker: (item: T) => Promise, +) { + let nextIndex = 0; + const workerCount = Math.min( + items.length, + Math.max(1, Math.floor(concurrency)), + ); + await Promise.all( + Array.from({ length: workerCount }, async () => { + while (nextIndex < items.length) { + const item = items[nextIndex]; + nextIndex += 1; + if (item !== undefined) await worker(item); + } + }), + ); +} diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index 07c37c2a7b..f21ded1886 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -1032,22 +1032,24 @@ describe("Builder CMS read client", () => { const read = readBuilderCmsContentEntries({ model: "blog_article", - limit: 500, + limit: 900, fetchImpl: fetchImpl as unknown as typeof fetch, }); await vi.waitFor(() => expect(resolvers.size).toBe(1)); resolvers.get(0)?.(); - await vi.waitFor(() => expect(resolvers.size).toBe(4)); - expect(maxActiveRequests).toBe(4); - for (const offset of [400, 300, 200, 100]) resolvers.get(offset)?.(); + await vi.waitFor(() => expect(resolvers.size).toBe(8)); + expect(maxActiveRequests).toBe(8); + for (const offset of [800, 700, 600, 500, 400, 300, 200, 100]) { + resolvers.get(offset)?.(); + } const result = await read; expect(result.entries.map((entry) => entry.id)).toEqual( - Array.from({ length: 500 }, (_, index) => `builder-entry-${index + 1}`), + Array.from({ length: 900 }, (_, index) => `builder-entry-${index + 1}`), ); - expect(maxActiveRequests).toBeLessThanOrEqual(4); + expect(maxActiveRequests).toBeLessThanOrEqual(8); }); it("stops at the first short projected page and ignores later in-flight pages", async () => { diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index dc740a1a7e..a22ffa1ce7 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -163,7 +163,7 @@ type BuilderMcpToolResult = { const BUILDER_CMS_DEFAULT_READ_LIMIT = 500; const BUILDER_CMS_MAX_READ_LIMIT = 10_000; const BUILDER_CMS_PAGE_SIZE = 100; -const BUILDER_CMS_PROJECTED_READ_CONCURRENCY = 4; +const BUILDER_CMS_PROJECTED_READ_CONCURRENCY = 8; const BUILDER_CMS_READ_RETRIES = 2; const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [ "id", diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 6d003bd5bb..8a8dd54fb3 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -55,7 +55,11 @@ import { type DocumentPropertyOptionColor, } from "../shared/properties.js"; import { sanitizeNormalizationFormula } from "../shared/properties.js"; -import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js"; +import { + bulkChunkSizeForColumnCount, + chunks, + processWithConcurrency, +} from "./_batch-utils.js"; export { bulkChunkSizeForColumnCount } from "./_batch-utils.js"; import { readBuilderCmsContentEntry, @@ -989,16 +993,6 @@ function idChunkSize() { return bulkChunkSizeForColumnCount(1); } -async function processInBatches( - items: T[], - concurrency: number, - worker: (item: T) => Promise, -) { - for (const batch of chunks(items, concurrency)) { - await Promise.all(batch.map((item) => worker(item))); - } -} - function builderBodyHydrationDelayMs() { if ( process.env.NODE_ENV === "production" || @@ -2288,6 +2282,7 @@ export async function processBuilderBodyHydrationQueue(args: { preloadedJobs?: ContentDatabaseBodyHydrationQueueRowDb[]; preloadBodies?: boolean; }) { + const diagnosticsStartedAt = Date.now(); const db = getDb(); const limit = normalizeHydrationLimit(args.limit); const now = new Date().toISOString(); @@ -2353,6 +2348,7 @@ export async function processBuilderBodyHydrationQueue(args: { }); })() : persistedJobs(limit)); + const jobsLoadedAt = Date.now(); let succeeded = 0; let failed = 0; @@ -2474,6 +2470,7 @@ export async function processBuilderBodyHydrationQueue(args: { claimedJobs.map((job) => job.sourceRowId), ); const bodyEntryById = new Map(); + const providerReadStartedAt = Date.now(); if ( !args.documentId && args.preloadBodies === true && @@ -2500,10 +2497,13 @@ export async function processBuilderBodyHydrationQueue(args: { } } } - await processInBatches( + const providerReadCompletedAt = Date.now(); + const bodyJobDurations: number[] = []; + await processWithConcurrency( claimedJobs, BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY, async (job) => { + const jobStartedAt = Date.now(); const attemptNow = new Date().toISOString(); try { const delayMs = builderBodyHydrationDelayMs(); @@ -2589,9 +2589,12 @@ export async function processBuilderBodyHydrationQueue(args: { updatedAt: attemptNow, }) .where(eq(schema.contentDatabaseItems.id, job.databaseItemId)); + } finally { + bodyJobDurations.push(Date.now() - jobStartedAt); } }, ); + const persistenceCompletedAt = Date.now(); const [remaining] = await db .select({ count: sql`COUNT(*)` }) .from(schema.contentDatabaseBodyHydrationQueue) @@ -2604,6 +2607,16 @@ export async function processBuilderBodyHydrationQueue(args: { succeeded, failed, remaining: Number(remaining?.count ?? 0), + diagnosticTimings: { + totalMs: Date.now() - diagnosticsStartedAt, + loadAndClaimMs: providerReadStartedAt - diagnosticsStartedAt, + jobsLoadMs: jobsLoadedAt - diagnosticsStartedAt, + providerReadMs: providerReadCompletedAt - providerReadStartedAt, + persistenceMs: persistenceCompletedAt - providerReadCompletedAt, + jobMinMs: bodyJobDurations.length > 0 ? Math.min(...bodyJobDurations) : 0, + jobMaxMs: bodyJobDurations.length > 0 ? Math.max(...bodyJobDurations) : 0, + bodyEntriesPreloaded: bodyEntryById.size, + }, }; } @@ -5938,6 +5951,7 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { }); importedEntriesByDocumentId.set(documentId, entry); } + const insertedItemIds = new Set(); await db.transaction(async (tx) => { for (const chunk of chunks( documentRows, @@ -5952,10 +5966,14 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { itemRows, bulkChunkSizeForColumnCount(10), )) { - await tx + const insertedItems = await tx .insert(schema.contentDatabaseItems) .values(chunk) - .onConflictDoNothing(); + .onConflictDoNothing() + .returning({ id: schema.contentDatabaseItems.id }); + for (const item of insertedItems) { + insertedItemIds.add(item.id); + } } await ensureDocumentsFilesMembership( tx, @@ -5965,9 +5983,13 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { ); }); - const importedItems: ContentDatabaseItem[] = itemRows.map( - (item, index) => { - const document = documentRows[index]; + const candidateDocumentById = new Map( + documentRows.map((document) => [document.id!, document]), + ); + const importedItems: ContentDatabaseItem[] = itemRows + .filter((item) => insertedItemIds.has(item.id!)) + .map((item) => { + const document = candidateDocumentById.get(item.documentId)!; return { id: item.id!, databaseId: args.database.id, @@ -5996,10 +6018,9 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: { version: null, }, }; - }, - ); + }); return { - imported: itemRows.length, + imported: insertedItemIds.size, importedEntriesByDocumentId, importedItems, }; diff --git a/templates/content/actions/_database-utils.ts b/templates/content/actions/_database-utils.ts index 6df715f57a..11e938a9a4 100644 --- a/templates/content/actions/_database-utils.ts +++ b/templates/content/actions/_database-utils.ts @@ -20,14 +20,24 @@ import { } from "../server/lib/documents.js"; import type { ContentDatabaseBodyHydration, + ContentDatabaseItem, ContentDatabaseMembership, ContentDatabaseResponse, ContentDatabaseTableQuery, + DocumentProperty, } from "../shared/api.js"; import { applyContentDatabaseTableQuery, contentDatabaseTableQueryUsesProperties, } from "../shared/database-query.js"; +import { + evaluatePropertyFormula, + isBlocksPropertyType, + isComputedPropertyType, + isPrimaryBlocksField, + parsePropertyValue, + type DocumentPropertyType, +} from "../shared/properties.js"; import { favoriteDocumentIds } from "./_content-favorites.js"; import { listContentOrganizationMemberships, @@ -52,6 +62,56 @@ export { getDocumentContextPath }; export const CONTENT_DATABASE_MAX_READ_LIMIT = 5_000; +const QUERY_PROJECTION_UNSUPPORTED_PROPERTY_TYPES = + new Set(["rollup"]); + +function boundedTableQueryProjectionPropertyIds( + query: ContentDatabaseTableQuery, + properties: DocumentProperty[], +) { + let propertyIds = new Set( + query.search.trim() + ? properties.map((property) => property.definition.id) + : [...query.filters, ...query.sorts] + .map((constraint) => constraint.key) + .filter((key) => key !== "name"), + ); + if ( + properties.some( + (property) => + propertyIds.has(property.definition.id) && + property.definition.type === "formula", + ) + ) { + propertyIds = new Set(properties.map((property) => property.definition.id)); + } + for (const property of properties) { + if ( + propertyIds.has(property.definition.id) && + QUERY_PROJECTION_UNSUPPORTED_PROPERTY_TYPES.has(property.definition.type) + ) { + return null; + } + } + return propertyIds; +} + +function projectedComputedPropertyValue( + type: DocumentPropertyType, + document: { + ownerEmail: string; + createdAt: string; + updatedAt: string; + }, +) { + if (type === "created_time") return document.createdAt; + if (type === "created_by" || type === "last_edited_by") { + return document.ownerEmail; + } + if (type === "last_edited_time") return document.updatedAt; + return null; +} + export const contentDatabaseTableQuerySchema = z .object({ search: z.string().max(500), @@ -442,6 +502,7 @@ export async function resolveContentDatabaseRead(args: { type ContentDatabasePageBuild = ContentDatabasePageResponse & { databaseRecord: typeof schema.contentDatabases.$inferSelect; properties: ContentDatabaseResponse["properties"]; + hydratedItemCount: number; }; export async function getContentDatabasePageResponse( @@ -615,18 +676,191 @@ export async function getContentDatabasePageResponse( ); } + const databaseProperties = await listPropertiesForDatabase(databaseId); + const boundedProjectionPropertyIds = + serverTableQuery && !database.systemRole + ? boundedTableQueryProjectionPropertyIds( + serverTableQuery, + databaseProperties, + ) + : null; + let itemsQuery = db .select() .from(schema.contentDatabaseItems) .where(visibleItemFilter) - .orderBy(asc(schema.contentDatabaseItems.position)) + .orderBy( + asc(schema.contentDatabaseItems.position), + asc(schema.contentDatabaseItems.createdAt), + asc(schema.contentDatabaseItems.id), + ) .$dynamic(); if (serverTableQuery) { itemsQuery = itemsQuery.limit(CONTENT_DATABASE_MAX_READ_LIMIT); } else if (limit !== null) { itemsQuery = itemsQuery.limit(limit).offset(offset); } - const items = await itemsQuery; + let items = await itemsQuery; + let boundedTableQueryTotal: number | null = null; + if (serverTableQuery && boundedProjectionPropertyIds) { + const candidateDocuments = await db + .select({ + id: schema.documents.id, + title: schema.documents.title, + ownerEmail: schema.documents.ownerEmail, + createdAt: schema.documents.createdAt, + updatedAt: schema.documents.updatedAt, + }) + .from(schema.documents) + .innerJoin( + schema.contentDatabaseItems, + eq(schema.contentDatabaseItems.documentId, schema.documents.id), + ) + .where( + and( + visibleItemFilter, + eq(schema.documents.ownerEmail, database.ownerEmail), + ), + ); + const candidateDocumentById = new Map( + candidateDocuments.map((document) => [document.id, document]), + ); + const candidateValues = + boundedProjectionPropertyIds.size > 0 + ? await db + .select({ + documentId: schema.documentPropertyValues.documentId, + propertyId: schema.documentPropertyValues.propertyId, + valueJson: schema.documentPropertyValues.valueJson, + }) + .from(schema.documentPropertyValues) + .innerJoin( + schema.contentDatabaseItems, + eq( + schema.contentDatabaseItems.documentId, + schema.documentPropertyValues.documentId, + ), + ) + .where( + and( + visibleItemFilter, + inArray(schema.documentPropertyValues.propertyId, [ + ...boundedProjectionPropertyIds, + ]), + ), + ) + : []; + const candidateValueByDocumentAndProperty = new Map( + candidateValues.map((value) => [ + `${value.documentId}\0${value.propertyId}`, + parsePropertyValue(value.valueJson), + ]), + ); + const queryProperties = databaseProperties.filter((property) => + boundedProjectionPropertyIds.has(property.definition.id), + ); + const additionalBlocksPropertyIds = queryProperties.flatMap((property) => + isBlocksPropertyType(property.definition.type) && + !isPrimaryBlocksField(property.definition.options) + ? [property.definition.id] + : [], + ); + const candidateBlockContents = + additionalBlocksPropertyIds.length > 0 + ? await db + .select({ + documentId: schema.documentBlockFieldContents.documentId, + propertyId: schema.documentBlockFieldContents.propertyId, + content: schema.documentBlockFieldContents.content, + }) + .from(schema.documentBlockFieldContents) + .innerJoin( + schema.contentDatabaseItems, + eq( + schema.contentDatabaseItems.documentId, + schema.documentBlockFieldContents.documentId, + ), + ) + .where( + and( + visibleItemFilter, + inArray( + schema.documentBlockFieldContents.propertyId, + additionalBlocksPropertyIds, + ), + ), + ) + : []; + const candidateBlockContentByDocumentAndProperty = new Map( + candidateBlockContents.map((row) => [ + `${row.documentId}\0${row.propertyId}`, + row.content ?? "", + ]), + ); + const candidateItems = items.flatMap((item, itemIndex) => { + const document = candidateDocumentById.get(item.documentId); + if (!document) return []; + const properties = queryProperties.map((property) => { + const type = property.definition.type; + const value = + type === "id" + ? itemIndex + 1 + : isBlocksPropertyType(type) + ? isPrimaryBlocksField(property.definition.options) + ? "" + : (candidateBlockContentByDocumentAndProperty.get( + `${document.id}\0${property.definition.id}`, + ) ?? "") + : isComputedPropertyType(type) + ? projectedComputedPropertyValue(type, document) + : (candidateValueByDocumentAndProperty.get( + `${document.id}\0${property.definition.id}`, + ) ?? null); + return { ...property, value }; + }); + const valuesByName = Object.fromEntries( + properties + .filter((property) => property.definition.type !== "formula") + .map((property) => [property.definition.name, property.value]), + ); + return [ + { + id: item.id, + databaseId: item.databaseId, + document: { title: document.title }, + properties: properties.map((property) => + property.definition.type === "formula" + ? { + ...property, + value: evaluatePropertyFormula( + property.definition.options.formula, + valuesByName, + ), + } + : property, + ), + } as ContentDatabaseItem, + ]; + }); + const constrainedCandidates = applyContentDatabaseTableQuery( + candidateItems, + databaseProperties, + serverTableQuery, + ); + boundedTableQueryTotal = constrainedCandidates.length; + const pageItemIds = new Set( + limit === null + ? constrainedCandidates.map((item) => item.id) + : constrainedCandidates + .slice(offset, offset + limit) + .map((item) => item.id), + ); + const itemById = new Map(items.map((item) => [item.id, item])); + items = [...pageItemIds].flatMap((itemId) => { + const item = itemById.get(itemId); + return item ? [item] : []; + }); + } const documents = items.length > 0 @@ -730,7 +964,6 @@ export async function getContentDatabasePageResponse( // every document field it consumes except the deliberately omitted body. documents as Array, ); - const databaseProperties = await listPropertiesForDatabase(databaseId); const filesProjection = await filesSystemPropertyProjection({ database, documents, @@ -800,17 +1033,20 @@ export async function getContentDatabasePageResponse( }); } - const constrainedItems = serverTableQuery - ? applyContentDatabaseTableQuery( - serializedCandidateItems, - responseProperties, - serverTableQuery, - ) - : serializedCandidateItems; + const constrainedItems = + serverTableQuery && boundedTableQueryTotal === null + ? applyContentDatabaseTableQuery( + serializedCandidateItems, + responseProperties, + serverTableQuery, + ) + : serializedCandidateItems; const serializedItems = - serverTableQuery && limit !== null - ? constrainedItems.slice(offset, offset + limit) - : constrainedItems; + boundedTableQueryTotal !== null + ? serializedCandidateItems + : serverTableQuery && limit !== null + ? constrainedItems.slice(offset, offset + limit) + : constrainedItems; const serializedDocumentIds = new Set( serializedItems.map((item) => item.document.id), @@ -880,16 +1116,20 @@ export async function getContentDatabasePageResponse( ? { offset, limit, - totalItems: serverTableQuery - ? constrainedItems.length - : totalVisibleItems, + totalItems: + boundedTableQueryTotal ?? + (serverTableQuery ? constrainedItems.length : totalVisibleItems), returnedItems: serializedItems.length, hasMore: offset + serializedItems.length < - (serverTableQuery ? constrainedItems.length : totalVisibleItems), + (boundedTableQueryTotal ?? + (serverTableQuery + ? constrainedItems.length + : totalVisibleItems)), } : undefined, tableQueryMode, + hydratedItemCount: documents.length, }; } diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index 3eb4ff4477..29b44829aa 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -17,7 +17,6 @@ import { type BuilderCmsReadResult, } from "./_builder-cms-read-client.js"; import type { BuilderCmsSourceEntry } from "./_builder-cms-source-adapter.js"; -import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { getContentDatabaseSourceAdapter } from "./_content-database-source-adapters.js"; import { databaseSourceExistsForTable, @@ -280,12 +279,6 @@ export default defineAction({ const now = new Date().toISOString(); const sourceType = (args.sourceType ?? "mock-local") as ContentDatabaseSourceType; - const timing = - sourceType === "builder-cms" - ? createBuilderSourceTiming("attach-content-database-source") - : null; - const measure = (name: string, operation: () => Promise) => - timing ? timing.measure(name, operation) : operation(); if (sourceType === "notion-database" && !args.sourceTable?.trim()) { throw new Error( "sourceTable must be a Notion data-source ID returned by list-notion-database-sources.", @@ -605,48 +598,40 @@ export default defineAction({ : []; const builderInitial = sourceType === "builder-cms" - ? await measure("provider-metadata", () => - readCompleteBuilderCmsAttachSource(sourceTable, { - fieldPaths: args.builderFieldPaths, - }), - ) + ? await readCompleteBuilderCmsAttachSource(sourceTable, { + fieldPaths: args.builderFieldPaths, + }) : null; - const sourceId = await measure("persist-source", () => - replaceSourceMetadata({ - database, - source: existingSource, - sourceType, - sourceName, - sourceTable, - now, - }), - ); + const sourceId = await replaceSourceMetadata({ + database, + source: existingSource, + sourceType, + sourceName, + sourceTable, + now, + }); const builderModelFields = builderInitial?.modelFields ?? []; const builderRead = builderInitial?.read ?? null; const builderEntries = builderRead?.state === "live" ? builderRead.entries : []; let importedEntriesByDocumentId = new Map(); if (builderRead?.state === "live") { - const importResult = await measure("import-first-page", () => - importBuilderCmsEntriesAsDatabaseItems({ - database, - entries: builderEntries, - now, - sourceTable, - existingSourceRows, - }), - ); + const importResult = await importBuilderCmsEntriesAsDatabaseItems({ + database, + entries: builderEntries, + now, + sourceTable, + existingSourceRows, + }); importedEntriesByDocumentId = importResult.importedEntriesByDocumentId; } - const refreshedSetup = await measure("read-imported-page", () => - sourceSetupPayload( - database.id, - initialBuilderAttachmentSetupOptions({ - builderRead, - importedEntriesByDocumentId, - }), - ), + const refreshedSetup = await sourceSetupPayload( + database.id, + initialBuilderAttachmentSetupOptions({ + builderRead, + importedEntriesByDocumentId, + }), ); const builderEntriesByDocumentId = builderRead?.state === "live" @@ -662,7 +647,7 @@ export default defineAction({ builderEntriesByDocumentId?.set(documentId, entry); } - await measure("persist-first-page", async () => { + await (async () => { await seedMockSourceFields({ sourceId, ownerEmail: database.ownerEmail, @@ -706,18 +691,11 @@ export default defineAction({ ...builderCmsAttachReadMetadata(builderRead), }); } - }); + })(); - const response = await measure("response", () => - getContentDatabaseResponse(database.id, { - limit: args.limit, - offset: args.offset, - }), - ); - if (timing) { - response.timings = timing.finish(); - timing.log("succeeded"); - } - return response; + return getContentDatabaseResponse(database.id, { + limit: args.limit, + offset: args.offset, + }); }, }); diff --git a/templates/content/actions/content-database-lifecycle.db.test.ts b/templates/content/actions/content-database-lifecycle.db.test.ts index c64d5b14fc..f138baa2bb 100644 --- a/templates/content/actions/content-database-lifecycle.db.test.ts +++ b/templates/content/actions/content-database-lifecycle.db.test.ts @@ -952,6 +952,120 @@ describe("content database soft-delete actions and reads", () => { ).rejects.toThrow(`Database "${databaseId}" not found`); }); + it("keeps a 584-row Date sort page-bounded before document and property hydration", async () => { + const { databaseId, databaseDocumentId } = await createDatabase({}); + const db = getDb(); + const now = new Date().toISOString(); + const datePropertyId = nextId("date_property"); + await db.insert(schema.documentPropertyDefinitions).values({ + id: datePropertyId, + ownerEmail: OWNER, + databaseId, + name: "Date", + type: "date", + visibility: "always_show", + optionsJson: "{}", + position: 0, + createdAt: now, + updatedAt: now, + }); + + const rows = Array.from({ length: 584 }, (_, index) => { + const documentId = nextId("date_row_doc"); + return { + documentId, + itemId: nextId("date_row_item"), + valueId: nextId("date_row_value"), + index, + date: new Date(Date.UTC(2024, 0, 1) + index * 86_400_000) + .toISOString() + .slice(0, 10), + }; + }); + await db.insert(schema.documents).values( + rows.map((row) => ({ + id: row.documentId, + ownerEmail: OWNER, + parentId: databaseDocumentId, + title: `Dated row ${row.index}`, + content: "", + position: row.index, + visibility: "private" as const, + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.contentDatabaseItems).values( + rows.map((row) => ({ + id: row.itemId, + ownerEmail: OWNER, + databaseId, + documentId: row.documentId, + position: row.index, + createdAt: now, + updatedAt: now, + })), + ); + await db.insert(schema.documentPropertyValues).values( + rows.map((row) => ({ + id: row.valueId, + ownerEmail: OWNER, + documentId: row.documentId, + propertyId: datePropertyId, + valueJson: JSON.stringify({ start: row.date, includeTime: false }), + createdAt: now, + updatedAt: now, + })), + ); + + const tableQuery = { + search: "", + filters: [], + sorts: [ + { + key: datePropertyId, + label: "Date", + direction: "desc" as const, + }, + ], + filterMode: "and" as const, + }; + const startedAt = performance.now(); + const response = await runWithRequestContext({ userEmail: OWNER }, () => + queryContentDatabaseItemsAction.run({ + databaseId, + limit: 1, + offset: 100, + tableQuery, + }), + ); + const durationMs = performance.now() - startedAt; + + expect(response.items.map((item) => item.document.title)).toEqual([ + "Dated row 483", + ]); + expect(response.pagination).toEqual({ + offset: 100, + limit: 1, + totalItems: 584, + returnedItems: 1, + hasMore: true, + }); + expect(durationMs).toBeLessThan(1_000); + + const { getContentDatabasePageResponse } = + await import("./_database-utils.js"); + const page = await runWithRequestContext({ userEmail: OWNER }, () => + getContentDatabasePageResponse(databaseId, { + limit: 1, + offset: 100, + tableQuery, + includeSources: false, + }), + ); + expect(page.hydratedItemCount).toBe(1); + }); + it("blocks direct document and property reads for soft-deleted database pages", async () => { const deletedAt = new Date().toISOString(); const { databaseId, databaseDocumentId } = await createDatabase({ diff --git a/templates/content/actions/preview-builder-source-review.ts b/templates/content/actions/preview-builder-source-review.ts index 96353446a3..bc620deea7 100644 --- a/templates/content/actions/preview-builder-source-review.ts +++ b/templates/content/actions/preview-builder-source-review.ts @@ -6,7 +6,6 @@ import type { PreviewBuilderSourceReviewRequest, PreviewBuilderSourceReviewResponse, } from "../shared/api.js"; -import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { getContentDatabaseSourceSnapshotForReview, resolveDatabaseForSourceMutation, @@ -37,83 +36,67 @@ export default defineAction({ run: async ( args: PreviewBuilderSourceReviewRequest, ): Promise => { - const timing = createBuilderSourceTiming("preview_builder_source_review"); - try { - const database = await timing.measure("access_and_database", async () => { - const database = await resolveDatabaseForSourceMutation(args); - if (!database) throw new Error("Database not found."); - await assertAccess("document", database.documentId, "editor"); - return database; - }); + const database = await resolveDatabaseForSourceMutation(args); + if (!database) throw new Error("Database not found."); + await assertAccess("document", database.documentId, "editor"); - const selectedDocumentIds = new Set(args.documentIds ?? []); - if (args.scope === "selected" && selectedDocumentIds.size === 0) { - throw new Error("Select at least one Builder row before reviewing it."); - } + const selectedDocumentIds = new Set(args.documentIds ?? []); + if (args.scope === "selected" && selectedDocumentIds.size === 0) { + throw new Error("Select at least one Builder row before reviewing it."); + } - const source = await timing.measure("snapshot_and_candidate_diff", () => - getContentDatabaseSourceSnapshotForReview( - database, - args.sourceId, - args.scope === "selected" ? args.documentIds : undefined, - ), - ); - if (!source || source.sourceType !== "builder-cms") { - throw new Error( - "Attach a Builder CMS source before reviewing updates.", - ); - } + const source = await getContentDatabaseSourceSnapshotForReview( + database, + args.sourceId, + args.scope === "selected" ? args.documentIds : undefined, + ); + if (!source || source.sourceType !== "builder-cms") { + throw new Error("Attach a Builder CMS source before reviewing updates."); + } - const allReviewableChanges = source.changeSets.filter((changeSet) => { - if ( - changeSet.direction !== "outbound" || - (changeSet.state !== "pending_push" && - changeSet.state !== "staged_revision" && - changeSet.state !== "approved") - ) { - return false; - } - return ( - args.scope === "all" || - (!!changeSet.documentId && - selectedDocumentIds.has(changeSet.documentId)) - ); - }); - const changeSets = [...allReviewableChanges] - .sort( - (left, right) => - reviewPreparePriority(left, source) - - reviewPreparePriority(right, source), - ) - .slice(0, BUILDER_SOURCE_REVIEW_PREPARE_LIMIT); - const authoritativeSource = await timing.measure( - "authoritative_target_rows", - () => withAuthoritativeBuilderTargetRows({ source, changeSets }), - ); - const review = - changeSets.length > 0 - ? buildBuilderSourceReviewPayload({ - source: authoritativeSource, - changeSets, - }) - : null; - if (review) { - review.totalRowCount = allReviewableChanges.length; - review.preparedRowLimit = changeSets.length; + const allReviewableChanges = source.changeSets.filter((changeSet) => { + if ( + changeSet.direction !== "outbound" || + (changeSet.state !== "pending_push" && + changeSet.state !== "staged_revision" && + changeSet.state !== "approved") + ) { + return false; } - - const response = { - sourceId: source.id, - sourceTable: source.sourceTable, - changeSetIds: changeSets.map((changeSet) => changeSet.id), - review, - timings: timing.finish(), - }; - timing.log("succeeded"); - return response; - } catch (error) { - timing.log("failed"); - throw error; + return ( + args.scope === "all" || + (!!changeSet.documentId && + selectedDocumentIds.has(changeSet.documentId)) + ); + }); + const changeSets = [...allReviewableChanges] + .sort( + (left, right) => + reviewPreparePriority(left, source) - + reviewPreparePriority(right, source), + ) + .slice(0, BUILDER_SOURCE_REVIEW_PREPARE_LIMIT); + const authoritativeSource = await withAuthoritativeBuilderTargetRows({ + source, + changeSets, + }); + const review = + changeSets.length > 0 + ? buildBuilderSourceReviewPayload({ + source: authoritativeSource, + changeSets, + }) + : null; + if (review) { + review.totalRowCount = allReviewableChanges.length; + review.preparedRowLimit = changeSets.length; } + + return { + sourceId: source.id, + sourceTable: source.sourceTable, + changeSetIds: changeSets.map((changeSet) => changeSet.id), + review, + }; }, }); diff --git a/templates/content/actions/preview-content-database-source-attach.ts b/templates/content/actions/preview-content-database-source-attach.ts index f0befa26ed..06555811fc 100644 --- a/templates/content/actions/preview-content-database-source-attach.ts +++ b/templates/content/actions/preview-content-database-source-attach.ts @@ -7,7 +7,6 @@ import type { ContentDatabaseItem, } from "../shared/api.js"; import { readBuilderCmsContentEntries } from "./_builder-cms-read-client.js"; -import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { builderCmsImportIds, resolveDatabaseForSourceMutation, @@ -25,29 +24,21 @@ export default defineAction({ http: { method: "GET" }, readOnly: true, run: async (args): Promise => { - const timing = createBuilderSourceTiming("preview-builder-attach"); - const database = await timing.measure("resolve-database", () => - resolveDatabaseForSourceMutation(args), - ); + const database = await resolveDatabaseForSourceMutation(args); if (!database) throw new Error("Database not found."); - await timing.measure("access", () => - assertAccess("document", database.documentId, "editor"), - ); + await assertAccess("document", database.documentId, "editor"); - const read = await timing.measure("provider-first-page", () => - readBuilderCmsContentEntries({ - model: args.sourceTable, - fieldPaths: args.fieldPaths, - maxPages: 1, - }), - ); + const read = await readBuilderCmsContentEntries({ + model: args.sourceTable, + fieldPaths: args.fieldPaths, + maxPages: 1, + }); if (read.state !== "live") { throw new Error( read.message ?? "Builder rows are not available for preview.", ); } - const mapStartedAt = timing.start(); const items: ContentDatabaseItem[] = read.entries.map((entry, index) => { const ids = builderCmsImportIds({ ownerEmail: database.ownerEmail, @@ -83,18 +74,13 @@ export default defineAction({ }, }; }); - timing.record("project-items", mapStartedAt); - - const response = { + return { databaseId: database.id, documentId: database.documentId, sourceTable: args.sourceTable, items, fetchedAt: read.fetchedAt, hasMore: read.progress.hasMore, - timings: timing.finish(), }; - timing.log("succeeded"); - return response; }, }); diff --git a/templates/content/actions/process-builder-body-hydration.ts b/templates/content/actions/process-builder-body-hydration.ts index 4469369351..24d22013b0 100644 --- a/templates/content/actions/process-builder-body-hydration.ts +++ b/templates/content/actions/process-builder-body-hydration.ts @@ -8,7 +8,6 @@ import type { ProcessBuilderBodyHydrationRequest, ProcessBuilderBodyHydrationResponse, } from "../shared/api.js"; -import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { processBuilderBodyHydrationQueue } from "./_database-source-utils.js"; export default defineAction({ @@ -22,38 +21,28 @@ export default defineAction({ run: async ( args: ProcessBuilderBodyHydrationRequest, ): Promise => { - const timing = createBuilderSourceTiming("process-builder-body-hydration"); const db = getDb(); - const [source] = await timing.measure("resolve-source", () => - db - .select({ - id: schema.contentDatabaseSources.id, - databaseDocumentId: schema.contentDatabases.documentId, - }) - .from(schema.contentDatabaseSources) - .innerJoin( - schema.contentDatabases, - eq( - schema.contentDatabases.id, - schema.contentDatabaseSources.databaseId, - ), - ) - .where(eq(schema.contentDatabaseSources.id, args.sourceId)), - ); + const [source] = await db + .select({ + id: schema.contentDatabaseSources.id, + databaseDocumentId: schema.contentDatabases.documentId, + }) + .from(schema.contentDatabaseSources) + .innerJoin( + schema.contentDatabases, + eq( + schema.contentDatabases.id, + schema.contentDatabaseSources.databaseId, + ), + ) + .where(eq(schema.contentDatabaseSources.id, args.sourceId)); if (!source) throw new Error(`Source "${args.sourceId}" not found`); - await timing.measure("access", () => - assertAccess("document", source.databaseDocumentId, "editor"), - ); - const result = await timing.measure("process-batch", () => - processBuilderBodyHydrationQueue({ - sourceId: args.sourceId, - documentId: args.documentId, - limit: args.limit, - preloadBodies: !args.documentId, - }), - ); - const response = { ...result, timings: timing.finish() }; - timing.log("succeeded"); - return response; + await assertAccess("document", source.databaseDocumentId, "editor"); + return processBuilderBodyHydrationQueue({ + sourceId: args.sourceId, + documentId: args.documentId, + limit: args.limit, + preloadBodies: !args.documentId, + }); }, }); diff --git a/templates/content/actions/query-content-database-items.ts b/templates/content/actions/query-content-database-items.ts index fe466ffc3d..777619db1a 100644 --- a/templates/content/actions/query-content-database-items.ts +++ b/templates/content/actions/query-content-database-items.ts @@ -5,7 +5,6 @@ import type { ContentDatabaseItemsPageResponse, ContentDatabaseUnavailableResponse, } from "../shared/api.js"; -import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { CONTENT_DATABASE_MAX_READ_LIMIT, contentDatabaseTableQuerySchema, @@ -40,34 +39,26 @@ export default defineAction({ }): Promise< ContentDatabaseItemsPageResponse | ContentDatabaseUnavailableResponse > => { - const timing = createBuilderSourceTiming("query-content-database-items"); - const resolved = await timing.measure("resolve-access", () => - resolveContentDatabaseRead({ - databaseId, - documentId, - }), - ); + const resolved = await resolveContentDatabaseRead({ + databaseId, + documentId, + }); if (!resolved.available) return resolved; - const page = await timing.measure("build-page", () => - getContentDatabasePageResponse(resolved.database.id, { - // This action is the bounded table replacement path; unlike the legacy - // database response, an omitted limit must not turn it into a full read. - limit: limit ?? 100, - offset, - tableQuery, - database: resolved.database, - }), - ); - const response = { + const page = await getContentDatabasePageResponse(resolved.database.id, { + // This action is the bounded table replacement path; unlike the legacy + // database response, an omitted limit must not turn it into a full read. + limit: limit ?? 100, + offset, + tableQuery, + database: resolved.database, + }); + return { items: page.items, source: page.source, sources: page.sources, pagination: page.pagination, tableQueryMode: page.tableQueryMode, - timings: timing.finish(), }; - timing.log("succeeded"); - return response; }, }); diff --git a/templates/content/actions/refresh-content-database-source.ts b/templates/content/actions/refresh-content-database-source.ts index 64674cca52..1c9f2f4ea9 100644 --- a/templates/content/actions/refresh-content-database-source.ts +++ b/templates/content/actions/refresh-content-database-source.ts @@ -3,7 +3,6 @@ import { assertAccess } from "@agent-native/core/sharing"; import { z } from "zod"; import type { ContentDatabaseSourceStatusResponse } from "../shared/api.js"; -import { createBuilderSourceTiming } from "./_builder-source-timings.js"; import { getContentDatabaseSourceAdapter } from "./_content-database-source-adapters.js"; import { claimBuilderCmsSourceRefresh, @@ -52,18 +51,11 @@ export default defineAction({ ), }), run: async (args): Promise => { - const timing = createBuilderSourceTiming("refresh-content-database-source"); - const database = await timing.measure("resolve-database", () => - resolveDatabaseForSourceMutation(args), - ); + const database = await resolveDatabaseForSourceMutation(args); if (!database) throw new Error("Database not found."); - await timing.measure("access", () => - assertAccess("document", database.documentId, "editor"), - ); + await assertAccess("document", database.documentId, "editor"); - const source = await timing.measure("resolve-source", () => - getExistingSourceForWrite(database.id, args.sourceId), - ); + const source = await getExistingSourceForWrite(database.id, args.sourceId); if (!source) { return { database: serializeDatabase(database), @@ -84,24 +76,20 @@ export default defineAction({ }); skippedOverlappingBuilderRefresh = !claimedSource; if (claimedSource) { - await timing - .measure("builder-resync", () => - resyncBuilderCmsSourceSnapshot({ - database, - source: claimedSource.source, - now, - runFullRefresh: args.fullRefresh === true, - finishPagination: args.finishBuilderPagination === true, - refreshClaimId: claimedSource.claimId, - }), - ) - .catch(async (error: unknown) => { - await releaseBuilderCmsSourceRefreshClaim({ - sourceId: source.id, - claimId: claimedSource.claimId, - }); - throw error; + await resyncBuilderCmsSourceSnapshot({ + database, + source: claimedSource.source, + now, + runFullRefresh: args.fullRefresh === true, + finishPagination: args.finishBuilderPagination === true, + refreshClaimId: claimedSource.claimId, + }).catch(async (error: unknown) => { + await releaseBuilderCmsSourceRefreshClaim({ + sourceId: source.id, + claimId: claimedSource.claimId, }); + throw error; + }); } } else if (source.sourceType === "local-table") { // Read-only federated secondary; its rows are re-read on demand, nothing @@ -142,11 +130,9 @@ export default defineAction({ } else { throw new Error(`Unsupported source type "${source.sourceType}".`); } - const snapshot = await timing.measure("status-snapshot", () => - args.sourceId - ? getContentDatabaseSourceSnapshotById(database, args.sourceId) - : getContentDatabaseSourceSnapshot(database), - ); + const snapshot = args.sourceId + ? await getContentDatabaseSourceSnapshotById(database, args.sourceId) + : await getContentDatabaseSourceSnapshot(database); const builderProgress = snapshot?.sourceType === "builder-cms" ? snapshot.metadata : null; @@ -158,7 +144,7 @@ export default defineAction({ ? builderProgress.lastReadFetchedEntryCount : undefined; - const response: ContentDatabaseSourceStatusResponse = { + return { database: serializeDatabase(database), mode: "source-backed", summary: snapshot @@ -171,9 +157,6 @@ export default defineAction({ : `${snapshot.sourceName} resynced locally; field mappings and row identity now reflect the current database snapshot.` : "Source metadata refreshed.", source: snapshot, - timings: timing.finish(), }; - timing.log("succeeded"); - return response; }, }); diff --git a/templates/content/actions/resync-content-database-source.db.test.ts b/templates/content/actions/resync-content-database-source.db.test.ts index 05877b430f..2251881ad1 100644 --- a/templates/content/actions/resync-content-database-source.db.test.ts +++ b/templates/content/actions/resync-content-database-source.db.test.ts @@ -1999,6 +1999,104 @@ it("records freshly imported Builder row identities even when title and URL keys expect(concurrentSourceRows).toHaveLength(2); }); +it("repairs a deterministic Builder item whose source-row link is missing", async () => { + builderReadMock.mode = "full"; + builderReadMock.calls = []; + const db = getDb(); + const now = new Date().toISOString(); + const databaseId = "db_interrupted_builder_attach"; + const databaseDocId = "doc_db_interrupted_builder_attach"; + const sourceId = "src_interrupted_builder_attach"; + const sourceTable = "collection-metadata-only"; + const entryId = "entry-metadata-only-1"; + const { documentId, itemId } = ( + await import("./_database-source-utils.js") + ).builderCmsImportIds({ + ownerEmail: OWNER, + databaseId, + sourceTable, + entryId, + }); + + await db.insert(schema.documents).values([ + { + id: databaseDocId, + spaceId: IMPORT_SPACE_ID, + ownerEmail: OWNER, + title: "Interrupted Builder attach", + createdAt: now, + updatedAt: now, + }, + { + id: documentId, + spaceId: IMPORT_SPACE_ID, + ownerEmail: OWNER, + parentId: databaseDocId, + title: "Metadata-only hydration", + content: "", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + spaceId: IMPORT_SPACE_ID, + ownerEmail: OWNER, + documentId: databaseDocId, + title: "Interrupted Builder attach", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: itemId, + ownerEmail: OWNER, + databaseId, + documentId, + position: 0, + bodyHydrationStatus: "pending", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: OWNER, + databaseId, + sourceType: "builder-cms", + sourceName: sourceTable, + sourceTable, + createdAt: now, + updatedAt: now, + }); + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, databaseId)); + const [source] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, sourceId)); + + await resync({ database, source, now, runFullRefresh: true }); + + const response = await ( + await import("./_database-utils.js") + ).getContentDatabaseResponse(databaseId); + const sourceRows = await db + .select({ + databaseItemId: schema.contentDatabaseSourceRows.databaseItemId, + documentId: schema.contentDatabaseSourceRows.documentId, + sourceRowId: schema.contentDatabaseSourceRows.sourceRowId, + }) + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, sourceId)); + + expect(response.items.map((item) => item.id)).toEqual([itemId]); + expect(sourceRows).toEqual([ + { databaseItemId: itemId, documentId, sourceRowId: entryId }, + ]); +}); + it("repairs a legacy organization database into its organization space", async () => { builderReadMock.mode = "full"; builderReadMock.calls = []; diff --git a/templates/content/shared/api.ts b/templates/content/shared/api.ts index c07add0acc..5cda67080d 100644 --- a/templates/content/shared/api.ts +++ b/templates/content/shared/api.ts @@ -726,7 +726,6 @@ export interface ContentDatabaseSourceStatusResponse { mode: "local" | "source-backed"; summary: string; source: ContentDatabaseSource | null; - timings?: BuilderActionTiming[]; } export interface BuilderCmsModelFieldSummary { @@ -808,7 +807,7 @@ export interface ContentDatabaseResponse { export type ContentDatabaseItemsPageResponse = Pick< ContentDatabaseResponse, "items" | "source" | "sources" | "pagination" | "tableQueryMode" -> & { timings?: BuilderActionTiming[] }; +>; export interface BuilderActionTiming { name: string; @@ -940,7 +939,6 @@ export interface BuilderCmsAttachPreviewResponse { items: ContentDatabaseItem[]; fetchedAt: string; hasMore: boolean; - timings?: BuilderActionTiming[]; } export interface ChangeContentDatabaseSourceRoleRequest { @@ -1294,7 +1292,6 @@ export interface PreviewBuilderSourceReviewResponse { sourceTable: string; changeSetIds: string[]; review: ContentDatabaseSourceReviewPayload | null; - timings?: BuilderActionTiming[]; } export interface PrepareBuilderSourceReviewResponse { @@ -1323,5 +1320,4 @@ export interface ProcessBuilderBodyHydrationResponse { succeeded: number; failed: number; remaining: number; - timings?: BuilderActionTiming[]; } From 560ed09a2301b4ba60395be5bab4511c38f1c018 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:42:10 -0400 Subject: [PATCH 19/21] Stabilize Builder attach preview and hydration claims --- .../actions/_builder-cms-read-client.test.ts | 39 +++++ .../actions/_builder-cms-read-client.ts | 15 +- .../content/actions/_database-source-utils.ts | 3 +- .../preview-content-database-source-attach.ts | 1 + .../resync-content-database-source.db.test.ts | 146 +++++++++++++++++- .../editor/database/DatabaseView.tsx | 11 +- 6 files changed, 205 insertions(+), 10 deletions(-) diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index f21ded1886..4663110853 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -456,6 +456,45 @@ describe("Builder CMS read client", () => { }); }); + it("allows the read-only attach preview to use Builder's cached projection", async () => { + process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PUBLIC_KEY" ? "public-key" : null, + ); + const fetchImpl = vi.fn(async (input: URL) => { + expect(input.searchParams.get("noCache")).toBeNull(); + expect(input.searchParams.get("cachebust")).toBeNull(); + expect(input.searchParams.get("includeUnpublished")).toBe("true"); + expect(input.searchParams.get("fields")).toContain("data.title"); + return new Response( + JSON.stringify({ + results: [ + { + id: "builder-preview-entry", + data: { title: "Cached preview" }, + }, + ], + }), + { status: 200 }, + ); + }); + + await expect( + readBuilderCmsContentEntries({ + model: "blog_article", + fieldPaths: ["data.title"], + allowCached: true, + maxPages: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toMatchObject({ + state: "live", + entries: [{ id: "builder-preview-entry", title: "Cached preview" }], + progress: { partial: false }, + }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it("can project Builder bodies in paged list reads for bulk hydration", async () => { process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; resolveBuilderCredentialMock.mockImplementation(async (key) => diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index a22ffa1ce7..155524820d 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -980,6 +980,7 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { model: string; fieldPaths?: readonly string[]; includeBodies?: boolean; + allowCached?: boolean; rawData?: boolean; limit?: number; maxPages?: number; @@ -998,11 +999,13 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { // Fidelity reads intentionally do neither: Builder's raw data object is the // clone contract, including unresolved references and every nested field. url.searchParams.set("enrich", args.rawData === true ? "false" : "true"); - url.searchParams.set("noCache", "true"); - url.searchParams.set( - "cachebust", - args.rawData === true ? String(Date.now()) : "true", - ); + if (args.allowCached !== true || args.rawData === true) { + url.searchParams.set("noCache", "true"); + url.searchParams.set( + "cachebust", + args.rawData === true ? String(Date.now()) : "true", + ); + } url.searchParams.set("includeUnpublished", "true"); if (args.rawData !== true) { url.searchParams.set( @@ -1306,6 +1309,7 @@ export async function readBuilderCmsContentEntries(args: { model: string; fieldPaths?: readonly string[]; includeBodies?: boolean; + allowCached?: boolean; rawData?: boolean; requirePrivateKey?: boolean; limit?: number; @@ -1341,6 +1345,7 @@ export async function readBuilderCmsContentEntries(args: { model: args.model, fieldPaths: args.fieldPaths, includeBodies: args.includeBodies, + allowCached: args.allowCached, rawData: args.rawData, limit: args.limit, maxPages: args.maxPages, diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 8a8dd54fb3..2f030886c2 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -2353,7 +2353,7 @@ export async function processBuilderBodyHydrationQueue(args: { let succeeded = 0; let failed = 0; const claimedJobs: ContentDatabaseBodyHydrationQueueRowDb[] = []; - for (const jobChunk of chunks(jobs, bulkChunkSizeForColumnCount(2))) { + for (const jobChunk of chunks(jobs, bulkChunkSizeForColumnCount(3))) { const claimFilters = jobChunk.map((job) => and( eq(schema.contentDatabaseBodyHydrationQueue.id, job.id), @@ -2361,6 +2361,7 @@ export async function processBuilderBodyHydrationQueue(args: { schema.contentDatabaseBodyHydrationQueue.sourceEntryJson, job.sourceEntryJson, ), + eq(schema.contentDatabaseBodyHydrationQueue.attempts, job.attempts), ), ); if (claimFilters.length === 0) continue; diff --git a/templates/content/actions/preview-content-database-source-attach.ts b/templates/content/actions/preview-content-database-source-attach.ts index 06555811fc..f857f0f6a4 100644 --- a/templates/content/actions/preview-content-database-source-attach.ts +++ b/templates/content/actions/preview-content-database-source-attach.ts @@ -31,6 +31,7 @@ export default defineAction({ const read = await readBuilderCmsContentEntries({ model: args.sourceTable, fieldPaths: args.fieldPaths, + allowCached: true, maxPages: 1, }); if (read.state !== "live") { diff --git a/templates/content/actions/resync-content-database-source.db.test.ts b/templates/content/actions/resync-content-database-source.db.test.ts index 2251881ad1..548bf464d7 100644 --- a/templates/content/actions/resync-content-database-source.db.test.ts +++ b/templates/content/actions/resync-content-database-source.db.test.ts @@ -3571,7 +3571,6 @@ it("does not let open-row hydration promotion downgrade a queued full Builder bo ), ) .where(eq(schema.documents.id, documentId)); - expect(after.content).toBe(fullBody); expect(after.status).toBe("hydrated"); expect(after.queued).toBeNull(); @@ -4123,6 +4122,151 @@ it("rebuilds an empty queued Builder body from the current source row before giv expect(after.attempts).toBeNull(); }); +it("lets only one overlapping worker claim and finish the same Builder body job", async () => { + const db = getDb(); + const now = new Date().toISOString(); + const databaseId = "db_hydration_overlapping_claim"; + const databaseDocId = "doc_db_hydration_overlapping_claim"; + const documentId = "doc_hydration_overlapping_claim"; + const itemId = "item_hydration_overlapping_claim"; + const sourceId = "src_hydration_overlapping_claim"; + const sourceRowId = "entry_hydration_overlapping_claim"; + const queueId = "queue_hydration_overlapping_claim"; + const body = "Only the worker holding the queue claim may write this body."; + const entry = { + id: sourceRowId, + model: "collection-hydration-overlapping-claim", + title: "Hydration overlapping claim", + urlPath: "/blog/hydration-overlapping-claim", + updatedAt: "2026-01-01T00:00:00.000Z", + sourceValues: { + "data.title": "Hydration overlapping claim", + "data.url": "/blog/hydration-overlapping-claim", + lastUpdated: "2026-01-01T00:00:00.000Z", + [BUILDER_CMS_BODY_CONTENT_KEY]: body, + }, + }; + + await db.insert(schema.documents).values([ + { + id: databaseDocId, + ownerEmail: OWNER, + title: "DB hydration overlapping claim", + createdAt: now, + updatedAt: now, + }, + { + id: documentId, + ownerEmail: OWNER, + parentId: databaseDocId, + title: "Hydration overlapping claim", + content: "", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + ownerEmail: OWNER, + documentId: databaseDocId, + title: "DB hydration overlapping claim", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: OWNER, + databaseId, + sourceType: "builder-cms", + sourceName: "collection-hydration-overlapping-claim", + sourceTable: "collection-hydration-overlapping-claim", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: itemId, + ownerEmail: OWNER, + databaseId, + documentId, + position: 0, + bodyHydrationStatus: "pending", + bodyHydrationError: null, + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSourceRows).values({ + id: "row_hydration_overlapping_claim", + ownerEmail: OWNER, + sourceId, + databaseItemId: itemId, + documentId, + sourceRowId, + sourceQualifiedId: `builder-cms://collection-hydration-overlapping-claim/${sourceRowId}`, + sourceDisplayKey: "Hydration overlapping claim", + sourceValuesJson: JSON.stringify(entry.sourceValues), + provenance: "Builder CMS read adapter", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseBodyHydrationQueue).values({ + id: queueId, + ownerEmail: OWNER, + sourceId, + databaseItemId: itemId, + documentId, + sourceRowId, + sourceTable: "collection-hydration-overlapping-claim", + sourceEntryJson: JSON.stringify(entry), + priority: 0, + attempts: 0, + createdAt: now, + updatedAt: now, + }); + const [queuedJob] = await db + .select() + .from(schema.contentDatabaseBodyHydrationQueue) + .where(eq(schema.contentDatabaseBodyHydrationQueue.id, queueId)); + + const results = await Promise.all([ + hydrateQueuedBodies({ sourceId, limit: 1, preloadedJobs: [queuedJob] }), + hydrateQueuedBodies({ sourceId, limit: 1, preloadedJobs: [queuedJob] }), + ]); + + const [after] = await db + .select({ + content: schema.documents.content, + status: schema.contentDatabaseItems.bodyHydrationStatus, + error: schema.contentDatabaseItems.bodyHydrationError, + queued: schema.contentDatabaseBodyHydrationQueue.id, + }) + .from(schema.documents) + .innerJoin( + schema.contentDatabaseItems, + eq(schema.contentDatabaseItems.documentId, schema.documents.id), + ) + .leftJoin( + schema.contentDatabaseBodyHydrationQueue, + eq( + schema.contentDatabaseBodyHydrationQueue.databaseItemId, + schema.contentDatabaseItems.id, + ), + ) + .where(eq(schema.documents.id, documentId)); + const remainingQueue = await db + .select({ id: schema.contentDatabaseBodyHydrationQueue.id }) + .from(schema.contentDatabaseBodyHydrationQueue) + .where(eq(schema.contentDatabaseBodyHydrationQueue.sourceId, sourceId)); + + expect(results.map((result) => result.processed).sort()).toEqual([0, 1]); + expect(results.map((result) => result.succeeded).sort()).toEqual([0, 1]); + expect(results.every((result) => result.failed === 0)).toBe(true); + expect(after.content).toBe(body); + expect(after.status).toBe("hydrated"); + expect(after.error).toBeNull(); + expect(after.queued).toBeNull(); + expect(remainingQueue).toEqual([]); +}); + it("repairs a stale remote body hash without overwriting a locally diverged document", async () => { const db = getDb(); const now = new Date().toISOString(); diff --git a/templates/content/app/components/editor/database/DatabaseView.tsx b/templates/content/app/components/editor/database/DatabaseView.tsx index 811183b25b..38c3575275 100644 --- a/templates/content/app/components/editor/database/DatabaseView.tsx +++ b/templates/content/app/components/editor/database/DatabaseView.tsx @@ -7569,6 +7569,7 @@ function DatabaseSettingsPanelSheet({ onReviewBuilderUpdate={onReviewBuilderUpdate} onSetBuilderLiveWrites={onSetBuilderLiveWrites} sourceActionPending={sourceActionPending} + builderAttachPreviewPending={builderAttachPreview.isFetching} sourcePendingOperations={sourcePendingOperations} /> ) : panel === "layout" ? ( @@ -7970,6 +7971,7 @@ function DatabaseSettingsSourcePanel({ onReviewBuilderUpdate, onSetBuilderLiveWrites, sourceActionPending, + builderAttachPreviewPending, sourcePendingOperations, }: { source: ContentDatabaseSource | null; @@ -8003,12 +8005,15 @@ function DatabaseSettingsSourcePanel({ onReviewBuilderUpdate: (sourceId: string) => void; onSetBuilderLiveWrites: (settings: BuilderSourceWriteSettingsInput) => void; sourceActionPending: boolean; + builderAttachPreviewPending: boolean; sourcePendingOperations: DatabaseSourcePendingOperations; }) { const { isCodeMode } = useCodeMode(); const navigate = useNavigate(); const builderSources = databaseAttachedBuilderSources(sources, source); const builderStatus = useBuilderStatus(); + const builderAttachPending = + sourceActionPending || builderAttachPreviewPending; const builderConfigured = builderStatus.status?.configured === true; const builderModelsQuery = useBuilderCmsModels(builderConfigured); const builderOrgName = builderStatus.status?.orgName ?? null; @@ -8313,7 +8318,7 @@ function DatabaseSettingsSourcePanel({ displayName: model.displayName, }} canEdit={canEdit} - pending={sourceActionPending} + pending={builderAttachPending} onAddDetails={() => onNavPush({ kind: "keyConfirm", @@ -8335,7 +8340,7 @@ function DatabaseSettingsSourcePanel({