diff --git a/.env.development b/.env.development index b993fe6bd1c..8980cbe347a 100644 --- a/.env.development +++ b/.env.development @@ -16,3 +16,6 @@ CAPACITOR_SERVER_URL=https://localhost:3000 # AI SERVER VITE_AI_URL=http://localhost:3111/ai + +# TREECRDT SYNC (optional) — direct ws/wss or http(s) discovery bootstrap; doc id matches thoughtspace (tsid) +# VITE_TREECRDT_SYNC_BASE_URL= diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index b26a5adad69..b7cb0464e00 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -104,6 +104,9 @@ jobs: name: BrowserStack runs-on: ubuntu-latest if: github.event_name != 'pull_request_target' || github.event.pull_request.changed_files > 0 + env: + BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }} + BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} # The repo-wide gate — see the concurrency comment at the top of the file for why it sits at the # job level (it must coexist with the per-PR workflow-level group; a workflow gets only one @@ -203,9 +206,11 @@ jobs: cache: 'yarn' - name: Install npm dependencies + if: env.BROWSERSTACK_USERNAME != '' && env.BROWSERSTACK_ACCESS_KEY != '' uses: ./.github/actions/install - name: Build + if: env.BROWSERSTACK_USERNAME != '' && env.BROWSERSTACK_ACCESS_KEY != '' run: yarn build - name: Generate tunnel token @@ -216,6 +221,7 @@ jobs: echo "TUNNEL_TOKEN=$TOKEN" >> "$GITHUB_ENV" - name: Serve + if: env.BROWSERSTACK_USERNAME != '' && env.BROWSERSTACK_ACCESS_KEY != '' env: HAS_TOKEN_GATE: ${{ steps.server_mode.outputs.has_token_gate }} run: | @@ -249,7 +255,12 @@ jobs: sleep 1 done + - name: Skip BrowserStack + if: env.BROWSERSTACK_USERNAME == '' || env.BROWSERSTACK_ACCESS_KEY == '' + run: echo "Skipping BrowserStack because credentials are not configured." + - name: Test iOS on BrowserStack + if: env.BROWSERSTACK_USERNAME != '' && env.BROWSERSTACK_ACCESS_KEY != '' run: yarn test:ios env: BROWSERSTACK_USERNAME: ${{secrets.BROWSERSTACK_USERNAME}} diff --git a/.gitignore b/.gitignore index 936a5eeb08b..b840c782767 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ desktop/gen/schemas # diff images output from failed Puppeteer snapshot tests __diff_output__ +# wa-sqlite assets (generated by treecrdt vite plugin) +public/wa-sqlite + ## PandaCSS styled-system styled-system-studio diff --git a/docs/folder-structure.md b/docs/folder-structure.md index d8e55b659c5..0550fcd1e96 100644 --- a/docs/folder-structure.md +++ b/docs/folder-structure.md @@ -8,12 +8,12 @@ The main directory structure is organized as follows. Tests are located in a sub - [`/src/actions`](../src/actions) — Redux reducers and action-creators are co-located. Prefer reducers when possible, as they are pure functions that are more easily testable and composable. Only define an action creator if it requires a side effect. Use [`util/reducerFlow`](../src/util/reducerFlow.ts) to compose reducers. - [`/src/commands`](../src/commands) — Keyboard, gesture, and toolbar commands (formerly `shortcuts`). One file per command, plus an `index.ts` barrel. See [commands.md](commands.md) for the architecture. - [`/src/components`](../src/components) — React components. -- [`/src/data-providers`](../src/data-providers) — Storage and sync backends implementing the [`DataProvider`](../src/data-providers/DataProvider.ts) interface. The live implementation is YJS in [`yjs/thoughtspace.ts`](../src/data-providers/yjs/thoughtspace.ts). See [persistence.md](persistence.md). +- [`/src/data-providers`](../src/data-providers) — Storage and sync backends implementing the [`DataProvider`](../src/data-providers/DataProvider.ts) interface. The live implementation is TreeCRDT in [`treecrdt/thoughtspace.ts`](../src/data-providers/treecrdt/thoughtspace.ts). See [persistence.md](persistence.md). - [`/src/device`](../src/device) — Device/DOM-level helpers for selection, scrolling, clipboard, focus, and platform detection. The selection wrapper [`device/selection.ts`](../src/device/selection.ts) is the single point of access to `window.getSelection()` (enforced by lint). See [cursor-and-caret.md](cursor-and-caret.md). - [`/src/e2e`](../src/e2e) — End-to-end test setup, including Puppeteer and iOS environments. See [testing.md](testing.md). - [`/src/hooks`](../src/hooks) — React hooks. -- [`/src/recipes`](../src/recipes) — Panda CSS recipes that define styled component variants. Prefer inline CSS (`css({ ... })`); only add a recipe here when the styles have variants or are shared by multiple components. -- [`/src/redux-enhancers`](../src/redux-enhancers) — Redux enhancers (e.g. the [`pushQueue`](../src/redux-enhancers/pushQueue.ts) that flushes state mutations to YJS). +- [`/src/recipes`](../src/recipes) — Panda CSS recipes that define styled component variants. New components should use these or inline styles. +- [`/src/redux-enhancers`](../src/redux-enhancers) — Redux enhancers (e.g. the [`pushQueue`](../src/redux-enhancers/pushQueue.ts) that flushes state mutations to thoughtspace persistence). - [`/src/redux-middleware`](../src/redux-middleware) — Redux middleware (e.g. the [`pullQueue`](../src/redux-middleware/pullQueue.ts) that loads thoughts on demand, or [`clearSelection`](../src/redux-middleware/clearSelection.ts) that clears the browser caret on cursor changes). - [`/src/selectors`](../src/selectors) — Pure functions that compute (and often memoize) slices from the Redux state. See [data-model.md](data-model.md) for the canonical traversal selectors. - [`/src/stores`](../src/stores) — Lightweight non-Redux ministores for ephemeral UI state. Examples: [`editingValue`](../src/stores/editingValue.ts) (the in-progress thought text), [`viewport`](../src/stores/viewport.ts), [`scrollTop`](../src/stores/scrollTop.ts), [`gesture`](../src/stores/gesture.ts), [`syncStatus`](../src/stores/syncStatus.ts), [`selectionRangeStore`](../src/stores/selectionRangeStore.ts). @@ -42,7 +42,7 @@ The main directory structure is organized as follows. Tests are located in a sub | Non-Redux UI state | [`stores/`](../src/stores) | | Browser DOM / selection / scroll APIs | [`device/`](../src/device) | | User-triggered command | [`commands/`](../src/commands) | -| Yjs persistence engine | [`data-providers/yjs/thoughtspace.ts`](../src/data-providers/yjs/thoughtspace.ts) | +| TreeCRDT persistence engine | [`data-providers/treecrdt/thoughtspace.ts`](../src/data-providers/treecrdt/thoughtspace.ts) | | Layout positioning math | [`hooks/usePositionedThoughts.ts`](../src/hooks/usePositionedThoughts.ts) | | Visible-thoughts traversal | [`selectors/linearizeTree.ts`](../src/selectors/linearizeTree.ts) | | Pure helper (no React, no Redux) | [`util/`](../src/util) | diff --git a/docs/glossary.md b/docs/glossary.md index be3bf03c24f..c8e0bb3c40d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -58,7 +58,7 @@ A flat reference of project-specific terms used in code and docs. For deeper con ## D -**DataProvider** — The single interface ([`DataProvider.ts`](../src/data-providers/DataProvider.ts)) for storage backends. `db` in [`yjs/thoughtspace.ts`](../src/data-providers/yjs/thoughtspace.ts) is the live implementation. +**DataProvider** — The single interface ([`DataProvider.ts`](../src/data-providers/DataProvider.ts)) for storage backends. The active implementation is exported through [`data-providers/thoughtspace.ts`](../src/data-providers/thoughtspace.ts). **dbQueue / freeQueue** — Two halves of the push-queue split. `dbQueue` writes batches with `local || remote` set; `freeQueue` releases entries from the in-memory cache. See [persistence.md → Push queue](persistence.md#push-queue-redux--yjs). @@ -192,7 +192,7 @@ A flat reference of project-specific terms used in code and docs. For deeper con **updatedBy** — `clientId` of the writer. Stamped on every Thought and Lexeme write so observers can filter out self-originated change events. -**updateThoughts** — Both an action ([`actions/updateThoughts.ts`](../src/actions/updateThoughts.ts)) that mutates Redux and queues a push, and the `DataProvider` entry point ([`yjs/thoughtspace.ts`](../src/data-providers/yjs/thoughtspace.ts)) that writes to Yjs. The action calls into the provider via the push queue. +**updateThoughts** — The action ([`actions/updateThoughts.ts`](../src/actions/updateThoughts.ts)) that mutates Redux and queues a push. The push queue persists those batches through the active data provider's `updateThoughts`. ## V diff --git a/docs/persistence.md b/docs/persistence.md index 25a246c3f40..6e5ff5368bf 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -11,7 +11,7 @@ Two queues bridge Redux and the local Yjs store: - **Push queue** ([`redux-enhancers/pushQueue.ts`](../src/redux-enhancers/pushQueue.ts)) drains `state.pushQueue` after every action and writes to Yjs. - **Pull queue** ([`redux-middleware/pullQueue.ts`](../src/redux-middleware/pullQueue.ts)) tracks visible thoughts and pulls any pending ones from Yjs. -The single point of integration with Yjs is the [`DataProvider`](../src/data-providers/DataProvider.ts) interface, implemented as `db` in [`data-providers/yjs/thoughtspace.ts`](../src/data-providers/yjs/thoughtspace.ts). +The single point of integration with persistence is the [`DataProvider`](../src/data-providers/DataProvider.ts) interface, implemented by the active thoughtspace backend. ## In-memory state (Redux) @@ -89,7 +89,7 @@ If IDB returns `AbortError` (the app was closed mid-sync), replication retries a - [`updateThought`](../src/data-providers/yjs/thoughtspace.ts) writes a single thought into its parent Y.Doc. If the thought has changed parents, it deletes the entry from the old parent Doc, updates the docKey in `docKeys` and in the Lexeme's `cx-${id}` map, and writes to the new parent. (These three writes span three Docs and are not atomic — there's a known risk of partial-failure inconsistency, called out in a code comment.) `childrenMap` is merged key-by-key as a nested `Y.Map`; other fields are written only if the value changed, to avoid emitting redundant CRDT updates. - [`updateLexeme`](../src/data-providers/yjs/thoughtspace.ts) adds and removes contexts via `cx-${id}` keys. -- [`updateThoughts`](../src/data-providers/yjs/thoughtspace.ts) is the public `DataProvider` entry point. It groups updates and deletes, then submits both groups to a `TaskQueue` with **concurrency 16** ([`util/taskQueue.ts`](../src/util/taskQueue.ts)). Updates run before deletes within a single batch. +- `DataProvider.updateThoughts` is the public persistence entry point for push-queue thought and lexeme batches. All update paths await `IndexeddbPersistence.whenSynced` (i.e. they resolve when the write hits IDB), but do not wait on the websocket. @@ -126,7 +126,7 @@ In effect, on `main` Yjs is used as a local-only document store. Local IndexedDB [`redux-enhancers/pushQueue.ts`](../src/redux-enhancers/pushQueue.ts) is a Redux store enhancer that runs after every reducer. It drains `state.pushQueue` (a list of `PushBatch` objects pushed there by [`updateThoughts`](../src/actions/updateThoughts.ts) and friends) and partitions it into: -- **`dbQueue`** — batches with `local || remote` set. Applied sequentially via `db.updateThoughts({ thoughtIndexUpdates, lexemeIndexUpdates, lexemeIndexUpdatesOld, schemaVersion })`. After each batch resolves, any `idbSynced` callback on the batch is invoked. +- **`dbQueue`** — batches with `local || remote` set. Applied sequentially through `thoughtspaceRuntime.persistPushQueueBatches`, which calls the active data provider's `updateThoughts`. After provider persistence finishes, any `idbSynced` callback on the original batch is invoked. - **`freeQueue`** — state-only batches whose `null` thought/lexeme entries indicate they should be released from the in-memory cache. Triggers `db.freeThought` / `db.freeLexeme`. The enhancer also caches a small set of critical settings (`CACHED_SETTINGS` in [`constants.ts`](../src/constants.ts)) into `localStorage` so that things like the Tutorial setting are available during the first paint before Yjs hydrates. The corresponding read path is [`selectors/getSetting.ts`](../src/selectors/getSetting.ts). @@ -135,10 +135,10 @@ Once Redux dispatches a thought update, the data flow is therefore: ``` reducer → state.pushQueue → pushQueue enhancer - → db.updateThoughts (thoughtspace.ts) - → updateQueue (TaskQueue, concurrency 16) - → Y.Doc.transact + IndexeddbPersistence - → idbSynced callback resolves + → thoughtspaceRuntime.persistPushQueueBatches + → DataProvider.updateThoughts + → active provider persistence + → idbSynced callback is invoked ``` ## Pull queue (Yjs → Redux) diff --git a/docs/testing.md b/docs/testing.md index e38e383630a..c6925e1c06c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -500,7 +500,7 @@ beforeEach(createTestApp) afterEach(cleanupTestApp) ``` -`initStore` clears the shared store, resets every [ministore](glossary.md#m) to its initial state, and enables fake timers. Ministores are module-level singletons that vitest isolates per test *file*, not per test, so resetting them in setup is what keeps one test's ephemeral UI state out of the next; a test that needs a ministore reset on its own can call `store.reset()` on it directly. `initStore({ persist: true })` skips both resets. `createTestApp` resets ministores the same way, before `initialize()` runs, and additionally mounts the React tree, initializes persistence and event handlers, and enables the test drag-and-drop backend. `cleanupTestApp` clears storage, the local YJS database, the store, and event handlers, and flushes pending timers. Do not share fixture state between tests or rely on test execution order. +`initStore` is async and enables fake timers. By default, it drops and reinitializes the in-memory TreeCRDT thoughtspace, clears the shared Redux store, and resets every [ministore](glossary.md#m) to its initial state. Ministores are module-level singletons that Vitest isolates per test file, not per test. `initStore({ persist: true })` skips the thoughtspace, Redux, and ministore resets. Pass it directly to `beforeEach(initStore)` so Vitest awaits it; wrappers must explicitly `await initStore()`. `createTestApp` resets ministores before `initialize({ storage: 'memory' })` runs, and additionally mounts the React tree, initializes persistence and event handlers, and enables the test drag-and-drop backend. `cleanupTestApp` clears storage, the TreeCRDT thoughtspace, the store, and event handlers, and flushes pending timers. Do not share fixture state between tests or rely on test execution order. ## Sanctioned Backdoors @@ -512,7 +512,7 @@ Integration tests are blackbox, but named helpers may take shortcuts during arra | Incidental app setup | Arrange | [`command`](../src/e2e/puppeteer/helpers/command.ts), [`openModal`](../src/e2e/puppeteer/helpers/openModal.ts), [`setTheme`](../src/e2e/puppeteer/helpers/setTheme.ts) | Use only when the command, modal entry point, or Settings navigation is not under test. | | Browser/driver limitation | Arrange | Puppeteer [`setSelection`](../src/e2e/puppeteer/helpers/setSelection.ts) and [`closeKeyboard`](../src/e2e/puppeteer/helpers/closeKeyboard.ts); iOS [`setSelection`](../src/e2e/iOS/helpers/setSelection.ts) | Simulate browser state the driver cannot reliably produce. The subsequent behavior under test must still use a real user entry point. | | Visual snapshot stabilization | Arrange | [`hide`](../src/e2e/puppeteer/helpers/hide.ts), [`hideVisibility`](../src/e2e/puppeteer/helpers/hideVisibility.ts), [`hideHUD`](../src/e2e/puppeteer/helpers/hideHUD.ts), [`showMousePointer`](../src/e2e/puppeteer/helpers/showMousePointer.ts), [`screenshot`](../src/e2e/puppeteer/helpers/screenshot.ts) | DOM/style mutation is allowed only to remove irrelevant nondeterminism or expose input position in a visual test. Do not hide the subject of the snapshot. | -| Test environment controls | Arrange | [`deviceEmulation`](../src/e2e/puppeteer/helpers/deviceEmulation.ts), [`setConnectionStatus`](../src/e2e/puppeteer/helpers/setConnectionStatus.ts), [`simulateDragAndDrop`](../src/e2e/puppeteer/helpers/simulateDragAndDrop.ts), [`scrollTo`](../src/e2e/puppeteer/helpers/scrollTo.ts), and reviewed helpers that set [`testFlags`](../src/e2e/testFlags.ts) | Use only for a condition that cannot be created reliably through normal input, explain why, and restore mutable controls in the corresponding `afterEach` or `afterAll` hook unless per-test page isolation resets them. The control must not change the semantic outcome under test. | +| Test environment controls | Arrange | [`deviceEmulation`](../src/e2e/puppeteer/helpers/deviceEmulation.ts), [`setConnectionStatus`](../src/e2e/puppeteer/helpers/setConnectionStatus.ts), [`simulateDragAndDrop`](../src/e2e/puppeteer/helpers/simulateDragAndDrop.ts), [`scrollTo`](../src/e2e/puppeteer/helpers/scrollTo.ts), the thoughtspace storage selection in [`puppeteer/setup.ts`](../src/e2e/puppeteer/setup.ts), and reviewed helpers that set [`testFlags`](../src/e2e/testFlags.ts) | Use only for a condition that cannot be created reliably through normal input, explain why, and restore mutable controls in the corresponding `afterEach` or `afterAll` hook unless per-test page isolation resets them. The control must not change the semantic outcome under test. | | Structural assertion | Assert | [`exportThoughts`](../src/e2e/puppeteer/helpers/exportThoughts.ts) | Export the thought tree as plaintext. Do not make additional assertions on Redux state. | | Non-visual synchronization | Wait | [`waitForContextHasChildWithValue`](../src/e2e/puppeteer/helpers/waitForContextHasChildWithValue.ts), [`waitForThoughtExistInDb`](../src/e2e/puppeteer/helpers/waitForThoughtExistInDb.ts), [`waitForState`](../src/e2e/puppeteer/helpers/waitForState.ts) | Use only when persistence or another prerequisite has no immediate visual signal. This is synchronization, not the test's assertion; assert the final user-visible result separately. | | Timing/environment spoofing | Arrange | [`reloadWithProductionTiming`](../src/e2e/puppeteer/helpers/reloadWithProductionTiming.ts) (spoofs `navigator.webdriver` to restore production animation timing) | Use only for a state that cannot exist under test timing (such as the loading phase). Justify in the helper's doc comment and state how the spoof is undone (per-test page isolation counts, but say so). Subsequent waits must still name conditions rather than replay production durations. | @@ -557,8 +557,8 @@ There are three helper directories. Use them before reaching for raw Redux dispa The helpers in [`../src/test-helpers/`](../src/test-helpers) cover store setup and operations that are otherwise verbose to write by hand: -- [`createTestApp`](../src/test-helpers/createTestApp.tsx) — mounts `` into the JSDOM environment via `@testing-library/react`, resets all ministores, runs `initialize()`, swaps in `react-dnd-test-backend`, opts into fake timers, and closes the welcome modal. Use this when a test touches the rendered app. Pair every call with `cleanupTestApp` (it clears `localStorage`, the local YJS db, the store, and event handlers). -- [`initStore`](../src/test-helpers/initStore.ts) — initializes the store without mounting the React tree, for store-level tests that don't need a DOM. +- [`createTestApp`](../src/test-helpers/createTestApp.tsx) — mounts `` into the JSDOM environment via `@testing-library/react`, runs `initialize({ storage: 'memory' })`, swaps in `react-dnd-test-backend`, opts into fake timers, and closes the welcome modal. Use this when a test touches the rendered app. Pair every call with `cleanupTestApp` (it clears `localStorage`, the TreeCRDT thoughtspace, the store, and event handlers). +- [`initStore`](../src/test-helpers/initStore.ts) — async store setup without mounting the React tree. Clears Redux state, resets ministores via `resetStores`, reinitializes the in-memory thoughtspace, and enables fake timers. Await it (or pass it directly to `beforeEach`). - [`importToContext`](../src/test-helpers/importToContext.ts) — seeds the store with a tree from a multi-line plaintext outline (the same format the `Import` modal accepts). Most fixture setup goes through this. - [`dispatch`](../src/test-helpers/dispatch.ts) — a thin wrapper that lets a test dispatch synchronously without re-typing `store.dispatch(...)` plumbing. - **Operate-by-value helpers.** Where a test would otherwise need to look up a `ThoughtId` to dispatch an action, prefer the value-keyed variants: @@ -566,7 +566,7 @@ The helpers in [`../src/test-helpers/`](../src/test-helpers) cover store setup a - **Read-by-value helpers.** [`getAllChildrenByContext`](../src/test-helpers/getAllChildrenByContext.ts), [`getChildrenRankedByContext`](../src/test-helpers/getChildrenRankedByContext.ts), [`getAllChildrenAsThoughtsByContext`](../src/test-helpers/getAllChildrenAsThoughtsByContext.ts), [`attributeByContext`](../src/test-helpers/attributeByContext.ts), [`contextToThought`](../src/test-helpers/contextToThought.ts). - [`expectPathToEqual`](../src/test-helpers/expectPathToEqual.ts) — Jest matcher that compares paths by their thought *values* rather than ids, so test failures are readable. - [`checkDataIntegrity`](../src/test-helpers/checkDataIntegrity.ts) — assertions that catch parent/child mismatches, missing Lexemes, and orphaned thoughts. Useful as a final assertion in mutation-heavy tests. -- [`dataProviderTest`](../src/test-helpers/dataProviderTest.ts) — the alternate `DataProvider` implementation used by tests that exercise the storage layer without going through Yjs. (See [persistence.md](persistence.md) for the live YJS provider.) +- [`dataProviderTest`](../src/test-helpers/dataProviderTest.ts) — shared assertions for storage providers that implement the data provider interface. ### `src/e2e/puppeteer/helpers/` — for Puppeteer tests @@ -598,6 +598,12 @@ Do not import Puppeteer helpers into iOS tests or assume identical driver behavi [testFlags](../src/e2e/testFlags.ts) are used to alter runtime behavior of the app during tests. This is generally forbidden, as the automated test environment should be as close as possible to production so that it is testing the same behavior the end user sees. But there are some conditions that are difficult or impossible to create through normal user behavior (e.g. network latency) or that can enhance test readability (e.g. visualizations) when runtime alteration is warranted. +### Thoughtspace storage + +Puppeteer preloads `testFlags.thoughtspaceStorage` before the application starts. Browser tests use in-memory storage by default, while persistence-specific suites call `usePersistentTreecrdtStorage` to use OPFS. The application entry point passes the selected storage explicitly to `initialize`, defaulting to persistent storage when no test override is present. + +Test durable persistence in a regular browser context. Private browsing storage is temporary: Safari Private Browsing falls back to memory and loses thoughts on reload, while Chromium Incognito keeps OPFS only until the private session ends. + ### Drag-and-drop visualization You can enable drop target visualization boxes by running `em.testFlags.simulateDrop = true` in the JS console or setting `testFlags.simulateDrop` to true in [src/e2e/testFlags.ts](../src/e2e/testFlags.ts). @@ -959,15 +965,15 @@ Test `enter` and `leave` on each of the following actions: ### Database operations and fake timers -`initStore` and `createTestApp` enable fake timers. When a test calls `initialize()` or performs database work directly, explicitly flush the resulting scheduled work before asserting: +`initStore` and `createTestApp` enable fake timers. When a test calls `initialize({ storage: 'memory' })` or performs database work directly, explicitly flush the resulting scheduled work before asserting: ```ts vi.useFakeTimers() -await initialize() +await initialize({ storage: 'memory' }) await vi.runAllTimersAsync() ``` -> It looks like we must use fake timers if we want the `store` state to be updated based on database operations (e.g., if we use `initialize()` to reload the state). I think this is because the `thoughtspace` operations are asynchronous and don't call the store operations prior to the test ending. (I'm not sure why we didn't get other errors that made this clear.) +> It looks like we must use fake timers if we want the `store` state to be updated based on database operations (e.g., if we use `initialize({ storage: 'memory' })` to reload the state). I think this is because the `thoughtspace` operations are asynchronous and don't call the store operations prior to the test ending. (I'm not sure why we didn't get other errors that made this clear.) https://github.com/cybersemics/em/pull/2741 @@ -1045,4 +1051,4 @@ Your only job at each step is: 3. Test for the regression. 4. Run `git bisect bad` if the regression is still present and `git bisect good` if it is gone. -Record the commit hash it gives you at the very end and you’ve found the source of the regression! Often I take one more step of testing the bad commit again and the commit right before it (should be good) just to be extra sure. If any good/bad determination was mistaken along the way then it will throw off the whole process and the final result will not be accurate. But if you are precise and methodical, you can search through hundreds of commits in a matter of minutes to find the offending commit. \ No newline at end of file +Record the commit hash it gives you at the very end and you’ve found the source of the regression! Often I take one more step of testing the bad commit again and the commit right before it (should be good) just to be extra sure. If any good/bad determination was mistaken along the way then it will throw off the whole process and the final result will not be accurate. But if you are precise and methodical, you can search through hundreds of commits in a matter of minutes to find the offending commit. diff --git a/eslint.config.js b/eslint.config.js index 0fe54507144..d09598e9f0c 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -105,6 +105,7 @@ export default [ '**/build/*', '**/docs/*', '**/functions/*', + 'public/wa-sqlite/**', ], }, { diff --git a/package.json b/package.json index d77bfd4880b..6e9802f7cda 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,6 @@ "resolutions": { "@pandacss/node@npm:0.47.0": "patch:@pandacss/node@npm%3A0.47.0#~/.yarn/patches/@pandacss-node-npm-0.47.0.patch", "page-lifecycle": "https://codeload.github.com/magic-akari/page-lifecycle/tar.gz/50b50421bdeab3d211a57e81a277f699638373b0", - "y-indexeddb": "https://codeload.github.com/raineorshine/y-indexeddb/tar.gz/60b960009085b1a988b5064ee35703229231531f", "use-latest": "^1.3.0", "use-isomorphic-layout-effect": "^1.2.1" }, @@ -109,6 +108,15 @@ "@redux-devtools/extension": "^4.0.0", "@stylistic/eslint-plugin-ts": "^4.4.1", "@tauri-apps/api": "^2.11.1", + "@treecrdt/auth": "0.1.2", + "@treecrdt/discovery": "0.1.0", + "@treecrdt/interface": "0.2.0", + "@treecrdt/riblt-wasm": "0.1.0", + "@treecrdt/sync": "0.1.2", + "@treecrdt/sync-protocol": "0.1.2", + "@treecrdt/sync-server-core": "0.1.2", + "@treecrdt/sync-sqlite": "0.1.2", + "@treecrdt/wa-sqlite": "0.4.2", "axios": "^1.19.0", "clipboard": "^2.0.11", "dnd-core": "^16.0.1", @@ -164,10 +172,7 @@ "workbox-strategies": "^7.4.1", "workbox-window": "^7.4.1", "xhtml-purifier": "^0.4.3", - "y-indexeddb": "⚠️ OVERRIDDEN BY 'resolutions' - Source: https://github.com/raineorshine/y-indexeddb#y-indexeddb-multiplex | Tarball: https://codeload.github.com/raineorshine/y-indexeddb/tar.gz/60b960009085b1a988b5064ee35703229231531f | Configured in: package.json#resolutions", - "y-protocols": "^1.0.7", - "yallist": "^5.0.0", - "yjs": "^13.6.32" + "yallist": "^5.0.0" }, "devDependencies": { "@babel/core": "^8.0.1", diff --git a/src/@types/Lexeme.ts b/src/@types/Lexeme.ts index 070536fc5fe..bdf8db2f151 100644 --- a/src/@types/Lexeme.ts +++ b/src/@types/Lexeme.ts @@ -6,7 +6,7 @@ interface Lexeme { contexts: ThoughtId[] created: Timestamp lastUpdated: Timestamp - /** The public key of the user defined by a hash of their private access token. See: clientId (yjs/index.ts). */ + /** The public key of the user defined by a hash of their private access token. See: clientId (thoughtspaceSession). */ updatedBy: string } diff --git a/src/@types/PushBatch.ts b/src/@types/PushBatch.ts index 6e9b80bed09..0831e9305c6 100644 --- a/src/@types/PushBatch.ts +++ b/src/@types/PushBatch.ts @@ -7,7 +7,7 @@ import ThoughtId from './ThoughtId' /** Defines a single batch of updates added to the push queue. */ interface PushBatch { - /** Callback for when the updates have been synced with IDB. */ + /** Callback invoked after provider persistence has completed. */ idbSynced?: () => void lexemeIndexUpdates: Index /** @@ -18,9 +18,8 @@ interface PushBatch { /** * Update the local device. * Default: true. - * If local and remote are false, null updates will still cause the YJS providers to be destroyed to free up memory. + * If local and remote are false, null updates only deallocate entries from Redux/provider cache. * In particular, this is used by the freeThoughts middleware. - * (The freeThoughts middleware calls the freeThoughts reducer when the cache limit has been reached. The reducer calls deleteThought with local:false and remote:false, which creates a batch that triggers freeThought/freeLexeme in the pushQueue). */ local?: boolean /** Contains the path of the pending thought to be deleted and all its siblings. Siblings may be resurrected from the pull, and the parent has already been deleted, so we need to store them to be deleted in flushDeletes. */ @@ -29,10 +28,12 @@ interface PushBatch { /** * Update the remote server. * Default: true. - * Set to false to free memory (See: local). + * Set to false together with local:false for cache-only deallocation (See: local). */ remote?: boolean thoughtIndexUpdates: Index + /** For treecrdt: per-moved-thought placement. Key = moved thought id, value = id of sibling after which to place (null = first). */ + movePlacements?: Index /** Arbitrary updates: use with caution! */ // eslint-disable-next-line @typescript-eslint/no-explicit-any updates?: Index diff --git a/src/@types/State.ts b/src/@types/State.ts index 7bd8272b547..7f266c236eb 100644 --- a/src/@types/State.ts +++ b/src/@types/State.ts @@ -129,7 +129,7 @@ interface State { noteOffset: number | null /** * Temporarily stores updates that need to be persisted. - * Passed to Yjs and cleared on every action. + * Passed to the data provider and cleared on every action. * See: /redux-enhancers/pushQueue.ts. */ pushQueue: PushBatch[] diff --git a/src/@types/Thought.ts b/src/@types/Thought.ts index 673bdb09b71..49390f4437f 100644 --- a/src/@types/Thought.ts +++ b/src/@types/Thought.ts @@ -19,7 +19,7 @@ interface Thought { rank: number /** Used to track if a space is required when merging two siblings/thoughts. */ splitSource?: ThoughtId - /** The public key of the user defined by a hash of their private access token. See: clientId (yjs/index.ts). */ + /** The public key of the user defined by a hash of their private access token. See: clientId (thoughtspaceSession). */ updatedBy: string value: string } diff --git a/src/@types/index.ts b/src/@types/index.ts index 3003daffa66..d19d67e824a 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -1,6 +1,14 @@ import { UnknownAction } from 'redux' +import type { WindowEm } from '../initialize' import Thunk from './Thunk' +/** Explicit pre-initialization view of window for preload scripts. */ +export type PreloadedEmWindow = { + em?: { + testFlags?: Partial + } +} + declare global { interface Document { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -8,7 +16,8 @@ declare global { } interface Window { - em: unknown + /** Fully initialized application namespace. Preload writers use {@link PreloadedEmWindow}. */ + em: WindowEm debug: (message: string) => void // FIX: Used only in puppeteer test environment. So need way to switch global context based on environment. delay: (ms: number) => Promise diff --git a/src/actions/__tests__/cursorUp.ts b/src/actions/__tests__/cursorUp.ts index 019d96afc86..da1888a802d 100644 --- a/src/actions/__tests__/cursorUp.ts +++ b/src/actions/__tests__/cursorUp.ts @@ -167,8 +167,8 @@ describe('normal view', () => { expectPathToEqual(state, prevThought(state, state.cursor!), ['a']) }) - it('move cursor from empty thought to previous thought in context sorted in descending order', () => { - initStore() + it('move cursor from empty thought to previous thought in context sorted in descending order', async () => { + await initStore() act(() => { store.dispatch([ diff --git a/src/actions/__tests__/importData.ts b/src/actions/__tests__/importData.ts index 8cb9a013259..0237e4fa5f1 100644 --- a/src/actions/__tests__/importData.ts +++ b/src/actions/__tests__/importData.ts @@ -12,7 +12,7 @@ import { newThoughtActionCreator as newThought } from '../newThought' /** Helper function that initializes the store, imports html into the root, and exports it as plaintext to make easily readable assertions. This is async because importFiles is async. */ const importExport = async (html: string, outputFormat: MimeType = 'text/plain') => { vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch(importDataActionCreator({ html })) await vi.runOnlyPendingTimersAsync() const exported = exportContext(store.getState(), HOME_PATH, outputFormat) @@ -79,7 +79,7 @@ it.skip('multi-line nested html tags', async () => { const actual = await importExport(paste, 'text/html') const expectedOutput = `
    -
  • __ROOT__${' '} +
  • ${HOME_TOKEN}${' '}
    • A
    • B
      • @@ -120,7 +120,7 @@ it.skip('text that contains em tag', async () => { const exported = await importExport(text, 'text/html') expect(exported.trim()).toBe( `
        -
      • __ROOT__${EMPTY_SPACE} +
      • ${HOME_TOKEN}${EMPTY_SPACE}
        • a${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
            @@ -192,7 +192,7 @@ it.skip('should paste plain text that contains formatting', async () => { const actual = await importExport(paste, 'text/html') expect(actual).toBe( `
              -
            • __ROOT__${EMPTY_SPACE} +
            • ${HOME_TOKEN}${EMPTY_SPACE}
              • a
              • b
                • @@ -208,7 +208,7 @@ it.skip('should paste plain text that contains formatting and bullet indicator i -b` const actual = await importExport(paste, 'text/html') const expectedHTML = `
                  -
                • __ROOT__${EMPTY_SPACE} +
                • ${HOME_TOKEN}${EMPTY_SPACE}
                  • a${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                      @@ -247,7 +247,7 @@ p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px 'Helvetica Neue'} const actual = await importExport(paste, 'text/html') const expectedOutput = `
                        -
                      • __ROOT__${EMPTY_SPACE} +
                      • ${HOME_TOKEN}${EMPTY_SPACE}
                        • A${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                            @@ -274,7 +274,7 @@ it.skip('should paste text properly that is copied from WebStorm', async () => { const actual = await importExport(paste, 'text/html') const expectedOutput = `
                              -
                            • __ROOT__${EMPTY_SPACE} +
                            • ${HOME_TOKEN}${EMPTY_SPACE}
                              • A${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                                  @@ -295,7 +295,7 @@ it.skip('should paste text properly that is copied from iOS notes.app', async () const actual = await importExport(paste, 'text/html') const expectedOutput = `
                                    -
                                  • __ROOT__${EMPTY_SPACE} +
                                  • ${HOME_TOKEN}${EMPTY_SPACE}
                                    • A${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                                        @@ -335,7 +335,7 @@ p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px 'Helvetica Neue'} const actual = await importExport(paste, 'text/html') const expectedOutput = `
                                          -
                                        • __ROOT__${EMPTY_SPACE} +
                                        • ${HOME_TOKEN}${EMPTY_SPACE}
                                          • A${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                                              @@ -376,7 +376,7 @@ p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px 'Helvetica Neue'} const actual = await importExport(paste, 'text/html') const expectedOutput = `
                                                -
                                              • __ROOT__${EMPTY_SPACE} +
                                              • ${HOME_TOKEN}${EMPTY_SPACE}
                                                • A${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                                                    @@ -400,7 +400,7 @@ it.skip('should paste text that contains formatting that is copied from iOS note const actual = await importExport(paste, 'text/html') const expectedOutput = `
                                                      -
                                                    • __ROOT__${EMPTY_SPACE} +
                                                    • ${HOME_TOKEN}${EMPTY_SPACE}
                                                      • A${EMPTY_SPACE}${EMPTY_SPACE}${EMPTY_SPACE}
                                                          @@ -1068,7 +1068,7 @@ it('empty parent', async () => { - x` vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({}), @@ -1108,7 +1108,7 @@ p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Helvetica; color: #000000} ` vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'a' }), @@ -1127,7 +1127,7 @@ p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Helvetica; color: #000000} it('paste em text with browser-injected meta charset as inline, not subthought', async () => { vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'a' }), @@ -1154,7 +1154,7 @@ it('paste em text with browser-injected meta charset as inline, not subthought', it('paste em text with formatted html and meta charset as inline', async () => { vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'a' }), @@ -1186,7 +1186,7 @@ it('insert single-line HTML copied from Windows desktop Chrome at end of thought ` vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'a' }), @@ -1206,7 +1206,7 @@ it('insert single-line HTML copied from Windows desktop Chrome at end of thought it('insert single-line HTML copied from Mac desktop Chrome at end of thought', async () => { const html = `foo` vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'a' }), @@ -1243,7 +1243,7 @@ bar

                                                          ` vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'a' }), @@ -1280,7 +1280,7 @@ p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 9.0px Helvetica; color: #000000} ` vi.useFakeTimers() - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) store.dispatch([ newThought({ value: 'x' }), diff --git a/src/actions/__tests__/importText.ts b/src/actions/__tests__/importText.ts index 5a61840c776..c5282c97367 100644 --- a/src/actions/__tests__/importText.ts +++ b/src/actions/__tests__/importText.ts @@ -452,7 +452,7 @@ it.skip('should strip tags whose font weight is less than or equal to 400', () = const paste = `Hello world. This is a test ` const actual = importExport(paste, 'text/html') const expectedOutput = `
                                                            -
                                                          • __ROOT__${EMPTY_SPACE} +
                                                          • ${HOME_TOKEN}${EMPTY_SPACE}
                                                            • Hello world. This is a test
                                                            @@ -465,7 +465,7 @@ it('should convert font weight to 700 if the font weight in a tag is greater tha const paste = `Hello world. This is a test` const actual = importExport(paste, 'text/html') const expectedOutput = `
                                                              -
                                                            • __ROOT__${EMPTY_SPACE} +
                                                            • ${HOME_TOKEN}${EMPTY_SPACE}
                                                              • Hello world. This is a test
                                                              @@ -478,7 +478,7 @@ it('should not strip whole tag unless other style apart from font-weight should const paste = `a` const actual = importExport(paste, 'text/html') const expectedOutput = `
                                                                -
                                                              • __ROOT__${EMPTY_SPACE} +
                                                              • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                • a
                                                                @@ -500,7 +500,7 @@ it('allow formatting tags', () => { const exported = exportContext(stateNew, [HOME_TOKEN], 'text/html') const expectedOutput = `
                                                                  -
                                                                • __ROOT__${EMPTY_SPACE} +
                                                                • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                  • guardians of the galaxy
                                                                  • guardians of the universe
                                                                    • @@ -630,7 +630,7 @@ it('import single line with style attributes', () => { const exported = exportContext(stateNew, [HOME_TOKEN], 'text/html') expect(exported).toBe(`
                                                                      -
                                                                    • __ROOT__${EMPTY_SPACE} +
                                                                    • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                      • Atonement
                                                                      @@ -645,7 +645,7 @@ it('import single line with style attributes and a single br tag', () => { const exported = exportContext(stateNew, [HOME_TOKEN], 'text/html') expect(exported).toBe(`
                                                                        -
                                                                      • __ROOT__${EMPTY_SPACE} +
                                                                      • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                        • Marcel Duchamp: The Art of the Possible
                                                                        diff --git a/src/actions/__tests__/longPress.ts b/src/actions/__tests__/longPress.ts index d472f627953..ae00807d5df 100644 --- a/src/actions/__tests__/longPress.ts +++ b/src/actions/__tests__/longPress.ts @@ -15,7 +15,7 @@ vi.mock('../../browser', async importOriginal => { beforeEach(initStore) it('blurs the focused editable when a drag begins, so the virtual keyboard closes (#4683)', async () => { - await initialize() + await initialize({ storage: 'memory' }) store.dispatch([importText({ text: '- a' }), setCursor(['a'])]) diff --git a/src/actions/__tests__/moveThought.ts b/src/actions/__tests__/moveThought.ts index a474da792b5..c71a75980c5 100644 --- a/src/actions/__tests__/moveThought.ts +++ b/src/actions/__tests__/moveThought.ts @@ -98,6 +98,54 @@ it('move within context (rank only)', () => { expect(getContexts(stateNew, 'a2')).toMatchObject([thoughtA2.id]) }) +it('rank adapter placement excludes the moved thought', () => { + const state = reducerFlow([newThought('a'), newThought('b'), newThought('c')])(initialState()) + + const thoughtA = contextToThought(state, ['a'])! + const thoughtB = contextToThought(state, ['b'])! + const thoughtC = contextToThought(state, ['c'])! + + const stateNew = moveThoughtAtFirstMatch({ + from: ['b'], + to: ['b'], + newRank: (thoughtB.rank + thoughtC.rank) / 2, + })(state) + + expect(stateNew.pushQueue.at(-1)?.movePlacements?.[thoughtB.id]).toBe(thoughtA.id) +}) + +it('explicit first placement is not inferred from rank', () => { + const state = reducerFlow([newThought('a'), newThought('b'), newThought('c')])(initialState()) + + const thoughtA = contextToThought(state, ['a'])! + const thoughtC = contextToThought(state, ['c'])! + + const stateNew = moveThoughtAtFirstMatch({ + from: ['c'], + to: ['c'], + newRank: thoughtA.rank + 0.5, + afterId: null, + })(state) + + expect(stateNew.pushQueue.at(-1)?.movePlacements).toHaveProperty(thoughtC.id) + expect(stateNew.pushQueue.at(-1)?.movePlacements?.[thoughtC.id]).toBeNull() +}) + +it('rejects placement after the moved thought', () => { + const state = reducerFlow([newThought('a'), newThought('b'), newThought('c')])(initialState()) + + const thoughtB = contextToThought(state, ['b'])! + + expect(() => + moveThoughtAtFirstMatch({ + from: ['b'], + to: ['b'], + newRank: thoughtB.rank, + afterId: thoughtB.id, + })(state), + ).toThrow('afterId must be null or a child of the destination context') +}) + it('move across contexts', () => { const steps = [ newThought('a'), diff --git a/src/actions/__tests__/settings.ts b/src/actions/__tests__/settings.ts new file mode 100644 index 00000000000..42b3982e1b7 --- /dev/null +++ b/src/actions/__tests__/settings.ts @@ -0,0 +1,20 @@ +import { EM_TOKEN, SETTINGS_TOKEN, SETTINGS_VALUE } from '../../constants' +import findDescendant from '../../selectors/findDescendant' +import { getAllChildrenAsThoughts } from '../../selectors/getChildren' +import initialState from '../../util/initialState' +import settings from '../settings' + +it('uses the bootstrapped Settings thought without creating a duplicate', () => { + const stateNew = settings(initialState(), { + key: 'Tutorial', + value: 'Off', + }) + + expect(findDescendant(stateNew, EM_TOKEN, SETTINGS_VALUE)).toBe(SETTINGS_TOKEN) + expect(findDescendant(stateNew, EM_TOKEN, [SETTINGS_VALUE, 'Tutorial', 'Off'])).toBeTruthy() + + const settingsChildren = getAllChildrenAsThoughts(stateNew, EM_TOKEN).filter( + thought => thought.value === SETTINGS_VALUE, + ) + expect(settingsChildren.map(thought => thought.id)).toEqual([SETTINGS_TOKEN]) +}) diff --git a/src/actions/__tests__/swapParent.ts b/src/actions/__tests__/swapParent.ts index f624f4383c9..57310a5e24e 100644 --- a/src/actions/__tests__/swapParent.ts +++ b/src/actions/__tests__/swapParent.ts @@ -324,6 +324,8 @@ describe('sort', () => { - d - b - a`) + + vi.useRealTimers() }) }) diff --git a/src/actions/createThought.ts b/src/actions/createThought.ts index 7d21f5b223b..0ac928e1e84 100644 --- a/src/actions/createThought.ts +++ b/src/actions/createThought.ts @@ -7,7 +7,7 @@ import Thought from '../@types/Thought' import ThoughtId from '../@types/ThoughtId' import Thunk from '../@types/Thunk' import updateThoughts from '../actions/updateThoughts' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import getLexeme from '../selectors/getLexeme' import getThoughtById from '../selectors/getThoughtById' import { registerActionMetadata } from '../util/actionMetadata.registry' diff --git a/src/actions/deleteThought.ts b/src/actions/deleteThought.ts index bac6ac62dc1..19863ed2b5c 100644 --- a/src/actions/deleteThought.ts +++ b/src/actions/deleteThought.ts @@ -9,7 +9,7 @@ import ThoughtId from '../@types/ThoughtId' import Thunk from '../@types/Thunk' import updateThoughts from '../actions/updateThoughts' import { HOME_PATH } from '../constants' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import { getChildrenRanked } from '../selectors/getChildren' import { getLexeme } from '../selectors/getLexeme' import getThoughtById from '../selectors/getThoughtById' diff --git a/src/actions/editThought.ts b/src/actions/editThought.ts index 1430654d2ef..d4820c75575 100644 --- a/src/actions/editThought.ts +++ b/src/actions/editThought.ts @@ -6,7 +6,7 @@ import State from '../@types/State' import Thought from '../@types/Thought' import ThoughtId from '../@types/ThoughtId' import Thunk from '../@types/Thunk' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import findDescendant from '../selectors/findDescendant' import { getAllChildren } from '../selectors/getChildren' import getLexeme from '../selectors/getLexeme' diff --git a/src/actions/freeThoughts.ts b/src/actions/freeThoughts.ts index 243da7ceced..26b7e6f5299 100644 --- a/src/actions/freeThoughts.ts +++ b/src/actions/freeThoughts.ts @@ -99,8 +99,7 @@ const freeThoughts = (state: State): State => { stateNew = deleteThought(stateNew, { thoughtId: deletableThought.id, pathParent: thoughtToPath(stateNew, deletableThought.parentId), - // Do not persist deletions; just delete from Redux state. - // The pushQueue will enhancer will detect this batch and deallocate YJS providers. + // Do not persist deletions; only free cached state/provider entries. local: false, remote: false, }) diff --git a/src/actions/importText.ts b/src/actions/importText.ts index 0dbfab96140..b8dae1a9f51 100644 --- a/src/actions/importText.ts +++ b/src/actions/importText.ts @@ -8,7 +8,7 @@ import editThought from '../actions/editThought' import setCursor from '../actions/setCursor' import updateThoughts from '../actions/updateThoughts' import { HOME_PATH } from '../constants' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import getTextContentFromHTML from '../device/getTextContentFromHTML' import { anyChild, findAnyChild, getAllChildren } from '../selectors/getChildren' import getThoughtById from '../selectors/getThoughtById' diff --git a/src/actions/indent.ts b/src/actions/indent.ts index 06cd3640a5a..a214545ff01 100644 --- a/src/actions/indent.ts +++ b/src/actions/indent.ts @@ -1,9 +1,11 @@ +import _ from 'lodash' import State from '../@types/State' import Thunk from '../@types/Thunk' import alert from '../actions/alert' import moveThought from '../actions/moveThought' import * as selection from '../device/selection' import findDescendant from '../selectors/findDescendant' +import { getChildrenRanked } from '../selectors/getChildren' import getNextRank from '../selectors/getNextRank' import isContextViewActive from '../selectors/isContextViewActive' import prevSibling from '../selectors/prevSibling' @@ -57,11 +59,17 @@ const indent = (state: State): State => { const cursorNew = appendToPath(parentOf(cursor), prev.id, head(cursor)) + // For treecrdt: afterId must be a sibling (child of new parent), not the parent. + // Tab indent should place as last child of prev, so use last child of prev; undefined if prev has no children. + const prevChildren = getChildrenRanked(state, prev.id) + const lastChildOfPrev = _.last(prevChildren) + return moveThought(state, { oldPath: cursor, newPath: cursorNew, ...(offset != null ? { offset } : null), newRank: getNextRank(state, prev.id), + afterId: lastChildOfPrev?.id ?? null, }) } diff --git a/src/actions/mergeThoughts.ts b/src/actions/mergeThoughts.ts index 96118844c03..7390943a95a 100644 --- a/src/actions/mergeThoughts.ts +++ b/src/actions/mergeThoughts.ts @@ -6,7 +6,7 @@ import State from '../@types/State' import Thought from '../@types/Thought' import Thunk from '../@types/Thunk' import { HOME_TOKEN } from '../constants' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import { getLexeme } from '../selectors/getLexeme' import getNextRank from '../selectors/getNextRank' import getThoughtById from '../selectors/getThoughtById' diff --git a/src/actions/moveThought.ts b/src/actions/moveThought.ts index e58ccc367dc..f490d164d65 100644 --- a/src/actions/moveThought.ts +++ b/src/actions/moveThought.ts @@ -4,11 +4,12 @@ import Path from '../@types/Path' import SimplePath from '../@types/SimplePath' import State from '../@types/State' import Thought from '../@types/Thought' +import ThoughtId from '../@types/ThoughtId' import Thunk from '../@types/Thunk' import mergeThoughts from '../actions/mergeThoughts' import rerank from '../actions/rerank' import updateThoughts from '../actions/updateThoughts' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import expandThoughts from '../selectors/expandThoughts' import { getChildrenRanked } from '../selectors/getChildren' import getSortPreference from '../selectors/getSortPreference' @@ -39,14 +40,32 @@ export interface MoveThoughtPayload { skipMerge?: boolean /** The new rank of the destination thought. This will be ignored if the thought is moved into a sorted context. */ newRank: number + /** + * ID of sibling after which to place in TreeCRDT. + * Explicit null means first child. + * Undefined means derive placement from newRank for legacy em rank-based callers. + */ + afterId?: ThoughtId | null +} + +/** Derives an explicit TreeCRDT afterId from em's temporary rank ordering. */ +const getMoveThoughtAfterIdByRank = ( + state: State, + destinationThoughtId: ThoughtId, + sourceThoughtId: ThoughtId, + newRank: number, +): ThoughtId | null => { + const after = getChildrenRanked(state, destinationThoughtId) + .filter(child => child.id !== sourceThoughtId && child.rank < newRank) + .at(-1) + + return after?.id ?? null } // @MIGRATION_TODO: use (sourceId and destinationId) or simplePath instead of passing paths. Should low level handle context view logic ?? /** Moves a thought from one context to another, or within the same context. */ -const moveThought = ( - state: State, - { oldPath, newPath, offset, skipRerank, skipMerge, newRank }: MoveThoughtPayload, -) => { +const moveThought = (state: State, payload: MoveThoughtPayload) => { + const { oldPath, newPath, offset, skipRerank, skipMerge, newRank, afterId } = payload // Uncaught TypeError: Cannot perform 'IsArray' on a proxy that has been revoked at Function.isArray (#417) const recentlyEdited = state.recentlyEdited // try { @@ -91,6 +110,17 @@ const moveThought = ( const sameContext = sourceParentThought.id === destinationThoughtId const childrenOfDestination = getChildrenRanked(state, destinationThoughtId) + const effectiveAfterId = + afterId !== undefined + ? afterId + : getMoveThoughtAfterIdByRank(state, destinationThoughtId, sourceThought.id, newRank) + + if ( + effectiveAfterId === sourceThought.id || + (effectiveAfterId !== null && !childrenOfDestination.some(child => child.id === effectiveAfterId)) + ) { + throw new Error(`moveThought: afterId must be null or a child of the destination context.`) + } /** * Find first normalized duplicate thought. @@ -190,6 +220,7 @@ const moveThought = ( lexemeIndexUpdates: {}, recentlyEdited, preventExpandThoughts: true, + movePlacements: { [sourceThought.id]: effectiveAfterId }, }) }, // update cursor if moved path is on the cursor diff --git a/src/actions/moveThoughtDown.ts b/src/actions/moveThoughtDown.ts index 250471fd09f..936df666391 100644 --- a/src/actions/moveThoughtDown.ts +++ b/src/actions/moveThoughtDown.ts @@ -68,6 +68,7 @@ const moveThoughtDown = (state: State): State => { newPath, ...(offset != null ? { offset } : null), newRank: rankNew, + afterId: nextThought ? nextThought.id : null, }) } diff --git a/src/actions/moveThoughtUp.ts b/src/actions/moveThoughtUp.ts index ebf9413f645..fc372d7ceda 100644 --- a/src/actions/moveThoughtUp.ts +++ b/src/actions/moveThoughtUp.ts @@ -5,6 +5,7 @@ import alert from '../actions/alert' import moveThought from '../actions/moveThought' import * as selection from '../device/selection' import findDescendant from '../selectors/findDescendant' +import { getChildrenRanked } from '../selectors/getChildren' import getNextRank from '../selectors/getNextRank' import getRankBefore from '../selectors/getRankBefore' import getThoughtBefore from '../selectors/getThoughtBefore' @@ -71,6 +72,9 @@ const moveThoughtUp = (state: State): State => { newPath, ...(offset != null ? { offset } : null), newRank: rankNew, + afterId: prevThought + ? (prevSibling(state, appendToPath(pathParent, prevThought.id))?.id ?? null) + : (getChildrenRanked(state, head(prevUnclePath!)).at(-1)?.id ?? null), }) } diff --git a/src/actions/outdent.ts b/src/actions/outdent.ts index fee3a80c004..ec77e129e4b 100644 --- a/src/actions/outdent.ts +++ b/src/actions/outdent.ts @@ -59,11 +59,13 @@ const outdent = (state: State): State => { const cursorNew: Path = appendToPath(parentOf(parentOf(cursor)), head(cursor)) + const parentPath = parentOf(simplifyPath(state, cursor)) return moveThought(state, { oldPath: cursor, newPath: cursorNew, ...(offset != null ? { offset } : null), - newRank: getRankAfter(state, parentOf(simplifyPath(state, cursor))), + newRank: getRankAfter(state, parentPath), + afterId: head(parentPath), }) } diff --git a/src/actions/pull.ts b/src/actions/pull.ts index f0b514beee0..0677ac16420 100644 --- a/src/actions/pull.ts +++ b/src/actions/pull.ts @@ -8,7 +8,7 @@ import Thunk from '../@types/Thunk' import { updateThoughtsActionCreator as updateThoughts } from '../actions/updateThoughts' import { HOME_TOKEN } from '../constants' import fetchDescendants from '../data-providers/data-helpers/fetchDescendants' -import db from '../data-providers/yjs/thoughtspace' +import db from '../data-providers/thoughtspace' import getDescendantThoughtIds from '../selectors/getDescendantThoughtIds' import getThoughtById from '../selectors/getThoughtById' import isPending from '../selectors/isPending' diff --git a/src/actions/repairThought.ts b/src/actions/repairThought.ts index c10796f0e37..b57703a8ed6 100644 --- a/src/actions/repairThought.ts +++ b/src/actions/repairThought.ts @@ -5,7 +5,7 @@ import ThoughtId from '../@types/ThoughtId' import Thunk from '../@types/Thunk' import { createThoughtActionCreator as createThought } from '../actions/createThought' import { updateThoughtsActionCreator as updateThoughts } from '../actions/updateThoughts' -import { replicateThought } from '../data-providers/yjs/thoughtspace' +import db from '../data-providers/thoughtspace' import getLexemeSelector from '../selectors/getLexeme' import isContextViewActive from '../selectors/isContextViewActive' import thoughtToPath from '../selectors/thoughtToPath' @@ -44,8 +44,7 @@ export const repairThoughtActionCreator = } // repair invalid parent else { - // replicating the parent should use the cached synced promise - replicateThought(thought.parentId, { background: true }).then(parent => { + db.getThoughtById(thought.parentId).then(parent => { if (parent) { const childKey = isFunction(thought.value) ? thought.value : thought.id if (!parent.childrenMap[childKey]) { diff --git a/src/actions/settings.ts b/src/actions/settings.ts index 1336dc7265a..b7d6316df4b 100644 --- a/src/actions/settings.ts +++ b/src/actions/settings.ts @@ -1,16 +1,29 @@ import _ from 'lodash' import State from '../@types/State' import Thunk from '../@types/Thunk' -import { EM_TOKEN } from '../constants' +import { EM_TOKEN, SETTINGS_TOKEN, SETTINGS_VALUE } from '../constants' import findDescendant from '../selectors/findDescendant' +import getPrevRank from '../selectors/getPrevRank' import { registerActionMetadata } from '../util/actionMetadata.registry' +import createThought from './createThought' import toggleAttribute from './toggleAttribute' /** Sets a setting thought. */ const settings = (state: State, { key, value }: { key: string; value: string }) => { - const emContext = ['Settings', key, value] + const emContext = [SETTINGS_VALUE, key, value] const exists = !!findDescendant(state, EM_TOKEN, emContext) - return exists ? state : toggleAttribute(state, { path: [EM_TOKEN], values: emContext }) + if (exists) return state + + const stateWithSettings = findDescendant(state, EM_TOKEN, SETTINGS_VALUE) + ? state + : createThought(state, { + id: SETTINGS_TOKEN, + path: [EM_TOKEN], + value: SETTINGS_VALUE, + rank: getPrevRank(state, EM_TOKEN), + }) + + return toggleAttribute(stateWithSettings, { path: [EM_TOKEN], values: emContext }) } /** Action-creator for settings. */ diff --git a/src/actions/sort.ts b/src/actions/sort.ts index a6cdac2db9f..9e711a60ca9 100644 --- a/src/actions/sort.ts +++ b/src/actions/sort.ts @@ -1,4 +1,5 @@ import _ from 'lodash' +import Index from '../@types/IndexType' import SortPreference from '../@types/SortPreference' import State from '../@types/State' import ThoughtId from '../@types/ThoughtId' @@ -30,9 +31,14 @@ const sort = (state: State, id: ThoughtId, sortPreference?: SortPreference): Sta if (Object.keys(thoughtIndexUpdates).length === 0) return state + const movePlacements: Index = keyValueBy(children, (child, i) => + child.id in thoughtIndexUpdates ? { [child.id]: i === 0 ? null : children[i - 1].id } : null, + ) + return updateThoughts(state, { thoughtIndexUpdates, lexemeIndexUpdates: {}, + movePlacements, preventExpandThoughts: true, }) } diff --git a/src/actions/toggleAbsoluteContext.ts b/src/actions/toggleAbsoluteContext.ts index 2c8749fdf2e..61c6c16310e 100644 --- a/src/actions/toggleAbsoluteContext.ts +++ b/src/actions/toggleAbsoluteContext.ts @@ -1,13 +1,10 @@ import State from '../@types/State' -import ThoughtId from '../@types/ThoughtId' import Thunk from '../@types/Thunk' -import { ABSOLUTE_TOKEN, HOME_TOKEN } from '../constants' +import { ABSOLUTE_TOKEN, HOME_TOKEN, TRANSIENT_THOUGHT_ID } from '../constants' import { registerActionMetadata } from '../util/actionMetadata.registry' import isHome from '../util/isHome' import timestamp from '../util/timestamp' -const TRANSIENT_THOUGHT_ID = 'TRANSIENT_THOUGHT' as ThoughtId - /** Toggles starting context. */ const toggleAbsoluteContext = (state: State): State => ({ ...state, diff --git a/src/actions/updateThoughts.ts b/src/actions/updateThoughts.ts index 519c5589915..b036d205baf 100644 --- a/src/actions/updateThoughts.ts +++ b/src/actions/updateThoughts.ts @@ -172,6 +172,7 @@ const updateThoughts = ( updates, pendingDeletes, preventExpandThoughts, + movePlacements, local = true, remote = true, idbSynced, @@ -230,6 +231,7 @@ const updateThoughts = ( lexemeIndexUpdates, lexemeIndexUpdatesOld, local, + movePlacements, pendingDeletes, recentlyEdited: recentlyEditedNew, remote, diff --git a/src/commands/__tests__/deleteEmptyThoughtOrOutdent.ts b/src/commands/__tests__/deleteEmptyThoughtOrOutdent.ts index d110b8f398f..8dcd627af47 100644 --- a/src/commands/__tests__/deleteEmptyThoughtOrOutdent.ts +++ b/src/commands/__tests__/deleteEmptyThoughtOrOutdent.ts @@ -36,7 +36,7 @@ describe('DOM', () => { // This ensures that the thought b exists so we can confirm later that it is deleted. const initialExportedData = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(initialExportedData).toBe(`- __ROOT__ + expect(initialExportedData).toBe(`- ${HOME_TOKEN} - a - b`) @@ -49,7 +49,7 @@ describe('DOM', () => { // This ensures that the thought b doesn't exist now. const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a`) }) }) diff --git a/src/commands/__tests__/deleteThought.ts b/src/commands/__tests__/deleteThought.ts index 7c19a711917..31f3bb9fbae 100644 --- a/src/commands/__tests__/deleteThought.ts +++ b/src/commands/__tests__/deleteThought.ts @@ -61,7 +61,7 @@ describe('mount', () => { // TODO: Fix test it.skip('delete pending descendants', async () => { timer.useFakeTimer() - initialize() + initialize({ storage: 'memory' }) await timer.runAllAsync() // c will be pending after refresh @@ -111,7 +111,7 @@ it.skip('delete pending descendants', async () => { // clear and call initialize again to reload from local db (simulating page refresh) store.dispatch(clear()) - initialize() + initialize({ storage: 'memory' }) await timer.runAllAsync() store.dispatch([setCursor(['a'])]) @@ -182,10 +182,10 @@ it.skip('delete pending descendants', async () => { }) }) -// TODO: y-indexeddb breaks tests so it is disabled +// TODO: IndexedDB in tests is disabled where it breaks fake-indexeddb it.skip('delete many pending descendants', async () => { timer.useFakeTimer() - initialize() + initialize({ storage: 'memory' }) await timer.runAllAsync() const text = ` @@ -268,7 +268,7 @@ it.skip('delete many pending descendants', async () => { // clear and call initialize again to reload from local db (simulating page refresh) store.dispatch(clear()) - initialize() + initialize({ storage: 'memory' }) await timer.runAllAsync() store.dispatch([setCursor(['Cybersemics'])]) diff --git a/src/commands/__tests__/generateThought.ts b/src/commands/__tests__/generateThought.ts index b81a480585a..cf6b8e9f4c4 100644 --- a/src/commands/__tests__/generateThought.ts +++ b/src/commands/__tests__/generateThought.ts @@ -17,8 +17,8 @@ import generateThought from '../generateThought' const mockFetch = vi.fn() global.fetch = mockFetch -beforeEach(() => { - initStore() +beforeEach(async () => { + await initStore() vi.clearAllMocks() // clearAllMocks does not drain queued mockResolvedValueOnce responses, which would otherwise leak into the next test mockFetch.mockReset() diff --git a/src/commands/__tests__/moveThought.ts b/src/commands/__tests__/moveThought.ts index 3f92713141f..ee6351d1e01 100644 --- a/src/commands/__tests__/moveThought.ts +++ b/src/commands/__tests__/moveThought.ts @@ -17,7 +17,7 @@ afterEach(cleanupTestApp) // TODO: TransactionInactiveError: A request was placed against a transaction which is currently not active, or which is finished. it.skip('merge up to pending destination descendant', async () => { timer.useFakeTimer() - initialize() + initialize({ storage: 'memory' }) await timer.runAllAsync() const text = ` @@ -42,7 +42,7 @@ it.skip('merge up to pending destination descendant', async () => { appStore.dispatch(clear()) await timer.runAllAsync() - initialize() + initialize({ storage: 'memory' }) await timer.runAllAsync() diff --git a/src/commands/__tests__/moveThoughtDown.ts b/src/commands/__tests__/moveThoughtDown.ts index 1dc1eb8ee28..7e61c01cbae 100644 --- a/src/commands/__tests__/moveThoughtDown.ts +++ b/src/commands/__tests__/moveThoughtDown.ts @@ -29,7 +29,7 @@ describe('moveThoughtDown', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - c - b`) @@ -81,7 +81,7 @@ describe('moveThoughtDown', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - c - b @@ -113,7 +113,7 @@ describe('moveThoughtDown', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - a2 - a1 @@ -142,7 +142,7 @@ describe('moveThoughtDown', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c`) diff --git a/src/commands/__tests__/moveThoughtUp.ts b/src/commands/__tests__/moveThoughtUp.ts index 85ee55bb8ca..1dff06c7663 100644 --- a/src/commands/__tests__/moveThoughtUp.ts +++ b/src/commands/__tests__/moveThoughtUp.ts @@ -29,7 +29,7 @@ describe('moveThoughtUp', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - c - b`) @@ -81,7 +81,7 @@ describe('moveThoughtUp', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - c - b @@ -113,7 +113,7 @@ describe('moveThoughtUp', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - a2 - a1 @@ -142,7 +142,7 @@ describe('moveThoughtUp', () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c`) diff --git a/src/commands/__tests__/newUncle.ts b/src/commands/__tests__/newUncle.ts index 4a23a18baf1..0fe4735a550 100644 --- a/src/commands/__tests__/newUncle.ts +++ b/src/commands/__tests__/newUncle.ts @@ -38,8 +38,8 @@ describe('multicursor', () => { - a - b - c - - - - + - ${''} + - ${''} - x`) }) @@ -66,7 +66,7 @@ describe('multicursor', () => { expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - - + - ${''} - c - d - e @@ -179,13 +179,13 @@ describe('multicursor', () => { expect(exportContext(store.getState(), [HOME_TOKEN], 'text/plain')).toEqual(`- ${HOME_TOKEN} - a - b - - + - ${''} - c - d - - + - ${''} - e - f - - `) + - ${''}`) store.dispatch(undo()) diff --git a/src/commands/__tests__/pin.ts b/src/commands/__tests__/pin.ts index ef19a6f37d0..2bd062170a8 100644 --- a/src/commands/__tests__/pin.ts +++ b/src/commands/__tests__/pin.ts @@ -30,7 +30,7 @@ it('toggle on when there is no =pin attribute', () => { executeCommand(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - =pin @@ -63,7 +63,7 @@ it('toggle on when =pin/false', () => { executeCommand(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - =pin @@ -96,7 +96,7 @@ it('remove =pin when toggling off', () => { executeCommand(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - c @@ -127,7 +127,7 @@ it('remove =pin/true when toggling off', () => { executeCommand(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - c @@ -162,7 +162,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - =pin @@ -203,7 +203,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c @@ -237,7 +237,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =pin - b @@ -271,7 +271,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c @@ -302,7 +302,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(pinCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - =pin diff --git a/src/commands/__tests__/pinAll.ts b/src/commands/__tests__/pinAll.ts index 27814cd4059..1011819ac40 100644 --- a/src/commands/__tests__/pinAll.ts +++ b/src/commands/__tests__/pinAll.ts @@ -29,7 +29,7 @@ it('toggle on when there is no =children attribute', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =children - =pin @@ -63,7 +63,7 @@ it('toggle on when there is an unrelated =children attribute', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - A - B - =children @@ -99,7 +99,7 @@ it('toggle on when =children/=pin is false', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =children - =pin @@ -135,7 +135,7 @@ it('remove =children when toggling off from =pin/true', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - c @@ -167,7 +167,7 @@ it('remove =children when toggling off from =pin', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - c @@ -203,7 +203,7 @@ it('remove =pin/false from all subthoughts when toggling on', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =children - =pin @@ -242,7 +242,7 @@ it('preserve unrelated =children attributes when toggling off', () => { executeCommand(pinAllCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =children - =bullet diff --git a/src/commands/__tests__/proseView.ts b/src/commands/__tests__/proseView.ts index 36388910ad5..83c1f82b5d8 100644 --- a/src/commands/__tests__/proseView.ts +++ b/src/commands/__tests__/proseView.ts @@ -25,7 +25,7 @@ it('toggle on prose view of parent of cursor (initial state without =view attrib executeCommand(proseViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =view - Prose @@ -50,7 +50,7 @@ it('toggle on prose view of parent of cursor (initial state with =view attribute executeCommand(proseViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =view - Prose @@ -75,7 +75,7 @@ it('toggle off prose view of parent of cursor', () => { executeCommand(proseViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c`) @@ -106,7 +106,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(proseViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =view - Prose @@ -152,7 +152,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(proseViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - a1 - a2 @@ -194,7 +194,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(proseViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =view - Prose diff --git a/src/commands/__tests__/repeat.ts b/src/commands/__tests__/repeat.ts index 3a21f5e3352..c27913174f4 100644 --- a/src/commands/__tests__/repeat.ts +++ b/src/commands/__tests__/repeat.ts @@ -21,8 +21,8 @@ vi.mock('../../util/throttleByAnimationFrame', () => ({ default: (f: (...args: any[]) => void) => f, })) -beforeEach(() => { - initStore() +beforeEach(async () => { + await initStore() // lastCommand is module-level state in commands.ts that is not reset by initStore resetLastCommand() }) @@ -43,7 +43,7 @@ it('execute the last command again', () => { executeCommandWithMulticursor(repeatCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - b - c - a`) @@ -67,7 +67,7 @@ it('repeat does not repeat itself', () => { executeCommandWithMulticursor(repeatCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - b - c - d @@ -88,7 +88,7 @@ it('do nothing when no command has been executed', () => { executeCommandWithMulticursor(repeatCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b`) }) @@ -113,7 +113,7 @@ it('ignore navigation commands', () => { expect(headValue(store.getState(), store.getState().cursor!)).toEqual('b') const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =pin - b @@ -138,7 +138,7 @@ it('ignore commands that do not dispatch an undoable action', () => { // pin is repeated, toggling it back off const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b`) }) @@ -166,7 +166,7 @@ it('repeat a command that handles the multiselect itself', () => { executeCommandWithMulticursor(repeatCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - ${''} - a - b @@ -193,7 +193,7 @@ it('ignore undo', () => { // pin is repeated rather than undone a second time const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =pin - b`) diff --git a/src/commands/__tests__/splitSentences.ts b/src/commands/__tests__/splitSentences.ts index 5e388035c28..1b10ec19337 100644 --- a/src/commands/__tests__/splitSentences.ts +++ b/src/commands/__tests__/splitSentences.ts @@ -1,7 +1,7 @@ import { importTextActionCreator as importText } from '../../actions/importText' import { newThoughtActionCreator as newThought } from '../../actions/newThought' import { executeCommand, executeCommandWithMulticursor } from '../../commands' -import { HOME_TOKEN } from '../../constants' +import { EMPTY_SPACE, HOME_TOKEN } from '../../constants' import exportContext from '../../selectors/exportContext' import store from '../../stores/app' import { addMulticursorAtFirstMatchActionCreator as addMulticursor } from '../../test-helpers/addMulticursorAtFirstMatch' @@ -26,7 +26,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - This is sentence one. - This is sentence two. - This is sentence three.`) @@ -42,7 +42,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - **This is sentence one.** - **This is sentence two.** - **This is sentence three.**`) @@ -55,7 +55,7 @@ describe('splitSentences', () => { const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/html') expect(exported).toBe(`
                                                                          -
                                                                        • __ROOT__ +
                                                                        • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                          • Hello.
                                                                          • World.
                                                                            • @@ -75,7 +75,7 @@ describe('splitSentences', () => { const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/html') expect(exported).toBe(`
                                                                              -
                                                                            • __ROOT__ +
                                                                            • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                              • font
                                                                              @@ -97,7 +97,7 @@ describe('splitSentences', () => { const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/html') expect(exported).toBe(`
                                                                                -
                                                                              • __ROOT__ +
                                                                              • ${HOME_TOKEN}${EMPTY_SPACE}
                                                                                • comma one
                                                                                • comma two
                                                                                  • @@ -119,7 +119,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - This is a single sentence.`) }) @@ -136,7 +136,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - Hello, world! - How are you? - I'm fine, thanks.`) @@ -155,7 +155,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - Gödel - Escher - Bach`) @@ -174,7 +174,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - me - you - he and she @@ -197,7 +197,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - Alice - the Lion`) }) @@ -216,7 +216,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - Standard`) }) @@ -233,7 +233,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - one - 1`) }) @@ -251,7 +251,7 @@ describe('splitSentences', () => { executeCommand(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - one - 1. - two. - three.`) @@ -275,7 +275,7 @@ describe('splitSentences', () => { executeCommandWithMulticursor(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - A. - This is A. - More A. @@ -303,7 +303,7 @@ describe('splitSentences', () => { executeCommandWithMulticursor(splitSentencesCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - One sentence only. - Two sentences here. - And the second one. diff --git a/src/commands/__tests__/swapNote.ts b/src/commands/__tests__/swapNote.ts index bc71fd5ac43..1acb40dfc39 100644 --- a/src/commands/__tests__/swapNote.ts +++ b/src/commands/__tests__/swapNote.ts @@ -34,7 +34,7 @@ describe('swapNote', () => { executeCommand(swapNoteCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =note - b`) @@ -56,7 +56,7 @@ describe('swapNote', () => { executeCommand(swapNoteCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c`) @@ -83,7 +83,7 @@ describe('swapNote', () => { executeCommandWithMulticursor(swapNoteCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =note - b diff --git a/src/commands/__tests__/toggleDone.ts b/src/commands/__tests__/toggleDone.ts index 8c46b3c39ad..7b94cc2ab1a 100644 --- a/src/commands/__tests__/toggleDone.ts +++ b/src/commands/__tests__/toggleDone.ts @@ -19,7 +19,7 @@ describe('toggleDone', () => { executeCommand(toggleDoneCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - `) }) @@ -36,7 +36,7 @@ describe('toggleDone', () => { ]) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - `) }) @@ -55,7 +55,7 @@ describe('toggleDone', () => { executeCommand(toggleDoneCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - =done @@ -78,7 +78,7 @@ describe('toggleDone', () => { executeCommand(toggleDoneCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c`) @@ -105,7 +105,7 @@ describe('toggleDone', () => { executeCommandWithMulticursor(toggleDoneCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - =done @@ -139,7 +139,7 @@ describe('toggleDone', () => { executeCommandWithMulticursor(toggleDoneCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - c diff --git a/src/commands/__tests__/toggleTableView.ts b/src/commands/__tests__/toggleTableView.ts index 037abd156db..90d5fcbaeb4 100644 --- a/src/commands/__tests__/toggleTableView.ts +++ b/src/commands/__tests__/toggleTableView.ts @@ -192,7 +192,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(toggleTableViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - =view - Table @@ -238,7 +238,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(toggleTableViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - a1 - a2 @@ -275,7 +275,7 @@ describe('multicursor', () => { executeCommandWithMulticursor(toggleTableViewCommand, { store }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toBe(`- __ROOT__ + expect(exported).toBe(`- ${HOME_TOKEN} - a - b - =view diff --git a/src/commands/__tests__/uncategorize.ts b/src/commands/__tests__/uncategorize.ts index 17fb6c7ffab..f59a907507a 100644 --- a/src/commands/__tests__/uncategorize.ts +++ b/src/commands/__tests__/uncategorize.ts @@ -48,7 +48,7 @@ describe('uncategorize', () => { }) it('persists undoing uncategorize of a duplicate uncle without a save error', async () => { - const { cleanup } = await initialize() + const { cleanup } = await initialize({ storage: 'memory' }) try { store.dispatch([ diff --git a/src/commands/__tests__/undo-redo.ts b/src/commands/__tests__/undo-redo.ts index 6d5b7673a5b..98c4ad1fe94 100644 --- a/src/commands/__tests__/undo-redo.ts +++ b/src/commands/__tests__/undo-redo.ts @@ -28,6 +28,7 @@ import { editThoughtByContextActionCreator as editThought } from '../../test-hel import getAllChildrenAsThoughtsByContext from '../../test-helpers/getAllChildrenAsThoughtsByContext' import initStore from '../../test-helpers/initStore' import { setCursorFirstMatchActionCreator as setCursor } from '../../test-helpers/setCursorFirstMatch' +import waitForThoughtspaceIdle from '../../test-helpers/waitForThoughtspaceIdle' import archiveCommand from '../archive' import deleteCommand from '../delete' import indentCommand from '../indent' @@ -46,7 +47,7 @@ beforeEach(initStore) */ describe('undo persistence', () => { it('persists undo thought change', async () => { - await initialize() + await initialize({ storage: 'memory' }) store.dispatch([ importText({ @@ -62,7 +63,7 @@ describe('undo persistence', () => { // clear and call initialize again to reload from local db (simulating page refresh) store.dispatch(clear()) - await initialize() + await initialize({ storage: 'memory' }) await vi.runAllTimersAsync() const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') @@ -76,6 +77,103 @@ describe('undo persistence', () => { await vi.runAllTimersAsync() vi.useRealTimers() }, 10000 /* increase timeout to give time for two calls to initialize() */) + + it('persists undo move placement after reload', async () => { + await initialize({ storage: 'memory' }) + + store.dispatch([ + importText({ + text: ` + - a + - b + - c + - d + - e`, + }), + setCursor(['a']), + addMulticursor(['a']), + addMulticursor(['b']), + addMulticursor(['c']), + ]) + + executeCommandWithMulticursor(moveThoughtDownCommand, { store }) + + expect(exportContext(store.getState(), [HOME_TOKEN], 'text/plain')).toEqual(`- ${HOME_TOKEN} + - d + - a + - b + - c + - e`) + + store.dispatch(undo()) + await waitForThoughtspaceIdle() + + store.dispatch(clear()) + + await initialize({ storage: 'memory' }) + await vi.runAllTimersAsync() + await waitForThoughtspaceIdle() + + expect(exportContext(store.getState(), [HOME_TOKEN], 'text/plain')).toEqual(`- ${HOME_TOKEN} + - a + - b + - c + - d + - e`) + }, 10000 /* increase timeout to give time for two calls to initialize() */) + + it('persists redo move placement after reload', async () => { + await initialize({ storage: 'memory' }) + + store.dispatch([ + importText({ + text: ` + - a + - b + - c + - d + - e`, + }), + setCursor(['a']), + addMulticursor(['a']), + addMulticursor(['b']), + addMulticursor(['c']), + ]) + + executeCommandWithMulticursor(moveThoughtDownCommand, { store }) + store.dispatch(undo()) + await waitForThoughtspaceIdle() + + expect(exportContext(store.getState(), [HOME_TOKEN], 'text/plain')).toEqual(`- ${HOME_TOKEN} + - a + - b + - c + - d + - e`) + + store.dispatch(redo()) + await waitForThoughtspaceIdle() + + expect(exportContext(store.getState(), [HOME_TOKEN], 'text/plain')).toEqual(`- ${HOME_TOKEN} + - d + - a + - b + - c + - e`) + + store.dispatch(clear()) + + await initialize({ storage: 'memory' }) + await vi.runAllTimersAsync() + await waitForThoughtspaceIdle() + + expect(exportContext(store.getState(), [HOME_TOKEN], 'text/plain')).toEqual(`- ${HOME_TOKEN} + - d + - a + - b + - c + - e`) + }, 10000 /* increase timeout to give time for two calls to initialize() */) }) describe('undo', () => { @@ -194,7 +292,7 @@ describe('undo', () => { }) it('cursor should restore correctly after undo archive', async () => { - await initialize() + await initialize({ storage: 'memory' }) store.dispatch([newThought({ value: 'a' }), setCursor(['a']), { type: 'archiveThought' }, undo()]) diff --git a/src/components/Content.tsx b/src/components/Content.tsx index 33e57e39b9f..1ff1e94238a 100644 --- a/src/components/Content.tsx +++ b/src/components/Content.tsx @@ -13,6 +13,7 @@ import { CONTENT_BOX_PADDING_RIGHT, HOME_PATH, LongPressState, + TRANSIENT_THOUGHT_ID, TUTORIAL2_STEP_SUCCESS, } from '../constants' import * as selection from '../device/selection' @@ -27,7 +28,7 @@ import EmptyThoughtspace from './EmptyThoughtspace' import LayoutTree from './LayoutTree' import Search from './Search' -const transientChildPath = ['TRANSIENT_THOUGHT_ID'] as SimplePath +const transientChildPath = [TRANSIENT_THOUGHT_ID] as SimplePath /* Transient Editable represents a child that is yet not in the state. diff --git a/src/components/DropEnd.tsx b/src/components/DropEnd.tsx index 23787bdd0d1..a2d9fdcc885 100644 --- a/src/components/DropEnd.tsx +++ b/src/components/DropEnd.tsx @@ -5,6 +5,7 @@ import { dropEndRecipe, dropHoverRecipe } from '../../styled-system/recipes' import DropThoughtZone from '../@types/DropThoughtZone' import Path from '../@types/Path' import { isTouch } from '../browser' +import { HOME_DISPLAY_VALUE } from '../constants' import testFlags from '../e2e/testFlags' import useDragAndDropSubThought from '../hooks/useDragAndDropSubThought' import attributeEquals from '../selectors/attributeEquals' @@ -48,6 +49,8 @@ const DropEnd = ({ const thoughtId = head(path) const isRootPath = isRoot(path) const value = useSelector(state => getThoughtById(state, thoughtId)?.value) ?? '' + // Simulated drag snapshots need a human-readable root label, but the canonical Home value remains HOME_TOKEN. + const displayValue = isRootPath ? HOME_DISPLAY_VALUE : value const dropHoverColorValue = useSelector(state => dropHoverColor(state, depth + 1)) const isParentTableCol1 = useSelector(state => @@ -132,7 +135,7 @@ const DropEnd = ({ > {isHovering ? '*' : ''} {last ? '$' : ''} - {strip(value)} + {strip(displayValue)} )} {(showDropHover || testFlags.simulateDrag) && ( diff --git a/src/components/Editable/useOnPaste.ts b/src/components/Editable/useOnPaste.ts index 34cbd4f4595..546ff7fff92 100644 --- a/src/components/Editable/useOnPaste.ts +++ b/src/components/Editable/useOnPaste.ts @@ -3,6 +3,7 @@ import React, { useCallback } from 'react' import { useDispatch } from 'react-redux' import SimplePath from '../../@types/SimplePath' import { importDataActionCreator as importData } from '../../actions/importData' +import { HOME_TOKEN } from '../../constants' import store from '../../stores/app' import equalPath from '../../util/equalPath' import strip from '../../util/strip' @@ -30,9 +31,12 @@ const useOnPaste = ({ // Handle raw thought import confirmation if ( typeof window !== 'undefined' && - plainText.startsWith(`{ + (plainText.startsWith(`{ "thoughtIndex": { - "__ROOT__": {`) && + "__ROOT__": {`) || + plainText.startsWith(`{ + "thoughtIndex": { + "${HOME_TOKEN}": {`)) && !window.confirm('Import raw thought state? Current state will be overwritten.') ) { e.preventDefault() diff --git a/src/components/Footer.tsx b/src/components/Footer.tsx index cb04f51ad04..8ad6813bcb3 100644 --- a/src/components/Footer.tsx +++ b/src/components/Footer.tsx @@ -9,7 +9,7 @@ import fontSizeDown from '../actions/fontSizeDown' import fontSizeUp from '../actions/fontSizeUp' import { showModalActionCreator as showModal } from '../actions/showModal' import { TUTORIAL2_STEP_SUCCESS } from '../constants' -import { tsid } from '../data-providers/yjs' +import { tsid } from '../data-providers/thoughtspaceSession' import scrollTo from '../device/scrollTo' import getSetting from '../selectors/getSetting' import isTutorial from '../selectors/isTutorial' diff --git a/src/components/GestureDiagram.tsx b/src/components/GestureDiagram.tsx index 926b9231d07..66a80ffb7af 100644 --- a/src/components/GestureDiagram.tsx +++ b/src/components/GestureDiagram.tsx @@ -1,3 +1,4 @@ +import { nanoid } from 'nanoid' import React, { useState } from 'react' import { css } from '../../styled-system/css' import { token } from '../../styled-system/tokens' @@ -5,7 +6,6 @@ import { SystemStyleObject } from '../../styled-system/types' import Direction from '../@types/Direction' import Gesture from '../@types/Gesture' import { GESTURE_GLOW_BLUR, GESTURE_GLOW_COLOR } from '../constants' -import createId from '../util/createId' interface GestureDiagramProps { arrowSize?: number @@ -366,7 +366,7 @@ const GestureDiagram = ({ useGradient = true, highlightColor, }: GestureDiagramProps) => { - const [id] = useState(createId()) + const [id] = useState(nanoid()) // match signaturePad shadow in TraceGesture component // TODO: Why isn't this working? diff --git a/src/components/RecentlyDeleted.tsx b/src/components/RecentlyDeleted.tsx index b54ff8ffa4a..2c68522f5f3 100644 --- a/src/components/RecentlyDeleted.tsx +++ b/src/components/RecentlyDeleted.tsx @@ -5,7 +5,7 @@ import { css } from '../../styled-system/css' import Thunk from '../@types/Thunk' import { pullActionCreator as pull } from '../actions/pull' import { pullAncestorsActionCreator as pullAncestors } from '../actions/pullAncestors' -import { getLexemeById } from '../data-providers/yjs/thoughtspace' +import db from '../data-providers/thoughtspace' import useDelayedState from '../hooks/useDelayedState' import getChildPath from '../selectors/getChildPath' import { getAllChildren } from '../selectors/getChildren' @@ -21,7 +21,7 @@ import ThoughtLink from './ThoughtLink' /** An action-creator that pulls all deleted thoughts, i.e. children of contexts of =archive. */ const pullDeleted = (): Thunk> => async (dispatch, getState) => { // pull the =archive lexeme - const lexeme = await getLexemeById(hashThought('=archive')) + const lexeme = await db.getLexemeById(hashThought('=archive')) // pull all ancestors of all contexts of =archive await dispatch(pullAncestors(lexeme?.contexts ?? [], { force: true, maxDepth: 0 })) const state = getState() diff --git a/src/components/Thought.tsx b/src/components/Thought.tsx index 803a70b0127..2bb80c1f6aa 100644 --- a/src/components/Thought.tsx +++ b/src/components/Thought.tsx @@ -170,10 +170,7 @@ const useCol1Alignment = ({ path, value, isTableCol1 }: UseCol1AlignParams) => { return cursorParentId ? getChildren(state, cursorParentId).map(t => t.value) : [] }, shallowEqual) - type TransitionStyle = { - transform: string - transition: string - } + type TransitionStyle = Pick const [alignmentTransition, setAlignmentTransition] = useState<{ bullet: TransitionStyle diff --git a/src/components/ThoughtspaceInUse.tsx b/src/components/ThoughtspaceInUse.tsx new file mode 100644 index 00000000000..fc34b8587df --- /dev/null +++ b/src/components/ThoughtspaceInUse.tsx @@ -0,0 +1,38 @@ +import { css, cx } from '../../styled-system/css' +import { anchorButtonRecipe } from '../../styled-system/recipes' +import type { ThoughtspaceAccessBlockedReason } from '../data-providers/thoughtspace' +import fastClick from '../util/fastClick' + +/** Bootstrap screen shown when the active thoughtspace cannot be opened safely in this tab. */ +const ThoughtspaceInUse = ({ reason }: { reason: ThoughtspaceAccessBlockedReason }) => ( +
                                                                                  +
                                                                                  +

                                                                                  {reason === 'already-open' ? 'em is already open' : 'em cannot safely open this thoughtspace'}

                                                                                  +

                                                                                  + {reason === 'already-open' + ? 'To protect your local data, em currently supports one tab per thoughtspace. Close the other tab, then retry.' + : 'This browser does not support the storage coordination required to protect your local data.'} +

                                                                                  + +
                                                                                  +
                                                                                  +) + +export default ThoughtspaceInUse diff --git a/src/components/__tests__/Bullet.ts b/src/components/__tests__/Bullet.ts index dd65f3b18de..37e96f6db2e 100644 --- a/src/components/__tests__/Bullet.ts +++ b/src/components/__tests__/Bullet.ts @@ -439,7 +439,7 @@ describe('expansion', () => { await act(vi.runOnlyPendingTimersAsync) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - c @@ -466,7 +466,7 @@ describe('expansion', () => { await act(() => vi.runAllTimersAsync()) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - =pin @@ -498,7 +498,7 @@ describe('expansion', () => { await act(() => vi.runAllTimersAsync()) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - =children - =pin @@ -628,7 +628,7 @@ describe('multiselect', () => { await clickWithModifiers(getBulletByContext(['b']), { shiftKey: true }) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - a - b - c`) diff --git a/src/components/__tests__/LetterCasePicker.ts b/src/components/__tests__/LetterCasePicker.ts index c4cd6532e8f..34757057db4 100644 --- a/src/components/__tests__/LetterCasePicker.ts +++ b/src/components/__tests__/LetterCasePicker.ts @@ -22,7 +22,7 @@ it('Set Lower Case to the current thought', async () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - hello everyone, this is rose. thanks for your help.`) }) @@ -36,7 +36,7 @@ it('Set Upper Case to the current thought', async () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - HELLO EVERYONE, THIS IS ROSE. THANKS FOR YOUR HELP.`) }) @@ -50,7 +50,7 @@ it('Set Sentence Case to the current thought', async () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - Hello everyone, this is rose. Thanks for your help.`) }) @@ -64,7 +64,7 @@ it('Set Title Case to the current thought', async () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - Hello Everyone, This Is Rose. Thanks for Your Help.`) }) @@ -83,7 +83,7 @@ it('Set Upper Case with multicursor selection', async () => { const state = store.getState() const exported = exportContext(state, [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - HELLO EVERYONE, THIS IS ROSE. THANKS FOR YOUR HELP. - GOODBYE EVERYONE, THIS IS MAX. THANKS FOR YOUR HELP.`) }) @@ -118,7 +118,7 @@ it('multicursor selection is preserved after applying Upper Case to one of two t expect(Object.keys(store.getState().multicursors)).toHaveLength(1) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - HELLO EVERYONE, THIS IS ROSE. THANKS FOR YOUR HELP. - Goodbye everyone, this is Max. Thanks for your help.`) }) @@ -140,7 +140,7 @@ it('multicursor selection is preserved after applying Upper Case to two of three expect(Object.keys(store.getState().multicursors)).toHaveLength(2) const exported = exportContext(store.getState(), [HOME_TOKEN], 'text/plain') - expect(exported).toEqual(`- __ROOT__ + expect(exported).toEqual(`- ${HOME_TOKEN} - HELLO EVERYONE, THIS IS ROSE. THANKS FOR YOUR HELP. - Goodbye everyone, this is Max. Thanks for your help. - SEE YOU SOON, THIS IS ANN. THANKS FOR YOUR HELP.`) diff --git a/src/components/modals/Devices.tsx b/src/components/modals/Devices.tsx index 4186eb17c85..b51a141ee0a 100644 --- a/src/components/modals/Devices.tsx +++ b/src/components/modals/Devices.tsx @@ -11,10 +11,10 @@ import Role from '../../@types/Role' import Share from '../../@types/Share' import { alertActionCreator as alert } from '../../actions/alert' import { isMac } from '../../browser' -import { accessToken as accessTokenCurrent, permissionsClientDoc, tsid } from '../../data-providers/yjs' -import permissionsModel from '../../data-providers/yjs/permissionsModel' +import permissionsModel from '../../data-providers/permissionsModel' +import { permissionsStore } from '../../data-providers/permissionsStore' +import { accessToken as accessTokenCurrent, tsid } from '../../data-providers/thoughtspaceSession' import * as selection from '../../device/selection' -import useSharedType from '../../hooks/useSharedType' import useStatus from '../../hooks/useStatus' import modalDescriptionClass from '../../recipes/modalDescriptionClass' import fastClick from '../../util/fastClick' @@ -26,8 +26,8 @@ import CopyClipboard from './../icons/CopyClipboard' import PencilIcon from './../icons/PencilIcon' import ModalComponent from './ModalComponent' -/** A hook that subscribes to the permissionsClientDoc. */ -const usePermissions = (): Index => useSharedType(permissionsClientDoc.getMap()) +/** A hook that subscribes to persisted device permissions. */ +const usePermissions = (): Index => permissionsStore.useSelector(s => s.entries) /** Gets the next available device name for a new device. Autoincrements by 1. */ const getNextDeviceName = (permissions: Index, start?: number): string => { diff --git a/src/components/modals/Export.tsx b/src/components/modals/Export.tsx index 90b80d48b56..250b45b0c0b 100644 --- a/src/components/modals/Export.tsx +++ b/src/components/modals/Export.tsx @@ -25,6 +25,7 @@ import { errorActionCreator as error } from '../../actions/error' import { isIOS, isMac, isTouch } from '../../browser' import { HOME_PATH, HOME_TOKEN } from '../../constants' import replicateTree from '../../data-providers/data-helpers/replicateTree' +import { thoughtspaceRuntime } from '../../data-providers/thoughtspace' import download from '../../device/download' import * as selection from '../../device/selection' import globals from '../../globals' @@ -144,20 +145,37 @@ const PullProvider: FC> = ({ ch () => { isMounted.current = true - const replications = simplePaths.map(simplePath => { - const id = head(simplePath) - - return replicateTree(id, { - // TODO: Warn the user if offline or not fully replicated - remote: false, - onThought: thought => { - if (!isMounted.current) return - setExportingThoughtsThrottled(thought) - }, + /** Waits for pending local persistence before reading the selected subtrees for export. */ + const startReplicationsAfterLocalWrites = async () => { + await thoughtspaceRuntime.waitForIdle() + if (!isMounted.current) return null + + const replications = simplePaths.map(simplePath => { + const id = head(simplePath) + + return replicateTree(id, { + // TODO: Warn the user if offline or not fully replicated + remote: false, + onThought: thought => { + if (!isMounted.current) return + setExportingThoughtsThrottled(thought) + }, + }) }) - }) - Promise.all(replications.map(replication => replication.promise)).then(thoughtIndices => { + return { + replications, + thoughtIndicesPromise: Promise.all(replications.map(replication => replication.promise)), + } + } + + const replicationsStartedPromise = startReplicationsAfterLocalWrites() + + void (async () => { + const startedReplications = await replicationsStartedPromise + if (!startedReplications) return + + const thoughtIndices = await startedReplications.thoughtIndicesPromise if (!isMounted.current) return setExportingThoughtsThrottled.flush() @@ -173,11 +191,13 @@ const PullProvider: FC> = ({ ch setExportedState(exportedState) setIsPulling(false) - }) + })() return () => { isMounted.current = false - replications.forEach(replication => replication.cancel()) + void replicationsStartedPromise.then(startedReplications => { + startedReplications?.replications.forEach(replication => replication.cancel()) + }) } }, // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/src/constants.ts b/src/constants.ts index 66cef437b26..9c45aee34d8 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -111,15 +111,24 @@ export const SCHEMA_THOUGHT_WITH_CHILDREN = 7 // store all children in the Thoug export const SCHEMA_LEMMA = 8 export const SCHEMA_LATEST = 8 -// store the root string as a token that is not likely to be written by the user (bad things will happen) -export const HOME_TOKEN = '__ROOT__' as ThoughtId +export const GLOBAL_ROOT_TOKEN = '00000000000000000000000000000000' as ThoughtId -export const ROOT_PARENT_ID = '__ROOT_PARENT_ID__' as ThoughtId +export const ROOT_PARENT_ID = GLOBAL_ROOT_TOKEN -// token for hidden system context -export const EM_TOKEN = '__EM__' as ThoughtId +export const HOME_TOKEN = '00000000000000000000000000000001' as ThoughtId -export const ABSOLUTE_TOKEN = '__ABSOLUTE__' as ThoughtId +// Display/export-only label for the fixed Home root. Do not store this as the root thought value. +export const HOME_DISPLAY_VALUE = '__ROOT__' + +export const EM_TOKEN = '00000000000000000000000000000002' as ThoughtId + +export const ABSOLUTE_TOKEN = '00000000000000000000000000000003' as ThoughtId + +// Fixed /EM/Settings identity used by system thought bootstrap and the TreeCRDT-backed thoughtspace. +export const SETTINGS_TOKEN = '00000000000000000000000000000004' as ThoughtId +export const SETTINGS_VALUE = 'Settings' + +export const TRANSIENT_THOUGHT_ID = '00000000000000000000000000ffffff' as ThoughtId export const ROOT_CONTEXTS = [HOME_TOKEN, ABSOLUTE_TOKEN] diff --git a/src/data-providers/DataProvider.ts b/src/data-providers/DataProvider.ts index ceee434b613..552646dd89f 100644 --- a/src/data-providers/DataProvider.ts +++ b/src/data-providers/DataProvider.ts @@ -13,11 +13,13 @@ export interface DataProvider { getLexemesByIds: (keys: string[]) => Promise<(Lexeme | undefined)[]> getThoughtById: (id: ThoughtId) => Promise getThoughtsByIds: (ids: ThoughtId[]) => Promise<(Thought | undefined)[]> + /** Resolved value is provider-specific; the treecrdt provider returns `readonly Operation[]` for local tree mutations. */ updateThoughts: (args: { thoughtIndexUpdates: Index lexemeIndexUpdates: Index lexemeIndexUpdatesOld: Index schemaVersion: number + movePlacements?: Index }) => Promise freeThought: (id: ThoughtId) => Promise freeLexeme: (key: string) => Promise diff --git a/src/data-providers/__tests__/permissionsStore.ts b/src/data-providers/__tests__/permissionsStore.ts new file mode 100644 index 00000000000..ef8186a3b5f --- /dev/null +++ b/src/data-providers/__tests__/permissionsStore.ts @@ -0,0 +1,22 @@ +import { initPermissionsStore } from '../permissionsStore' + +const { get, set } = vi.hoisted(() => ({ + get: vi.fn(), + set: vi.fn(), +})) + +vi.mock('idb-keyval', () => ({ get, set })) + +afterEach(() => { + vi.unstubAllEnvs() +}) + +it('retries after a failed permissions load', async () => { + vi.stubEnv('MODE', 'production') + const error = new Error('load failed') + get.mockRejectedValueOnce(error).mockResolvedValueOnce(undefined) + + await expect(initPermissionsStore()).rejects.toBe(error) + await expect(initPermissionsStore()).resolves.toBeUndefined() + expect(get).toHaveBeenCalledTimes(2) +}) diff --git a/src/data-providers/data-helpers/fetchDescendants.ts b/src/data-providers/data-helpers/fetchDescendants.ts index 362efd72086..0ba8d0499bc 100644 --- a/src/data-providers/data-helpers/fetchDescendants.ts +++ b/src/data-providers/data-helpers/fetchDescendants.ts @@ -16,7 +16,7 @@ import keyValueBy from '../../util/keyValueBy' import never from '../../util/never' import nonNull from '../../util/nonNull' import { DataProvider } from '../DataProvider' -import { clientId } from '../yjs' +import { clientId } from '../thoughtspaceSession' // default maxDepth before thoughts become pending const MAX_DEPTH = 100 @@ -74,6 +74,10 @@ const isMetaDescendant = (state: State, thought: Thought) => const isThoughtExpanded = (state: State, thoughtId: ThoughtId) => !!state.expanded[hashPath(thoughtToPath(state, thoughtId))] +/** Returns the real child ThoughtId for an attribute child indexed by value. */ +const attributeChildId = (thought: Thought | undefined | null, attr: string): ThoughtId | undefined => + thought?.childrenMap[attr] + /** * Returns buffered lexemeIndex and thoughtIndex for all descendants using async iterables. * @@ -172,11 +176,12 @@ async function* fetchDescendants( const isExpandedOrPinned = // we need to check directly for =pin, since it is a sibling and thus not part of accumulatedThoughts yet + // attributeChildId returns the real child ThoughtId; =pin is only a childrenMap lookup key // technically =pin/false is a false positive here, and will cause some thoughts not to be buffered that should, but it is rare // we need to determine if this thought should be buffered now, and cannot wait for the =pin child to load isExpanded || !!isThoughtExpanded(updatedState, thought.parentId) || - !!parent?.childrenMap?.['=pin'] || + !!attributeChildId(parent, '=pin') || parent?.value.endsWith(EXPAND_THOUGHT_CHAR) // if either the max depth or the max number of thoughts are reached, mark the thought as pending and do not add enqueue children (i.e. buffering) @@ -198,7 +203,7 @@ async function* fetchDescendants( if (isPending) { // enqueue =pin even if the thought is buffered // when =pin/true is loaded, then this thought will be marked as expanded and its children can be loaded - const pinId = thought.childrenMap?.['=pin'] + const pinId = attributeChildId(thought, '=pin') if (pinId) { thoughtIdQueue.add([pinId]) } @@ -231,7 +236,7 @@ async function* fetchDescendants( // If =pin is encountered, we need to load its children before yielding. // Otherwise thoughts with =pin/false will flash expanded while it waits for "false" to load. // See: https://github.com/cybersemics/em/issues/3268 - const pinIds = thoughts.map(thought => thought.childrenMap['=pin']).filter(nonNull) + const pinIds = thoughts.map(thought => attributeChildId(thought, '=pin')).filter(nonNull) let pinIdsValidated: ThoughtId[] = [] let pinChildrenIdsValidated: ThoughtId[] = [] const pinnedThoughtsRaw = await provider.getThoughtsByIds(pinIds) diff --git a/src/data-providers/data-helpers/replicateTree.ts b/src/data-providers/data-helpers/replicateTree.ts index e7eba824375..7cee381df09 100644 --- a/src/data-providers/data-helpers/replicateTree.ts +++ b/src/data-providers/data-helpers/replicateTree.ts @@ -2,13 +2,12 @@ import Index from '../../@types/IndexType' import Thought from '../../@types/Thought' import ThoughtId from '../../@types/ThoughtId' import taskQueue from '../../util/taskQueue' -import { replicateChildren, replicateThought } from '../yjs/thoughtspace' +import db from '../thoughtspace' -/** Replicates an entire subtree, starting at a given thought. Replicates in the background (not populating the Redux state). Does not wait for Websocket to sync. */ +/** Replicates an entire subtree, starting at a given thought. Replicates in the background (not populating the Redux state). */ const replicateTree = ( id: ThoughtId, { - remote, onThought, }: { /** Sync with Websocket. Default: true. */ @@ -29,30 +28,36 @@ const replicateTree = ( let abort = false /** Creates a task to replicate all children of the given id and add them to the thoughtIndex. Queues up grandchildren replication. */ - const replicateDescendantsRecursive = async (id: ThoughtId) => { + const replicateDescendantsRecursive = async (parentId: ThoughtId) => { if (abort) return - const children = await replicateChildren(id, { background: true, remote }) + const parentThought = await db.getThoughtById(parentId) + if (abort || !parentThought) return + + const childIds = Object.values(parentThought.childrenMap) + if (childIds.length === 0) return + + const children = await db.getThoughtsByIds(childIds) if (abort) return - children?.forEach(child => { - thoughtIndexAccum[child.id] = child - onThought?.(child, thoughtIndexAccum) + children.forEach(child => { + if (child) { + thoughtIndexAccum[child.id] = child + onThought?.(child, thoughtIndexAccum) - queue.add({ - function: () => replicateDescendantsRecursive(child.id), - description: `replicateTree: ${child.id}`, - }) + queue.add({ + function: () => replicateDescendantsRecursive(child.id), + description: `replicateTree: ${child.id}`, + }) + } }) } /** Replicates the starting thoughts and all descendants by populating the initial replication queue and waiting for all tasks to resolve. */ const replicateDescendants = async () => { - // kick off the descendant replication by enqueueing a task for start thought's children queue.add([ - // replicate the starting thought individually (should already be cached) { function: async () => { - const startThought = await replicateThought(id, { background: true, remote }) + const startThought = await db.getThoughtById(id) if (!startThought) { throw new Error(`Thought ${id} not replicated. Either replication is broken or this is a timing issue.`) @@ -63,7 +68,6 @@ const replicateTree = ( }, description: `replicateTree: ${id} (starting thought)`, }, - // replicate the starting thought's children { function: () => replicateDescendantsRecursive(id), description: `replicateTree: ${id}`, diff --git a/src/data-providers/permissionsModel.ts b/src/data-providers/permissionsModel.ts new file mode 100644 index 00000000000..d1feca1f4de --- /dev/null +++ b/src/data-providers/permissionsModel.ts @@ -0,0 +1,74 @@ +import { nanoid } from 'nanoid' +import Index from '../@types/IndexType' +import Routes from '../@types/Routes' +import Share from '../@types/Share' +import { alertActionCreator as alert } from '../actions/alert' +import { clearActionCreator } from '../actions/clear' +import store from '../stores/app' +import storage from '../util/storage' +import timestamp from '../util/timestamp' +import { permissionsStore, persistPermissions } from './permissionsStore' +import db from './thoughtspace' +import { accessTokenLocal } from './thoughtspaceSession' + +/** Snapshot of device permissions keyed by access token. */ +const entries = (): Index => permissionsStore.getState().entries + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const permissionsModel: { [key in keyof Routes['share']]: any } = { + add: ({ name, role }: Pick) => { + const accessToken = nanoid() + permissionsStore.update({ + entries: { + ...entries(), + [accessToken]: { + created: timestamp(), + name: name || '', + role, + }, + }, + }) + void persistPermissions() + store.dispatch(alert(`Added ${name ? `"${name}"` : 'device'}`)) + return { accessToken } + }, + delete: async (accessToken: string, { name }: { name?: string } = {}) => { + const prev = entries() + const next = { ...prev } + delete next[accessToken] + permissionsStore.update({ entries: next }) + await persistPermissions() + + if (accessToken !== accessTokenLocal) { + store.dispatch(alert(`Removed ${name ? `"${name}"` : 'device'}`)) + } else if (Object.keys(next).length > 1) { + store.dispatch([clearActionCreator(), alert(`Removed this device from the thoughtspace`)]) + } else { + storage.clear() + await db.clear() + store.dispatch(clearActionCreator()) + + // TODO: Do a full reset without refreshing the page. + window.location.reload() + } + }, + update: (accessToken: string, { name, role }: Share) => { + const e = entries() + const permission = e[accessToken]! + permissionsStore.update({ + entries: { + ...e, + [accessToken]: { + ...(permission || null), + created: timestamp(), + ...(name ? { name } : null), + ...(role ? { role } : null), + }, + }, + }) + void persistPermissions() + store.dispatch(alert(`${name ? ` "${name}"` : 'Device '} updated`)) + }, +} + +export default permissionsModel diff --git a/src/data-providers/permissionsStore.ts b/src/data-providers/permissionsStore.ts new file mode 100644 index 00000000000..48a56bb4d8d --- /dev/null +++ b/src/data-providers/permissionsStore.ts @@ -0,0 +1,48 @@ +import { get, set } from 'idb-keyval' +import Index from '../@types/IndexType' +import Share from '../@types/Share' +import reactMinistore from '../stores/react-ministore' +import { tsid } from './thoughtspaceSession' + +type PermissionsState = { entries: Index } + +/** Key for the idb-keyval permissions blob scoped to the active tsid. */ +const storageKey = (): string => `em-permissions:${tsid}` + +/** Device permissions for the thoughtspace (indexed by access token). */ +export const permissionsStore = reactMinistore({ entries: {} }) + +let loadPromise: Promise | null = null + +/** Loads persisted permissions from IndexedDB (no-op in unit tests). */ +export const initPermissionsStore = async (): Promise => { + if (import.meta.env.MODE === 'test') { + return + } + if (loadPromise) return loadPromise + const promise = (async () => { + const data = await get>(storageKey()) + if (data && typeof data === 'object' && !Array.isArray(data)) { + permissionsStore.update({ entries: data }) + } + })() + loadPromise = promise + void promise.catch(() => { + if (loadPromise === promise) loadPromise = null + }) + return promise +} + +/** Persists current permissions (skipped in unit tests). */ +export const persistPermissions = async (): Promise => { + if (import.meta.env.MODE === 'test') { + return + } + await set(storageKey(), permissionsStore.getState().entries) +} + +export default { + permissionsStore, + initPermissionsStore, + persistPermissions, +} diff --git a/src/data-providers/thoughtspace.ts b/src/data-providers/thoughtspace.ts new file mode 100644 index 00000000000..8645481e3e0 --- /dev/null +++ b/src/data-providers/thoughtspace.ts @@ -0,0 +1,54 @@ +import type Index from '../@types/IndexType' +import type Lexeme from '../@types/Lexeme' +import type Thought from '../@types/Thought' +import type ThoughtUpdates from '../@types/ThoughtUpdates' +import type { DataProvider } from './DataProvider' +import createTreecrdtThoughtspace from './treecrdt/runtime' + +export type PersistThoughtspaceBatch = Parameters[0] & { + local?: boolean +} + +/** Storage lifetime requested from the active thoughtspace provider. */ +export type ThoughtspaceStorage = 'memory' | 'persistent' + +export type ThoughtspaceMaterializationSnapshot = { + schemaVersion: number + thoughtIndex: Index + lexemeIndex: Index +} + +export type ThoughtspaceMaterializationBridge = { + getSnapshot: () => ThoughtspaceMaterializationSnapshot + apply: (updates: ThoughtUpdates) => void | Promise +} + +export type ThoughtspaceRuntimeInitOptions = { + storage: ThoughtspaceStorage + materialization?: ThoughtspaceMaterializationBridge +} + +export type ThoughtspaceAccessBlockedReason = 'already-open' | 'unsupported' + +export type ThoughtspaceAccessResult = + { status: 'acquired' } | { status: 'blocked'; reason: ThoughtspaceAccessBlockedReason } + +/** App-facing lifecycle interface for the active thoughtspace implementation. */ +export interface ThoughtspaceRuntime { + /** Acquires any runtime-specific access required before opening the interactive thoughtspace. */ + acquireAccess: () => Promise + init: (options: ThoughtspaceRuntimeInitOptions) => Promise<{ clientId: string }> + drop: () => Promise + waitForIdle: () => Promise + persistPushQueueBatches: (batches: readonly PersistThoughtspaceBatch[]) => Promise +} + +const treecrdtThoughtspace = createTreecrdtThoughtspace() + +/** The active data provider backing the current app thoughtspace. */ +export const db: DataProvider = treecrdtThoughtspace.db + +/** The active thoughtspace runtime implementation. */ +export const thoughtspaceRuntime: ThoughtspaceRuntime = treecrdtThoughtspace + +export default db diff --git a/src/data-providers/thoughtspaceSession.ts b/src/data-providers/thoughtspaceSession.ts new file mode 100644 index 00000000000..1b196be58bc --- /dev/null +++ b/src/data-providers/thoughtspaceSession.ts @@ -0,0 +1,39 @@ +/* eslint-disable import/prefer-default-export */ +import { nanoid } from 'nanoid' +import storage from '../util/storage' + +/** Secret access token for this device. */ +export const accessTokenLocal = storage.getItem('accessToken', () => nanoid(21)) + +/** Unique thoughtspace id for this device (default doc id). Share via ?share={docId} when using sync. */ +export const tsidLocal = storage.getItem('tsid', () => nanoid(21)) + +/** Share link doc id from URL when present. */ +export const tsidShared = new URLSearchParams(window.location?.search).get('share') +const accessTokenShared = new URLSearchParams(window.location?.search).get('auth') + +export const tsid = tsidShared || tsidLocal +export const accessToken = accessTokenShared || accessTokenLocal + +/** Public key derived from the access token. Not set until clientIdReady resolves. */ +export let clientId = '' + +/** Encodes binary data in base64. */ +async function bufferToBase64(buffer: ArrayBuffer) { + const base64url = await new Promise(resolve => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.readAsDataURL(new Blob([buffer])) + }) + return base64url.slice(base64url.indexOf(',') + 1) +} + +/** Resolves when clientId is available to use synchronously. */ +export const clientIdReady = ( + crypto.subtle + ? crypto.subtle.digest('SHA-256', new TextEncoder().encode(accessToken)).then(bufferToBase64) + : Promise.resolve(nanoid()) +).then(s => { + clientId = s + return s +}) diff --git a/src/data-providers/treecrdt/__tests__/materializationContext.ts b/src/data-providers/treecrdt/__tests__/materializationContext.ts new file mode 100644 index 00000000000..826e662a2a4 --- /dev/null +++ b/src/data-providers/treecrdt/__tests__/materializationContext.ts @@ -0,0 +1,96 @@ +import type { MaterializationListener } from '@treecrdt/interface/engine' +import { createTreecrdtClient } from '@treecrdt/wa-sqlite' +import type ThoughtId from '../../../@types/ThoughtId' +import type Timestamp from '../../../@types/Timestamp' +import { EM_TOKEN } from '../../../constants' +import type { DataProvider } from '../../DataProvider' +import type { enqueueMaterializedThoughtsToStore as EnqueueMaterializedThoughtsToStore } from '../sync/applyMaterializedThoughtsToStore' +import createTreecrdtDataProvider from '../thoughtspace' + +const { enqueueMaterializedThoughtsToStore } = vi.hoisted(() => ({ + enqueueMaterializedThoughtsToStore: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock('../sync', async importOriginal => { + const actual = await importOriginal() + return { ...actual, enqueueMaterializedThoughtsToStore } +}) + +const THOUGHT_ID = '00000000000000000000000000000201' as ThoughtId + +/** Creates a minimal thought fixture for the materialization context regression. */ +const thought = (value: string) => ({ + id: THOUGHT_ID, + parentId: EM_TOKEN, + value, + rank: 0, + childrenMap: {}, + created: 1 as Timestamp, + lastUpdated: 1 as Timestamp, + updatedBy: 'test', +}) + +/** Persists one thought through the public provider. */ +const persistThought = (db: Pick, value: string) => + db.updateThoughts({ + thoughtIndexUpdates: { [THOUGHT_ID]: thought(value) }, + lexemeIndexUpdates: {}, + lexemeIndexUpdatesOld: {}, + schemaVersion: 0, + }) + +it('retains the originating materialization context after rebinding the provider', async () => { + const clientOne = await createTreecrdtClient({ + storage: { type: 'memory' }, + runtime: { type: 'direct' }, + docId: 'materialization-context-one', + }) + const clientTwo = await createTreecrdtClient({ + storage: { type: 'memory' }, + runtime: { type: 'direct' }, + docId: 'materialization-context-two', + }) + const provider = createTreecrdtDataProvider() + const bridgeOne = { + getSnapshot: () => ({ schemaVersion: 0, thoughtIndex: {}, lexemeIndex: {} }), + apply: vi.fn(), + } + const bridgeTwo = { + getSnapshot: () => ({ schemaVersion: 0, thoughtIndex: {}, lexemeIndex: {} }), + apply: vi.fn(), + } + + let onMaterializedOne: MaterializationListener | undefined + vi.spyOn(clientOne, 'onMaterialized').mockImplementation(listener => { + onMaterializedOne = listener + return () => undefined + }) + + try { + await provider.bindClient(clientOne, new Uint8Array(32).fill(1), bridgeOne) + await persistThought(provider.db, 'client one') + + provider.resetBinding(new Error('switch client binding')) + await provider.bindClient(clientTwo, new Uint8Array(32).fill(2), bridgeTwo) + await persistThought(provider.db, 'client two') + + onMaterializedOne?.({ + headSeq: 1, + changes: [{ kind: 'payload', node: THOUGHT_ID, payload: null }], + }) + + expect(enqueueMaterializedThoughtsToStore).toHaveBeenCalledTimes(1) + const [, context] = enqueueMaterializedThoughtsToStore.mock.calls[0] as unknown as Parameters< + typeof EnqueueMaterializedThoughtsToStore + > + + expect(context.bridge).toBe(bridgeOne) + expect(context.client).toBe(clientOne) + expect(context.db).not.toBe(provider.db) + await expect(context.db.getThoughtById(THOUGHT_ID)).resolves.toMatchObject({ value: 'client one' }) + await expect(provider.db.getThoughtById(THOUGHT_ID)).resolves.toMatchObject({ value: 'client two' }) + } finally { + await clientOne.drop() + await clientTwo.drop() + } +}) diff --git a/src/data-providers/treecrdt/__tests__/runtime.ts b/src/data-providers/treecrdt/__tests__/runtime.ts new file mode 100644 index 00000000000..5bd157a6bb5 --- /dev/null +++ b/src/data-providers/treecrdt/__tests__/runtime.ts @@ -0,0 +1,199 @@ +import { EM_TOKEN } from '../../../constants' +import { tsid } from '../../thoughtspaceSession' +import createTreecrdtThoughtspace from '../runtime' + +const { mockAcquireTreecrdtSessionLock, mockCreateTreecrdtClient } = vi.hoisted(() => ({ + mockAcquireTreecrdtSessionLock: vi.fn(), + mockCreateTreecrdtClient: vi.fn(), +})) + +vi.mock('../sessionLock', () => ({ default: mockAcquireTreecrdtSessionLock })) +vi.mock('@treecrdt/wa-sqlite', async importOriginal => { + const actual = await importOriginal() + return { ...actual, createTreecrdtClient: mockCreateTreecrdtClient } +}) + +type TreecrdtModule = typeof import('@treecrdt/wa-sqlite') + +let createRealTreecrdtClient!: TreecrdtModule['createTreecrdtClient'] + +const emptyUpdates = { + thoughtIndexUpdates: {}, + lexemeIndexUpdates: {}, + lexemeIndexUpdatesOld: {}, + schemaVersion: 0, +} + +beforeAll(async () => { + const actual = await vi.importActual('@treecrdt/wa-sqlite') + createRealTreecrdtClient = actual.createTreecrdtClient +}) + +beforeEach(() => { + mockCreateTreecrdtClient.mockImplementation(createRealTreecrdtClient) +}) + +afterEach(() => { + mockAcquireTreecrdtSessionLock.mockReset() + mockCreateTreecrdtClient.mockReset() +}) + +it.each([ + ['acquired', { status: 'acquired' }], + ['unavailable', { status: 'blocked', reason: 'already-open' }], + ['unsupported', { status: 'blocked', reason: 'unsupported' }], +] as const)('maps the %s session-lock status to thoughtspace access', async (lockStatus, access) => { + mockAcquireTreecrdtSessionLock.mockResolvedValue(lockStatus) + const treecrdtThoughtspace = createTreecrdtThoughtspace() + + await expect(treecrdtThoughtspace.acquireAccess()).resolves.toEqual(access) + expect(mockAcquireTreecrdtSessionLock).toHaveBeenCalledWith() +}) + +it('maps em persistent storage to TreeCRDT OPFS client options', async () => { + const stopAfterOptions = new Error('stop after capturing client options') + mockCreateTreecrdtClient.mockRejectedValueOnce(stopAfterOptions) + const treecrdtThoughtspace = createTreecrdtThoughtspace() + + await expect( + treecrdtThoughtspace.init({ + storage: 'persistent', + }), + ).rejects.toBe(stopAfterOptions) + expect(mockCreateTreecrdtClient).toHaveBeenCalledWith({ + storage: { + type: 'opfs', + filename: `/treecrdt-em-${tsid}.db`, + fallback: 'memory', + }, + runtime: { type: 'dedicated-worker' }, + docId: tsid, + }) +}) + +it('creates the client lazily', async () => { + const treecrdtThoughtspace = createTreecrdtThoughtspace() + mockAcquireTreecrdtSessionLock.mockResolvedValueOnce('acquired') + + expect(mockCreateTreecrdtClient).not.toHaveBeenCalled() + await expect(treecrdtThoughtspace.acquireAccess()).resolves.toEqual({ status: 'acquired' }) + expect(mockCreateTreecrdtClient).not.toHaveBeenCalled() + + await treecrdtThoughtspace.init({ storage: 'memory' }) + expect(mockCreateTreecrdtClient).toHaveBeenCalledTimes(1) + expect(mockCreateTreecrdtClient).toHaveBeenCalledWith({ + storage: { type: 'memory' }, + runtime: { type: 'direct' }, + docId: tsid, + }) + + await treecrdtThoughtspace.drop() +}) + +it('coalesces concurrent initialization into one client', async () => { + const treecrdtThoughtspace = createTreecrdtThoughtspace() + const firstInit = treecrdtThoughtspace.init({ storage: 'memory' }) + const secondInit = treecrdtThoughtspace.init({ storage: 'memory' }) + + await expect(Promise.all([firstInit, secondInit])).resolves.toHaveLength(2) + expect(mockCreateTreecrdtClient).toHaveBeenCalledTimes(1) + + await treecrdtThoughtspace.drop() +}) + +it('serializes an in-flight init, drop, and following init', async () => { + let releaseClient!: () => void + let markClientStarted!: () => void + const clientStarted = new Promise(resolve => { + markClientStarted = resolve + }) + const clientReleased = new Promise(resolve => { + releaseClient = resolve + }) + mockCreateTreecrdtClient.mockImplementationOnce(async options => { + markClientStarted() + await clientReleased + return createRealTreecrdtClient(options) + }) + + const treecrdtThoughtspace = createTreecrdtThoughtspace() + const firstInit = treecrdtThoughtspace.init({ storage: 'memory' }) + await clientStarted + const drop = treecrdtThoughtspace.drop() + const secondInit = treecrdtThoughtspace.init({ storage: 'memory' }) + + expect(mockCreateTreecrdtClient).toHaveBeenCalledTimes(1) + + releaseClient() + await Promise.all([firstInit, drop, secondInit]) + + expect(mockCreateTreecrdtClient).toHaveBeenCalledTimes(2) + await expect(treecrdtThoughtspace.db.getThoughtById(EM_TOKEN)).resolves.toMatchObject({ id: EM_TOKEN }) + + await treecrdtThoughtspace.drop() +}) + +it('rejects queued startup writes when initialization fails and uses a fresh gate on retry', async () => { + const initError = new Error('client initialization failed') + mockCreateTreecrdtClient.mockRejectedValueOnce(initError) + + const treecrdtThoughtspace = createTreecrdtThoughtspace() + const queuedWrite = treecrdtThoughtspace.db.updateThoughts(emptyUpdates) + const queuedWriteExpectation = expect(queuedWrite).rejects.toBe(initError) + + await expect(treecrdtThoughtspace.init({ storage: 'memory' })).rejects.toBe(initError) + await queuedWriteExpectation + + await treecrdtThoughtspace.init({ storage: 'memory' }) + await expect(treecrdtThoughtspace.db.updateThoughts(emptyUpdates)).resolves.toEqual([]) + await treecrdtThoughtspace.drop() +}) + +it('rejects writes queued before each settled drop and creates a fresh gate for init', async () => { + const treecrdtThoughtspace = createTreecrdtThoughtspace() + + const firstWrite = treecrdtThoughtspace.db.updateThoughts(emptyUpdates) + const firstWriteExpectation = expect(firstWrite).rejects.toThrow( + 'TreeCRDT client binding cleared before initialization.', + ) + await Promise.all([treecrdtThoughtspace.drop(), firstWriteExpectation]) + + const secondWrite = treecrdtThoughtspace.db.updateThoughts(emptyUpdates) + const secondWriteExpectation = expect(secondWrite).rejects.toThrow( + 'TreeCRDT client binding cleared before initialization.', + ) + await Promise.all([treecrdtThoughtspace.drop(), secondWriteExpectation]) + + await treecrdtThoughtspace.init({ storage: 'memory' }) + await expect(treecrdtThoughtspace.db.updateThoughts(emptyUpdates)).resolves.toEqual([]) + await treecrdtThoughtspace.drop() +}) + +it('discards a terminal client when drop reports an error', async () => { + const client = await createRealTreecrdtClient({ + storage: { type: 'memory' }, + runtime: { type: 'direct' }, + }) + const dropError = new Error('client drop failed') + const originalDrop = client.drop.bind(client) + // Model wa-sqlite 0.4: drop may report an error after making the client terminal. + vi.spyOn(client, 'drop').mockImplementationOnce(async () => { + await originalDrop() + throw dropError + }) + const close = vi.spyOn(client, 'close') + mockCreateTreecrdtClient.mockResolvedValueOnce(client) + + const treecrdtThoughtspace = createTreecrdtThoughtspace() + await treecrdtThoughtspace.init({ storage: 'memory' }) + await expect(treecrdtThoughtspace.drop()).rejects.toBe(dropError) + expect(() => treecrdtThoughtspace.db.getThoughtById('missing' as never)).toThrow( + 'TreeCRDT DataProvider: init not called', + ) + expect(close).not.toHaveBeenCalled() + + await expect(treecrdtThoughtspace.init({ storage: 'memory' })).resolves.toEqual({ + clientId: expect.any(String), + }) + await treecrdtThoughtspace.drop() +}) diff --git a/src/data-providers/treecrdt/__tests__/sessionLock.ts b/src/data-providers/treecrdt/__tests__/sessionLock.ts new file mode 100644 index 00000000000..e391c135951 --- /dev/null +++ b/src/data-providers/treecrdt/__tests__/sessionLock.ts @@ -0,0 +1,65 @@ +type LockCallback = (lock: Lock | null) => Promise | unknown + +const TEST_TSID = 'test-thoughtspace' +const originalLocks = Object.getOwnPropertyDescriptor(navigator, 'locks') + +vi.mock('../../thoughtspaceSession', () => ({ tsid: 'test-thoughtspace' })) + +/** Installs a controllable Web Locks implementation for the current test. */ +const setLocks = (request: (...args: unknown[]) => Promise): void => { + Object.defineProperty(navigator, 'locks', { + configurable: true, + value: { request }, + }) +} + +afterEach(() => { + localStorage.clear() + vi.resetModules() + + if (originalLocks) { + Object.defineProperty(navigator, 'locks', originalLocks) + } else { + Reflect.deleteProperty(navigator, 'locks') + } +}) + +it('holds an exclusive lock for this thoughtspace for the lifetime of the page', async () => { + localStorage.setItem('tsid', TEST_TSID) + const request = vi.fn((...args: unknown[]) => { + const callback = args[2] as LockCallback + return Promise.resolve(callback({ mode: 'exclusive', name: String(args[0]) } as Lock)) + }) + setLocks(request) + + const { acquireTreecrdtSessionLock } = await import('../sessionLock') + + await expect(acquireTreecrdtSessionLock()).resolves.toBe('acquired') + await expect(acquireTreecrdtSessionLock()).resolves.toBe('acquired') + expect(request).toHaveBeenCalledWith( + `em-treecrdt-session:${TEST_TSID}`, + { ifAvailable: true, mode: 'exclusive' }, + expect.any(Function), + ) + expect(request).toHaveBeenCalledTimes(1) +}) + +it('reports when another tab already owns the thoughtspace', async () => { + const request = vi.fn((...args: unknown[]) => { + const callback = args[2] as LockCallback + return Promise.resolve(callback(null)) + }) + setLocks(request) + + const { acquireTreecrdtSessionLock } = await import('../sessionLock') + + await expect(acquireTreecrdtSessionLock()).resolves.toBe('unavailable') +}) + +it('fails closed when Web Locks are unavailable', async () => { + Reflect.deleteProperty(navigator, 'locks') + + const { acquireTreecrdtSessionLock } = await import('../sessionLock') + + await expect(acquireTreecrdtSessionLock()).resolves.toBe('unsupported') +}) diff --git a/src/data-providers/treecrdt/__tests__/testThoughtspace.ts b/src/data-providers/treecrdt/__tests__/testThoughtspace.ts new file mode 100644 index 00000000000..8b30f1d1e34 --- /dev/null +++ b/src/data-providers/treecrdt/__tests__/testThoughtspace.ts @@ -0,0 +1,214 @@ +import { createTreecrdtClient } from '@treecrdt/wa-sqlite' +import type ThoughtId from '../../../@types/ThoughtId' +import type Timestamp from '../../../@types/Timestamp' +import { EM_TOKEN, SETTINGS_TOKEN, SETTINGS_VALUE } from '../../../constants' +import hashThought from '../../../util/hashThought' +import type { DataProvider } from '../../DataProvider' +import createTreecrdtThoughtspace from '../runtime' +import createTreecrdtDataProvider, { createIndexedChildrenMap } from '../thoughtspace' + +/** Initializes an isolated in-memory TreeCRDT client and thoughtspace for unit tests. */ +const treecrdt = createTreecrdtThoughtspace() +const treecrdtThoughtspace = treecrdt.db + +/** Initializes the bound in-memory test runtime. */ +const initTestThoughtspace = async (): Promise => { + await treecrdt.init({ storage: 'memory' }) +} + +const PIN_ID = '00000000000000000000000000000101' as ThoughtId +const FALSE_ID = '00000000000000000000000000000102' as ThoughtId +const PIN_DUPLICATE_ID = '00000000000000000000000000000103' as ThoughtId +const PARENT_ID = '00000000000000000000000000000110' as ThoughtId +const OTHER_PARENT_ID = '00000000000000000000000000000111' as ThoughtId +const THOUGHT_A_ID = '00000000000000000000000000000112' as ThoughtId +const THOUGHT_Y_ID = '00000000000000000000000000000113' as ThoughtId +const THOUGHT_B_ID = '00000000000000000000000000000114' as ThoughtId +const THOUGHT_X_ID = '00000000000000000000000000000115' as ThoughtId + +/** Creates a minimal thought fixture for provider-level ordering tests. */ +const thought = (id: ThoughtId, parentId: ThoughtId, value: string, rank: number) => ({ + id, + parentId, + value, + rank, + childrenMap: {}, + created: 1 as Timestamp, + lastUpdated: 1 as Timestamp, + updatedBy: 'test', +}) + +/** Persists thoughts through the real TreeCRDT data provider. */ +const persistThoughtsTo = ( + db: Pick, + thoughts: ReturnType[], + movePlacements?: Record, +) => + db.updateThoughts({ + thoughtIndexUpdates: Object.fromEntries(thoughts.map(thought => [thought.id, thought])), + lexemeIndexUpdates: {}, + lexemeIndexUpdatesOld: {}, + schemaVersion: 0, + movePlacements, + }) + +/** Persists thoughts through the shared test thoughtspace. */ +const persistThoughts = ( + thoughts: ReturnType[], + movePlacements?: Record, +) => persistThoughtsTo(treecrdtThoughtspace, thoughts, movePlacements) + +afterEach(async () => { + await treecrdt.drop() +}) + +it('seeds fixed system thoughts in the TreeCRDT provider', async () => { + await initTestThoughtspace() + + const em = await treecrdtThoughtspace.getThoughtById(EM_TOKEN) + expect(em?.childrenMap[SETTINGS_TOKEN]).toBe(SETTINGS_TOKEN) + + const settings = await treecrdtThoughtspace.getThoughtById(SETTINGS_TOKEN) + expect(settings).toMatchObject({ + id: SETTINGS_TOKEN, + parentId: EM_TOKEN, + value: SETTINGS_VALUE, + }) + + const settingsLexeme = await treecrdtThoughtspace.getLexemeById(hashThought(SETTINGS_VALUE)) + expect(settingsLexeme?.contexts).toEqual([SETTINGS_TOKEN]) +}) + +it('does not delete persisted lexemes when freeing cache', async () => { + await initTestThoughtspace() + + const settingsKey = hashThought(SETTINGS_VALUE) + await treecrdtThoughtspace.freeLexeme(settingsKey) + + const settingsLexeme = await treecrdtThoughtspace.getLexemeById(settingsKey) + expect(settingsLexeme?.contexts).toEqual([SETTINGS_TOKEN]) +}) + +it('does not require an initialized TreeCRDT client when freeing lexeme cache', async () => { + await expect(treecrdtThoughtspace.freeLexeme(hashThought('missing'))).resolves.toBeUndefined() +}) + +it('uses indexed attribute values as childrenMap keys without changing TreeCRDT node ids', async () => { + const valueById = { + [PIN_ID]: '=pin', + [PIN_DUPLICATE_ID]: '=pin', + } + + const childrenMap = createIndexedChildrenMap([PIN_ID, PIN_DUPLICATE_ID, FALSE_ID], valueById) + + expect(childrenMap['=pin']).toBe(PIN_ID) + expect(childrenMap[PIN_DUPLICATE_ID]).toBe(PIN_DUPLICATE_ID) + expect(childrenMap[FALSE_ID]).toBe(FALSE_ID) + expect(childrenMap.false).toBeUndefined() + expect(Object.values(childrenMap)).toEqual([PIN_ID, PIN_DUPLICATE_ID, FALSE_ID]) +}) + +it('falls back to rank placement when explicit afterId is stale', async () => { + await initTestThoughtspace() + + await persistThoughts([thought(PARENT_ID, EM_TOKEN, 'parent', 0), thought(OTHER_PARENT_ID, EM_TOKEN, 'other', 1)]) + await persistThoughts([thought(THOUGHT_A_ID, PARENT_ID, 'a', 0)]) + await persistThoughts([thought(THOUGHT_Y_ID, PARENT_ID, 'y', 1)]) + await persistThoughts([thought(THOUGHT_B_ID, PARENT_ID, 'b', 2)]) + await persistThoughts([thought(THOUGHT_X_ID, PARENT_ID, 'x', 3)]) + + await persistThoughts([thought(THOUGHT_Y_ID, OTHER_PARENT_ID, 'y', 0)], { + [THOUGHT_Y_ID]: null, + }) + + await expect( + persistThoughts([thought(THOUGHT_X_ID, PARENT_ID, 'x', 1)], { + [THOUGHT_X_ID]: THOUGHT_Y_ID, + }), + ).resolves.toBeDefined() + + const parent = await treecrdtThoughtspace.getThoughtById(PARENT_ID) + expect(Object.values(parent?.childrenMap ?? {})).toEqual([THOUGHT_A_ID, THOUGHT_X_ID, THOUGHT_B_ID]) +}) + +it('excludes the moving thought from stale rank placement', async () => { + await initTestThoughtspace() + + await persistThoughts([thought(PARENT_ID, EM_TOKEN, 'parent', 0), thought(OTHER_PARENT_ID, EM_TOKEN, 'other', 1)]) + await persistThoughts([thought(THOUGHT_X_ID, PARENT_ID, 'x', 0)]) + await persistThoughts([thought(THOUGHT_Y_ID, PARENT_ID, 'y', 1)]) + await persistThoughts([thought(THOUGHT_A_ID, PARENT_ID, 'a', 2)]) + + await persistThoughts([thought(THOUGHT_Y_ID, OTHER_PARENT_ID, 'y', 0)], { + [THOUGHT_Y_ID]: null, + }) + + await expect( + persistThoughts([thought(THOUGHT_X_ID, PARENT_ID, 'x', 1)], { + [THOUGHT_X_ID]: THOUGHT_Y_ID, + }), + ).resolves.toBeDefined() + + const parent = await treecrdtThoughtspace.getThoughtById(PARENT_ID) + expect(Object.values(parent?.childrenMap ?? {})).toEqual([THOUGHT_X_ID, THOUGHT_A_ID]) +}) + +// https://github.com/cybersemics/em/pull/4325#issuecomment-5248342036 +it('reads sibling order once per thought when inserting a wide batch', async () => { + const client = await createTreecrdtClient({ + storage: { type: 'memory' }, + runtime: { type: 'direct' }, + }) + const provider = createTreecrdtDataProvider() + + try { + await provider.bindClient(client, new Uint8Array(32).fill(1)) + await persistThoughtsTo(provider.db, [thought(PARENT_ID, EM_TOKEN, 'parent', 0)]) + + const childIds = Array.from({ length: 40 }, (_, index) => (index + 512).toString(16).padStart(32, '0') as ThoughtId) + const childrenSpy = vi.spyOn(client.tree, 'children') + + await persistThoughtsTo( + provider.db, + childIds.map((id, index) => thought(id, PARENT_ID, `child-${index}`, index + 0.5)), + ) + + expect(childrenSpy).toHaveBeenCalledTimes(childIds.length) + await expect(client.tree.children(PARENT_ID)).resolves.toEqual(childIds) + } finally { + await client.drop() + } +}) + +it('queues writes issued before initialization and applies them to the bound client', async () => { + let writeSettled = false + const write = persistThoughts([thought(PARENT_ID, EM_TOKEN, 'queued', 0)]).finally(() => { + writeSettled = true + }) + + await Promise.resolve() + expect(writeSettled).toBe(false) + + await initTestThoughtspace() + await expect(write).resolves.toBeDefined() + await expect(treecrdtThoughtspace.getThoughtById(PARENT_ID)).resolves.toMatchObject({ value: 'queued' }) +}) + +it('keeps separately created thoughtspace instances isolated', async () => { + const first = createTreecrdtThoughtspace() + const second = createTreecrdtThoughtspace() + + try { + await first.init({ storage: 'memory' }) + await second.init({ storage: 'memory' }) + + await persistThoughtsTo(first.db, [thought(PARENT_ID, EM_TOKEN, 'first', 0)]) + await persistThoughtsTo(second.db, [thought(PARENT_ID, EM_TOKEN, 'second', 0)]) + + await expect(first.db.getThoughtById(PARENT_ID)).resolves.toMatchObject({ value: 'first' }) + await expect(second.db.getThoughtById(PARENT_ID)).resolves.toMatchObject({ value: 'second' }) + } finally { + await first.drop() + await second.drop() + } +}) diff --git a/src/data-providers/treecrdt/__tests__/thoughtspacePayload.ts b/src/data-providers/treecrdt/__tests__/thoughtspacePayload.ts new file mode 100644 index 00000000000..dfae42bbfc6 --- /dev/null +++ b/src/data-providers/treecrdt/__tests__/thoughtspacePayload.ts @@ -0,0 +1,17 @@ +import { encodeThoughtPayload } from '../payload' + +it('keeps em rank out of the serialized TreeCRDT thought payload', () => { + const encoded = encodeThoughtPayload({ + value: 'a', + created: 1, + lastUpdated: 2, + updatedBy: 'test', + }) + + expect(JSON.parse(new TextDecoder().decode(encoded))).toEqual({ + value: 'a', + created: 1, + lastUpdated: 2, + updatedBy: 'test', + }) +}) diff --git a/src/data-providers/treecrdt/__tests__/writeBarrier.ts b/src/data-providers/treecrdt/__tests__/writeBarrier.ts new file mode 100644 index 00000000000..a627aa084a5 --- /dev/null +++ b/src/data-providers/treecrdt/__tests__/writeBarrier.ts @@ -0,0 +1,86 @@ +import { + createTreecrdtLocalWriteOptions, + isTreecrdtLocalMaterialization, + waitForTreecrdtWriteBarrier, + withTreecrdtWriteBarrier, +} from '../writeBarrier' + +it('waits for TreeCRDT writes queued while waiting for idle', async () => { + const order: string[] = [] + let finishFirst!: () => void + + const first = withTreecrdtWriteBarrier(async () => { + order.push('first:start') + await new Promise(resolve => { + finishFirst = resolve + }) + order.push('first:end') + }) + + const wait = waitForTreecrdtWriteBarrier().then(() => { + order.push('idle') + }) + + const second = withTreecrdtWriteBarrier(async () => { + order.push('second') + }) + + await Promise.resolve() + finishFirst() + await Promise.all([first, second, wait]) + + expect(order).toEqual(['first:start', 'first:end', 'second', 'idle']) +}) + +it('surfaces TreeCRDT write failures when waiting for idle', async () => { + const err = new Error('write failed') + + await expect( + withTreecrdtWriteBarrier(async () => { + throw err + }), + ).rejects.toThrow('write failed') + + await expect(waitForTreecrdtWriteBarrier()).rejects.toThrow('write failed') + await expect(waitForTreecrdtWriteBarrier()).resolves.toBeUndefined() +}) + +it('identifies only this tab local TreeCRDT materialization events', () => { + const first = createTreecrdtLocalWriteOptions() + const second = createTreecrdtLocalWriteOptions() + + expect(first.writeId).toBeDefined() + expect(second.writeId).toBeDefined() + expect(second.writeId).not.toBe(first.writeId) + + expect( + isTreecrdtLocalMaterialization({ + headSeq: 1, + changes: [{ kind: 'payload', node: 'local-a', payload: null, source: { writeIds: [first.writeId!] } }], + }), + ).toBe(true) + expect( + isTreecrdtLocalMaterialization({ + headSeq: 1, + changes: [{ kind: 'payload', node: 'remote-a', payload: null, source: { writeIds: ['remote-write'] } }], + }), + ).toBe(false) + expect( + isTreecrdtLocalMaterialization({ + headSeq: 1, + changes: [], + }), + ).toBe(false) + expect( + isTreecrdtLocalMaterialization({ + headSeq: 1, + changes: [{ kind: 'payload', node: 'local-a', payload: null }], + }), + ).toBe(false) + expect( + isTreecrdtLocalMaterialization({ + headSeq: 1, + changes: [{ kind: 'payload', node: 'remote-a', payload: null }], + }), + ).toBe(false) +}) diff --git a/src/data-providers/treecrdt/attributeChildren.ts b/src/data-providers/treecrdt/attributeChildren.ts new file mode 100644 index 00000000000..5ee76f85e0d --- /dev/null +++ b/src/data-providers/treecrdt/attributeChildren.ts @@ -0,0 +1,224 @@ +/* eslint-disable import/prefer-default-export */ +import type { Change } from '@treecrdt/interface/engine' +import type { TreecrdtClient } from '@treecrdt/wa-sqlite' +import type Index from '../../@types/IndexType' +import type ThoughtId from '../../@types/ThoughtId' +import { GLOBAL_ROOT_TOKEN } from '../../constants' +import isAttribute from '../../util/isAttribute' +import { decodeThoughtPayload } from './payload' + +/** Application-owned child value index used to restore em's attribute-keyed childrenMap contract. */ +const TABLE = 'em_attribute_children' +const META_TABLE = 'em_attribute_children_meta' +const INDEX_VERSION = '1' +const schemaReady = new WeakSet() + +const CREATE_TABLE_SQL = `CREATE TABLE IF NOT EXISTS ${TABLE} ( + child_id TEXT PRIMARY KEY NOT NULL, + parent_id TEXT NOT NULL, + value TEXT NOT NULL +);` + +const CREATE_PARENT_INDEX_SQL = `CREATE INDEX IF NOT EXISTS idx_${TABLE}_parent ON ${TABLE} (parent_id);` + +const CREATE_META_TABLE_SQL = `CREATE TABLE IF NOT EXISTS ${META_TABLE} ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL +);` + +/** + * Injects bound parameters into SQL for `runner.exec`, which does not accept bind args. + * Only `?1` ... `?n` placeholders are supported, in order. + */ +function bindParams(sql: string, params: (string | number | null)[]): string { + let out = sql + for (let i = params.length - 1; i >= 0; i--) { + const p = params[i] + const lit = p === null ? 'NULL' : typeof p === 'number' ? String(p) : `'${String(p).replace(/'/g, "''")}'` + out = out.replace(new RegExp(`\\?${i + 1}\\b`, 'g'), lit) + } + return out +} + +/** Ensures the derived attribute-child index schema exists. */ +export async function ensureAttributeChildrenSchema(client: TreecrdtClient): Promise { + if (schemaReady.has(client)) return + await client.runner.exec(CREATE_TABLE_SQL) + await client.runner.exec(CREATE_PARENT_INDEX_SQL) + await client.runner.exec(CREATE_META_TABLE_SQL) + schemaReady.add(client) +} + +/** Returns the indexed attribute values for a parent's direct children, keyed by child id. */ +export async function getAttributeChildrenByParent( + client: TreecrdtClient, + parentId: ThoughtId, +): Promise> { + await ensureAttributeChildrenSchema(client) + const text = await client.runner.getText( + `SELECT json_group_array(json_object('childId', child_id, 'value', value)) FROM ${TABLE} WHERE parent_id = ?1`, + [parentId], + ) + if (!text) return {} + + const rows = JSON.parse(text) as ({ childId: ThoughtId; value: string } | null)[] + const valueByChildId: Index = {} + for (const row of rows) { + if (row?.childId && row.value) valueByChildId[row.childId] = row.value + } + return valueByChildId +} + +/** Deletes one child from the attribute-child index. */ +export async function deleteAttributeChild(client: TreecrdtClient, childId: ThoughtId): Promise { + await ensureAttributeChildrenSchema(client) + await client.runner.exec(bindParams(`DELETE FROM ${TABLE} WHERE child_id = ?1`, [childId])) +} + +/** Upserts an attribute child into the derived index. */ +export async function upsertAttributeChild( + client: TreecrdtClient, + parentId: ThoughtId, + childId: ThoughtId, + value: string, +): Promise { + await ensureAttributeChildrenSchema(client) + + const sql = `INSERT INTO ${TABLE} (child_id, parent_id, value) VALUES (?1, ?2, ?3) + ON CONFLICT(child_id) DO UPDATE SET parent_id = excluded.parent_id, value = excluded.value` + await client.runner.exec(bindParams(sql, [childId, parentId, value])) +} + +/** Upserts or removes a child from the derived attribute-child index based on a known current value. */ +export async function syncAttributeChild( + client: TreecrdtClient, + parentId: ThoughtId, + childId: ThoughtId, + value: string, +): Promise { + if (isAttribute(value)) { + await upsertAttributeChild(client, parentId, childId, value) + } else { + await deleteAttributeChild(client, childId) + } +} + +/** Updates the indexed parent for a moved child, if the child is indexed. */ +export async function moveAttributeChild( + client: TreecrdtClient, + parentId: ThoughtId, + childId: ThoughtId, +): Promise { + await ensureAttributeChildrenSchema(client) + await client.runner.exec(bindParams(`UPDATE ${TABLE} SET parent_id = ?1 WHERE child_id = ?2`, [parentId, childId])) +} + +/** Reindexes a single child from TreeCRDT's current materialized state. */ +export async function reindexAttributeChild(client: TreecrdtClient, childId: ThoughtId): Promise { + await ensureAttributeChildrenSchema(client) + const [payloadBytes, parentIdRaw] = await Promise.all([client.tree.getPayload(childId), client.tree.parent(childId)]) + + if (!payloadBytes || parentIdRaw === null) { + await deleteAttributeChild(client, childId) + return + } + + const payload = decodeThoughtPayload(payloadBytes) + await syncAttributeChild(client, parentIdRaw as ThoughtId, childId, payload.value) +} + +/** Deletes all derived attribute-child rows. */ +async function deleteAllAttributeChildren(client: TreecrdtClient): Promise { + await ensureAttributeChildrenSchema(client) + await client.runner.exec(`DELETE FROM ${TABLE}`) +} + +/** Returns true when the persisted attribute-child index has been initialized for this schema version. */ +async function isAttributeChildrenIndexReady(client: TreecrdtClient): Promise { + await ensureAttributeChildrenSchema(client) + const version = await client.runner.getText(`SELECT value FROM ${META_TABLE} WHERE key = ?1`, [ + 'attribute_children_index_version', + ]) + return version === INDEX_VERSION +} + +/** Marks the persisted attribute-child index initialized for this schema version. */ +async function setAttributeChildrenIndexReady(client: TreecrdtClient): Promise { + const sql = `INSERT INTO ${META_TABLE} (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value` + await client.runner.exec(bindParams(sql, ['attribute_children_index_version', INDEX_VERSION])) +} + +/** Rebuilds the derived attribute-child index by walking the materialized TreeCRDT tree once. */ +async function rebuildAttributeChildrenIndex(client: TreecrdtClient): Promise { + await ensureAttributeChildrenSchema(client) + await deleteAllAttributeChildren(client) + + const parentQueue = [GLOBAL_ROOT_TOKEN] + for (let i = 0; i < parentQueue.length; i++) { + const parentId = parentQueue[i] + const childIds = (await client.tree.children(parentId)) as ThoughtId[] + + for (const childId of childIds) { + await reindexAttributeChild(client, childId) + parentQueue.push(childId) + } + } + + await setAttributeChildrenIndexReady(client) +} + +/** Initializes the persisted attribute-child index once, then keeps it for fast cold-start reads. */ +export async function ensureAttributeChildrenIndexReady(client: TreecrdtClient): Promise { + if (!(await isAttributeChildrenIndexReady(client))) { + await rebuildAttributeChildrenIndex(client) + } +} + +/** Updates the derived attribute-child index for a materialized TreeCRDT change batch. */ +export async function refreshAttributeChildrenFromChanges( + client: TreecrdtClient, + changes: readonly Change[], +): Promise { + await ensureAttributeChildrenSchema(client) + + for (const ch of changes) { + const childId = ch.node as ThoughtId + switch (ch.kind) { + case 'insert': + case 'restore': + if (ch.payload && ch.parentAfter) { + await syncAttributeChild(client, ch.parentAfter as ThoughtId, childId, decodeThoughtPayload(ch.payload).value) + } else { + await deleteAttributeChild(client, childId) + } + break + case 'move': + if (ch.parentBefore !== ch.parentAfter) { + await moveAttributeChild(client, ch.parentAfter as ThoughtId, childId) + } + break + case 'payload': { + if (!ch.payload) { + await deleteAttributeChild(client, childId) + break + } + const payload = decodeThoughtPayload(ch.payload) + if (!isAttribute(payload.value)) { + await deleteAttributeChild(client, childId) + break + } + const parentIdRaw = await client.tree.parent(childId) + if (parentIdRaw === null) { + await deleteAttributeChild(client, childId) + } else { + await upsertAttributeChild(client, parentIdRaw as ThoughtId, childId, payload.value) + } + break + } + case 'delete': + await deleteAttributeChild(client, childId) + break + } + } +} diff --git a/src/data-providers/treecrdt/lexemes.ts b/src/data-providers/treecrdt/lexemes.ts new file mode 100644 index 00000000000..8a7bfe00e25 --- /dev/null +++ b/src/data-providers/treecrdt/lexemes.ts @@ -0,0 +1,89 @@ +/* eslint-disable import/prefer-default-export */ +import type { TreecrdtClient } from '@treecrdt/wa-sqlite' +import type Lexeme from '../../@types/Lexeme' + +/** Application-owned lexeme rows in the same SQLite DB as TreeCRDT (not part of the CRDT tree). */ +const TABLE = 'em_lexemes' +const schemaReady = new WeakSet() + +const DDL = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + id TEXT PRIMARY KEY NOT NULL, + payload_json TEXT NOT NULL +); +` + +/** + * Injects bound parameters into SQL for `runner.exec`, which does not accept bind args. + * Only `?1` … `?n` placeholders are supported, in order. + */ +function bindParams(sql: string, params: (string | number | null)[]): string { + let out = sql + for (let i = params.length - 1; i >= 0; i--) { + const p = params[i] + const lit = p === null ? 'NULL' : typeof p === 'number' ? String(p) : `'${String(p).replace(/'/g, "''")}'` + out = out.replace(new RegExp(`\\?${i + 1}\\b`, 'g'), lit) + } + return out +} + +/** Serializes a Lexeme to JSON for storage in `payload_json`. */ +function serializeLexeme(lexeme: Lexeme): string { + return JSON.stringify(lexeme) +} + +/** Parses a stored `payload_json` string into a Lexeme. */ +function parseLexemeJson(text: string): Lexeme { + return JSON.parse(text) as Lexeme +} + +/** Ensures the lexeme table exists. Safe to call on every init. */ +export async function ensureLexemesSchema(client: TreecrdtClient): Promise { + if (schemaReady.has(client)) return + await client.runner.exec(DDL) + schemaReady.add(client) +} + +/** Loads one lexeme by id (lexeme key / hash). */ +export async function getLexemeById(client: TreecrdtClient, id: string): Promise { + await ensureLexemesSchema(client) + const text = await client.runner.getText(`SELECT payload_json FROM ${TABLE} WHERE id = ?1`, [id]) + if (!text) return undefined + return parseLexemeJson(text) +} + +/** Loads lexemes for the given ids; order matches `ids`. */ +export async function getLexemesByIds(client: TreecrdtClient, ids: string[]): Promise<(Lexeme | undefined)[]> { + if (ids.length === 0) return [] + await ensureLexemesSchema(client) + const placeholders = ids.map(() => '?').join(',') + const sql = `SELECT json_group_array(json_object('id', id, 'lexeme', json(payload_json))) FROM ${TABLE} WHERE id IN (${placeholders})` + const text = await client.runner.getText(sql, ids) + if (!text) return ids.map(() => undefined) + const rows = JSON.parse(text) as ({ id: string; lexeme: Lexeme } | null)[] + const map = new Map() + for (const row of rows) { + if (row && row.id) map.set(row.id, row.lexeme) + } + return ids.map(id => map.get(id)) +} + +/** Inserts or replaces a lexeme row. */ +export async function upsertLexeme(client: TreecrdtClient, id: string, lexeme: Lexeme): Promise { + await ensureLexemesSchema(client) + const sql = `INSERT INTO ${TABLE} (id, payload_json) VALUES (?1, ?2) + ON CONFLICT(id) DO UPDATE SET payload_json = excluded.payload_json` + await client.runner.exec(bindParams(sql, [id, serializeLexeme(lexeme)])) +} + +/** Deletes a lexeme row by id. */ +export async function deleteLexeme(client: TreecrdtClient, id: string): Promise { + await ensureLexemesSchema(client) + await client.runner.exec(bindParams(`DELETE FROM ${TABLE} WHERE id = ?1`, [id])) +} + +/** Deletes all lexeme rows (table must already be ensured for callers that need it). */ +export async function deleteAllLexemes(client: TreecrdtClient): Promise { + await ensureLexemesSchema(client) + await client.runner.exec(`DELETE FROM ${TABLE}`) +} diff --git a/src/data-providers/treecrdt/payload.ts b/src/data-providers/treecrdt/payload.ts new file mode 100644 index 00000000000..eacd3fe90b0 --- /dev/null +++ b/src/data-providers/treecrdt/payload.ts @@ -0,0 +1,20 @@ +export type ThoughtPayload = { + value: string + created: number + lastUpdated: number + updatedBy: string + archived?: number +} + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +/** Encodes a thought payload to bytes. */ +export function encodeThoughtPayload(payload: ThoughtPayload): Uint8Array { + return encoder.encode(JSON.stringify(payload)) +} + +/** Decodes bytes to a thought payload. */ +export function decodeThoughtPayload(bytes: Uint8Array): ThoughtPayload { + return JSON.parse(decoder.decode(bytes)) as ThoughtPayload +} diff --git a/src/data-providers/treecrdt/runtime.ts b/src/data-providers/treecrdt/runtime.ts new file mode 100644 index 00000000000..50eff53ac04 --- /dev/null +++ b/src/data-providers/treecrdt/runtime.ts @@ -0,0 +1,226 @@ +import type { Operation } from '@treecrdt/interface' +import { type ClientOptions, type TreecrdtClient, createTreecrdtClient } from '@treecrdt/wa-sqlite' +import type { DataProvider } from '../DataProvider' +import { initPermissionsStore } from '../permissionsStore' +import type { + ThoughtspaceAccessResult, + ThoughtspaceRuntime, + ThoughtspaceRuntimeInitOptions, + ThoughtspaceStorage, +} from '../thoughtspace' +import { clientIdReady, tsid } from '../thoughtspaceSession' +import acquireTreecrdtSessionLock from './sessionLock' +import { getMaterializedThoughtsToStoreVersion, waitForMaterializedThoughtsToStore } from './sync/materializationQueue' +import createTreecrdtWebSocketSync from './sync/treecrdtWebSocketSync' +import createTreecrdtDataProvider from './thoughtspace' +import { getTreecrdtWriteBarrierVersion, waitForTreecrdtWriteBarrier, withTreecrdtWriteBarrier } from './writeBarrier' + +type PersistTreecrdtBatch = Parameters[0] & { + local?: boolean +} + +/** One app-scoped TreeCRDT thoughtspace with its bound data provider and lifecycle. */ +interface TreecrdtThoughtspace extends ThoughtspaceRuntime { + readonly db: DataProvider +} + +const TREECRDT_IDLE_TIMEOUT = 30000 + +/** Rejects if provider idle work never settles. */ +const withIdleTimeout = (promise: Promise): Promise => { + let timeoutId: ReturnType | undefined + + return Promise.race([ + promise, + new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`TreeCRDT idle wait timed out after ${TREECRDT_IDLE_TIMEOUT}ms`)) + }, TREECRDT_IDLE_TIMEOUT) + }), + ]).finally(() => { + if (timeoutId) clearTimeout(timeoutId) + }) +} + +/** Converts the app client id to TreeCRDT's 32-byte replica id. */ +const clientIdToReplicaId = (clientId: string): Uint8Array => + clientId.length === 44 + ? Uint8Array.from(atob(clientId), c => c.charCodeAt(0)) + : (() => { + const bytes = new TextEncoder().encode(clientId) + const replicaId = new Uint8Array(32) + replicaId.set(bytes.subarray(0, 32)) + return replicaId + })() + +/** Converts the selected storage lifetime to TreeCRDT client options. */ +const getTreecrdtClientOptions = (storage: ThoughtspaceStorage): ClientOptions => ({ + storage: + storage === 'memory' + ? { type: 'memory' } + : { + type: 'opfs', + filename: `/treecrdt-em-${tsid}.db`, + fallback: 'memory', + }, + runtime: { type: storage === 'memory' ? 'direct' : 'dedicated-worker' }, + docId: tsid, +}) + +/** Waits until both local writes and materialization refreshes are stable. */ +const waitForStableIdle = async (): Promise => { + let writeVersion: number + let materializationVersion: number + do { + writeVersion = getTreecrdtWriteBarrierVersion() + materializationVersion = getMaterializedThoughtsToStoreVersion() + await waitForTreecrdtWriteBarrier() + await waitForMaterializedThoughtsToStore() + } while ( + writeVersion !== getTreecrdtWriteBarrierVersion() || + materializationVersion !== getMaterializedThoughtsToStoreVersion() + ) +} + +/** Creates an inert TreeCRDT client owner whose storage is selected during initialization. */ +const createTreecrdtThoughtspace = (): TreecrdtThoughtspace => { + type InitResult = { clientId: string } + + let client: TreecrdtClient | null = null + let unsubscribeMaterialization: (() => void) | null = null + let lifecycleTail: Promise = Promise.resolve() + let initPromise: Promise | null = null + let dropPromise: Promise | null = null + const provider = createTreecrdtDataProvider() + const websocketSync = createTreecrdtWebSocketSync() + + /** Applies the app's single-tab policy before the TreeCRDT client is opened. */ + const acquireAccess = async (): Promise => { + const lockStatus = await acquireTreecrdtSessionLock() + + return lockStatus === 'acquired' + ? { status: 'acquired' } + : { + status: 'blocked', + reason: lockStatus === 'unavailable' ? 'already-open' : 'unsupported', + } + } + + /** Detaches the provider and releases all resources owned by this thoughtspace. */ + const dropClient = async (): Promise => { + const errors: unknown[] = [] + /** Records cleanup failures without skipping later owned resources. */ + const captureError = async (work: () => void | Promise): Promise => { + try { + await work() + } catch (error) { + errors.push(error) + } + } + + provider.resetBinding(new Error('TreeCRDT client binding cleared before initialization.')) + + const unsubscribe = unsubscribeMaterialization + unsubscribeMaterialization = null + const clientToDrop = client + + await captureError(websocketSync.stop) + await captureError(() => unsubscribe?.()) + + await captureError(() => clientToDrop?.drop()) + + // wa-sqlite clients are terminal after drop settles, including when teardown reports an error. + if (client === clientToDrop) client = null + if (errors.length > 0) throw errors[0] + } + + /** Serializes teardown after any preceding initialization. */ + const drop = (): Promise => { + if (dropPromise) return dropPromise + + initPromise = null + const promise = lifecycleTail.then(dropClient) + dropPromise = promise + /** Clears this drop's single-flight slot without disturbing a newer queued drop. */ + const clearCurrentDrop = () => { + if (dropPromise === promise) dropPromise = null + } + lifecycleTail = promise.then(clearCurrentDrop, clearCurrentDrop) + return promise + } + + const db: DataProvider = { ...provider.db, clear: drop } + + /** Persists push queue batches through the bound provider and forwards local ops to remote sync. */ + const persistPushQueueBatches = (batches: readonly PersistTreecrdtBatch[]): Promise => + withTreecrdtWriteBarrier(async () => { + for (const batch of batches) { + const { local: isLocal, ...updates } = batch + const maybeOps = await db.updateThoughts(updates) + if (isLocal && Array.isArray(maybeOps) && maybeOps.length > 0) { + void websocketSync.pushLocalOps(maybeOps as readonly Operation[]) + } + } + }) + + /** Opens and binds one client. Lifecycle serialization provides retryable single-flight behavior. */ + const initializeClient = async (options: ThoughtspaceRuntimeInitOptions): Promise => { + let nextClient: TreecrdtClient | null = null + let nextUnsubscribeMaterialization: (() => void) | null = null + + try { + if (client) throw new Error('TreeCRDT client cleanup is incomplete. Retry drop before initialization.') + const clientId = await clientIdReady + await initPermissionsStore() + nextClient = await createTreecrdtClient(getTreecrdtClientOptions(options.storage)) + nextUnsubscribeMaterialization = await provider.bindClient( + nextClient, + clientIdToReplicaId(clientId), + options.materialization, + ) + await websocketSync.tryStartFromEnv(nextClient) + + client = nextClient + unsubscribeMaterialization = nextUnsubscribeMaterialization + if (options.storage === 'persistent' && nextClient.storage === 'memory') { + console.warn( + 'Persistent thoughtspace storage is unavailable. em is using temporary in-memory storage; changes will be lost when this page reloads or closes.', + ) + } + return { clientId } + } catch (error) { + provider.resetBinding(error) + nextUnsubscribeMaterialization?.() + await nextClient?.close() + throw error + } + } + + /** Coalesces adjacent init calls and preserves their order relative to drop. */ + const init = (options: ThoughtspaceRuntimeInitOptions): Promise => { + if (initPromise) return initPromise + + dropPromise = null + const promise = lifecycleTail.then(() => initializeClient(options)) + initPromise = promise + lifecycleTail = promise.then( + () => undefined, + () => undefined, + ) + void promise.catch(() => { + if (initPromise === promise) initPromise = null + }) + return promise + } + + return { + db, + acquireAccess, + init, + drop, + waitForIdle: (): Promise => withIdleTimeout(waitForStableIdle()), + persistPushQueueBatches, + } +} + +export default createTreecrdtThoughtspace diff --git a/src/data-providers/treecrdt/sessionLock.ts b/src/data-providers/treecrdt/sessionLock.ts new file mode 100644 index 00000000000..abb63b724e5 --- /dev/null +++ b/src/data-providers/treecrdt/sessionLock.ts @@ -0,0 +1,40 @@ +import { Capacitor } from '@capacitor/core' +import { tsid } from '../thoughtspaceSession' + +export type TreecrdtSessionLockStatus = 'acquired' | 'unavailable' | 'unsupported' + +let statusPromise: Promise | null = null + +/** + * Acquires an origin-wide, page-lifetime lock for a single-tab TreeCRDT thoughtspace. + * + * The callback deliberately remains pending. The browser releases the Web Lock automatically when + * the page is closed or navigated away from, allowing another tab to acquire access. + */ +export const acquireTreecrdtSessionLock = (): Promise => { + // Native apps cannot open a second browser tab, so exclusive access is already guaranteed. + if (Capacitor.isNativePlatform()) { + return Promise.resolve('acquired') + } + + if (!navigator.locks) return Promise.resolve('unsupported') + if (statusPromise) return statusPromise + + statusPromise = new Promise(resolve => { + void navigator.locks + .request(`em-treecrdt-session:${tsid}`, { ifAvailable: true, mode: 'exclusive' }, async lock => { + if (!lock) { + resolve('unavailable') + return + } + + resolve('acquired') + await new Promise(() => undefined) + }) + .catch(() => resolve('unsupported')) + }) + + return statusPromise +} + +export default acquireTreecrdtSessionLock diff --git a/src/data-providers/treecrdt/sync/__tests__/materializationQueue.ts b/src/data-providers/treecrdt/sync/__tests__/materializationQueue.ts new file mode 100644 index 00000000000..828aef7253d --- /dev/null +++ b/src/data-providers/treecrdt/sync/__tests__/materializationQueue.ts @@ -0,0 +1,41 @@ +import { enqueueMaterializedThoughtsToStoreWork, waitForMaterializedThoughtsToStore } from '../materializationQueue' + +it('waits for materialization work queued while waiting for idle', async () => { + const order: string[] = [] + let finishFirst!: () => void + + const first = enqueueMaterializedThoughtsToStoreWork(async () => { + order.push('first:start') + await new Promise(resolve => { + finishFirst = resolve + }) + order.push('first:end') + }) + + const wait = waitForMaterializedThoughtsToStore().then(() => { + order.push('idle') + }) + + const second = enqueueMaterializedThoughtsToStoreWork(async () => { + order.push('second') + }) + + await Promise.resolve() + finishFirst() + await Promise.all([first, second, wait]) + + expect(order).toEqual(['first:start', 'first:end', 'second', 'idle']) +}) + +it('surfaces materialization failures when waiting for idle', async () => { + const err = new Error('materialization failed') + + await expect( + enqueueMaterializedThoughtsToStoreWork(async () => { + throw err + }), + ).rejects.toThrow('materialization failed') + + await expect(waitForMaterializedThoughtsToStore()).rejects.toThrow('materialization failed') + await expect(waitForMaterializedThoughtsToStore()).resolves.toBeUndefined() +}) diff --git a/src/data-providers/treecrdt/sync/__tests__/materializationThoughtUpdates.ts b/src/data-providers/treecrdt/sync/__tests__/materializationThoughtUpdates.ts new file mode 100644 index 00000000000..aa8c5151362 --- /dev/null +++ b/src/data-providers/treecrdt/sync/__tests__/materializationThoughtUpdates.ts @@ -0,0 +1,140 @@ +import type Index from '../../../../@types/IndexType' +import type Thought from '../../../../@types/Thought' +import type ThoughtId from '../../../../@types/ThoughtId' +import type Timestamp from '../../../../@types/Timestamp' +import { HOME_TOKEN, ROOT_PARENT_ID } from '../../../../constants' +import initialState from '../../../../util/initialState' +import type { DataProvider } from '../../../DataProvider' +import { refreshThoughtsFromMaterializationChanges } from '../materializationThoughtUpdates' + +const A_ID = 'a-id' as ThoughtId +const B_ID = 'b-id' as ThoughtId +const C_ID = 'c-id' as ThoughtId +const LEFT_ID = 'left-id' as ThoughtId +const RIGHT_ID = 'right-id' as ThoughtId + +/** Creates a childrenMap that preserves the provided insertion order for Object.values. */ +const childrenMap = (children: ThoughtId[]): Index => + Object.fromEntries(children.map(childId => [childId, childId])) + +/** Creates a minimal Thought for materialization projection tests. */ +const thought = ( + id: ThoughtId, + value: string, + rank: number, + parentId: ThoughtId, + children: ThoughtId[] = [], +): Thought => ({ + id, + value, + rank, + parentId, + childrenMap: childrenMap(children), + created: 0 as Timestamp, + lastUpdated: 0 as Timestamp, + updatedBy: '', +}) + +/** Creates the minimal thoughtspace provider surface needed by refreshThoughtsFromMaterializationChanges. */ +const fakeProvider = (thoughts: Index): DataProvider => ({ + clear: async () => undefined, + getLexemeById: async () => undefined, + getLexemesByIds: async keys => keys.map(() => undefined), + getThoughtById: async (id: ThoughtId) => thoughts[id], + getThoughtsByIds: async ids => ids.map(id => thoughts[id]), + updateThoughts: async () => undefined, + freeThought: async () => undefined, + freeLexeme: async () => undefined, +}) + +/** Converts test state to the provider-facing materialization snapshot. */ +const materializationSnapshot = (state: ReturnType) => ({ + schemaVersion: state.schemaVersion, + thoughtIndex: state.thoughts.thoughtIndex, + lexemeIndex: state.thoughts.lexemeIndex, +}) + +it('projects TreeCRDT sibling order into compatibility ranks', async () => { + const oldParent = thought(HOME_TOKEN, HOME_TOKEN, 0, ROOT_PARENT_ID, [A_ID, B_ID, C_ID]) + const newParent = thought(HOME_TOKEN, HOME_TOKEN, 0, ROOT_PARENT_ID, [C_ID, A_ID, B_ID]) + const thoughtA = thought(A_ID, 'a', 0, HOME_TOKEN) + const thoughtB = thought(B_ID, 'b', 1, HOME_TOKEN) + const thoughtC = thought(C_ID, 'c', 2, HOME_TOKEN) + const state = { + ...initialState(), + thoughts: { + thoughtIndex: { + [HOME_TOKEN]: oldParent, + [A_ID]: thoughtA, + [B_ID]: thoughtB, + [C_ID]: thoughtC, + }, + lexemeIndex: {}, + }, + } + + const result = await refreshThoughtsFromMaterializationChanges( + [{ kind: 'move', node: C_ID, parentBefore: HOME_TOKEN, parentAfter: HOME_TOKEN }], + fakeProvider({ + [HOME_TOKEN]: newParent, + [A_ID]: thoughtA, + [B_ID]: thoughtB, + [C_ID]: thoughtC, + }), + materializationSnapshot(state), + ) + + const updates = Object.fromEntries(result.thoughts.map(nextThought => [nextThought.id, nextThought])) + + expect(Object.values(updates[HOME_TOKEN].childrenMap)).toEqual([C_ID, A_ID, B_ID]) + expect(updates[C_ID].rank).toBe(0) + expect(updates[A_ID].rank).toBe(1) + expect(updates[B_ID].rank).toBe(2) +}) + +it('projects TreeCRDT sibling order for both parents after a cross-parent move', async () => { + const oldLeft = thought(LEFT_ID, 'left', 0, HOME_TOKEN, [A_ID, B_ID]) + const oldRight = thought(RIGHT_ID, 'right', 1, HOME_TOKEN, [C_ID]) + const newLeft = thought(LEFT_ID, 'left', 0, HOME_TOKEN, [B_ID]) + const newRight = thought(RIGHT_ID, 'right', 1, HOME_TOKEN, [C_ID, A_ID]) + const thoughtAOld = thought(A_ID, 'a', 0, LEFT_ID) + const thoughtANew = thought(A_ID, 'a', 1, RIGHT_ID) + const thoughtB = thought(B_ID, 'b', 1, LEFT_ID) + const thoughtC = thought(C_ID, 'c', 0, RIGHT_ID) + const state = { + ...initialState(), + thoughts: { + thoughtIndex: { + [LEFT_ID]: oldLeft, + [RIGHT_ID]: oldRight, + [A_ID]: thoughtAOld, + [B_ID]: thoughtB, + [C_ID]: thoughtC, + }, + lexemeIndex: {}, + }, + } + + const result = await refreshThoughtsFromMaterializationChanges( + [{ kind: 'move', node: A_ID, parentBefore: LEFT_ID, parentAfter: RIGHT_ID }], + fakeProvider({ + [LEFT_ID]: newLeft, + [RIGHT_ID]: newRight, + [A_ID]: thoughtANew, + [B_ID]: thoughtB, + [C_ID]: thoughtC, + }), + materializationSnapshot(state), + ) + + const updates = Object.fromEntries(result.thoughts.map(nextThought => [nextThought.id, nextThought])) + + expect(Object.values(updates[LEFT_ID].childrenMap)).toEqual([B_ID]) + expect(Object.values(updates[RIGHT_ID].childrenMap)).toEqual([C_ID, A_ID]) + expect(updates[B_ID].rank).toBe(0) + expect(updates[C_ID].rank).toBe(0) + expect(updates[A_ID]).toMatchObject({ + parentId: RIGHT_ID, + rank: 1, + }) +}) diff --git a/src/data-providers/treecrdt/sync/__tests__/treecrdtWebSocketSync.ts b/src/data-providers/treecrdt/sync/__tests__/treecrdtWebSocketSync.ts new file mode 100644 index 00000000000..5f8b0c0941f --- /dev/null +++ b/src/data-providers/treecrdt/sync/__tests__/treecrdtWebSocketSync.ts @@ -0,0 +1,49 @@ +import type { Operation } from '@treecrdt/interface' +import type { TreecrdtClient } from '@treecrdt/wa-sqlite' +import createTreecrdtWebSocketSync from '../treecrdtWebSocketSync' + +const { connectTreecrdtWebSocketSync, getTreecrdtSyncBaseUrl } = vi.hoisted(() => ({ + connectTreecrdtWebSocketSync: vi.fn(), + getTreecrdtSyncBaseUrl: vi.fn(), +})) + +vi.mock('@treecrdt/sync', () => ({ connectTreecrdtWebSocketSync })) +vi.mock('../config', () => ({ getTreecrdtSyncBaseUrl })) + +/** Creates a minimal WebSocket sync handle for lifecycle assertions. */ +const createMockSyncHandle = () => ({ + close: vi.fn().mockResolvedValue(undefined), + pushLocalOps: vi.fn().mockResolvedValue(undefined), + startLive: vi.fn().mockResolvedValue(undefined), + syncOnce: vi.fn().mockResolvedValue(undefined), +}) + +it('isolates handles and local ops between thoughtspace instances', async () => { + const firstHandle = createMockSyncHandle() + const secondHandle = createMockSyncHandle() + getTreecrdtSyncBaseUrl.mockReturnValue('https://sync.example.test') + connectTreecrdtWebSocketSync.mockResolvedValueOnce(firstHandle).mockResolvedValueOnce(secondHandle) + + const first = createTreecrdtWebSocketSync() + const second = createTreecrdtWebSocketSync() + const firstClient = {} as TreecrdtClient + const secondClient = {} as TreecrdtClient + const firstOp = {} as Operation + const secondOp = {} as Operation + + await first.start(firstClient) + await second.start(secondClient) + await first.pushLocalOps([firstOp]) + await second.pushLocalOps([secondOp]) + + expect(connectTreecrdtWebSocketSync).toHaveBeenNthCalledWith(1, firstClient, expect.any(Object)) + expect(connectTreecrdtWebSocketSync).toHaveBeenNthCalledWith(2, secondClient, expect.any(Object)) + expect(firstHandle.pushLocalOps).toHaveBeenCalledWith([firstOp]) + expect(secondHandle.pushLocalOps).toHaveBeenCalledWith([secondOp]) + + await first.stop() + expect(firstHandle.close).toHaveBeenCalledTimes(1) + expect(secondHandle.close).not.toHaveBeenCalled() + + await second.stop() +}) diff --git a/src/data-providers/treecrdt/sync/applyMaterializedThoughtsToStore.ts b/src/data-providers/treecrdt/sync/applyMaterializedThoughtsToStore.ts new file mode 100644 index 00000000000..8c6f596e4dc --- /dev/null +++ b/src/data-providers/treecrdt/sync/applyMaterializedThoughtsToStore.ts @@ -0,0 +1,81 @@ +/* eslint-disable import/prefer-default-export -- bridge module */ +import type { MaterializationEvent } from '@treecrdt/interface/engine' +import type { TreecrdtClient } from '@treecrdt/wa-sqlite' +import type Index from '../../../@types/IndexType' +import type Thought from '../../../@types/Thought' +import type { ThoughtspaceMaterializationBridge } from '../../thoughtspace' +import { refreshAttributeChildrenFromChanges } from '../attributeChildren' +import { waitForTreecrdtWriteBarrier } from '../writeBarrier' +import { enqueueMaterializedThoughtsToStoreWork } from './materializationQueue' +import { type MaterializationStore, refreshThoughtsFromMaterializationChanges } from './materializationThoughtUpdates' + +/** Dependencies captured when a client registers its materialization listener. */ +type MaterializationContext = Readonly<{ + bridge: ThoughtspaceMaterializationBridge + client: TreecrdtClient + db: MaterializationStore +}> + +/** + * After remote TreeCRDT ops are materialized into SQLite, refresh the app-facing thoughtspace in one batch. + * This is used for cross-tab and server sync events; same-tab local writes are already applied optimistically. + */ +export async function applyMaterializedThoughtsToStore( + event: MaterializationEvent, + { bridge, client, db }: MaterializationContext, +): Promise { + if (event.changes.length === 0) return + + // Local writes and materialization callbacks can race. Wait for queued em -> TreeCRDT writes before reading + // SQLite back into app state, otherwise a remote refresh can reapply stale rows over newer optimistic state. + await waitForTreecrdtWriteBarrier() + + await refreshAttributeChildrenFromChanges(client, event.changes) + + const snapshot = bridge.getSnapshot() + const { deletedIds, thoughts, lexemeIndexUpdates } = await refreshThoughtsFromMaterializationChanges( + event.changes, + db, + snapshot, + ) + + if (Object.keys(lexemeIndexUpdates).length > 0) { + await db.updateThoughts({ + thoughtIndexUpdates: {}, + lexemeIndexUpdates, + lexemeIndexUpdatesOld: {}, + schemaVersion: snapshot.schemaVersion, + }) + } + + const thoughtIndexUpdates: Index = {} + + for (const id of deletedIds) { + thoughtIndexUpdates[id] = null + } + + for (const latest of thoughts) { + const thoughtInState = snapshot.thoughtIndex[latest.id] + const parentInState = snapshot.thoughtIndex[latest.parentId] + // Pending is not part of the TreeCRDT payload. Preserve the local UI flag until auth/sync handling owns it. + const pending = thoughtInState?.pending || parentInState?.pending + const latestWithPending = { + ...latest, + ...(pending ? { pending } : null), + } + + thoughtIndexUpdates[latest.id] = latestWithPending + } + + if (Object.keys(thoughtIndexUpdates).length > 0 || Object.keys(lexemeIndexUpdates).length > 0) { + await bridge.apply({ thoughtIndex: thoughtIndexUpdates, lexemeIndex: lexemeIndexUpdates }) + } +} + +/** Serializes materialization refreshes so overlapping async events cannot apply out of order. */ +export function enqueueMaterializedThoughtsToStore( + event: MaterializationEvent, + context: MaterializationContext, +): Promise { + return enqueueMaterializedThoughtsToStoreWork(() => applyMaterializedThoughtsToStore(event, context)) +} diff --git a/src/data-providers/treecrdt/sync/config.ts b/src/data-providers/treecrdt/sync/config.ts new file mode 100644 index 00000000000..e8751e51995 --- /dev/null +++ b/src/data-providers/treecrdt/sync/config.ts @@ -0,0 +1,8 @@ +/* eslint-disable import/prefer-default-export -- small config module */ +/** Sync bootstrap or direct WebSocket URL (`ws://`, `wss://`, or `http(s)://` discovery). Set via Vite: `VITE_TREECRDT_SYNC_BASE_URL`. */ +export function getTreecrdtSyncBaseUrl(): string | undefined { + const raw = import.meta.env.VITE_TREECRDT_SYNC_BASE_URL + if (raw == null || typeof raw !== 'string') return undefined + const trimmed = raw.trim() + return trimmed === '' ? undefined : trimmed +} diff --git a/src/data-providers/treecrdt/sync/index.ts b/src/data-providers/treecrdt/sync/index.ts new file mode 100644 index 00000000000..d865cae6368 --- /dev/null +++ b/src/data-providers/treecrdt/sync/index.ts @@ -0,0 +1,6 @@ +/* eslint-disable import/prefer-default-export -- barrel re-exports */ +export { getTreecrdtSyncBaseUrl } from './config' +export { + applyMaterializedThoughtsToStore, + enqueueMaterializedThoughtsToStore, +} from './applyMaterializedThoughtsToStore' diff --git a/src/data-providers/treecrdt/sync/materializationQueue.ts b/src/data-providers/treecrdt/sync/materializationQueue.ts new file mode 100644 index 00000000000..2d7cc614381 --- /dev/null +++ b/src/data-providers/treecrdt/sync/materializationQueue.ts @@ -0,0 +1,37 @@ +let materializedThoughtsToStoreQueue = Promise.resolve() +let materializedThoughtsToStoreError: unknown = null +let materializedThoughtsToStoreVersion = 0 + +/** Serializes materialization refresh work and records failures for the idle barrier. */ +export function enqueueMaterializedThoughtsToStoreWork(work: () => Promise): Promise { + materializedThoughtsToStoreVersion += 1 + const apply = materializedThoughtsToStoreQueue.then(work) + materializedThoughtsToStoreQueue = apply.catch(err => { + materializedThoughtsToStoreError = err + }) + return apply +} + +/** Monotonically increases whenever materialization refresh work is queued. */ +export const getMaterializedThoughtsToStoreVersion = (): number => materializedThoughtsToStoreVersion + +/** Waits for queued materialization refreshes to finish and surfaces the first refresh error. */ +export async function waitForMaterializedThoughtsToStore(): Promise { + let pending: Promise + do { + pending = materializedThoughtsToStoreQueue + await pending + } while (pending !== materializedThoughtsToStoreQueue) + + if (materializedThoughtsToStoreError) { + const err = materializedThoughtsToStoreError + materializedThoughtsToStoreError = null + throw err + } +} + +export default { + enqueueMaterializedThoughtsToStoreWork, + getMaterializedThoughtsToStoreVersion, + waitForMaterializedThoughtsToStore, +} diff --git a/src/data-providers/treecrdt/sync/materializationThoughtUpdates.ts b/src/data-providers/treecrdt/sync/materializationThoughtUpdates.ts new file mode 100644 index 00000000000..1908e2b8c3d --- /dev/null +++ b/src/data-providers/treecrdt/sync/materializationThoughtUpdates.ts @@ -0,0 +1,199 @@ +import type { Change } from '@treecrdt/interface/engine' +import type Index from '../../../@types/IndexType' +import type Lexeme from '../../../@types/Lexeme' +import type Thought from '../../../@types/Thought' +import type ThoughtId from '../../../@types/ThoughtId' +import type Timestamp from '../../../@types/Timestamp' +import { ABSOLUTE_TOKEN, EM_TOKEN, GLOBAL_ROOT_TOKEN, HOME_TOKEN, ROOT_PARENT_ID } from '../../../constants' +import hashThought from '../../../util/hashThought' +import type { DataProvider } from '../../DataProvider' +import type { ThoughtspaceMaterializationSnapshot } from '../../thoughtspace' + +/** Data-provider operations needed to persist materialized TreeCRDT changes. */ +export type MaterializationStore = Pick + +export type MaterializationThoughtRefresh = { + /** Thought ids removed from the tree. */ + deletedIds: ThoughtId[] + /** Thoughts to merge into app state after materialization. */ + thoughts: Thought[] + /** Lexeme rows for the refreshed thoughts' values. */ + lexemeIndexUpdates: Index +} + +const ROOT_THOUGHT_IDS = new Set([GLOBAL_ROOT_TOKEN, ROOT_PARENT_ID, HOME_TOKEN, EM_TOKEN, ABSOLUTE_TOKEN]) + +/** True when a thought should be represented as a Lexeme context. */ +const isLexemeContextThought = (thought: Thought | undefined): thought is Thought => + !!thought && !ROOT_THOUGHT_IDS.has(thought.id) + +/** Returns the latest timestamp while preserving the branded Timestamp type. */ +const maxTimestamp = (...values: (Timestamp | number | undefined)[]): Timestamp => + Math.max(...values.map(value => value || 0)) as Timestamp + +/** Gets the latest lexeme from already staged updates, app state, or the local derived lexeme table. */ +const getCurrentLexeme = async ( + key: string, + updates: Index, + snapshot: ThoughtspaceMaterializationSnapshot, + db: MaterializationStore, +): Promise => { + if (updates[key] === null) return undefined + return updates[key] || snapshot.lexemeIndex[key] || (await db.getLexemeById(key)) +} + +/** Adds the thought id to the locally derived lexeme for the thought value. */ +const addLexemeContext = async ( + updates: Index, + snapshot: ThoughtspaceMaterializationSnapshot, + db: MaterializationStore, + thought: Thought, +): Promise => { + if (!isLexemeContextThought(thought)) return + + const key = hashThought(thought.value) + const lexeme = await getCurrentLexeme(key, updates, snapshot, db) + const contexts = [...(lexeme?.contexts || []).filter(id => id !== thought.id), thought.id] + updates[key] = { + contexts, + created: lexeme?.created || thought.created, + lastUpdated: maxTimestamp(lexeme?.lastUpdated, thought.lastUpdated), + updatedBy: thought.updatedBy || lexeme?.updatedBy || '', + } +} + +/** Removes the thought id from the locally derived lexeme for the previous thought value. */ +const removeLexemeContext = async ( + updates: Index, + snapshot: ThoughtspaceMaterializationSnapshot, + db: MaterializationStore, + thought: Thought | undefined, +): Promise => { + if (!isLexemeContextThought(thought)) return + + const key = hashThought(thought.value) + const lexeme = await getCurrentLexeme(key, updates, snapshot, db) + if (!lexeme) return + + const contexts = lexeme.contexts.filter(id => id !== thought.id) + updates[key] = + contexts.length === 0 + ? null + : { + ...lexeme, + contexts, + lastUpdated: maxTimestamp(lexeme.lastUpdated, thought.lastUpdated), + updatedBy: thought.updatedBy || lexeme.updatedBy, + } +} + +/** Applies TreeCRDT sibling order to em's temporary rank projection for one parent. */ +const addTreeOrderRankProjection = async ( + updates: Index, + db: MaterializationStore, + parentId: ThoughtId, +): Promise => { + const parent = await db.getThoughtById(parentId) + if (!parent) return + + updates[parent.id] = parent + + const orderedChildIds = Object.values(parent.childrenMap || {}) + for (const [rank, childId] of orderedChildIds.entries()) { + const child = await db.getThoughtById(childId) + if (!child) continue + updates[child.id] = { + ...child, + rank, + } + } +} + +/** Collects affected ids from materialization changes, loads fresh thoughts + lexemes from the provider. */ +export async function refreshThoughtsFromMaterializationChanges( + changes: Change[], + db: MaterializationStore, + snapshot: ThoughtspaceMaterializationSnapshot, +): Promise { + const deleted = new Set() + const touched = new Set() + const orderParents = new Set() + for (const ch of changes) { + switch (ch.kind) { + case 'insert': + touched.add(ch.node as ThoughtId) + touched.add(ch.parentAfter as ThoughtId) + orderParents.add(ch.parentAfter as ThoughtId) + break + case 'move': + touched.add(ch.node as ThoughtId) + if (ch.parentBefore) { + touched.add(ch.parentBefore as ThoughtId) + orderParents.add(ch.parentBefore as ThoughtId) + } + touched.add(ch.parentAfter as ThoughtId) + orderParents.add(ch.parentAfter as ThoughtId) + break + case 'delete': + deleted.add(ch.node as ThoughtId) + if (ch.parentBefore) { + touched.add(ch.parentBefore as ThoughtId) + orderParents.add(ch.parentBefore as ThoughtId) + } + break + case 'restore': + touched.add(ch.node as ThoughtId) + if (ch.parentAfter) { + touched.add(ch.parentAfter as ThoughtId) + orderParents.add(ch.parentAfter as ThoughtId) + } + break + case 'payload': + touched.add(ch.node as ThoughtId) + break + } + } + + for (const id of deleted) { + touched.delete(id) + } + + const thoughts: Thought[] = [] + const thoughtIndexUpdates: Index = {} + const lexemeIndexUpdates: Index = {} + + for (const id of touched) { + const thought = await db.getThoughtById(id) + if (!thought) continue + thoughtIndexUpdates[thought.id] = thought + orderParents.add(thought.parentId) + const previous = snapshot.thoughtIndex[id] + if (previous && previous.value !== thought.value) { + await removeLexemeContext(lexemeIndexUpdates, snapshot, db, previous) + } + await addLexemeContext(lexemeIndexUpdates, snapshot, db, thought) + } + + // Current em selectors still sort by numeric rank. For remote/order-only TreeCRDT changes, derive a local rank + // projection from the authoritative TreeCRDT child order without exposing TreeCRDT's internal order keys. + // TODO: Remove when read-side selectors consume provider-backed sibling order instead of rank projection. + for (const parentId of orderParents) { + await addTreeOrderRankProjection(thoughtIndexUpdates, db, parentId) + } + + for (const id of deleted) { + delete thoughtIndexUpdates[id] + } + + thoughts.push(...Object.values(thoughtIndexUpdates)) + + for (const id of deleted) { + await removeLexemeContext(lexemeIndexUpdates, snapshot, db, snapshot.thoughtIndex[id]) + } + + return { + deletedIds: [...deleted], + thoughts, + lexemeIndexUpdates, + } +} diff --git a/src/data-providers/treecrdt/sync/treecrdtWebSocketSync.ts b/src/data-providers/treecrdt/sync/treecrdtWebSocketSync.ts new file mode 100644 index 00000000000..b1c24894004 --- /dev/null +++ b/src/data-providers/treecrdt/sync/treecrdtWebSocketSync.ts @@ -0,0 +1,73 @@ +import type { Operation } from '@treecrdt/interface' +import { type TreecrdtWebSocketSync, connectTreecrdtWebSocketSync } from '@treecrdt/sync' +import type { TreecrdtClient } from '@treecrdt/wa-sqlite' +import { getTreecrdtSyncBaseUrl } from './config' + +/** Creates WebSocket sync state owned by one TreeCRDT thoughtspace and document. */ +const createTreecrdtWebSocketSync = () => { + let syncHandle: TreecrdtWebSocketSync | null = null + + /** Stops live sync and closes this thoughtspace's WebSocket. */ + const stop = async (): Promise => { + if (syncHandle) { + await syncHandle.close() + syncHandle = null + } + } + + /** Connects to the sync server, runs catch-up, then live subscription. No-op if no base URL. */ + const start = async (client: TreecrdtClient): Promise => { + const baseUrl = getTreecrdtSyncBaseUrl() + if (!baseUrl) return + + await stop() + + const handle = await connectTreecrdtWebSocketSync(client, { + baseUrl, + fetch, + onLiveError: err => { + console.error('TreeCRDT WebSocket sync live subscription error', err) + }, + }) + + try { + await handle.syncOnce() + await handle.startLive() + } catch (err) { + await handle.close() + throw err + } + + syncHandle = handle + } + + /** Starts sync when `VITE_TREECRDT_SYNC_BASE_URL` is set; skips in test; logs warnings on failure. */ + const tryStartFromEnv = async (client: TreecrdtClient): Promise => { + if (import.meta.env.MODE === 'test') return + if (!getTreecrdtSyncBaseUrl()) return + try { + await start(client) + } catch (err) { + console.warn('TreeCRDT WebSocket sync failed to start', err) + } + } + + /** Uploads local edits through this thoughtspace's active WebSocket handle. */ + const pushLocalOps = async (ops: readonly Operation[]): Promise => { + if (ops.length === 0 || !syncHandle) return + try { + await syncHandle.pushLocalOps(ops) + } catch (err) { + console.warn('TreeCRDT pushLocalOps failed', err) + } + } + + return { + pushLocalOps, + start, + stop, + tryStartFromEnv, + } +} + +export default createTreecrdtWebSocketSync diff --git a/src/data-providers/treecrdt/systemThoughtIds.ts b/src/data-providers/treecrdt/systemThoughtIds.ts new file mode 100644 index 00000000000..3eaf309c045 --- /dev/null +++ b/src/data-providers/treecrdt/systemThoughtIds.ts @@ -0,0 +1,7 @@ +import { ABSOLUTE_TOKEN, EM_TOKEN, HOME_TOKEN } from '../../constants' + +export const SYSTEM_ROOT_THOUGHT_IDS = [HOME_TOKEN, EM_TOKEN, ABSOLUTE_TOKEN] as const + +export default { + SYSTEM_ROOT_THOUGHT_IDS, +} diff --git a/src/data-providers/treecrdt/thoughtspace.ts b/src/data-providers/treecrdt/thoughtspace.ts new file mode 100644 index 00000000000..1cfc7192a8e --- /dev/null +++ b/src/data-providers/treecrdt/thoughtspace.ts @@ -0,0 +1,444 @@ +import type { Operation } from '@treecrdt/interface' +import type { TreecrdtClient } from '@treecrdt/wa-sqlite' +import type Index from '../../@types/IndexType' +import type Lexeme from '../../@types/Lexeme' +import type Thought from '../../@types/Thought' +import type ThoughtId from '../../@types/ThoughtId' +import type Timestamp from '../../@types/Timestamp' +import { EM_TOKEN, GLOBAL_ROOT_TOKEN, ROOT_PARENT_ID, SETTINGS_TOKEN, SETTINGS_VALUE } from '../../constants' +import testFlags from '../../e2e/testFlags' +import { childrenMapKey } from '../../util/createChildrenMap' +import hashThought from '../../util/hashThought' +import isAttribute from '../../util/isAttribute' +import sleep from '../../util/sleep' +import type { DataProvider } from '../DataProvider' +import type { ThoughtspaceMaterializationBridge } from '../thoughtspace' +import { + deleteAttributeChild, + ensureAttributeChildrenIndexReady, + getAttributeChildrenByParent, + upsertAttributeChild, +} from './attributeChildren' +import { + deleteAllLexemes, + deleteLexeme as deleteLexemeRow, + ensureLexemesSchema, + getLexemeById as getLexemeByIdSql, + getLexemesByIds as getLexemesByIdsSql, + upsertLexeme, +} from './lexemes' +import { decodeThoughtPayload, encodeThoughtPayload } from './payload' +import { enqueueMaterializedThoughtsToStore } from './sync' +import { SYSTEM_ROOT_THOUGHT_IDS } from './systemThoughtIds' +import { createTreecrdtLocalWriteOptions, isTreecrdtLocalMaterialization } from './writeBarrier' + +type TreecrdtPlacement = { type: 'first' } | { type: 'last' } | { type: 'after'; after: ThoughtId } + +type TreecrdtClientIdentity = Readonly<{ + client: TreecrdtClient + replicaId: Uint8Array +}> + +type TreecrdtClientDataProvider = Pick< + DataProvider, + 'getLexemeById' | 'getLexemesByIds' | 'getThoughtById' | 'getThoughtsByIds' | 'updateThoughts' +> & + Required> + +/** Creates the private provider-readiness state used by writes that race startup. */ +const createProviderReadiness = () => { + let resolve!: (db: TreecrdtClientDataProvider) => void + let reject!: (reason: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + + // A public write observes this rejection. This catch only prevents an unhandled rejection when no write was waiting. + void promise.catch(() => undefined) + + return { promise, reject, resolve } +} + +/** Creates em's childrenMap read-model index while preserving TreeCRDT's strict child ids as values. */ +export const createIndexedChildrenMap = ( + childIds: ThoughtId[], + attributeValueByChildId: Index, +): Index => { + const childrenMap: Index = {} + for (const childId of childIds) { + const value = attributeValueByChildId[childId] + childrenMap[value ? childrenMapKey(childrenMap, { id: childId, value }) : childId] = childId + } + return childrenMap +} + +/** Injects delayed TreeCRDT reads for e2e tests that exercise slow local materialization after refresh. */ +const waitForTestReplicationDelay = async (): Promise => { + if (testFlags.replicationDelay > 0) { + await sleep(testFlags.replicationDelay) + } +} + +/** Fetches a thought by ID from the given TreeCRDT client. */ +const getThoughtByIdFromClient = async (client: TreecrdtClient, id: ThoughtId): Promise => { + const payloadBytes = await client.tree.getPayload(id) + if (payloadBytes === null) return undefined + + const payload = decodeThoughtPayload(payloadBytes) + + const parentIdRaw = await client.tree.parent(id) + const parentId: ThoughtId = parentIdRaw === null ? (ROOT_PARENT_ID as ThoughtId) : (parentIdRaw as ThoughtId) + const siblingIds = parentIdRaw === null ? [] : await client.tree.children(parentIdRaw) + const rank = parentIdRaw === null ? 0 : Math.max(0, siblingIds.indexOf(id)) + + const childIds = (await client.tree.children(id)) as ThoughtId[] + const childrenMap = createIndexedChildrenMap(childIds, await getAttributeChildrenByParent(client, id)) + + const thought: Thought = { + id, + value: payload.value, + rank, + created: payload.created as Timestamp, + lastUpdated: payload.lastUpdated as Timestamp, + updatedBy: payload.updatedBy, + parentId, + childrenMap, + ...(payload.archived !== undefined && { archived: payload.archived as Timestamp }), + } + + return thought +} + +/** Converts em's root parent id to TreeCRDT's global root id. */ +const treeParentId = (id: ThoughtId): ThoughtId => (id === ROOT_PARENT_ID ? GLOBAL_ROOT_TOKEN : id) + +/** + * Derives TreeCRDT relative placement from em's numeric rank payload. + * This is the compatibility bridge while the app still treats rank as canonical display order. + * TODO: Remove when create/import/newThought paths pass explicit placement and selectors read provider-backed order. + */ +const getRankPlacement = async ( + client: TreecrdtClient, + parentId: ThoughtId, + thoughtId: ThoughtId, + rank: number, +): Promise => { + const childIds = await client.tree.children(parentId) + const afterId = childIds.reduce( + (previousId, childId, index) => (childId !== thoughtId && index < rank ? (childId as ThoughtId) : previousId), + undefined, + ) + + return afterId ? { type: 'after', after: afterId } : { type: 'first' } +} + +/** Resolves caller-provided TreeCRDT placement, falling back to rank when old callers or stale siblings omit it. */ +const getTreecrdtPlacement = async ( + client: TreecrdtClient, + thoughtId: ThoughtId, + thought: Thought, + movePlacements?: Index, + options?: { requireExplicit?: boolean }, +): Promise => { + const parentId = treeParentId(thought.parentId) + + if (!movePlacements || !Object.prototype.hasOwnProperty.call(movePlacements, thoughtId)) { + if (options?.requireExplicit) { + throw new Error(`TreeCRDT move for ${thoughtId} requires explicit placement.`) + } + return getRankPlacement(client, parentId, thoughtId, thought.rank) + } + + const afterId = movePlacements[thoughtId] + if (afterId == null) return { type: 'first' } + if (afterId === thoughtId) throw new Error(`TreeCRDT move for ${thoughtId} cannot be placed after itself.`) + + const childIds = await client.tree.children(parentId) + if (!childIds.includes(afterId)) { + return getRankPlacement(client, parentId, thoughtId, thought.rank) + } + + return { type: 'after', after: afterId } +} + +/** Applies thought index updates and move placements to one exact TreeCRDT client. */ +const updateThoughtsForClient = async ( + { client, replicaId }: TreecrdtClientIdentity, + { thoughtIndexUpdates, lexemeIndexUpdates, movePlacements }: Parameters[0], +): Promise => { + const ops: Operation[] = [] + + for (const [id, lexeme] of Object.entries(lexemeIndexUpdates)) { + if (lexeme === null) { + await deleteLexemeRow(client, id) + } else { + await upsertLexeme(client, id, lexeme) + } + } + + const updates: Index = {} + const deletes: ThoughtId[] = [] + + for (const [id, thought] of Object.entries(thoughtIndexUpdates)) { + const thoughtId = id as ThoughtId + if (thought === null) { + deletes.push(thoughtId) + } else { + updates[thoughtId] = thought + } + } + + for (const id of deletes) { + ops.push(await client.local.delete(replicaId, id, createTreecrdtLocalWriteOptions())) + await deleteAttributeChild(client, id) + } + + for (const [id, thought] of Object.entries(updates)) { + const thoughtId = id as ThoughtId + const payloadBytes = encodeThoughtPayload({ + value: thought.value, + created: thought.created, + lastUpdated: thought.lastUpdated, + updatedBy: thought.updatedBy, + ...(thought.archived !== undefined && { archived: thought.archived }), + }) + + const exists = await client.tree.exists(thoughtId) + const parentId = treeParentId(thought.parentId) + + if (!exists) { + const placement = await getTreecrdtPlacement(client, thoughtId, thought, movePlacements) + ops.push( + await client.local.insert( + replicaId, + parentId, + thoughtId, + placement, + payloadBytes, + createTreecrdtLocalWriteOptions(), + ), + ) + if (isAttribute(thought.value)) { + await upsertAttributeChild(client, parentId, thoughtId, thought.value) + } + } else { + const existing = await getThoughtByIdFromClient(client, thoughtId) + if (!existing) continue + + const parentChanged = existing.parentId !== thought.parentId + const valueChanged = existing.value !== thought.value + const orderChanged = thoughtId in (movePlacements || {}) + if (parentChanged || orderChanged) { + const placement = await getTreecrdtPlacement(client, thoughtId, thought, movePlacements, { + requireExplicit: true, + }) + ops.push(await client.local.move(replicaId, thoughtId, parentId, placement, createTreecrdtLocalWriteOptions())) + } + + const payloadChanged = + existing.value !== thought.value || + existing.created !== thought.created || + existing.lastUpdated !== thought.lastUpdated || + existing.updatedBy !== thought.updatedBy || + existing.archived !== thought.archived + + if (payloadChanged) { + ops.push(await client.local.payload(replicaId, thoughtId, payloadBytes, createTreecrdtLocalWriteOptions())) + } + + if (parentChanged || valueChanged) { + if (isAttribute(thought.value)) { + await upsertAttributeChild(client, parentId, thoughtId, thought.value) + } else if (isAttribute(existing.value)) { + await deleteAttributeChild(client, thoughtId) + } + } + } + } + + return ops +} + +/** Replaces all stored lexemes in one exact TreeCRDT client. */ +const updateLexemeIndexForClient = async (client: TreecrdtClient, lexemeIndex: Index): Promise => { + await deleteAllLexemes(client) + for (const [id, lexeme] of Object.entries(lexemeIndex)) { + await upsertLexeme(client, id, lexeme) + } +} + +const ROOT_PAYLOAD = encodeThoughtPayload({ + value: GLOBAL_ROOT_TOKEN, + created: 0, + lastUpdated: 0, + updatedBy: '', +}) + +/** Seeds TreeCRDT storage for an em thoughtspace. */ +const initializeThoughtspaceStorage = async (client: TreecrdtClient, replicaId: Uint8Array): Promise => { + await ensureLexemesSchema(client) + // Ensure root has payload so getThoughtById can use the generic path. + await client.local.payload(replicaId, GLOBAL_ROOT_TOKEN, ROOT_PAYLOAD, createTreecrdtLocalWriteOptions()) + for (const id of SYSTEM_ROOT_THOUGHT_IDS) { + if (!(await client.tree.exists(id))) { + const now = Date.now() + await client.local.insert( + replicaId, + GLOBAL_ROOT_TOKEN, + id, + { type: 'last' }, + encodeThoughtPayload({ + value: id, + created: now, + lastUpdated: now, + updatedBy: '', + }), + createTreecrdtLocalWriteOptions(), + ) + } + } + + let settingsId: ThoughtId | null = null + for (const childId of await client.tree.children(EM_TOKEN)) { + const payloadBytes = await client.tree.getPayload(childId) + if (!payloadBytes) continue + const payload = decodeThoughtPayload(payloadBytes) + if (payload.value === SETTINGS_VALUE) { + settingsId = childId as ThoughtId + break + } + } + + if ( + !settingsId && + (await client.tree.exists(SETTINGS_TOKEN)) && + (await client.tree.parent(SETTINGS_TOKEN)) === EM_TOKEN + ) { + settingsId = SETTINGS_TOKEN + } + + if (!settingsId) { + const now = Date.now() + await client.local.insert( + replicaId, + EM_TOKEN, + SETTINGS_TOKEN, + { type: 'last' }, + encodeThoughtPayload({ + value: SETTINGS_VALUE, + created: now, + lastUpdated: now, + updatedBy: '', + }), + createTreecrdtLocalWriteOptions(), + ) + settingsId = SETTINGS_TOKEN + } + + if (settingsId) { + const now = Date.now() + await upsertLexeme(client, hashThought(SETTINGS_VALUE), { + contexts: [settingsId], + created: now as Timestamp, + lastUpdated: now as Timestamp, + updatedBy: '', + }) + } + + await ensureAttributeChildrenIndexReady(client) +} + +/** Creates a data provider whose operations are permanently bound to one TreeCRDT client. */ +const createClientDataProvider = ({ client, replicaId }: TreecrdtClientIdentity): TreecrdtClientDataProvider => ({ + getLexemeById: key => getLexemeByIdSql(client, key), + getLexemesByIds: keys => getLexemesByIdsSql(client, keys), + getThoughtById: id => getThoughtByIdFromClient(client, id), + getThoughtsByIds: async ids => { + await waitForTestReplicationDelay() + return Promise.all(ids.map(id => getThoughtByIdFromClient(client, id))) + }, + updateThoughts: updates => updateThoughtsForClient({ client, replicaId }, updates), + updateLexemeIndex: lexemeIndex => updateLexemeIndexForClient(client, lexemeIndex), +}) + +/** + * Creates the stable app-facing TreeCRDT data provider. + * + * The runtime supplies its client during initialization. Writes that race initialization wait for it; a failed + * initialization or drop rejects those writes so a later initialization can start cleanly. + */ +const createTreecrdtDataProvider = () => { + let activeDb: TreecrdtClientDataProvider | null = null + let providerReadiness = createProviderReadiness() + + /** Returns the active client provider for reads that are only valid after runtime initialization. */ + const getActiveDb = (): TreecrdtClientDataProvider => { + if (!activeDb) throw new Error('TreeCRDT DataProvider: init not called') + return activeDb + } + + /** Dispatches public writes to the client provider that becomes ready for them. */ + const updateThoughts: DataProvider['updateThoughts'] = async updates => + (await providerReadiness.promise).updateThoughts(updates) + + /** Clears the current client provider, rejects startup writes, and creates fresh readiness state. */ + const resetBinding = (reason: unknown): void => { + providerReadiness.reject(reason) + activeDb = null + providerReadiness = createProviderReadiness() + } + + const db = { + name: 'treecrdt', + getLexemeById: key => getActiveDb().getLexemeById(key), + getLexemesByIds: keys => getActiveDb().getLexemesByIds(keys), + getThoughtById: id => getActiveDb().getThoughtById(id), + getThoughtsByIds: ids => getActiveDb().getThoughtsByIds(ids), + updateThoughts, + // Freeing cache entries remains a no-op before initialization. + freeThought: async _id => undefined, + freeLexeme: async _key => undefined, + updateLexemeIndex: lexemeIndex => getActiveDb().updateLexemeIndex(lexemeIndex), + } satisfies Omit + + /** Seeds the supplied client, creates its provider, and then releases queued startup writes. */ + const bindClient = async ( + client: TreecrdtClient, + replicaId: Uint8Array, + materialization?: ThoughtspaceMaterializationBridge, + ): Promise<() => void> => { + if (activeDb) throw new Error('TreeCRDT DataProvider: client already bound') + await initializeThoughtspaceStorage(client, replicaId) + + const clientDb = createClientDataProvider({ client, replicaId }) + const materializationContext = materialization ? { bridge: materialization, client, db: clientDb } : null + + const unsubscribeMaterialized = client.onMaterialized(event => { + // Local writes are already reflected optimistically. Other materialization uses the exact provider and client + // that created this subscription, even if the app later binds a new client. + if (isTreecrdtLocalMaterialization(event) || !materializationContext) return + + void enqueueMaterializedThoughtsToStore(event, materializationContext).catch(err => + console.error('TreeCRDT materialized UI sync failed', err), + ) + }) + + activeDb = clientDb + providerReadiness.resolve(clientDb) + let subscribed = true + return () => { + if (!subscribed) return + subscribed = false + unsubscribeMaterialized() + } + } + + return { + db, + bindClient, + resetBinding, + } +} + +export default createTreecrdtDataProvider diff --git a/src/data-providers/treecrdt/writeBarrier.ts b/src/data-providers/treecrdt/writeBarrier.ts new file mode 100644 index 00000000000..030f786dcca --- /dev/null +++ b/src/data-providers/treecrdt/writeBarrier.ts @@ -0,0 +1,72 @@ +import type { LocalWriteOptions, MaterializationEvent } from '@treecrdt/interface/engine' + +let pendingTreecrdtWrite = Promise.resolve() +let pendingTreecrdtWriteError: unknown = null +let pendingTreecrdtWriteVersion = 0 +let localWriteCounter = 0 + +const localWriteSourceId = + typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function' + ? crypto.randomUUID() + : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}` + +const localWriteIdPrefix = `em-local:${localWriteSourceId}:` + +/** + * Queues em -> TreeCRDT persistence work and exposes an idle barrier for materialization refreshes. + * This is a local ordering guard, not a CRDT requirement; it keeps app-state refreshes from racing local persistence. + */ +export function withTreecrdtWriteBarrier(work: () => Promise): Promise { + pendingTreecrdtWriteVersion += 1 + const run = pendingTreecrdtWrite.then(work, work) + pendingTreecrdtWrite = run.then( + () => undefined, + err => { + pendingTreecrdtWriteError = err + }, + ) + return run +} + +/** Monotonically increases whenever TreeCRDT persistence work is queued. */ +export const getTreecrdtWriteBarrierVersion = (): number => pendingTreecrdtWriteVersion + +/** Waits until TreeCRDT persistence is idle, including work queued while waiting. */ +export async function waitForTreecrdtWriteBarrier(): Promise { + let pending: Promise + do { + pending = pendingTreecrdtWrite + await pending + } while (pending !== pendingTreecrdtWrite) + + if (pendingTreecrdtWriteError) { + const err = pendingTreecrdtWriteError + pendingTreecrdtWriteError = null + throw err + } +} + +/** Creates local write metadata used to identify materialization events already applied optimistically by the app. */ +export function createTreecrdtLocalWriteOptions(): LocalWriteOptions { + localWriteCounter += 1 + return { writeId: `${localWriteIdPrefix}${localWriteCounter}` } +} + +/** True when a materialization event was produced by this tab's own optimistic TreeCRDT write. */ +export const isTreecrdtLocalMaterialization = (event: MaterializationEvent): boolean => { + return ( + event.changes.length > 0 && + event.changes.every(change => { + const writeIds = change.source?.writeIds + return !!writeIds?.length && writeIds.every(writeId => writeId.startsWith(localWriteIdPrefix)) + }) + ) +} + +export default { + createTreecrdtLocalWriteOptions, + getTreecrdtWriteBarrierVersion, + isTreecrdtLocalMaterialization, + waitForTreecrdtWriteBarrier, + withTreecrdtWriteBarrier, +} diff --git a/src/data-providers/yjs/documentNameEncoder.ts b/src/data-providers/yjs/documentNameEncoder.ts deleted file mode 100644 index 28c12da3da5..00000000000 --- a/src/data-providers/yjs/documentNameEncoder.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-disable import/prefer-default-export */ -import Index from '../../@types/IndexType' -import ThoughtId from '../../@types/ThoughtId' - -type DocType = 'doclog' | 'permissions' | 'thought' | 'lexeme' - -// DocType abbreviations to save storage space -const docTypeAbbrev: Index = { - t: 'thought', - l: 'lexeme', -} - -/** Generates a documentName for a thought. */ -export const encodeThoughtDocumentName = (tsid: string, key: string) => `${tsid}/t/${key}` - -/** Generates a documentName for a lexeme. */ -export const encodeLexemeDocumentName = (tsid: string, key: string) => `${tsid}/l/${key}` - -/** Generates a permissions documentName. */ -export const encodePermissionsDocumentName = (tsid: string) => `${tsid}/permissions` - -/** Extracts the parts from a document name. */ -export const parseDocumentName = ( - documentName: string, -): { - /** Defined for all documents. Set to 'permissions' for server permissions doc. */ - tsid: string - /** Defined for all documents except subdocs. */ - type: DocType | undefined - /** Only defined for Thought and Lexeme documents. */ - id: ThoughtId | string | undefined - /** Only defined for doclog subdocs. */ - blockId: string | undefined -} => { - const [tsid, type, id, blockId] = documentName.split('/') - // the server permissions doc has a simple docName of 'permissions', so we need to set the type manually instead of extracting it from the docName - const typeNormalized = tsid === 'permissions' ? 'permissions' : docTypeAbbrev[type] || type || '' - return { tsid, type: typeNormalized, id, blockId } -} diff --git a/src/data-providers/yjs/index.ts b/src/data-providers/yjs/index.ts deleted file mode 100644 index 7d06deb2bd3..00000000000 --- a/src/data-providers/yjs/index.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* eslint-disable import/prefer-default-export */ -import { nanoid } from 'nanoid' -import { IndexeddbPersistence } from 'y-indexeddb' -import * as Y from 'yjs' -import storage from '../../util/storage' -import { encodePermissionsDocumentName } from './documentNameEncoder' - -// stores the permissions for the entire thoughtspace as Index (indexed by access token) -// only accessible by owner -export const permissionsClientDoc = new Y.Doc() - -// Define a secret access token for this device. -// Used to authenticate a connection to the y-websocket server. -export const accessTokenLocal = storage.getItem('accessToken', () => nanoid(21)) - -// Define a unique tsid (thoughtspace id) that is used as the default yjs doc id. -// This can be shared with ?share={docId} when connected to a y-websocket server. -export const tsidLocal = storage.getItem('tsid', () => nanoid(21)) - -// Access a shared document when the URL contains share=DOCID&. -// Otherwise use the tsid stored on the device. -// window.location may be undefined in puppeteer if this module is imported before happy-dom is set up, so guard against undefined despite the truthy type. -export const tsidShared = new URLSearchParams(window.location?.search).get('share') -const accessTokenShared = new URLSearchParams(window.location?.search).get('auth') - -export const tsid = tsidShared || tsidLocal -export const accessToken = accessTokenShared || accessTokenLocal - -/** A public key that is a secure hash of the access token. Not available until clientIDReady resolves. */ -export let clientId = '' - -/** Encodes binary data in base64. */ -async function bufferToBase64(buffer: ArrayBuffer) { - // use a FileReader to generate a base64 data URI: - const base64url = await new Promise(resolve => { - const reader = new FileReader() - reader.onload = () => resolve(reader.result as string) - reader.readAsDataURL(new Blob([buffer])) - }) - // remove the `data:...;base64,` part from the start - return base64url.slice(base64url.indexOf(',') + 1) -} - -/** Resolves when the clientId is available to use synchronously. */ -export const clientIdReady = ( - crypto.subtle - ? crypto.subtle.digest('SHA-256', new TextEncoder().encode(accessToken)).then(bufferToBase64) - : // fall back to nanoid if crypto.subtle is not available - Promise.resolve(nanoid()) -).then(s => { - clientId = s - return s -}) - -// Disable IndexedDB during tests because of TransactionInactiveError in fake-indexeddb. -if (import.meta.env.MODE !== 'test') { - new IndexeddbPersistence(encodePermissionsDocumentName(tsid), permissionsClientDoc) -} diff --git a/src/data-providers/yjs/permissionsModel.ts b/src/data-providers/yjs/permissionsModel.ts deleted file mode 100644 index c4ce12815cb..00000000000 --- a/src/data-providers/yjs/permissionsModel.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { nanoid } from 'nanoid' -import Routes from '../../@types/Routes' -import Share from '../../@types/Share' -import { alertActionCreator as alert } from '../../actions/alert' -import { clearActionCreator } from '../../actions/clear' -import { accessTokenLocal, permissionsClientDoc } from '../../data-providers/yjs/index' -import { clear } from '../../data-providers/yjs/thoughtspace' -import store from '../../stores/app' -import storage from '../../util/storage' -import timestamp from '../../util/timestamp' - -const permissionsMap = permissionsClientDoc.getMap() - -// permissions model that waps permissionsClientDoc -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const permissionsModel: { [key in keyof Routes['share']]: any } = { - add: ({ name, role }: Pick) => { - const accessToken = nanoid() - permissionsMap.set(accessToken, { - created: timestamp(), - name: name || '', - role, - }) - store.dispatch(alert(`Added ${name ? `"${name}"` : 'device'}`)) - return { accessToken } - }, - delete: (accessToken: string, { name }: { name?: string } = {}) => { - permissionsMap.delete(accessToken) - - // removed other device - if (accessToken !== accessTokenLocal) { - store.dispatch(alert(`Removed ${name ? `"${name}"` : 'device'}`)) - } - // removed current device when there are others - else if (permissionsMap.size > 1) { - store.dispatch([clearActionCreator(), alert(`Removed this device from the thoughtspace`)]) - } - // remove last device - else { - storage.clear() - clear() - store.dispatch(clearActionCreator()) - - // TODO: Do a full reset without refreshing the page. - window.location.reload() - } - }, - update: (accessToken: string, { name, role }: Share) => { - const permission = permissionsMap.get(accessToken)! - permissionsMap.set(accessToken, { - ...(permission || null), - created: timestamp(), - ...(name ? { name } : null), - ...(role ? { role } : null), - }) - store.dispatch(alert(`${name ? ` "${name}"` : 'Device '} updated`)) - }, -} - -export default permissionsModel diff --git a/src/data-providers/yjs/thoughtspace.ts b/src/data-providers/yjs/thoughtspace.ts deleted file mode 100644 index 6260e8bdcd2..00000000000 --- a/src/data-providers/yjs/thoughtspace.ts +++ /dev/null @@ -1,1136 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -// there are multiple function callling it self (recursive) so we just disable the lint error -/* eslint-disable @typescript-eslint/no-use-before-define */ -import { IndexeddbPersistence, clearDocument } from 'y-indexeddb' -import * as Y from 'yjs' -import Index from '../../@types/IndexType' -import Lexeme from '../../@types/Lexeme' -import Path from '../../@types/Path' -import PushBatch from '../../@types/PushBatch' -import Thought from '../../@types/Thought' -import ThoughtId from '../../@types/ThoughtId' -import Timestamp from '../../@types/Timestamp' -import ValueOf from '../../@types/ValueOf' -import { UpdateThoughtsOptions } from '../../actions/updateThoughts' -import { ABSOLUTE_TOKEN, EM_TOKEN, HOME_TOKEN, ROOT_CONTEXTS, ROOT_PARENT_ID } from '../../constants' -import testFlags from '../../e2e/testFlags' -import groupObjectBy from '../../util/groupObjectBy' -import hashThought from '../../util/hashThought' -import mergeBatch from '../../util/mergeBatch' -import nonNull from '../../util/nonNull' -import sleep from '../../util/sleep' -import taskQueue, { TaskQueue } from '../../util/taskQueue' -import throttleConcat from '../../util/throttleConcat' -import { DataProvider } from '../DataProvider' -import { encodeLexemeDocumentName, encodeThoughtDocumentName, parseDocumentName } from './documentNameEncoder' - -/********************************************************************** - * Types - **********************************************************************/ - -/** A thought that is persisted to storage. */ -interface ThoughtDb { - // archived - a?: Timestamp - // created - c: Timestamp - // lastUpdated - l: Timestamp - // childrenMap - m: Index - // parentId - p: ThoughtId - // rank - r: number - // updatedBy - u: string - // value - v: string -} - -/** A Lexeme database type that defines contexts as separate keys. */ -type LexemeDb = { - // created - c: Timestamp - // lastUpdated - l: Timestamp - // updatedBy - u: string - // contexts - x: Index -} & { - // mapped to docKey to allow co-location of children in db - [key in `cx-${string}`]: string | null -} - -// YMap takes a generic type representing the union of values -// Individual values must be explicitly type cast, e.g. thoughtMap.get('m') as Y.Map -type ThoughtYjs = ValueOf> | Y.Map -type LexemeYjs = ValueOf> | ThoughtId - -/** A partial YMapEvent that can be more easily constructed than a complete YMapEvent. */ -interface SimpleYMapEvent { - target: Y.Map - transaction: { - origin: any - } -} - -/** Creates a promise that is resolved with promise.resolve and rejected with promise.reject. */ -interface ResolvablePromise extends Promise { - resolve: (arg: T) => void - reject: (err: E) => void -} - -export interface ThoughtspaceOptions { - accessToken: string - /** Used to seed docKeys, otherwise replicateThought triggered from initializeCursor will fail. */ - cursor: Path | null - isLexemeLoaded: (key: string, lexeme: Lexeme | undefined) => Promise - isThoughtLoaded: (thought: Thought | undefined) => Promise - onThoughtIDBSynced: (thought: Thought | undefined, options: { background: boolean }) => void - onError: (message: string, ...objects: any[]) => void - onProgress: (args: { replicationProgress?: number; savingProgress?: number }) => void - onThoughtChange: (thought: Thought) => void - onThoughtReplicated: (id: ThoughtId, thought: Thought | undefined) => void - onUpdateThoughts: (args: UpdateThoughtsOptions) => void - tsid: string - tsidShared: string | null -} - -type ThoughtspaceConfig = ThoughtspaceOptions & { - updateQueue: TaskQueue -} - -/********************************************************************** - * Constants - **********************************************************************/ - -/** Number of milliseconds after which to retry a failed IndexeddbPersistence sync. */ -const IDB_ERROR_RETRY = 1000 - -/** Number of milliseconds to throttle dispatching updateThoughts on thought/lexeme change. */ -const UPDATE_THOUGHTS_THROTTLE = 100 - -/** Maps ThoughtDb keys to Thought keys. */ -// const thoughtKeyFromDb = { -// l: 'lastUpdated', -// m: 'childrenMap', -// p: 'parentId', -// r: 'rank', -// u: 'updatedBy', -// v: 'value', -// a: 'archived', -// } as const - -/** Maps Thought keys to ThoughtDb keys. */ -const thoughtKeyToDb = { - lastUpdated: 'l', - childrenMap: 'm', - parentId: 'p', - rank: 'r', - updatedBy: 'u', - value: 'v', - archived: 'a', -} as const - -/** Maps Lexeme keys to LexemeDb keys. */ -const lexemeKeyToDb = { - created: 'c', - lastUpdated: 'l', - contexts: 'x', - updatedBy: 'u', -} as const - -/** Maps LexemeDb keys to Lexeme keys. */ -const lexemeKeyFromDb = { - c: 'created', - l: 'lastUpdated', - x: 'contexts', - u: 'updatedBy', -} as const - -/********************************************************************** - * Helper Functions - **********************************************************************/ - -/** Attaches a resolve function to a promise. */ -const resolvable = () => { - let _resolve: (value: T) => void - let _reject: (err: E) => void - const promise = new Promise((resolve, reject) => { - _resolve = resolve - _reject = reject - }) - const p = promise as ResolvablePromise - p.resolve = _resolve! - p.reject = _reject! - return promise as ResolvablePromise -} - -/** Dispatches updateThoughts with all updates in the throttle period. */ -const updateThoughtsThrottled = throttleConcat((batches: PushBatch[]) => { - const merged = batches.reduce(mergeBatch, { - thoughtIndexUpdates: {}, - lexemeIndexUpdates: {}, - lexemeIndexUpdatesOld: {}, - }) - - // dispatch on next tick, since the leading edge is synchronous and can be triggered during a reducer - setTimeout(() => { - config.then(({ onUpdateThoughts: updateThoughts }) => - updateThoughts?.({ ...merged, local: false, remote: false, repairCursor: true }), - ) - }) -}, UPDATE_THOUGHTS_THROTTLE) - -/** Convert a Thought to a ThoughtDb for efficient storage. */ -const thoughtToDb = (thought: Thought): ThoughtDb => ({ - c: thought.created, - l: thought.lastUpdated, - m: thought.childrenMap, - p: thought.parentId, - r: thought.rank, - u: thought.updatedBy, - v: thought.value, - ...(thought.archived ? { a: thought.archived } : null), -}) - -/********************************************************************** - * Module variables - **********************************************************************/ - -// Map of all YJS thought Docs loaded into memory. -// Keyed by docKey (See docKeys below). -const thoughtDocs = new Map() -const thoughtPersistence = new Map() -// Thoughts retained until freeThought is called. These are thoughts that are replicated in the foreground and kept in Redux State. -const thoughtRetained = new Set() -const thoughtIDBSynced = new Map>() -const thoughtWebsocketSynced = new Map>() - -const lexemeDocs = new Map() -const lexemePersistence = new Map() -// Lexemes retained until freeLexeme is called. These are lexemes that are replicated in the foreground and kept in Redux State. -const lexemeRetained: Set = new Set() -const lexemeIDBSynced = new Map>() -const lexemeWebsocketSynced = new Map>() - -/** Map all known thought ids to document keys. This allows us to co-locate children in a single Doc without changing the DataProvider API. Currently the thought's parentId is used, and a special ROOT_PARENT_ID value for the root and em contexts. */ -const docKeys: Map = new Map([...ROOT_CONTEXTS, EM_TOKEN].map(id => [id, ROOT_PARENT_ID])) - -/********************************************************************** - * Module variables - **********************************************************************/ - -/** The thoughtspace config that is resolved after init is called. Used to pass objects and callbacks into the thoughtspace from the UI. After they are initialized, they can be accessed synchronously on the module-level config variable. This avoids timing issues with concurrent replicateChildren calls that need conflict to check if the doc already exists. */ -const config = resolvable() - -/** Cache the config for synchronous access. This is needed by replicateChildren to set thoughtDocs synchronously, otherwise it will not be idempotent. */ -let configCache: ThoughtspaceConfig - -/** Initialize the thoughtspace with event handlers and selectors to call back to the UI. */ -export const init = async (options: ThoughtspaceOptions) => { - const { - isLexemeLoaded, - isThoughtLoaded, - onError, - onProgress, - onThoughtChange, - onThoughtIDBSynced, - onThoughtReplicated, - onUpdateThoughts, - } = options - - const accessToken = await options.accessToken - const tsid = await options.tsid - const tsidShared = await options.tsidShared - const cursor = await options.cursor - - // generate docKeys for cursor, otherwise replicateThought will fail - if (cursor) { - cursor.forEach((id, i) => docKeys.set(id, cursor[i - 1] ?? HOME_TOKEN)) - } - - // limit the number of thoughts and lexemes that are updated in the Y.Doc at once - const updateQueue = taskQueue({ - // concurrency above 16 make the % go in bursts as batches of tasks are processed and awaited all at once - // this may vary based on # of cores and network conditions - concurrency: 16, - onStep: ({ completed, expected, total }) => { - const estimatedTotal = expected || total - onProgress({ savingProgress: completed / estimatedTotal }) - }, - onEnd: () => { - onProgress({ savingProgress: 1 }) - }, - onError: (err: Error) => { - onError( - `Oops! That's embarrassing. I was not able to save the last change. You should restart the app to avoid additional data loss. Error: ${err.message}`, - err, - ) - }, - }) - - configCache = { - accessToken, - cursor, - isLexemeLoaded, - isThoughtLoaded, - onError, - onProgress, - onThoughtChange, - onThoughtIDBSynced, - onThoughtReplicated, - onUpdateThoughts, - tsid, - tsidShared, - updateQueue, - } - - config.resolve(configCache) -} - -/********************************************************************** - * Methods - **********************************************************************/ - -/** Updates a yjs thought doc. Converts childrenMap to a nested Y.Map for proper children merging. Resolves when transaction is committed and IDB is synced (not when websocket is synced). */ -// NOTE: Ids are added to the thought log in updateThoughts for efficiency. If updateThought is ever called outside of updateThoughts, we will need to push individual thought ids here. -export const updateThought = async (id: ThoughtId, thought: Thought): Promise => { - let docKey = docKeys.get(id) - let lexemeOldIDBSynced: Promise | undefined - let thoughtOldIDBSynced: Promise | undefined - if (docKey) { - // When a thought changes parents, we need to delete it from the old parent Doc and update the docKey. - // Unfortunately, transactions on two different Documents are not atomic, so there is a possibility that one will fail and the other will succeed, resulting in an invalid tree. - if (docKey !== thought.parentId) { - const lexemeKey = hashThought(thought.value) - const lexemeDoc = lexemeDocs.get(lexemeKey) - if (!lexemeDoc && id !== HOME_TOKEN && id !== EM_TOKEN) { - // TODO: Why does throwing an error get suppressed? - console.error(`updateThought: Missing Lexeme doc for thought ${id}`) - return - } - - // delete from old parent - const thoughtDocOld = thoughtDocs.get(docKey) - thoughtDocOld?.transact(() => { - const yChildren = thoughtDocOld.getMap>('children') - yChildren.delete(id) - docKey = thought.parentId - docKeys.set(id, docKey) - }, thoughtDocOld.clientID) - - // subscribe to thoughtPersistence directly since thoughtIDBSynced can await websocketSynced on new devices - thoughtOldIDBSynced = thoughtPersistence.get(docKey)?.whenSynced - - // update Lexeme context docKey - if (lexemeDoc) { - lexemeDoc.transact(() => { - const lexemeMap = lexemeDoc.getMap() - lexemeMap.set(`cx-${id}`, thought.parentId) - }, lexemeDoc.clientID) - // subscribe to lexemePersistence directly since lexemeIDBSynced can await websocketSynced on new devices - lexemeOldIDBSynced = lexemePersistence.get(lexemeKey)?.whenSynced - } - } - } else { - docKey = thought.parentId - docKeys.set(id, docKey) - Object.values(thought.childrenMap).forEach(childId => { - docKeys.set(childId, id) - }) - } - - // Get the thought Doc if it has been cached, or initiate a replication. - // Do not wait for thought to full replicate. - const thoughtDoc = - thoughtDocs.get(docKey) || - (await new Promise(resolve => { - replicateThought(id, { onDoc: resolve }) - })) - - // subscribe to thoughtPersistence directly since thoughtIDBSynced can await websocketSynced on new devices - const thoughtNewIdbSynced = thoughtPersistence.get(docKey)?.whenSynced.catch((err: Error) => { - // AbortError happens if the app is closed during replication. - // Not sure if the timeout will be preserved, but at least we can retry. - if (err.name === 'AbortError' || err.message.includes('[AbortError]')) { - setTimeout(() => { - updateThought(id, thought) - }, IDB_ERROR_RETRY) - return - } - config.then(({ onError }) => { - onError?.(`Error saving thought ${id}: ${err.message}`, err) - }) - }) - - thoughtDoc.transact(() => { - // Set parent docKey directly on the thought Doc. - // This is needed to traverse up the ancestor path of tangential contexts. - const yThought = thoughtDoc.getMap('thought') - const parentDocKey = - thought.parentId === ROOT_PARENT_ID ? null : (docKeys.get(thought.parentId) as ThoughtId | undefined) - if (parentDocKey === undefined) { - // TODO: Since pushQueue batchns are no longer merged, this occurs consistently in the CI, but not on local machine. - console.error(`updateThought: Missing docKey for parent ${thought.parentId} of thought ${id}`) - } else { - yThought.set('docKey', parentDocKey) - } - - const yChildren = thoughtDoc.getMap>('children') - if (!yChildren.has(id)) { - yChildren.set(id, new Y.Map()) - } - const thoughtMap = yChildren.get(id)! - const thoughtDb = thoughtToDb(thought) - ;(Object.keys(thoughtDb) as (keyof ThoughtDb)[]).forEach(key => { - // merge childrenMap Y.Map - if (key === thoughtKeyToDb.childrenMap) { - const value = thoughtDb[key] - let childrenMap = thoughtMap.get('childrenMap') as Y.Map - - // create new Y.Map for new thought - if (!childrenMap) { - childrenMap = new Y.Map() - thoughtMap.set(thoughtKeyToDb.childrenMap, childrenMap) - } - - // delete children from the yjs thought that are no longer in the state thought - childrenMap.forEach((childKey: string, childId: string) => { - if (!value[childId]) { - childrenMap.delete(childId) - } - }) - - // add children that are not in the yjs thought - Object.entries(thoughtDb[thoughtKeyToDb.childrenMap]).forEach(([key, childId]) => { - if (!childrenMap.has(key)) { - childrenMap.set(key, childId) - } - }) - } - // other keys - else { - const value = thoughtDb[key] - // Only set a value if it has changed. - // Otherwise YJS adds another update. - if (value !== thoughtMap.get(key)) { - thoughtMap.set(key, value) - } - } - }) - }, thoughtDoc.clientID) - - await Promise.all([thoughtNewIdbSynced, thoughtOldIDBSynced, lexemeOldIDBSynced]) -} - -/** Updates a yjs lexeme doc. Converts contexts to a nested Y.Map for proper context merging. Resolves when transaction is committed and IDB is synced (not when websocket is synced). */ -// NOTE: Keys are added to the lexeme log in updateLexemes for efficiency. If updateLexeme is ever called outside of updateLexemes, we will need to push individual keys here. -export const updateLexeme = async ( - key: string, - lexemeNew: Lexeme, - /** The old Lexeme to determine context deletions. Should be undefined only if Lexeme is completely new. */ - // TODO: Pass the diffed contexts all the way through from updateThoughts. - // The YJS Lexeme should be the same as the old Lexeme in State, since they are synced. - // If the Lexeme has not yet been loaded from YJS, then we can ignore deletions, as a Lexeme normally cannot be deleted before it has been loaded. Unless the user creates and deletes the Lexeme so quickly that IDB is still loading (?). - // In light of all that, it would be better to get the deletions directly from the reducer. - lexemeOld: Lexeme | undefined, -): Promise => { - if (!lexemeDocs.has(key)) { - // TODO: Why is replication awaited here, but not in updateThought? - await replicateLexeme(key) - } - const lexemeDoc = lexemeDocs.get(key) - const contextsOld = new Set(lexemeOld?.contexts) - - // The Lexeme may be deleted if the user creates and deletes a thought very quickly - if (!lexemeDoc) return - - // subscribe to lexemePersistence directly since lexemeIDBSynced can await websocketSynced on new devices - const idbSynced = lexemePersistence.get(key)?.whenSynced.catch((err: Error) => { - // AbortError happens if the app is closed during replication. - // Not sure if the timeout will be preserved, but at least we can retry. - if (err.name === 'AbortError' || err.message.includes('[AbortError]')) { - setTimeout(() => { - updateLexeme(key, lexemeNew, lexemeOld) - }, IDB_ERROR_RETRY) - return - } - config.then(({ onError }) => { - const message = `Error saving lexeme: ${err.message}` - console.error(message, lexemeNew) - onError?.(message, err) - }) - }) - - lexemeDoc.transact(() => { - const lexemeMap = lexemeDoc.getMap() - ;(Object.keys(lexemeNew) as (keyof Lexeme)[]).forEach(key => { - if (key === 'contexts') { - const value = lexemeNew[key] - const contextsNew = new Set(value) - - value.forEach(cxid => { - if (!contextsOld.has(cxid)) { - const docKey = docKeys.get(cxid) - if (!docKey) { - throw new Error(`updateLexeme: Missing docKey for context ${cxid} in Lexeme.`) - } - lexemeMap.set(`cx-${cxid}`, docKey) - } - }) - - // delete contexts that have been deleted, i.e. exist in lexemeOld but not lexemeNew - lexemeOld?.contexts.forEach(cxid => { - if (!contextsNew.has(cxid)) { - lexemeMap.delete(`cx-${cxid}`) - } - }) - } else { - const value = lexemeNew[key] - // Only set a value if it has changed. - // Otherwise YJS adds another update. - if (value !== lexemeMap.get(lexemeKeyToDb[key])) { - lexemeMap.set(lexemeKeyToDb[key], value) - } - } - }) - }, lexemeDoc.clientID) - - await idbSynced -} - -/** Handles the Thought observe event. Ignores events from self. */ -const onThoughtChange = (id: ThoughtId) => (e: SimpleYMapEvent) => { - const thoughtDoc = e.target.doc! - if (e.transaction.origin === thoughtDoc.clientID) return - - const thought = getThought(thoughtDoc, id) - if (!thought) return - - // update docKeys of children - Object.values(thought.childrenMap).forEach(childId => { - docKeys.set(childId, id) - }) - - config.then(({ onThoughtChange }) => onThoughtChange?.(thought)) -} - -/** Handles the Lexeme observe event. Ignores events from self. */ -const onLexemeChange = (e: SimpleYMapEvent) => { - const lexemeDoc = e.target.doc! - if (e.transaction.origin === lexemeDoc.clientID) return - - const lexeme = getLexeme(lexemeDoc) - if (!lexeme) return - - // we can assume id is defined since lexeme doc guids are always in the format `${tsid}/lexeme/${id}` - const { id: key } = parseDocumentName(lexemeDoc.guid) as { id: string } - - updateThoughtsThrottled({ - thoughtIndexUpdates: {}, - lexemeIndexUpdates: { - [key]: lexeme, - }, - lexemeIndexUpdatesOld: {}, - }) -} - -/** - * Replicates a thought from the persistence layers to state, IDB, and the Websocket server. If already replicating or replicated, resolves as soon as data is available (depends on background/remote params). The Doc can be updated concurrently while replicating. - * - * Precondition: docKey of id must be cached. - * - * Warning: It is not recommended to run replicateThought in background mode. The Doc is not cached in background mode, so calling replicateThought on multiple siblings will result in multiple replications of the parent. - */ -export const replicateThought = async ( - id: ThoughtId, - { - background, - onDoc, - remote = true, - }: { - /** - * Replicate in the background, meaning: - * - Only update Redux state if thought is visible. - * - Do not cache Doc or providers in memory. - * - Destroy providers after sync. - * - If remote is also true, does not resolve until websocket replication is complete (e.g. replicationController). - */ - background?: boolean - /** Callback with the doc as soon as it has been instantiated. */ - onDoc?: (doc: Y.Doc) => void - /** Sync with websocket server. Set to false during export. Default: true. */ - remote?: boolean - } = {}, -): Promise => { - const docKey = docKeys.get(id) - if (!docKey) { - throw new Error(`replicateThought: Missing docKey for thought ${id}`) - } - const children = await replicateChildren(docKey, { background, onDoc, remote }) - const child = children?.find(child => child.id === id) - return child -} - -/** - * Replicates all thoughts contained within a Thought doc. - * - * @see replicateThought - */ -export const replicateChildren = async ( - docKey: string, - { - background, - onDoc, - remote = true, - }: { - background?: boolean - onDoc?: (doc: Y.Doc) => void - remote?: boolean - } = {}, -): Promise => { - // Inject test delay if configured (for e2e testing slow data loading) - if (!!testFlags.replicationDelay) { - await sleep(testFlags.replicationDelay) - } - - // Only await the config promise once. Otherwise the initial call to replicateChildren for a given docKey will not set thoughtDocs synchronously, and we will lose memoization of concurrent calls. - if (!configCache) { - await config - } - const { onError, onThoughtIDBSynced, tsid } = configCache - const documentName = encodeThoughtDocumentName(tsid, docKey) - const doc = thoughtDocs.get(docKey) || new Y.Doc({ guid: documentName }) - onDoc?.(doc) - - // Foreground replication retains the thought in the cache even when replication completes. - // The thought will only be removed after freeThoughts is called. - if (!background) { - thoughtRetained.add(docKey) - } - - // If the doc is cached, return as soon as the appropriate providers are synced. - // Disable IDB during tests because of TransactionInactiveError in fake-indexeddb. - // Disable websocket during tests because of infinite loop in sinon runAllAsync. - if (thoughtDocs.get(docKey)) { - // The Doc exists, but it may not be populated yet if replication has not completed. - // Wait for the appropriate replication to complete before accessing children. - if (background && remote) { - await thoughtWebsocketSynced.get(docKey) - } else { - await thoughtIDBSynced.get(docKey) - } - - const children = getChildren(doc) - - // TODO: There may be a bug in freeThought, because we should not have to recreate the docKeys if the doc is already cached. - // Without this, a missing docKey error will occur if a thought is re-loaded after being deallocated. - children?.forEach(child => { - docKeys.set(child.id, docKey) - Object.values(child.childrenMap).forEach(grandchildId => { - docKeys.set(grandchildId, child.id) - }) - }) - - return children - } - - // set up idb and websocket persistence and subscribe to changes - const persistence = new IndexeddbPersistence(documentName, doc) - const idbSynced = persistence.whenSynced - .then(() => { - const children = getChildren(doc) - - // if idb is empty, then we have to wait for websocketSynced before we can get the docKey - const parentDocKey = - docKey === ROOT_PARENT_ID - ? null - : docKey === HOME_TOKEN || docKey === EM_TOKEN - ? ROOT_PARENT_ID - : doc.getMap('thought').get('docKey') - if (parentDocKey) { - docKeys.set(docKey as ThoughtId, parentDocKey) - } - - // update docKeys of children and grandchildren - children?.forEach(child => { - docKeys.set(child.id, docKey) - Object.values(child.childrenMap).forEach(grandchildId => { - docKeys.set(grandchildId, child.id) - }) - }) - - children?.forEach(child => { - onThoughtIDBSynced?.(child, { background: !!background }) - }) - }) - .catch((err: Error) => { - // AbortError happens if the app is closed during replication. - // Not sure if the timeout will be preserved, but we can at least try to re-replicate. - if (err.name === 'AbortError' || err.message.includes('[AbortError]')) { - freeThought(docKey) - setTimeout(() => { - replicateChildren(docKey, { background, onDoc, remote }) - }, IDB_ERROR_RETRY) - return - } - onError?.(`Error loading thought ${docKey} from IndexedDB: ${err.message}`, err) - }) - - // Cache docs, promises, and providers - // Must be done synchronously, before waiting for idbSynced or websocketSynced, so that the cached objects are available immediately for concurrent calls to replicateChildren. - thoughtDocs.set(docKey, doc) - thoughtIDBSynced.set(docKey, idbSynced) - thoughtPersistence.set(docKey, persistence) - - // always wait for IDB to sync - await idbSynced - - // foreground - if (!background) { - // Subscribe to changes after first sync to ensure that pending is set properly. - // If thought is updated as non-pending first (i.e. before pull), then mergeUpdates will not set pending by design. - const yChildren = doc.getMap>('children') - const childrenEntries = [...(yChildren.entries() as IterableIterator<[ThoughtId, Y.Map]>)] - childrenEntries.forEach(([childId, thoughtMap]) => { - thoughtMap.observe(onThoughtChange(childId)) - }) - } - const children = getChildren(doc) - - // If the thought is not retained by foreground replication, deallocate it. - tryDeallocateThought(docKey) - - return children -} - -/** Replicates a Lexeme from the persistence layers to state, IDB, and the Websocket server. Does nothing if the Lexeme is already replicated, or is being replicated. Otherwise creates a new, empty YDoc that can be updated concurrently while syncing. */ -export const replicateLexeme = async ( - key: string, - { - background, - }: { - /** - * Do not store thought doc in memory. - * Do not update thoughtIndex. - * Destroy IndexedDBPersistence after sync. - */ - background?: boolean - } = {}, -): Promise => { - // special contexts do not have Lexemes - // Redux state will store dummy Lexemes with empty contexts, but there is no reason to try to replicate them - if (key === HOME_TOKEN || key === EM_TOKEN || key === ABSOLUTE_TOKEN) return undefined - - // Do not await config if it is already cached. Otherwise the initial call to replicateLexeme will not set thoughtDocs synchronously and concurrent calls to replicateLexeme will not be idempotent. - if (!configCache) { - await config - } - const { onError, tsid } = configCache - const documentName = encodeLexemeDocumentName(tsid, key) - const doc = lexemeDocs.get(key) || new Y.Doc({ guid: documentName }) - const lexemeMap = doc.getMap() - - // Foreground replication retains the lexeme in the cache even when replication completes. - // The lexeme will only be removed after freeLexeme is called. - if (!background) { - lexemeRetained.add(key) - } - - // If the doc is cached, return as soon as the appropriate providers are synced. - // Disable IDB during tests because of TransactionInactiveError in fake-indexeddb. - // Disable websocket during tests because of infinite loop in sinon runAllAsync. - if (lexemeDocs.get(key)) { - if (background) { - await lexemeWebsocketSynced.get(key) - } else { - await lexemeIDBSynced.get(key) - } - - return getLexeme(doc) - } - - // set up idb and websocket persistence and subscribe to changes - const persistence = new IndexeddbPersistence(documentName, doc) - - // if replicating in the background, destroy the IndexeddbProvider once synced - const idbSynced = persistence.whenSynced.catch((err: Error) => { - // AbortError happens if the app is closed during replication. - // Not sure if the timeout will be preserved, but we can at least try to re-replicate. - if (err.name === 'AbortError' || err.message.includes('[AbortError]')) { - freeLexeme(key) - setTimeout(() => { - replicateLexeme(key, { background }) - }, IDB_ERROR_RETRY) - return - } - onError?.(`Error loading lexeme ${key}: ${err.message}`, err) - }) as Promise - - // Cache docs, promises, and providers - // Must be done synchronously, before waiting for idbSynced or websocketSynced, so that the cached objects are available immediately for concurrent calls to replicateChildren. - lexemeDocs.set(key, doc) - lexemeIDBSynced.set(key, idbSynced) - lexemePersistence.set(key, persistence) - - // always wait for IDB to sync - await idbSynced - - // foreground - if (!background) { - // subscribe to changes after idbSynced since foreground replicated lexemes are already updated through pull - lexemeMap.observe(onLexemeChange) - } - - // get the Lexeme before we destroy the Doc - const lexeme = getLexeme(doc) - const lexemeRaw = lexemeMap.toJSON() as Index - - // set docKey from Lexeme context to allow tangential contexts to be loaded - ;(Object.keys(lexemeRaw) as (keyof LexemeDb | `cx-${string}`)[]).forEach(key => { - const cxid = key.split('cx-')[1] as ThoughtId | undefined - - if (cxid) { - const docKey = lexemeRaw[key as `cx-${string}`] as ThoughtId - docKeys.set(cxid, docKey) - } - }) - - // If the lexeme is not retained by foreground replication, deallocate it. - tryDeallocateLexeme(key) - - return lexeme -} - -/** Gets all children from a thought Y.Doc. Returns undefined if the doc does not exist. */ -const getChildren = (thoughtDoc: Y.Doc | undefined): Thought[] | undefined => { - if (!thoughtDoc) return undefined - - // If docKey is not set, then the doc does not exist. - // It is important to return undefined here instead of an empty array so that the caller (specifically, replicateChildren) can distinguish between a non-existent thought and a synced thought with no children. It uses that to force remote replication to wait for websocketSynced on the initial replication. - const yThought = thoughtDoc.getMap('thought') - if (!yThought.has('docKey')) return undefined - - const yChildren = thoughtDoc.getMap>('children') - - return [...(yChildren.keys() as IterableIterator)].map(id => getThought(thoughtDoc, id)).filter(nonNull) -} - -/** Gets a Thought from a thought Y.Doc. */ -const getThought = (thoughtDoc: Y.Doc | undefined, id: ThoughtId): Thought | undefined => { - if (!thoughtDoc) return - const yChildren = thoughtDoc.getMap>('children') - const thoughtMap = yChildren.get(id) - if (!thoughtMap || thoughtMap.size === 0) return - const thoughtRaw = thoughtMap.toJSON() as Omit & { - // TODO: Why is childrenMap sometimes a YMap and sometimes a plain object? - // toJSON is not recursive so we need to toJSON childrenMap as well - // It is possible that this was fixed in later versions of yjs after v13.5.41 - [thoughtKeyToDb.childrenMap]: Y.Map | Index - } - return { - childrenMap: - thoughtRaw[thoughtKeyToDb.childrenMap] instanceof Y.Map - ? (thoughtRaw[thoughtKeyToDb.childrenMap] as Y.Map).toJSON() - : (thoughtRaw[thoughtKeyToDb.childrenMap] as Index), - created: thoughtRaw.c, - id, - lastUpdated: thoughtRaw.l, - parentId: thoughtRaw.p, - rank: thoughtRaw.r, - updatedBy: thoughtRaw.u, - value: thoughtRaw.v, - ...(thoughtRaw.a ? { archived: thoughtRaw.a } : null), - } -} - -/** Gets a Lexeme from a lexeme Y.Doc. */ -// SIDE EFFECT: Sets docKeys for contexts. -const getLexeme = (lexemeDoc: Y.Doc | undefined): Lexeme | undefined => { - if (!lexemeDoc) return - const lexemeMap = lexemeDoc.getMap() - if (lexemeMap.size === 0) return - const lexemeRaw = lexemeMap.toJSON() as Index - - // convert LexemeDb to Lexeme - // Lexeme is bult up one key at a time, so accum is a Partial while the final value is assumed to be a complete Lexeme - const lexeme = (Object.keys(lexemeRaw) as (keyof LexemeDb | `cx-${string}`)[]).reduce>( - (acc, key) => { - const cxid = key.split('cx-')[1] as ThoughtId | undefined - - // Set docKey from Lexeme context to allow tangential contexts to be loaded. - if (cxid) { - return { - ...acc, - contexts: [...(acc.contexts || []), cxid], - } - } else { - const keyNonContext = key as Exclude - const value = lexemeRaw[keyNonContext] - return { - ...acc, - [lexemeKeyFromDb[keyNonContext]]: value, - } - } - }, - { contexts: [] }, - ) as Lexeme - - return lexeme -} - -/** Waits until the thought finishes replicating, then deallocates the cached thought and associated providers (without permanently deleting the persisted data). */ -// Note: freeThought and deleteThought are the only places where we use the id as the docKey directly. -// This is because we want to free all of the thought's children, not the thought's siblings, which are contained in the parent Doc accessed via docKeys. -export const freeThought = async (docKey: string): Promise => { - thoughtRetained.delete(docKey) - - // wait for idb replication, otherwise the deletion may not be saved to disk - await thoughtIDBSynced.get(docKey) - - // TODO: How to prevent background replication from getting interrupted by editing? If a user edits a thought while its lexeme is being replicated in the background, then the provider will be destroyed and replication will halt. It should not affect the replication cursors, but will require a refresh to resume. - // However, we cannot wait for websocketSynced when offline. - // await thoughtWebsocketSynced.get(docKey) - - // if the thought is retained again, it means it has been replicated in the foreground, and tryDeallocateThought will be a noop. - await tryDeallocateThought(docKey) -} - -/** Deallocates the cached thought and associated providers (without permanently deleting the persisted data). If the thought is retained, noop. Call freeThought to both safely unretain the thought and trigger deallocation when replication completes. */ -const tryDeallocateThought = async (docKey: string): Promise => { - if (thoughtRetained.has(docKey)) return - - // Destroying the doc does not remove top level shared type observers, so we need to unobserve onLexemeChange. - // YJS logs an error if the event handler does not exist, which can occur when rapidly deleting thoughts. - // Unfortunately there is no way to catch this, since YJS logs it directly to the console, so we have to override the YJS internals. - // https://github.com/yjs/yjs/blob/5db1eed181b70cb6a6d7eab66c7e6d752f70141a/src/utils/EventHandler.js#L58 - // const yChildren = thoughtDocs.get(id)?.getMap>('children') - // yChildren?.forEach(thoughtMap => { - // const listeners = thoughtMap?._eH.l.slice(0) || [] - // if (listeners.some(l => l === onThoughtChange)) { - // thoughtMap?.unobserve(onThoughtChange) - // } - // }) - - // Remove children docKeys. - // They may have already been deleted by deleteThought, but we need to also delete them here to handle thought deallocation independent from delete. - // TODO: Why is not safe to remove the thought docKey here? Doing that causes replication on a new device to throw "Missing docKey for thought". - const thoughtDoc = thoughtDocs.get(docKey) - const yChildren = thoughtDoc?.getMap>('children') - yChildren?.forEach(thoughtMap => { - const childId = thoughtMap.get('id') as ThoughtId - docKeys.delete(childId) - }) - - // Destroy doc and websocket provider. - // IndexedDB provider is automatically destroyed when the Doc is destroyed - thoughtDocs.get(docKey)?.destroy() - - // delete from cache - thoughtDocs.delete(docKey) - thoughtPersistence.delete(docKey) - thoughtIDBSynced.delete(docKey) - thoughtWebsocketSynced.delete(docKey) -} - -/** Deletes a thought and clears the doc from IndexedDB. Resolves when local database is deleted. */ -const deleteThought = async (docKey: string): Promise => { - // freeThought and deleteThought are the only places where we use the id as the docKey directly. - // This is because we want to free all of the thought's children, not the thought's siblings, which are contained in the parent Doc accessed via docKeys. - - const { tsid } = await config - const persistence = thoughtPersistence.get(docKey) - - // delete thought from parent - const docKeyParent = docKeys.get(docKey as ThoughtId) - if (docKeyParent) { - const docParent = thoughtDocs.get(docKeyParent) - const yChildren = docParent?.getMap>('children') - yChildren?.delete(docKey) - } - - // delete children docKeys here since freeThought will no longer have access to the deleted children - docKeys.delete(docKey as ThoughtId) - const children = getChildren(thoughtDocs.get(docKey)) - children?.forEach(child => { - docKeys.delete(child.id) - }) - - try { - // if there is no persistence in memory (e.g. because the thought has not been loaded or has been deallocated by freeThought), then we need to manually delete it from the db - const deleted = persistence ? persistence.clearData() : clearDocument(encodeThoughtDocumentName(tsid, docKey)) - await freeThought(docKey) - await deleted - } catch (e: any) { - // Ignore NotFoundError, which indicates that the object stores have already been deleted. - // This is currently expected on load, when the thoughtReplicationCursor is synced with the doclog - // TODO: Update the thoughtReplicationCursor immediateley rather than waiting till the next reload (is the order of updates preserved even when integrating changes from other clients?) - if (e.name !== 'NotFoundError') { - throw e - } - } -} - -/** Waits until the lexeme finishes replicating, then deallocates the cached lexeme and associated providers (without permanently deleting the persisted data). */ -export const freeLexeme = async (key: string): Promise => { - lexemeRetained.delete(key) - await lexemeIDBSynced.get(key) - - // TODO: See freeThought for problems with awaiting websocketSynced. - // await lexemeWebsocketSynced.get(key) - - // if the lexeme is retained again, it means it has been replicated in the foreground, and tryDeallocateLexeme will be a noop. - await tryDeallocateLexeme(key) -} - -/** Deallocates the cached lexeme and associated providers (without permanently deleting the persisted data). If the lexeme is retained, noop. Call freeLexeme to both safely unretain the lexeme and trigger deallocation when replication completes. */ -const tryDeallocateLexeme = async (key: string): Promise => { - if (lexemeRetained.has(key)) return - - // Destroying the doc does not remove top level shared type observers, so we need to unobserve onLexemeChange. - // YJS logs an error if the event handler does not exist, which can occur when rapidly deleting thoughts. - // Unfortunately there is no way to catch this, since YJS logs it directly to the console, so we have to override the YJS internals. - // https://github.com/yjs/yjs/blob/5db1eed181b70cb6a6d7eab66c7e6d752f70141a/src/utils/EventHandler.js#L58 - const lexemeMap: Y.Map | undefined = lexemeDocs.get(key)?.getMap() - const listeners = lexemeMap?._eH.l.slice(0) || [] - if (listeners.some(l => l === onLexemeChange)) { - lexemeMap?.unobserve(onLexemeChange) - } - - // IndeeddbPersistence is automatically destroyed when the Doc is destroyed - lexemeDocs.get(key)?.destroy() - lexemeDocs.delete(key) - lexemePersistence.delete(key) - lexemeIDBSynced.delete(key) - lexemeWebsocketSynced.delete(key) -} - -/** Deletes a Lexeme and clears the doc from IndexedDB. The server-side doc will eventually get deleted by the doclog replicationController. Resolves when the local database is deleted. */ -const deleteLexeme = async (key: string): Promise => { - const { tsid } = await config - const persistence = lexemePersistence.get(key) - - // When deleting a Lexeme, clear out the contexts first to ensure that if a new Lexeme with the same key gets created, it doesn't accidentally pull the old contexts. - const lexemeOld = getLexeme(lexemeDocs.get(key) || persistence?.doc) - if (lexemeOld) { - await updateLexeme(key, { ...lexemeOld, contexts: [] }, lexemeOld) - } - - try { - // if there is no persistence in memory (e.g. because the thought has not been loaded or has been deallocated by freeThought), then we need to manually delete it from the db - const deleted = persistence ? persistence.clearData() : clearDocument(encodeLexemeDocumentName(tsid, key)) - await freeLexeme(key) - await deleted - } catch (e: any) { - // See: deleteThought NotFoundError handler - if (e.name !== 'NotFoundError') { - throw e - } - } -} - -/** Updates shared thoughts and lexemes. Resolves when IDB is synced (not when websocket is synced). */ -// Note: Does not await updates, but that could be added. -export const updateThoughts = async ({ - thoughtIndexUpdates, - lexemeIndexUpdates, - lexemeIndexUpdatesOld, -}: { - thoughtIndexUpdates: Index - lexemeIndexUpdates: Index - lexemeIndexUpdatesOld: Index - schemaVersion: number -}) => { - const { updateQueue } = await config - - // group thought updates and deletes so that we can use the db bulk functions - const { update: thoughtUpdates, delete: thoughtDeletes } = groupObjectBy(thoughtIndexUpdates, (id, thought) => - thought ? 'update' : 'delete', - ) as { - update?: Index - delete?: Index - } - - // group lexeme updates and deletes so that we can use the db bulk functions - const { update: lexemeUpdates, delete: lexemeDeletes } = groupObjectBy(lexemeIndexUpdates, (id, lexeme) => - lexeme ? 'update' : 'delete', - ) as { - update?: Index - delete?: Index - } - - // update - await updateQueue.add([ - ...Object.entries(thoughtUpdates || {}).map( - ([id, thought]) => - () => - updateThought(id as ThoughtId, thought), - ), - ...Object.entries(lexemeUpdates || {}).map( - ([key, lexeme]) => - () => - updateLexeme(key, lexeme, lexemeIndexUpdatesOld[key]), - ), - ]) - - // delete - await updateQueue.add([ - ...(Object.keys(thoughtDeletes || {}) as ThoughtId[]).map(id => () => deleteThought(id)), - ...Object.keys(lexemeDeletes || {}).map(key => () => deleteLexeme(key)), - ]) -} - -/** Clears all thoughts and lexemes from the db. */ -export const clear = async () => { - const deleteThoughtPromises = Array.from(thoughtDocs, ([id]) => deleteThought(id as ThoughtId)) - const deleteLexemePromises = Array.from(lexemeDocs, ([key]) => deleteLexeme(key)) - - await Promise.all([...deleteThoughtPromises, ...deleteLexemePromises]) - - // TODO: reset to initialState, otherwise a missing ROOT error will occur when thought observe is triggered - // const state = initialState() - // const thoughtIndexUpdates = keyValueBy(state.thoughts.thoughtIndex, (id, thought) => ({ - // [id]: thoughtToDb(thought), - // })) - // const lexemeIndexUpdates = state.thoughts.lexemeIndex - - // await updateThoughts({ - // thoughtIndexUpdates, - // lexemeIndexUpdates, - // lexemeIndexUpdatesOld: {}, - // schemaVersion: SCHEMA_LATEST, - // }) -} - -/** Gets a thought from the thoughtIndex. Replicates the thought if not already done. */ -export const getLexemeById = (key: string) => replicateLexeme(key) - -/** Gets multiple thoughts from the lexemeIndex by key. */ -export const getLexemesByIds = (keys: string[]): Promise<(Lexeme | undefined)[]> => Promise.all(keys.map(getLexemeById)) - -/** Gets a thought from the thoughtIndex. Replicates the thought if not already done. */ -export const getThoughtById = async (id: ThoughtId): Promise => { - await replicateThought(id) - const docKey = docKeys.get(id) - return getThought(thoughtDocs.get(docKey!), id) -} - -/** Gets multiple contexts from the thoughtIndex by ids. O(n). */ -export const getThoughtsByIds = (ids: ThoughtId[]): Promise<(Thought | undefined)[]> => - Promise.all(ids.map(getThoughtById)) - -const db: DataProvider = { - clear, - freeLexeme, - freeThought, - getLexemeById, - getLexemesByIds, - getThoughtById, - getThoughtsByIds, - updateThoughts, -} - -export default db diff --git a/src/e2e/iOS/helpers/paste.ts b/src/e2e/iOS/helpers/paste.ts index b6dc13a7e25..d71e438c038 100644 --- a/src/e2e/iOS/helpers/paste.ts +++ b/src/e2e/iOS/helpers/paste.ts @@ -1,5 +1,4 @@ import { HOME_TOKEN } from '../../../constants.js' -import { WindowEm } from '../../../initialize.js' async function paste(text: string): Promise async function paste(pathUnranked: string[], text: string): Promise @@ -13,7 +12,7 @@ async function paste(pathUnranked: string | string[], text?: string): Promise { - const em = window.em as WindowEm + const em = window.em em.testHelpers.importToContext(_pathUnranked, _text) // Wait for React to render the DOM changes diff --git a/src/e2e/puppeteer-environment.ts b/src/e2e/puppeteer-environment.ts index ff8be73cbde..84bb8641cc6 100644 --- a/src/e2e/puppeteer-environment.ts +++ b/src/e2e/puppeteer-environment.ts @@ -8,15 +8,11 @@ const PuppeteerEnvironment: Environment = { async setup(global, options) { builtinEnvironments['happy-dom'].setup(global, options) - // Disable Chrome features that crash GitHub Actions with "Protocol error (Target.createTarget): Target closed." - // See: https://stackoverflow.com/a/66994528/480608 // List of Chromium switches: https://peter.sh/experiments/chromium-command-line-switches/ const args = [ '--deterministic-fetch', '--disable-dev-shm-usage', - '--disable-features=IsolateOrigins', '--disable-setuid-sandbox', - '--disable-site-isolation-trials', '--no-first-run', '--no-sandbox', '--no-zygote', diff --git a/src/e2e/puppeteer/__tests__/ContextView.ts b/src/e2e/puppeteer/__tests__/ContextView.ts index d89a464ee51..c32ae1b72a4 100644 --- a/src/e2e/puppeteer/__tests__/ContextView.ts +++ b/src/e2e/puppeteer/__tests__/ContextView.ts @@ -6,8 +6,10 @@ import scrollBy from '../helpers/scrollBy' import scrollIntoView from '../helpers/scrollIntoView' import waitForEditable from '../helpers/waitForEditable' import waitForThoughtExistInDb from '../helpers/waitForThoughtExistInDb' +import { usePersistentTreecrdtStorage } from '../setup' vi.setConfig({ testTimeout: 20000 }) +usePersistentTreecrdtStorage() // using a puppeteer test since I can't get refresh to work in RTL tests it('load buffered ancestors of contexts when context view is activated', async () => { diff --git a/src/e2e/puppeteer/__tests__/caret.ts b/src/e2e/puppeteer/__tests__/caret.ts index 8ebb2b32fea..e86e34e48a6 100644 --- a/src/e2e/puppeteer/__tests__/caret.ts +++ b/src/e2e/puppeteer/__tests__/caret.ts @@ -20,6 +20,7 @@ import waitForSelector from '../helpers/waitForSelector' import waitForThoughtExistInDb from '../helpers/waitForThoughtExistInDb' import waitUntil from '../helpers/waitUntil' import { page } from '../session' +import { usePersistentTreecrdtStorage } from '../setup' vi.setConfig({ testTimeout: 20000, hookTimeout: 20000 }) @@ -131,29 +132,6 @@ describe('all platforms', () => { expect(offset).toBe(0) }) - it('when cursor is null, clicking on a thought after refreshing page, caret should be set on first click', async () => { - const importText = ` - - a - - b` - - await paste(importText) - await clickThought('a') - - // Set cursor to null - await click('#content') - - await waitForThoughtExistInDb('a') - await waitForThoughtExistInDb('b') - - await refresh() - - await waitForEditable('b') - await clickThought('b') - - const textContext = await getSelection().focusNode?.textContent - expect(textContext).toBe('b') - }) - // https://github.com/cybersemics/em/issues/1568 it('caret at the end of a thought should be preserved on indent and outdent', async () => { const importText = ` @@ -298,6 +276,33 @@ describe('all platforms', () => { }) }) +describe('persistent storage', () => { + usePersistentTreecrdtStorage() + + it('when cursor is null, clicking on a thought after refreshing page, caret should be set on first click', async () => { + const importText = ` + - a + - b` + + await paste(importText) + await clickThought('a') + + // Set cursor to null + await click('#content') + + await waitForThoughtExistInDb('a') + await waitForThoughtExistInDb('b') + + await refresh() + + await waitForEditable('b') + await clickThought('b') + + const textContext = await getSelection().focusNode?.textContent + expect(textContext).toBe('b') + }) +}) + it('clicking backspace when the caret is at the end of a thought should delete a character.', async () => { const importText = ` - first diff --git a/src/e2e/puppeteer/__tests__/cursor.ts b/src/e2e/puppeteer/__tests__/cursor.ts index 69fd2e34bdf..bf4e48ccece 100644 --- a/src/e2e/puppeteer/__tests__/cursor.ts +++ b/src/e2e/puppeteer/__tests__/cursor.ts @@ -7,8 +7,10 @@ import press from '../helpers/press' import refresh from '../helpers/refresh' import waitForEditable from '../helpers/waitForEditable' import waitUntil from '../helpers/waitUntil' +import { usePersistentTreecrdtStorage } from '../setup' vi.setConfig({ testTimeout: 20000, hookTimeout: 20000 }) +usePersistentTreecrdtStorage() /** Returns the persistent tree node that contains the editable with the given value. */ const getTreeNode = async (value: string) => { diff --git a/src/e2e/puppeteer/__tests__/drag-and-drop.ts b/src/e2e/puppeteer/__tests__/drag-and-drop.ts index 0c2be1b7c43..177c4bfcf19 100644 --- a/src/e2e/puppeteer/__tests__/drag-and-drop.ts +++ b/src/e2e/puppeteer/__tests__/drag-and-drop.ts @@ -1,5 +1,4 @@ import path from 'path' -import { WindowEm } from '../../../initialize' import sleep from '../../../util/sleep' import configureSnapshots from '../configureSnapshots' import clickThought from '../helpers/clickThought' @@ -513,9 +512,8 @@ describe('hover expansion', () => { await hideHUD() // inject MOCK_EXPAND_HOVER_DELAY - const em = window.em as WindowEm await page.evaluate(value => { - em.testFlags.expandHoverDelay = value + window.em.testFlags.expandHoverDelay = value }, MOCK_EXPAND_HOVER_DELAY) }) diff --git a/src/e2e/puppeteer/__tests__/pull.ts b/src/e2e/puppeteer/__tests__/pull.ts index f6d920c6195..598bab29127 100644 --- a/src/e2e/puppeteer/__tests__/pull.ts +++ b/src/e2e/puppeteer/__tests__/pull.ts @@ -5,8 +5,10 @@ import press from '../helpers/press' import refresh from '../helpers/refresh' import waitForEditable from '../helpers/waitForEditable' import waitForThoughtExistInDb from '../helpers/waitForThoughtExistInDb' +import { usePersistentTreecrdtStorage } from '../setup' vi.setConfig({ testTimeout: 20000 }) +usePersistentTreecrdtStorage() // TODO: Fix thought buffering after switch to YJS it.skip('load a child after a parent is expanded', async () => { diff --git a/src/e2e/puppeteer/__tests__/render-thoughts.ts b/src/e2e/puppeteer/__tests__/render-thoughts.ts index ce1e98a96d6..c89fef80d31 100644 --- a/src/e2e/puppeteer/__tests__/render-thoughts.ts +++ b/src/e2e/puppeteer/__tests__/render-thoughts.ts @@ -1,4 +1,5 @@ import path from 'path' +import { HOME_TOKEN } from '../../../constants' import configureSnapshots from '../configureSnapshots' import click from '../helpers/click' import clickThought from '../helpers/clickThought' @@ -292,7 +293,7 @@ describe('Superscripts', () => { // get exported html and compress all indentation (whitespace before/after newline) const output = (await exportThoughts({ mimeType: 'text/html' })).replace(/\s*\n\s*/g, '') - const expected = `
                                                                                  • __ROOT__
                                                                                    • This is a Thisthought
                                                                                      • =note
                                                                                        • This is a note
                                                                                  ` + const expected = `
                                                                                  • ${HOME_TOKEN}
                                                                                    • This is a Thisthought
                                                                                      • =note
                                                                                        • This is a note
                                                                                  ` expect(output).toBe(expected) }) diff --git a/src/e2e/puppeteer/__tests__/scroll.ts b/src/e2e/puppeteer/__tests__/scroll.ts index bbceae4c090..137b55bf0ac 100644 --- a/src/e2e/puppeteer/__tests__/scroll.ts +++ b/src/e2e/puppeteer/__tests__/scroll.ts @@ -1,4 +1,4 @@ -import { WindowEm } from '../../../initialize' +import type { PreloadedEmWindow } from '../../../@types' import clickThought from '../helpers/clickThought' import getEditingText from '../helpers/getEditingText' import paste from '../helpers/paste' @@ -7,11 +7,12 @@ import waitForEditable from '../helpers/waitForEditable' import waitForThoughtExistInDb from '../helpers/waitForThoughtExistInDb' import waitUntil from '../helpers/waitUntil' import { page } from '../session' +import { usePersistentTreecrdtStorage } from '../setup' -const em = window.em as WindowEm const MOCK_REPLICATION_DELAY = 100 -vi.setConfig({ testTimeout: 20000, hookTimeout: 20000 }) +vi.setConfig({ testTimeout: 60000, hookTimeout: 20000 }) +usePersistentTreecrdtStorage() describe('scrollCursorIntoView', () => { it('should scroll cursor into view after page refresh with delayed replicateChildren', async () => { @@ -49,6 +50,18 @@ describe('scrollCursorIntoView', () => { await waitForThoughtExistInDb('t') + // Simulate slow TreeCRDT reads during app startup after refresh. + await page.evaluateOnNewDocument(value => { + const preloadedWindow = window as unknown as PreloadedEmWindow + preloadedWindow.em = { + ...preloadedWindow.em, + testFlags: { + ...preloadedWindow.em?.testFlags, + replicationDelay: value, + }, + } + }, MOCK_REPLICATION_DELAY) + await refresh() // Wait for page to be ready after refresh @@ -58,12 +71,6 @@ describe('scrollCursorIntoView', () => { const initialScrollY = await page.evaluate(() => window.scrollY) expect(initialScrollY).toBe(0) - // Set test delay for data replication after refresh - // This simulates the regression case where thoughts are loaded slowly from the database - await page.evaluate(value => { - em.testFlags.replicationDelay = value - }, MOCK_REPLICATION_DELAY) - // Wait for the cursor to be restored to thought 't' await waitForEditable('t') diff --git a/src/e2e/puppeteer/__tests__/sidebar.ts b/src/e2e/puppeteer/__tests__/sidebar.ts index 59c471f96f8..422c78b5113 100644 --- a/src/e2e/puppeteer/__tests__/sidebar.ts +++ b/src/e2e/puppeteer/__tests__/sidebar.ts @@ -13,11 +13,13 @@ expect.extend({ toMatchImageSnapshot: configureSnapshots({ fileName: path.basename(__filename).replace('.ts', '') }), }) -vi.setConfig({ testTimeout: 20000, hookTimeout: 20000 }) +vi.setConfig({ testTimeout: 60000, hookTimeout: 20000 }) /** Screenshot without the toolbar. */ const screenshotWithoutToolbarIcons = async () => { await hideVisibility('[data-testid="toolbar-icon"]') + // New-thought command alerts are transient and unrelated to the sidebar snapshot. + await hideVisibility('[data-testid="alert"]') return screenshot() } diff --git a/src/e2e/puppeteer/__tests__/startup.ts b/src/e2e/puppeteer/__tests__/startup.ts index 9b03c7cce60..1e6751aae6d 100644 --- a/src/e2e/puppeteer/__tests__/startup.ts +++ b/src/e2e/puppeteer/__tests__/startup.ts @@ -1,24 +1,16 @@ -import type { WindowEm } from '../../../initialize' +import type { PreloadedEmWindow } from '../../../@types' import getEditingText from '../helpers/getEditingText' import { page } from '../session' -type StartupWindow = Window & { - em?: { - testFlags?: { - preventInitialize?: boolean - } - } -} - vi.setConfig({ testTimeout: 30000, hookTimeout: 20000 }) it('handles keyboard commands while thoughtspace initialization is delayed', async () => { await page.evaluateOnNewDocument(() => { - const startupWindow = window as StartupWindow - startupWindow.em = { - ...(startupWindow.em ?? {}), + const preloadedWindow = window as unknown as PreloadedEmWindow + preloadedWindow.em = { + ...preloadedWindow.em, testFlags: { - ...startupWindow.em?.testFlags, + ...preloadedWindow.em?.testFlags, preventInitialize: true, }, } @@ -26,7 +18,7 @@ it('handles keyboard commands while thoughtspace initialization is delayed', asy await page.reload({ waitUntil: 'domcontentloaded' }) - await page.waitForFunction(() => typeof (window.em as WindowEm).testFlags.initialize === 'function') + await page.waitForFunction(() => typeof window.em.testFlags.initialize === 'function') await page.waitForFunction(() => !document.querySelector('[aria-label=modal]')) await page.waitForSelector('[aria-label=empty-thoughtspace]') expect(await getEditingText()).toBeUndefined() @@ -38,8 +30,8 @@ it('handles keyboard commands while thoughtspace initialization is delayed', asy expect(await getEditingText()).toBe('') } finally { await page.evaluate(async () => { - const em = window.em as WindowEm - await em.testFlags.initialize?.() + const em = window.em + await em.testFlags.initialize?.({ storage: 'memory' }) em.testFlags.preventInitialize = false }) } diff --git a/src/e2e/puppeteer/__tests__/treecrdt-memory-fallback.ts b/src/e2e/puppeteer/__tests__/treecrdt-memory-fallback.ts new file mode 100644 index 00000000000..2721908d16b --- /dev/null +++ b/src/e2e/puppeteer/__tests__/treecrdt-memory-fallback.ts @@ -0,0 +1,33 @@ +import type { ConsoleMessage } from 'puppeteer' +import keyboard from '../helpers/keyboard' +import press from '../helpers/press' +import waitForThoughtExistInDb from '../helpers/waitForThoughtExistInDb' +import { page } from '../session' +import { usePersistentTreecrdtStorage } from '../setup' + +vi.setConfig({ testTimeout: 60000 }) +usePersistentTreecrdtStorage() + +it('keeps the thoughtspace writable when persistent storage falls back to memory', async () => { + const warnings: string[] = [] + /** Captures the storage fallback warning emitted during initialization. */ + const captureWarning = (message: ConsoleMessage) => { + if (message.type() === 'warn') warnings.push(message.text()) + } + page.on('console', captureWarning) + + // Exceed SQLite's path capacity so the real dedicated-worker OPFS open fails deterministically. + await page.evaluateOnNewDocument(() => localStorage.setItem('tsid', 'x'.repeat(512))) + await page.reload({ waitUntil: 'load' }) + await page.evaluate(() => window.em.testHelpers.waitForInitialized()) + + expect(warnings).toContain( + 'Persistent thoughtspace storage is unavailable. em is using temporary in-memory storage; changes will be lost when this page reloads or closes.', + ) + + await press('Enter') + await keyboard.type('fallback write') + await waitForThoughtExistInDb('fallback write') + + page.off('console', captureWarning) +}) diff --git a/src/e2e/puppeteer/__tests__/treecrdt-single-tab.ts b/src/e2e/puppeteer/__tests__/treecrdt-single-tab.ts new file mode 100644 index 00000000000..cca9a419886 --- /dev/null +++ b/src/e2e/puppeteer/__tests__/treecrdt-single-tab.ts @@ -0,0 +1,116 @@ +import type { ConsoleMessage, Page } from 'puppeteer' +import exportThoughts from '../helpers/exportThoughts' +import paste from '../helpers/paste' +import refresh from '../helpers/refresh' +import waitForThoughtExistInDb from '../helpers/waitForThoughtExistInDb' +import { page, setPage } from '../session' +import { createTreecrdtTestPage, usePersistentTreecrdtStorage } from '../setup' + +vi.setConfig({ testTimeout: 60000 }) +const thoughtspaceStorage = usePersistentTreecrdtStorage() + +const PERSISTENCE_ERROR = /sqlite3_open_v2|SQL logic error|database is locked|Thoughtspace persistence failed|TreeCRDT/i + +/** Waits for startup hydration to attach the persisted thought to the home context. */ +const waitForHydratedThought = (target: Page, value: string): Promise => + target.waitForFunction(expected => !!window.em.getThoughtByContext([expected]), { timeout: 10000 }, value) + +/** Captures page failures that would otherwise be easy to miss behind the bootstrap screen. */ +const captureRuntimeErrors = (target: Page, errors: string[]): void => { + target.on('pageerror', error => errors.push(error instanceof Error ? error.message : String(error))) + target.on('console', (message: ConsoleMessage) => { + if (message.type() === 'error' && PERSISTENCE_ERROR.test(message.text())) errors.push(message.text()) + }) +} + +it('keeps one active tab across refreshes and successive tab handoffs', async () => { + const errors: string[] = [] + const first = page + + await paste('persisted in the first tab') + expect(await exportThoughts()).toContain('persisted in the first tab') + await waitForThoughtExistInDb('persisted in the first tab') + + await refresh() + await waitForHydratedThought(first, 'persisted in the first tab') + expect(await exportThoughts()).toContain('persisted in the first tab') + + const sessionId = await first.evaluate(() => localStorage.getItem('tsid')) + if (!sessionId) throw new Error('Expected the Puppeteer session to define a tsid') + const second = await createTreecrdtTestPage(first.browserContext(), thoughtspaceStorage) + const third = await createTreecrdtTestPage(first.browserContext(), thoughtspaceStorage) + captureRuntimeErrors(second, errors) + captureRuntimeErrors(third, errors) + + await second.goto(first.url(), { waitUntil: 'load' }) + await third.goto(first.url(), { waitUntil: 'load' }) + expect(await second.evaluate(() => localStorage.getItem('tsid'))).toBe(sessionId) + expect(await third.evaluate(() => localStorage.getItem('tsid'))).toBe(sessionId) + await second.waitForSelector('[aria-label=thoughtspace-in-use]') + await third.waitForSelector('[aria-label=thoughtspace-in-use]') + + expect(await second.$('#content')).toBeNull() + expect(await third.$('#content')).toBeNull() + expect(errors).toEqual([]) + + await first.close() + await second.bringToFront() + setPage(second) + + await second.waitForFunction( + async lockName => { + const snapshot = await navigator.locks.query() + return !snapshot.held?.some(lock => lock.name === lockName) + }, + {}, + `em-treecrdt-session:${sessionId}`, + ) + + await Promise.all([ + second.waitForNavigation({ waitUntil: 'load' }), + second.evaluate(() => (document.querySelector('[aria-label=retry-thoughtspace]') as HTMLElement | null)?.click()), + ]) + await second.evaluate(() => window.em.testHelpers.waitForInitialized()) + await second.waitForSelector('#content') + expect(await second.evaluate(() => localStorage.getItem('tsid'))).toBe(sessionId) + await waitForHydratedThought(second, 'persisted in the first tab') + + expect(await exportThoughts()).toContain('persisted in the first tab') + expect(errors).toEqual([]) + + await refresh() + await waitForHydratedThought(second, 'persisted in the first tab') + + expect(await exportThoughts()).toContain('persisted in the first tab') + expect(errors).toEqual([]) + + await third.bringToFront() + await Promise.all([ + third.waitForNavigation({ waitUntil: 'load' }), + third.evaluate(() => (document.querySelector('[aria-label=retry-thoughtspace]') as HTMLElement | null)?.click()), + ]) + await third.waitForSelector('[aria-label=thoughtspace-in-use]') + expect(await third.$('#content')).toBeNull() + + await second.close() + setPage(third) + + await third.waitForFunction( + async lockName => { + const snapshot = await navigator.locks.query() + return !snapshot.held?.some(lock => lock.name === lockName) + }, + {}, + `em-treecrdt-session:${sessionId}`, + ) + + await Promise.all([ + third.waitForNavigation({ waitUntil: 'load' }), + third.evaluate(() => (document.querySelector('[aria-label=retry-thoughtspace]') as HTMLElement | null)?.click()), + ]) + await third.evaluate(() => window.em.testHelpers.waitForInitialized()) + await waitForHydratedThought(third, 'persisted in the first tab') + + expect(await exportThoughts()).toContain('persisted in the first tab') + expect(errors).toEqual([]) +}) diff --git a/src/e2e/puppeteer/__tests__/ui.ts b/src/e2e/puppeteer/__tests__/ui.ts index c798c6f4f5f..16e760533e3 100644 --- a/src/e2e/puppeteer/__tests__/ui.ts +++ b/src/e2e/puppeteer/__tests__/ui.ts @@ -4,6 +4,7 @@ import { KnownDevices } from 'puppeteer' import openCommandCenterCommand from '../../../commands/openCommandCenter' import configureSnapshots from '../configureSnapshots' import clickThought from '../helpers/clickThought' +import emulate from '../helpers/emulate' import gesture from '../helpers/gesture' import hide from '../helpers/hide' import hideHUD from '../helpers/hideHUD' @@ -12,7 +13,6 @@ import press from '../helpers/press' import screenshot from '../helpers/screenshot' import setTheme from '../helpers/setTheme' import waitForSelector from '../helpers/waitForSelector' -import { page } from '../session' expect.extend({ toMatchImageSnapshot: configureSnapshots({ fileName: path.basename(__filename).replace('.ts', '') }), @@ -39,7 +39,7 @@ it('DesktopCommandUniverse', async () => { }) it('GestureMenu', async () => { - await page.emulate(KnownDevices['iPhone 15 Pro']) + await emulate(KnownDevices['iPhone 15 Pro']) await hideHUD() @@ -66,7 +66,7 @@ it('GestureMenu', async () => { }) it('CommandCenter', async () => { - await page.emulate(KnownDevices['iPhone 15 Pro']) + await emulate(KnownDevices['iPhone 15 Pro']) // the undo button toggles between active and inactive states for some reason. Hence hide the HUD to ensure the undo button is not visible. await hideHUD() diff --git a/src/e2e/puppeteer/__tests__/undo.ts b/src/e2e/puppeteer/__tests__/undo.ts index 206ea43cb81..6ed0fb4e960 100644 --- a/src/e2e/puppeteer/__tests__/undo.ts +++ b/src/e2e/puppeteer/__tests__/undo.ts @@ -2,6 +2,7 @@ import { KnownDevices } from 'puppeteer' import newThoughtCommand from '../../../commands/newThought' import clickThought from '../helpers/clickThought' import command from '../helpers/command' +import emulate from '../helpers/emulate' import exportThoughts from '../helpers/exportThoughts' import gesture from '../helpers/gesture' import getEditingText from '../helpers/getEditingText' @@ -193,7 +194,7 @@ it('Re-render cursor thought on undo', async () => { // We have to test this in puppeteer because chained commands are executed as separate commands at a higher level than action-creators and undone with an ad hoc mergeNext property on the action. it('Undo Select All + Categorize chained command in one step', async () => { - await page.emulate(KnownDevices['iPhone 15 Pro']) + await emulate(KnownDevices['iPhone 15 Pro']) // create thoughts a, b, c await gesture(newThoughtCommand) diff --git a/src/e2e/puppeteer/helpers/closeKeyboard.ts b/src/e2e/puppeteer/helpers/closeKeyboard.ts index 38678e402c0..4b6324a7b56 100644 --- a/src/e2e/puppeteer/helpers/closeKeyboard.ts +++ b/src/e2e/puppeteer/helpers/closeKeyboard.ts @@ -1,6 +1,6 @@ import { page } from '../session' /** Closes the virtual keyboard by blurring the active element, simulating the native Done button. */ -const closeKeyboard = () => page.evaluate(() => (document.activeElement as HTMLElement)?.blur()) +const closeKeyboard = () => page.evaluate(() => (document.activeElement as HTMLElement | null)?.blur()) export default closeKeyboard diff --git a/src/e2e/puppeteer/helpers/command.ts b/src/e2e/puppeteer/helpers/command.ts index d6fb2ec81f5..26e59cae577 100644 --- a/src/e2e/puppeteer/helpers/command.ts +++ b/src/e2e/puppeteer/helpers/command.ts @@ -1,13 +1,12 @@ import CommandId from '../../../@types/CommandId' -import { WindowEm } from '../../../initialize' import { page } from '../session' -const em = window.em as WindowEm - /** Executes a command by id. Use in tests when the specific shortcut used to execute the command doesn't matter. This decouples the tests from the shortcuts and makes the tests more readable. */ -const command = async (id: CommandId) => - page.evaluate(id => { +const command = async (id: CommandId) => { + await page.evaluate(id => { + const em = window.em em.testHelpers.executeCommandById(id) }, id) +} export default command diff --git a/src/e2e/puppeteer/helpers/emulate.ts b/src/e2e/puppeteer/helpers/emulate.ts index 0ba547f7adb..230ab2d08a2 100644 --- a/src/e2e/puppeteer/helpers/emulate.ts +++ b/src/e2e/puppeteer/helpers/emulate.ts @@ -1,7 +1,12 @@ import { Device } from 'puppeteer' import { page } from '../session' +import waitForBrowserSettled from './waitForBrowserSettled' -/** Holds down a key on the keyboad. */ -const emulate = async (device: Device) => page.emulate(device) +/** Emulates a device after the app is already mounted. */ +const emulate = async (device: Device) => { + await page.emulate(device) + // Emulation changes viewport, touch, and media state. Wait for layout/effects before gestures or snapshots. + await waitForBrowserSettled() +} export default emulate diff --git a/src/e2e/puppeteer/helpers/exportThoughts.ts b/src/e2e/puppeteer/helpers/exportThoughts.ts index 0733c2a85ef..53ed986a2d3 100644 --- a/src/e2e/puppeteer/helpers/exportThoughts.ts +++ b/src/e2e/puppeteer/helpers/exportThoughts.ts @@ -1,11 +1,8 @@ import MimeType from '../../../@types/MimeType' import { HOME_TOKEN } from '../../../constants' -import { WindowEm } from '../../../initialize' import removeHome from '../../../util/removeHome' import { page } from '../session' -const em = window.em as WindowEm - /** * Export the current state of thoughts as plain text. * This allows puppeteer tests to verify thought structure without using snapshots. @@ -14,7 +11,7 @@ const exportThoughts = async ( { mimeType = 'text/plain' }: { mimeType: MimeType } = { mimeType: 'text/plain' }, ): Promise => { const exported = await page.evaluate( - (HOME_TOKEN, mimeType) => em.exportContext([HOME_TOKEN], mimeType), + (HOME_TOKEN, mimeType) => window.em.exportContext([HOME_TOKEN], mimeType), HOME_TOKEN, mimeType, ) diff --git a/src/e2e/puppeteer/helpers/openModal.ts b/src/e2e/puppeteer/helpers/openModal.ts index cbfe478fff7..5fe10323752 100644 --- a/src/e2e/puppeteer/helpers/openModal.ts +++ b/src/e2e/puppeteer/helpers/openModal.ts @@ -1,13 +1,11 @@ import ModalType from '../../../@types/Modal' -import { WindowEm } from '../../../initialize' import { page } from '../session' import waitUntil from './waitUntil' -const em = window.em as WindowEm - /** Directly opens a Modal and waits for it to finish loading. */ const openModal = async (id: ModalType): Promise => { await page.evaluate(id => { + const em = window.em em.store.dispatch({ type: 'showModal', id }) }, id) diff --git a/src/e2e/puppeteer/helpers/paste.ts b/src/e2e/puppeteer/helpers/paste.ts index 457928f1a89..06cee86cc43 100644 --- a/src/e2e/puppeteer/helpers/paste.ts +++ b/src/e2e/puppeteer/helpers/paste.ts @@ -1,9 +1,6 @@ import { HOME_TOKEN } from '../../../constants' -import { WindowEm } from '../../../initialize' import { page } from '../session' -const em = window.em as WindowEm - async function paste(text: string): Promise async function paste(pathUnranked: string[], text: string): Promise @@ -21,7 +18,7 @@ async function paste(pathUnranked: string | string[], text?: string): Promise { - const testHelpers = em.testHelpers + const testHelpers = window.em.testHelpers testHelpers.importToContext(_pathUnranked, _text) }, _pathUnranked, diff --git a/src/e2e/puppeteer/helpers/refresh.ts b/src/e2e/puppeteer/helpers/refresh.ts index 605545e6d7d..d21c9dcd564 100644 --- a/src/e2e/puppeteer/helpers/refresh.ts +++ b/src/e2e/puppeteer/helpers/refresh.ts @@ -1,6 +1,12 @@ import { page } from '../session' -/** Refreshes the page. */ -const refresh = () => page.evaluate(() => window.location.reload()) +/** Reloads the page and waits for the thoughtspace to load. */ +const refresh = async (): Promise => { + await page.evaluate(async () => { + await window.em.testHelpers.waitForThoughtspaceRuntimeIdle() + }) + await page.reload({ waitUntil: 'load' }) + await page.evaluate(() => window.em.testHelpers.waitForInitialized()) +} export default refresh diff --git a/src/e2e/puppeteer/helpers/scrollTo.ts b/src/e2e/puppeteer/helpers/scrollTo.ts index 727f578d79d..5b7ba1a8096 100644 --- a/src/e2e/puppeteer/helpers/scrollTo.ts +++ b/src/e2e/puppeteer/helpers/scrollTo.ts @@ -1,11 +1,10 @@ -import { WindowEm } from '../../../initialize' import { page } from '../session' /** Scrolls instantly to the given position. Cancels the pending trailing scrollCursorIntoView throttle (400ms) first so it does not scroll the cursor back into view afterwards. */ const scrollTo = async (x: number, y: number) => { await page.evaluate( (x: number, y: number) => { - const em = window.em as WindowEm + const em = window.em em.testFlags.throttledScrollCursorIntoView?.cancel() window.scrollTo(x, y) }, diff --git a/src/e2e/puppeteer/helpers/setTheme.ts b/src/e2e/puppeteer/helpers/setTheme.ts index 84b8d0b333a..1ec71fcd785 100644 --- a/src/e2e/puppeteer/helpers/setTheme.ts +++ b/src/e2e/puppeteer/helpers/setTheme.ts @@ -1,15 +1,13 @@ -import { WindowEm } from '../../../initialize' import sleep from '../../../util/sleep' import { page } from '../session' -const em = window.em as WindowEm - /** Set color theme to light or dark by directly dispatching settings action. */ const setTheme = async (theme: 'Light' | 'Dark'): Promise => { // TODO await sleep(200) await page.evaluate(theme => { + const em = window.em em.store.dispatch({ type: 'settings', key: 'Theme', value: theme }) }, theme) diff --git a/src/e2e/puppeteer/helpers/simulateDragAndDrop.ts b/src/e2e/puppeteer/helpers/simulateDragAndDrop.ts index a2c7f1a4166..fe04e4d73d5 100644 --- a/src/e2e/puppeteer/helpers/simulateDragAndDrop.ts +++ b/src/e2e/puppeteer/helpers/simulateDragAndDrop.ts @@ -1,4 +1,3 @@ -import { WindowEm } from '../../../initialize' import { page } from '../session' interface Options { @@ -9,14 +8,13 @@ interface Options { /** Keeps every drop hover that becomes visible during the current drag mounted, so multiple drop hovers can be compared in a single snapshot. See: https://github.com/cybersemics/em/issues/3115. */ pinDropHovers?: boolean } -const em = window.em as WindowEm - /** Sets testFlags for simulating drag and drop process. */ const simulateDragAndDrop = async ({ drag, drop, pinDropHovers }: Options): Promise => { await new Promise(resolve => setTimeout(resolve, 100)) await page.evaluate( (drag, drop, pinDropHovers) => { + const em = window.em em.testFlags.simulateDrag = !!drag em.testFlags.simulateDrop = !!drop em.testFlags.pinDropHovers = !!pinDropHovers diff --git a/src/e2e/puppeteer/helpers/waitForBrowserSettled.ts b/src/e2e/puppeteer/helpers/waitForBrowserSettled.ts new file mode 100644 index 00000000000..90cdc9c30f3 --- /dev/null +++ b/src/e2e/puppeteer/helpers/waitForBrowserSettled.ts @@ -0,0 +1,12 @@ +import { page } from '../session' + +/** Waits for browser layout, paint, and queued macrotasks to settle after DOM-affecting e2e actions. */ +const waitForBrowserSettled = async (): Promise => { + await page.evaluate(async () => { + await new Promise(requestAnimationFrame) + await new Promise(requestAnimationFrame) + await new Promise(resolve => setTimeout(resolve)) + }) +} + +export default waitForBrowserSettled diff --git a/src/e2e/puppeteer/helpers/waitForContextHasChildWithValue.ts b/src/e2e/puppeteer/helpers/waitForContextHasChildWithValue.ts index c874e49996d..3074e3ef358 100644 --- a/src/e2e/puppeteer/helpers/waitForContextHasChildWithValue.ts +++ b/src/e2e/puppeteer/helpers/waitForContextHasChildWithValue.ts @@ -1,6 +1,5 @@ import Context from '../../../@types/Context' import Thought from '../../../@types/Thought' -import { WindowEm } from '../../../initialize' import { page } from '../session' /** @@ -9,9 +8,8 @@ import { page } from '../session' const waitForContextHasChildWithValue = async (context: Context, childValue: string) => page.waitForFunction( (context: Context, childValue: string) => - (window.em as WindowEm) - .getAllChildrenAsThoughts(context) - .some((thought: Thought) => thought.value === childValue) && (window.em as WindowEm).getLexeme(childValue), + window.em.getAllChildrenAsThoughts(context).some((thought: Thought) => thought.value === childValue) && + window.em.getLexeme(childValue), {}, context, childValue, diff --git a/src/e2e/puppeteer/helpers/waitForState.ts b/src/e2e/puppeteer/helpers/waitForState.ts index 225a4d35448..1a9e5bf50c3 100644 --- a/src/e2e/puppeteer/helpers/waitForState.ts +++ b/src/e2e/puppeteer/helpers/waitForState.ts @@ -1,8 +1,5 @@ -import { WindowEm } from '../../../initialize' import { page } from '../session' -const em = window.em as WindowEm - /** * Wait until value of the state for the given property path equals the given value. */ @@ -11,7 +8,7 @@ const waitForState = async (path: string, value: any) => { await page.evaluate( async (path, value) => { await new Promise(resolve => { - const { getState, _ } = em.testHelpers + const { getState, _ } = window.em.testHelpers /** Listen state changes. */ const stateListener = () => { diff --git a/src/e2e/puppeteer/helpers/waitForThoughtExistInDb.ts b/src/e2e/puppeteer/helpers/waitForThoughtExistInDb.ts index d367d0ee64f..354b30620f2 100644 --- a/src/e2e/puppeteer/helpers/waitForThoughtExistInDb.ts +++ b/src/e2e/puppeteer/helpers/waitForThoughtExistInDb.ts @@ -1,18 +1,16 @@ -import { WindowEm } from '../../../initialize' import { page } from '../session' -const em = window.em as WindowEm - -/** Wait for the given thought value to exist in the database. */ +/** Waits for thoughtspace initialization and the given thought value to exist in the database. */ const waitForThoughtExistInDb = async (value: string) => { await page.evaluate(async value => { - await new Promise(resolve => { - const testHelpers = em.testHelpers + const testHelpers = window.em.testHelpers + await testHelpers.waitForInitialized() + await new Promise(resolve => { /** Polls for Lexeme in IndexedDB. */ function pollForLexeme(value: string) { setTimeout(async () => { - const thoughtFromDB = await testHelpers.getLexemeFromIndexedDB(value) + const thoughtFromDB = await testHelpers.getLexemeFromThoughtspace(value) if (thoughtFromDB) { resolve(thoughtFromDB) } else { diff --git a/src/e2e/puppeteer/setup.ts b/src/e2e/puppeteer/setup.ts index 1ff76298977..50030315caf 100644 --- a/src/e2e/puppeteer/setup.ts +++ b/src/e2e/puppeteer/setup.ts @@ -1,5 +1,9 @@ +/* eslint-disable import/prefer-default-export */ import chalk from 'chalk' -import { Browser, BrowserContext, ConsoleMessage, Device } from 'puppeteer' +import { Browser, BrowserContext, ConsoleMessage, Device, Page } from 'puppeteer' +import type { PreloadedEmWindow } from '../../@types' +import type { ThoughtspaceStorage } from '../../data-providers/thoughtspace' +import createId from '../../util/createId' import deviceEmulation from './helpers/deviceEmulation' import { page, setPage } from './session' @@ -9,6 +13,43 @@ declare module global { } let context: BrowserContext +let activeThoughtspaceStorage: ThoughtspaceStorage = 'memory' + +/** Selects thoughtspace storage before a Puppeteer page starts the app. */ +const preloadThoughtspaceStorage = (target: Page, storage: ThoughtspaceStorage) => + target.evaluateOnNewDocument(storage => { + const preloadedWindow: PreloadedEmWindow = window + preloadedWindow.em = { + ...preloadedWindow.em, + testFlags: { + ...preloadedWindow.em?.testFlags, + thoughtspaceStorage: storage, + }, + } + }, storage) + +/** Opens an additional page with the requested thoughtspace storage. */ +export const createTreecrdtTestPage = async ( + browserContext: BrowserContext, + storage: ThoughtspaceStorage, +): Promise => { + const target = await browserContext.newPage() + await preloadThoughtspaceStorage(target, storage) + return target +} + +/** Use persistent OPFS storage for tests that verify reload/materialization from storage. */ +export const usePersistentTreecrdtStorage = (): ThoughtspaceStorage => { + beforeAll(() => { + activeThoughtspaceStorage = 'persistent' + }) + + afterAll(() => { + activeThoughtspaceStorage = 'memory' + }) + + return 'persistent' +} /** Opens em in a new incognito window in Puppeteer. */ const setup = async ({ @@ -35,6 +76,20 @@ const setup = async ({ await page.emulate(emulatedDevice) } + const sessionId = createId() + + await page.evaluateOnNewDocument(sessionId => { + if (!sessionStorage.getItem('__em_puppeteer_storage_initialized')) { + localStorage.clear() + sessionStorage.setItem('__em_puppeteer_storage_initialized', '1') + } + + localStorage.setItem('tsid', sessionId) + localStorage.setItem('accessToken', sessionId) + }, sessionId) + + await preloadThoughtspaceStorage(page, activeThoughtspaceStorage) + page.on('dialog', async dialog => dialog.accept()) // forward puppeteer logs to console logs @@ -78,8 +133,18 @@ const setup = async ({ beforeEach(setup, 60000) +// TreeCRDT teardown can drain OPFS writes from import-heavy tests before dropping storage. afterEach(async () => { if (page) { + await page + .evaluate(async () => { + await window.em?.testHelpers?.waitForThoughtspaceRuntimeIdle?.() + await window.em?.testHelpers?.dropThoughtspace?.() + }) + .catch(() => { + // Ignore teardown errors when a failing test has already closed or navigated the page. + }) + await page.close().catch(() => { // Ignore errors when closing the page. }) @@ -90,4 +155,4 @@ afterEach(async () => { // Ignore errors when closing the context. }) } -}) +}, 60000) diff --git a/src/e2e/testFlags.ts b/src/e2e/testFlags.ts index eb27642b31d..5252b38115c 100644 --- a/src/e2e/testFlags.ts +++ b/src/e2e/testFlags.ts @@ -1,4 +1,5 @@ import { DebouncedFunc } from 'lodash' +import type { ThoughtspaceStorage } from '../data-providers/thoughtspace' type TestFlags = { logActions: boolean @@ -10,7 +11,9 @@ type TestFlags = { /** Prevent automatic app initialization on page load. */ preventInitialize: boolean /** Starts app initialization when preventInitialize is enabled. */ - initialize: (() => Promise) | null + initialize: ((options: { storage: ThoughtspaceStorage }) => Promise) | null + /** Overrides production thoughtspace storage during test startup. */ + thoughtspaceStorage: ThoughtspaceStorage | null /** Keep every drop hover that becomes visible during the current drag mounted, so multiple drop hovers can be compared in a single snapshot. */ pinDropHovers: boolean /** Render drop-hover elements as blocks of color. */ @@ -21,19 +24,17 @@ type TestFlags = { throttledScrollCursorIntoView: DebouncedFunc<(y: number, height: number) => void> | null } -const preloadedTestFlags = - typeof window === 'undefined' - ? null - : ((window.em as { testFlags?: Partial } | undefined)?.testFlags ?? null) +const preloadedTestFlags = typeof window === 'undefined' ? null : (window.em?.testFlags ?? null) /** Test flags that are injected into window.em.testFlags. */ const testFlags: TestFlags = { logActions: false, logMultigesture: false, expandHoverDelay: null, - replicationDelay: 0, + replicationDelay: preloadedTestFlags?.replicationDelay ?? 0, preventInitialize: preloadedTestFlags?.preventInitialize ?? false, initialize: null, + thoughtspaceStorage: preloadedTestFlags?.thoughtspaceStorage ?? null, pinDropHovers: false, simulateDrag: false, simulateDrop: false, diff --git a/src/hooks/__tests__/useDragLeave.ts b/src/hooks/__tests__/useDragLeave.ts index cc1dc99a20b..c706a97e77f 100644 --- a/src/hooks/__tests__/useDragLeave.ts +++ b/src/hooks/__tests__/useDragLeave.ts @@ -24,8 +24,8 @@ const startHovering = () => { /** Advances past the hook's 50ms debounce. */ const flushDebounce = () => act(() => vi.advanceTimersByTimeAsync(100)) -beforeEach(() => { - initStore() +beforeEach(async () => { + await initStore() vi.useFakeTimers() }) diff --git a/src/hooks/useDragAndDropThought.tsx b/src/hooks/useDragAndDropThought.tsx index 99d1e77790d..e226cb302dd 100644 --- a/src/hooks/useDragAndDropThought.tsx +++ b/src/hooks/useDragAndDropThought.tsx @@ -34,6 +34,7 @@ import isBefore from '../selectors/isBefore' import isContextViewActive from '../selectors/isContextViewActive' import isMulticursorPath from '../selectors/isMulticursorPath' import pathToThought from '../selectors/pathToThought' +import prevSibling from '../selectors/prevSibling' import simplifyPath from '../selectors/simplifyPath' import store from '../stores/app' import selectionRangeStore from '../stores/selectionRangeStore' @@ -242,6 +243,7 @@ const drop = (props: ThoughtContainerProps, monitor: DropTargetMonitor) => { oldPath: thoughtFrom, newPath, newRank: prevPath ? getRankAfter(state, prevPath) : getRankBefore(state, props.simplePath), + afterId: prevPath ? head(prevPath) : (prevSibling(state, props.simplePath)?.id ?? null), }), ) } diff --git a/src/hooks/useSharedType.ts b/src/hooks/useSharedType.ts deleted file mode 100644 index ab5e303f6ca..00000000000 --- a/src/hooks/useSharedType.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { useCallback, useEffect, useState } from 'react' -import { shallowEqual } from 'react-redux' -import * as Y from 'yjs' -import Index from '../@types/IndexType' - -// Infer the generic type of a specific YEvent such as YMapEvent or YArrayEvent -// This is needed because YEvent is not generic. -type ExtractYEvent = T extends Y.YMapEvent | Y.YArrayEvent ? U : never - -/** Subscribes to a yjs shared type, e.g. Y.Map. Performs shallow comparison between new and old state and only updates if shallow value has changed. */ -const useSharedType = (yobj: Y.AbstractType): Index> => { - const [state, setState] = useState>>(yobj.toJSON()) - - const updateState = useCallback( - async () => { - const stateNew: Index> = yobj.toJSON() - setState((stateOld: Index>) => (!shallowEqual(stateNew, stateOld) ? stateNew : stateOld)) - }, - // eslint-disable-next-line react-hooks/exhaustive-deps - [], - ) - - useEffect( - () => { - yobj.observe(updateState) - return () => { - yobj.unobserve(updateState) - } - }, - // eslint-disable-next-line react-hooks/exhaustive-deps - [], - ) - - return state -} - -export default useSharedType diff --git a/src/index.tsx b/src/index.tsx index 3266e83fda1..ea6babcfeb3 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -2,6 +2,8 @@ import './util/consoleProxy' import { createRoot } from 'react-dom/client' import App from './components/App' +import ThoughtspaceInUse from './components/ThoughtspaceInUse' +import { thoughtspaceRuntime } from './data-providers/thoughtspace' import testFlags from './e2e/testFlags' import './index.css' import { initialize } from './initialize' @@ -9,14 +11,26 @@ import { register } from './serviceWorkerRegistration' import store from './stores/app' import initEvents from './util/initEvents' -initEvents(store) - -if (!testFlags.preventInitialize) { - void initialize() -} - const container = document.getElementById('root') const root = createRoot(container!) -root.render() +/** Acquires thoughtspace access before initializing or rendering the interactive app. */ +const bootstrap = async (): Promise => { + const access = await thoughtspaceRuntime.acquireAccess() + + if (access.status === 'blocked') { + root.render() + return + } + + initEvents(store) + + if (!testFlags.preventInitialize) { + void initialize({ storage: testFlags.thoughtspaceStorage ?? 'persistent' }) + } + + root.render() +} + +void bootstrap() register() diff --git a/src/initialize.ts b/src/initialize.ts index b0335427748..3b47accdd09 100644 --- a/src/initialize.ts +++ b/src/initialize.ts @@ -2,27 +2,19 @@ import _ from 'lodash' import moize from 'moize' import CommandId from './@types/CommandId' import Context from './@types/Context' -import Lexeme from './@types/Lexeme' import MimeType from './@types/MimeType' -import PushBatch from './@types/PushBatch' import State from './@types/State' -import Thought from './@types/Thought' import ThoughtId from './@types/ThoughtId' import Thunk from './@types/Thunk' -import { errorActionCreator as error } from './actions/error' import { importFilesActionCreator as importFiles } from './actions/importFiles' import { initThoughtsActionCreator as initThoughts } from './actions/initThoughts' import { loadFromUrlActionCreator as loadFromUrl } from './actions/loadFromUrl' import { preloadSourcesActionCreator as preloadSources } from './actions/preloadSources' import { pullActionCreator as pull } from './actions/pull' -import { repairThoughtActionCreator as repairThought } from './actions/repairThought' import { setCursorActionCreator as setCursor } from './actions/setCursor' import { updateThoughtsActionCreator } from './actions/updateThoughts' import { commandById, executeCommand } from './commands' -import { HOME_TOKEN } from './constants' -import getLexemeHelper from './data-providers/data-helpers/getLexeme' -import { accessToken, clientIdReady, tsid, tsidShared } from './data-providers/yjs' -import db, { init as initThoughtspace, replicateLexeme, replicateThought } from './data-providers/yjs/thoughtspace' +import db, { type ThoughtspaceStorage, thoughtspaceRuntime } from './data-providers/thoughtspace' import * as selection from './device/selection' import testFlags from './e2e/testFlags' import contextToThoughtId from './selectors/contextToThoughtId' @@ -41,14 +33,9 @@ import prettyPath from './test-helpers/prettyPath' import hashThought from './util/hashThought' import initEvents from './util/initEvents' import isRoot from './util/isRoot' -import mergeBatch from './util/mergeBatch' import owner from './util/owner' -import throttleConcat from './util/throttleConcat' import urlDataSource from './util/urlDataSource' -/** Number of milliseconds to throttle dispatching updateThoughts on thought/lexeme change. */ -const UPDATE_THOUGHTS_THROTTLE = 100 - /** * Decode cursor from url, pull and initialize the cursor. */ @@ -70,92 +57,41 @@ const initializeCursor = async () => { } } -/** Dispatches updateThoughts with all updates in the throttle period. */ -const updateThoughtsThrottled = throttleConcat((batches: PushBatch[]) => { - const merged = batches.reduce(mergeBatch, { - thoughtIndexUpdates: {}, - lexemeIndexUpdates: {}, - lexemeIndexUpdatesOld: {}, - }) - - // dispatch on next tick, since the leading edge is synchronous and can be triggered during a reducer - setTimeout(() => { - store.dispatch(updateThoughtsActionCreator({ ...merged, local: false, remote: false, repairCursor: true })) - }) -}, UPDATE_THOUGHTS_THROTTLE) +type InitializeOptions = { storage: ThoughtspaceStorage } /** Initialize local db and window events. */ -export const initialize = async () => { +const initializeInternal = async ({ storage }: InitializeOptions) => { initOfflineStatusStore(/* websocket */) const eventHandlers = initEvents(store) - await initThoughtspace({ - cursor: decodeThoughtsUrl(store.getState()).path, - accessToken, - /** Returns true if the Thought or its parent is in State. */ - isThoughtLoaded: async (thought: Thought | undefined): Promise => { - const state = store.getState() - return !!(thought && (getThoughtById(state, thought.parentId) || getThoughtById(state, thought.id))) - }, - /** Returns true if the Lexeme or one of its contexts are in State. */ - isLexemeLoaded: async (key: string, lexeme: Lexeme | undefined): Promise => { - const state = store.getState() - return !!((lexeme && getLexeme(state, key)) || lexeme?.contexts.some(cxid => getThoughtById(state, cxid))) - }, - onError: (message, object) => { - store.dispatch(error({ value: message })) - }, - onProgress: syncStatusStore.update, - onThoughtChange: (thought: Thought) => { - store.dispatch((dispatch, getState) => { - // if parent is pending, the thought must be marked pending. - // Note: Do not clear pending from the parent, because other children may not be loaded. - // The next pull should handle that automatically. - // TODO: Do we need to use fresh State when updateThoughtsThrottled resolves? - const state = getState() - const thoughtInState = getThoughtById(state, thought.id) - const parentInState = getThoughtById(state, thought.parentId) - const pending = thoughtInState?.pending || parentInState?.pending - - updateThoughtsThrottled({ - thoughtIndexUpdates: { - [thought.id]: { - ...thought, - ...(pending ? { pending } : null), - }, - }, - lexemeIndexUpdates: {}, - lexemeIndexUpdatesOld: {}, - }) - }) - }, - onThoughtIDBSynced: (thought, { background }) => { - // If the websocket is still connecting for the first time when IDB is synced and non-empty, change the status to reconnecting to dismiss "Connecting..." and render the available thoughts. See: EmptyThoughtspace.tsx. - if (!background && thought?.id === HOME_TOKEN) { - const hasRootChildren = Object.keys(thought?.childrenMap || {}).length > 0 - if (hasRootChildren) { - offlineStatusStore.update(statusOld => - statusOld === 'preconnecting' || statusOld === 'connecting' ? 'reconnecting' : statusOld, - ) + const { clientId } = await thoughtspaceRuntime.init({ + storage, + materialization: { + getSnapshot: () => { + const state = store.getState() + return { + schemaVersion: state.schemaVersion, + thoughtIndex: state.thoughts.thoughtIndex, + lexemeIndex: state.thoughts.lexemeIndex, } - } + }, + apply: ({ thoughtIndex, lexemeIndex }) => { + store.dispatch( + updateThoughtsActionCreator({ + thoughtIndexUpdates: thoughtIndex, + lexemeIndexUpdates: lexemeIndex, + local: false, + remote: false, + repairCursor: true, + }), + ) + }, }, - onThoughtReplicated: (id, thought) => { - store.dispatch(repairThought(id, thought)) - }, - onUpdateThoughts: options => { - store.dispatch(updateThoughtsActionCreator(options)) - }, - tsid, - tsidShared, }) // load local state unless loading a public context or source url // await initDB() - // initialize clientId before dispatching any actions that create new thoughts - const clientId = await clientIdReady - const src = urlDataSource() const thoughtsLocalPromise = owner() === '~' @@ -180,6 +116,28 @@ export const initialize = async () => { return eventHandlers } +let initializationPromise: ReturnType | null = null +let resolveInitializationStarted: (() => void) | null = null + +/** Allows readiness waiters to arrive before access acquisition finishes. */ +const initializationStartedPromise = new Promise(resolve => { + resolveInitializationStarted = resolve +}) + +/** Initialize local db and window events. */ +export const initialize = (options: InitializeOptions): ReturnType => { + initializationPromise = initializeInternal(options) + resolveInitializationStarted?.() + resolveInitializationStarted = null + return initializationPromise +} + +/** Waits for app initialization to finish. Used by e2e tests before interacting with exposed helpers. */ +export const waitForInitialized = async (): Promise => { + if (!initializationPromise) await initializationStartedPromise + await initializationPromise +} + testFlags.initialize = initialize /** Partially apply state to a function. */ @@ -204,9 +162,12 @@ const testHelpers = { executeCommandById: (id: CommandId) => { executeCommand(commandById(id)) }, + dropThoughtspace: thoughtspaceRuntime.drop, + waitForInitialized, + waitForThoughtspaceRuntimeIdle: thoughtspaceRuntime.waitForIdle, setSelection: selection.set, importToContext: withDispatch(importToContext), - getLexemeFromIndexedDB: (value: string) => getLexemeHelper(db, value), + getLexemeFromThoughtspace: (value: string) => db.getLexemeById(hashThought(value)), getState: store.getState, _: _, } @@ -263,8 +224,6 @@ const windowEm = { return store.subscribe(onState) }, prettyPath, - replicateThought, - replicateLexeme: (value: string) => replicateLexeme(hashThought(value)), store, offlineStatusStore, syncStatusStore, diff --git a/src/redux-enhancers/pushQueue.ts b/src/redux-enhancers/pushQueue.ts index 123f3d35f8a..8a0fe37763d 100644 --- a/src/redux-enhancers/pushQueue.ts +++ b/src/redux-enhancers/pushQueue.ts @@ -5,7 +5,7 @@ import PushBatch from '../@types/PushBatch' import State from '../@types/State' import ThoughtId from '../@types/ThoughtId' import { CACHED_SETTINGS, EM_TOKEN } from '../constants' -import db from '../data-providers/yjs/thoughtspace' +import db, { thoughtspaceRuntime } from '../data-providers/thoughtspace' import contextToThoughtId from '../selectors/contextToThoughtId' import { getChildrenRanked } from '../selectors/getChildren' import getThoughtById from '../selectors/getThoughtById' @@ -48,7 +48,7 @@ const cacheSetting = (name: keyof typeof cachedSettingsIds, value: string | null } } -/** Merges state.pushQueue batches and pushes them to Yjs, frees memory from state-only batches, and caches settings. */ +/** Pushes database batches, frees provider cache for state-only batches, and caches settings. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any const pushQueue: StoreEnhancer = (createStore: StoreEnhancerStoreCreator) => @@ -85,32 +85,35 @@ const pushQueue: StoreEnhancer = } }) - /** - * Push updates to database sequentially. - */ + /** Pushes queued updates to the active thoughtspace provider sequentially. */ const applyDbQueue = async () => { - for (const batch of dbQueue ?? []) { - await db.updateThoughts({ + await thoughtspaceRuntime.persistPushQueueBatches( + (dbQueue ?? []).map(batch => ({ thoughtIndexUpdates: batch.thoughtIndexUpdates, lexemeIndexUpdates: batch.lexemeIndexUpdates, lexemeIndexUpdatesOld: batch.lexemeIndexUpdatesOld, - schemaVersion: batch.updates?.schemaVersion, - }) - } + schemaVersion: batch.updates?.schemaVersion ?? 0, + movePlacements: batch.movePlacements, + local: batch.local, + })), + ) } - applyDbQueue().then(() => { - dbQueue?.forEach(batch => batch.idbSynced?.()) - }) + void applyDbQueue() + .then(() => { + dbQueue?.forEach(batch => batch.idbSynced?.()) + }) + .catch(err => { + console.error('Thoughtspace persistence failed', err) + }) } - const freeBatch = (freeQueue || []).reduce(mergeBatch, { + const freeBatch = (freeQueue || []).reduce(mergeBatch, { thoughtIndexUpdates: {}, lexemeIndexUpdates: {}, lexemeIndexUpdatesOld: {}, }) - // free up memory of thoughts that have been deleted Object.entries(freeBatch.thoughtIndexUpdates).forEach(([id, thoughtUpdate]) => { if (!thoughtUpdate) { db.freeThought?.(id as ThoughtId) diff --git a/src/redux-enhancers/storageCache.ts b/src/redux-enhancers/storageCache.ts index d938cacaf11..1d6c33f89ba 100644 --- a/src/redux-enhancers/storageCache.ts +++ b/src/redux-enhancers/storageCache.ts @@ -4,7 +4,7 @@ import CommandId from '../@types/CommandId' import State from '../@types/State' import StorageCache from '../@types/StorageCache' import ValueOf from '../@types/ValueOf' -import { tsidShared } from '../data-providers/yjs' +import { tsidShared } from '../data-providers/thoughtspaceSession' import { getStateSetting } from '../selectors/getSetting' import getUserToolbar from '../selectors/getUserToolbar' import keyValueBy from '../util/keyValueBy' diff --git a/src/redux-enhancers/undoRedoEnhancer.ts b/src/redux-enhancers/undoRedoEnhancer.ts index c7b20fbe3b4..86b224eea30 100644 --- a/src/redux-enhancers/undoRedoEnhancer.ts +++ b/src/redux-enhancers/undoRedoEnhancer.ts @@ -7,12 +7,15 @@ import Index from '../@types/IndexType' import Lexeme from '../@types/Lexeme' import Patch from '../@types/Patch' import State from '../@types/State' +import Thought from '../@types/Thought' import ThoughtId from '../@types/ThoughtId' import { editThoughtPayload } from '../actions/editThought' import editableRender from '../actions/editableRender' import updateThoughts from '../actions/updateThoughts' +import { getChildrenRanked } from '../selectors/getChildren' import getThoughtById from '../selectors/getThoughtById' import { isNavigation, isUndoable } from '../util/actionMetadata.registry' +import equalArrays from '../util/equalArrays' import headValue from '../util/headValue' import reducerFlow from '../util/reducerFlow' import stripTags from '../util/stripTags' @@ -88,6 +91,66 @@ function getEditThoughtDirection(action: UnknownAction): EditThoughtDirection { * editing (allowInnerHTMLChange is false), so undoing a formatting/letter-case edit appears to do nothing. */ const statePropertiesToOmit: (keyof State)[] = ['alert', 'cursorCleared', 'editableNonce', 'pushQueue'] +/** Reconstructs TreeCRDT move updates and placement metadata from the final state produced by an undo/redo patch. */ +const restoreMoveUpdatesFromThoughtUpdates = ( + state: State, + oldState: State, + thoughtIndexUpdates: Index, +): { + thoughtIndexUpdates: Index + movePlacements: Index +} => { + const touchedParentIds = Object.entries(thoughtIndexUpdates).reduce>((acc, [id, thought]) => { + const thoughtId = id as ThoughtId + if (!thought) return acc + + const oldThought = getThoughtById(oldState, thoughtId) + const moved = oldThought && (oldThought.parentId !== thought.parentId || oldThought.rank !== thought.rank) + if (!moved) return acc + + acc.add(oldThought.parentId) + acc.add(thought.parentId) + return acc + }, new Set()) + + const { thoughtIndexUpdates: moveThoughtIndexUpdates, movePlacements } = [...touchedParentIds].reduce<{ + thoughtIndexUpdates: Index + movePlacements: Index + }>( + (acc, parentId) => { + const oldChildren = getChildrenRanked(oldState, parentId).map(child => child.id) + const children = getChildrenRanked(state, parentId) + const childIds = children.map(child => child.id) + if (equalArrays(oldChildren, childIds)) return acc + + children.forEach((child, i) => { + const childThought = getThoughtById(state, child.id) + if (!childThought) return + + acc.thoughtIndexUpdates[child.id] = childThought + acc.movePlacements[child.id] = i === 0 ? null : childIds[i - 1] + }) + + return acc + }, + { thoughtIndexUpdates: {}, movePlacements: {} }, + ) + + const moveThoughtIds = new Set(Object.keys(movePlacements)) + const nonMoveThoughtIndexUpdates = Object.entries(thoughtIndexUpdates).reduce>( + (acc, [id, thought]) => (moveThoughtIds.has(id) ? acc : { ...acc, [id]: thought }), + {}, + ) + + return { + thoughtIndexUpdates: { + ...nonMoveThoughtIndexUpdates, + ...moveThoughtIndexUpdates, + }, + movePlacements, + } +} + /** * Manually recreate the pushQueue for thought and thought index updates from patches. */ @@ -103,7 +166,7 @@ const restorePushQueueFromPatches = (state: State, oldState: State, patch: Patch [lexemeKey]: state.thoughts.lexemeIndex[lexemeKey] || null, } }, {}) - const thoughtIndexUpdates = thoughtIndexChanges.reduce((acc, { path }) => { + const thoughtIndexUpdates = thoughtIndexChanges.reduce>((acc, { path }) => { const id = path.slice('/thoughts/thoughtIndex/'.length).split('/')[0] return { ...acc, @@ -123,10 +186,15 @@ const restorePushQueueFromPatches = (state: State, oldState: State, patch: Patch cursor: state.cursor, editingValue: state.cursor ? headValue(state, state.cursor) : null, } + const moveUpdates = restoreMoveUpdatesFromThoughtUpdates(state, oldState, thoughtIndexUpdates) return { ...state, - pushQueue: updateThoughts({ lexemeIndexUpdates, thoughtIndexUpdates })(oldStateWithUpdatedCursor).pushQueue, + pushQueue: updateThoughts({ + lexemeIndexUpdates, + thoughtIndexUpdates: moveUpdates.thoughtIndexUpdates, + ...(Object.keys(moveUpdates.movePlacements).length > 0 ? { movePlacements: moveUpdates.movePlacements } : null), + })(oldStateWithUpdatedCursor).pushQueue, } } diff --git a/src/redux-enhancers/validateStateEnhancer.ts b/src/redux-enhancers/validateStateEnhancer.ts index 8b80bbdae47..6a92c278946 100644 --- a/src/redux-enhancers/validateStateEnhancer.ts +++ b/src/redux-enhancers/validateStateEnhancer.ts @@ -1,6 +1,7 @@ import { Action, Store, StoreEnhancer, StoreEnhancerStoreCreator } from 'redux' import State from '../@types/State' -import { HOME_TOKEN } from '../constants' +import { EM_TOKEN, HOME_TOKEN } from '../constants' +import { tsidShared } from '../data-providers/thoughtspaceSession' import isTutorial from '../selectors/isTutorial' import equalPath from '../util/equalPath' @@ -11,19 +12,25 @@ import equalPath from '../util/equalPath' const validateNextState = (nextState: State, action: Action): void => { const { isLoading, showModal, thoughts } = nextState - // Try to catch the __EM__ with empty childrenMap bug + // Try to catch the EM_TOKEN with empty childrenMap bug // https://github.com/cybersemics/em/issues/2223 + const emThought = thoughts.thoughtIndex[EM_TOKEN] if ( // childrenMap is expected to be empty on the loading screen, welcome screen, and beginning of tutorial !isLoading && showModal !== 'welcome' && !isTutorial(nextState) && + // Shared links skip the welcome flow that normally creates EM/Settings, so EM may stay empty in a valid shared doc. + !tsidShared && + // guard against EM thought not yet loaded + emThought && + !emThought.pending && // after that, it should never be empty - Object.keys(thoughts.thoughtIndex.__EM__.childrenMap).length === 0 + Object.keys(emThought.childrenMap).length === 0 ) { console.error(action) throw new Error( - '__EM__ with empty childrenMap detected. This should never happen after the welcome screen is closed.', + 'EM_TOKEN with empty childrenMap detected. This should never happen after the welcome screen is closed.', ) } else if (equalPath(nextState.cursor, [HOME_TOKEN])) { console.error(action) diff --git a/src/redux-middleware/__tests__/multicursorAlertMiddleware.ts b/src/redux-middleware/__tests__/multicursorAlertMiddleware.ts index 64a7ccc5a24..ccd67645270 100644 --- a/src/redux-middleware/__tests__/multicursorAlertMiddleware.ts +++ b/src/redux-middleware/__tests__/multicursorAlertMiddleware.ts @@ -18,7 +18,7 @@ vi.mock('../../browser', async importOriginal => { beforeEach(initStore) it('shows the Command Center on mobile when a multicursor is active', async () => { - await initialize() + await initialize({ storage: 'memory' }) store.dispatch([ importText({ @@ -36,7 +36,7 @@ it('shows the Command Center on mobile when a multicursor is active', async () = }) it('does not show the Command Center when undoing a multicursor delete while the Undo Slider is active', async () => { - await initialize() + await initialize({ storage: 'memory' }) store.dispatch([ importText({ diff --git a/src/redux-middleware/__tests__/pullQueue.ts b/src/redux-middleware/__tests__/pullQueue.ts index 9deddca9790..49f90a36ded 100644 --- a/src/redux-middleware/__tests__/pullQueue.ts +++ b/src/redux-middleware/__tests__/pullQueue.ts @@ -5,10 +5,10 @@ import { clearActionCreator as clear } from '../../actions/clear' import { importTextActionCreator as importText } from '../../actions/importText' import { newThoughtActionCreator as newThought } from '../../actions/newThought' import { HOME_TOKEN } from '../../constants' -import { DataProvider } from '../../data-providers/DataProvider' +import type { DataProvider } from '../../data-providers/DataProvider' import getContext from '../../data-providers/data-helpers/getContext' import getThoughtByIdFromDB from '../../data-providers/data-helpers/getThoughtById' -import db from '../../data-providers/yjs/thoughtspace' +import db from '../../data-providers/thoughtspace' import store from '../../stores/app' import contextToThought from '../../test-helpers/contextToThought' import createTestApp, { cleanupTestApp, refreshTestApp } from '../../test-helpers/createTestApp' diff --git a/src/redux-middleware/__tests__/pushQueue.ts b/src/redux-middleware/__tests__/pushQueue.ts index 1692b5ad1aa..32c7000c142 100644 --- a/src/redux-middleware/__tests__/pushQueue.ts +++ b/src/redux-middleware/__tests__/pushQueue.ts @@ -1,7 +1,7 @@ import { act } from 'react' import { importTextActionCreator as importText } from '../../actions/importText' import getLexemeFromProvider from '../../data-providers/data-helpers/getLexeme' -import db from '../../data-providers/yjs/thoughtspace' +import db from '../../data-providers/thoughtspace' import getLexemeFromState from '../../selectors/getLexeme' import store from '../../stores/app' import contextToThought from '../../test-helpers/contextToThought' diff --git a/src/redux-middleware/freeThoughts.ts b/src/redux-middleware/freeThoughts.ts index d21e19c68e2..6dab4616974 100644 --- a/src/redux-middleware/freeThoughts.ts +++ b/src/redux-middleware/freeThoughts.ts @@ -7,11 +7,10 @@ import { freeThoughtsActionCreator as freeThoughts } from '../actions/freeThough import { FREE_THOUGHTS_THROTTLE } from '../constants' import globals from '../globals' -/** Checks if the thought cache has exceeded its memory limit. If so, dispatches freeThoughts which frees memory in the thoughtIndex, lexemeIndex, and YJS providers. */ +/** Checks if the thought cache has exceeded its memory limit. If so, dispatches freeThoughts which frees Redux indexes and provider cache. */ const checkThreshold: Thunk = (dispatch, getState): void => { const state = getState() if (Object.keys(state.thoughts.thoughtIndex).length > globals.freeThoughtsThreshold) { - // Note: YJS docs and providers are deallocated in the pushQueue enhancer based on the updates generated by the freeThoughts reducer. dispatch(freeThoughts()) } } diff --git a/src/redux-middleware/pullQueue.ts b/src/redux-middleware/pullQueue.ts index 17cd3b12102..48eda061bee 100644 --- a/src/redux-middleware/pullQueue.ts +++ b/src/redux-middleware/pullQueue.ts @@ -9,7 +9,7 @@ import { AuthenticateAction } from '../actions/authenticate' import { pullActionCreator as pull } from '../actions/pull' import { pullAncestorsActionCreator as pullAncestors } from '../actions/pullAncestors' import { EM_TOKEN, HOME_TOKEN } from '../constants' -import db from '../data-providers/yjs/thoughtspace' +import db from '../data-providers/thoughtspace' import { getChildren } from '../selectors/getChildren' import getContexts from '../selectors/getContexts' import getThoughtById from '../selectors/getThoughtById' @@ -218,7 +218,10 @@ const pullQueueMiddleware: ThunkMiddleware = ({ getState, dispatch }) => // reset internal pullQueue when clear action is dispatched if (isAction(action) && action.type === 'clear') { + updatePullQueueDebounced.cancel() + flushPullQueueThrottled.cancel() pullQueue = initialPullQueue() + lastExpandedPullQueue = {} } // Update pullQueue and flush on authenticate to force a remote fetch and make remote-only updates. // Otherwise, because thoughts are previously loaded from local storage which turns off pending on the root context, a normal pull will short circuit and remote thoughts will not be loaded. diff --git a/src/stores/offlineStatusStore.ts b/src/stores/offlineStatusStore.ts index 6cfa35a44ab..6bd686eda57 100644 --- a/src/stores/offlineStatusStore.ts +++ b/src/stores/offlineStatusStore.ts @@ -30,7 +30,7 @@ const startConnecting = () => { }, offlineTimeout) } -/** Initializes the yjs data provider. */ +/** Initializes offline / connection status (e.g. preconnect timer). */ export const init = () => { // if (websocket.status === 'connected') { // offlineStatusStore.update('connected') diff --git a/src/stores/syncStatus.ts b/src/stores/syncStatus.ts index 629f429f3a1..30a22f5be85 100644 --- a/src/stores/syncStatus.ts +++ b/src/stores/syncStatus.ts @@ -1,6 +1,6 @@ import reactMinistore from './react-ministore' -/** A store that tracks state related to syncing. Updated by yjs/thouguhtspace. */ +/** A store that tracks state related to syncing. Updated by the treecrdt thoughtspace data provider. */ const syncStatusStore = reactMinistore<{ /** Tracks if the pullQueue is currently pulling. */ isPulling: boolean diff --git a/src/test-helpers/createTestApp.tsx b/src/test-helpers/createTestApp.tsx index bee34320dfb..892661c8f7a 100644 --- a/src/test-helpers/createTestApp.tsx +++ b/src/test-helpers/createTestApp.tsx @@ -5,11 +5,12 @@ import { TestBackend } from 'react-dnd-test-backend' import Await from '../@types/Await' import { clearActionCreator as clear } from '../actions/clear' import App from '../components/App' -import * as db from '../data-providers/yjs/thoughtspace' +import db from '../data-providers/thoughtspace' import { initialize } from '../initialize' import store from '../stores/app' import { resetStores } from '../stores/ministore' import storage from '../util/storage' +import waitForThoughtspaceIdle from './waitForThoughtspaceIdle' let cleanup: Await>['cleanup'] @@ -24,7 +25,7 @@ const createTestApp = async ({ tutorial }: { tutorial?: boolean } = {}) => { resetStores() // calls initEvents, which must be manually cleaned up - const init = await initialize() + const init = await initialize({ storage: 'memory' }) cleanup = init.cleanup // const root = document.body.appendChild(document.createElement('div')) @@ -47,6 +48,7 @@ const createTestApp = async ({ tutorial }: { tutorial?: boolean } = {}) => { ]) await vi.runOnlyPendingTimersAsync() + await waitForThoughtspaceIdle() // make DND ref available for drag and drop tests. document.DND = dndRef.current @@ -66,10 +68,11 @@ export const cleanupTestApp = async () => { store.dispatch(clear({ full: true })) - // run out timers before db.clear, otherwise pending calls to replicateThought may resolve after thoughts have been deleted, triggering the "Missing docKey for thought" error + // run out timers before provider clear, otherwise pending persistence calls may resolve after thoughts have been deleted. await vi.runAllTimersAsync() + await waitForThoughtspaceIdle() - db.clear() + await db.clear() await vi.runAllTimersAsync() // set url back to home @@ -82,11 +85,14 @@ export const cleanupTestApp = async () => { /** Refresh the test app. */ export const refreshTestApp = async () => { await act(async () => { + await waitForThoughtspaceIdle() await store.dispatch(clear()) - await initialize() + await initialize({ storage: 'memory' }) + await waitForThoughtspaceIdle() }) await act(vi.runOnlyPendingTimersAsync) + await waitForThoughtspaceIdle() } /** Clear existing event listeners(e.g. keyboard, gestures), but without clearing the app. */ diff --git a/src/test-helpers/dataProviderTest.ts b/src/test-helpers/dataProviderTest.ts index 833701fffc8..bc620fc775b 100644 --- a/src/test-helpers/dataProviderTest.ts +++ b/src/test-helpers/dataProviderTest.ts @@ -6,12 +6,12 @@ import Thought from '../@types/Thought' import ThoughtId from '../@types/ThoughtId' import importText from '../actions/importText' import { ABSOLUTE_TOKEN, EM_TOKEN, HOME_TOKEN } from '../constants' -import { DataProvider } from '../data-providers/DataProvider' +import type { DataProvider } from '../data-providers/DataProvider' import fetchDescendants from '../data-providers/data-helpers/fetchDescendants' import getContext from '../data-providers/data-helpers/getContext' import getLexeme from '../data-providers/data-helpers/getLexeme' import getThoughtById from '../data-providers/data-helpers/getThoughtById' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import hashThought from '../util/hashThought' import initialState from '../util/initialState' import keyValueBy from '../util/keyValueBy' @@ -38,7 +38,7 @@ const getThoughtIdsForContexts = async (provider: DataProvider, contexts: Contex (await Promise.all(contexts.map(cx => getContext(provider, cx)))).map(thought => thought!.id) /** - * Returns many descandants fot the given contexts,. + * Returns many descendants for the given contexts. */ const fetchDescendantsByContext = async ( provider: DataProvider, @@ -107,7 +107,7 @@ const importThoughts = (text: string) => { } } -/** Runs tests for a module that conforms to the data-provider API. */ +/** Runs tests for a module that conforms to the data provider API. */ const dataProviderTest = (provider: DataProvider) => { test('getLexemeById', async () => { const nothought = await provider.getLexemeById('12345') @@ -163,58 +163,71 @@ const dataProviderTest = (provider: DataProvider) => { }) test('getThoughtById', async () => { - const nocontext = await getThoughtById(provider, 'test' as ThoughtId) + const testId = '00000000000000000000000000de5700' as ThoughtId + const childId1 = '000000000000000000000000000c0001' as ThoughtId + const childId2 = '000000000000000000000000000c0002' as ThoughtId + const childId3 = '000000000000000000000000000c0003' as ThoughtId + const parentId = '0000000000000000000000000000a001' as ThoughtId + + const nocontext = await getThoughtById(provider, testId) expect(nocontext).toBeUndefined() const thought: Thought = { - id: 'test' as ThoughtId, + id: testId, childrenMap: { - child1: 'child1' as ThoughtId, - child2: 'child2' as ThoughtId, - child3: 'child3' as ThoughtId, + [childId1]: childId1, + [childId2]: childId2, + [childId3]: childId3, }, created: timestamp(), lastUpdated: timestamp(), - parentId: 'parentId' as ThoughtId, + parentId, rank: 0, updatedBy: clientId, value: 'test', } - await provider.updateThought?.('test' as ThoughtId, undefined, thought) + await provider.updateThought?.(testId, undefined, thought) - const dbThought = await getThoughtById(provider, 'test' as ThoughtId) + const dbThought = await getThoughtById(provider, testId) expect(dbThought).toEqual(thought) }) test('getThoughtsByIds', async () => { + const idX = '0000000000000000000000000000000a' as ThoughtId + const idA = '0000000000000000000000000000000b' as ThoughtId + const childId1 = '000000000000000000000000000c0011' as ThoughtId + const childId2 = '000000000000000000000000000c0012' as ThoughtId + const parentId1 = '0000000000000000000000000000a011' as ThoughtId + const parentId2 = '0000000000000000000000000000a012' as ThoughtId + const thoughtX: Thought = { - id: 'testIdX' as ThoughtId, + id: idX, childrenMap: { - child1: 'child1' as ThoughtId, - child2: 'child2' as ThoughtId, + [childId1]: childId1, + [childId2]: childId2, }, created: timestamp(), lastUpdated: timestamp(), updatedBy: clientId, value: 'x', rank: 0, - parentId: 'parent1' as ThoughtId, + parentId: parentId1, } const thoughtA: Thought = { - id: 'testIdA' as ThoughtId, + id: idA, childrenMap: {}, created: timestamp(), lastUpdated: timestamp(), updatedBy: clientId, value: 'a', rank: 1, - parentId: 'parent2' as ThoughtId, + parentId: parentId2, } - await provider.updateThought?.('testIdX' as ThoughtId, undefined, thoughtX) - await provider.updateThought?.('testIdA' as ThoughtId, undefined, thoughtA) + await provider.updateThought?.(idX, undefined, thoughtX) + await provider.updateThought?.(idA, undefined, thoughtA) const dbThoughts = await provider.getThoughtsByIds([thoughtX.id, thoughtA.id]) expect(dbThoughts).toEqual([thoughtX, thoughtA]) @@ -248,45 +261,56 @@ const dataProviderTest = (provider: DataProvider) => { }) test('updateThoughtIndex', async () => { + const idX = '0000000000000000000000000000d00a' as ThoughtId + const idY = '0000000000000000000000000000d00b' as ThoughtId + const childId1 = '000000000000000000000000000c0021' as ThoughtId + const childId2 = '000000000000000000000000000c0022' as ThoughtId + const childId3 = '000000000000000000000000000c0023' as ThoughtId + const childId4 = '000000000000000000000000000c0024' as ThoughtId + const childId5 = '000000000000000000000000000c0025' as ThoughtId + const childId6 = '000000000000000000000000000c0026' as ThoughtId + const parentId1 = '0000000000000000000000000000a021' as ThoughtId + const parentId2 = '0000000000000000000000000000a022' as ThoughtId + const thoughtX: Thought = { - id: 'idX' as ThoughtId, + id: idX, childrenMap: { - child1: 'child1' as ThoughtId, - child2: 'child2' as ThoughtId, - child3: 'child3' as ThoughtId, + [childId1]: childId1, + [childId2]: childId2, + [childId3]: childId3, }, created: timestamp(), value: 'x', - parentId: 'parent1' as ThoughtId, + parentId: parentId1, rank: 0, lastUpdated: timestamp(), updatedBy: clientId, } const thoughtY: Thought = { - id: 'idY' as ThoughtId, + id: idY, childrenMap: { - child4: 'child4' as ThoughtId, - child5: 'child5' as ThoughtId, - child6: 'child6' as ThoughtId, + [childId4]: childId4, + [childId5]: childId5, + [childId6]: childId6, }, created: timestamp(), value: 'y', rank: 1, - parentId: 'parent2' as ThoughtId, + parentId: parentId2, lastUpdated: timestamp(), updatedBy: clientId, } await provider.updateThoughtIndex?.({ - idX: thoughtX, - idY: thoughtY, + [idX]: thoughtX, + [idY]: thoughtY, }) - const contextX = await getThoughtById(provider, 'idX' as ThoughtId) + const contextX = await getThoughtById(provider, idX) expect(contextX).toEqual(thoughtX) - const contextY = await getThoughtById(provider, 'idY' as ThoughtId) + const contextY = await getThoughtById(provider, idY) expect(contextY).toEqual(thoughtY) }) diff --git a/src/test-helpers/initStore.ts b/src/test-helpers/initStore.ts index 228b6083382..9b8a83649bd 100644 --- a/src/test-helpers/initStore.ts +++ b/src/test-helpers/initStore.ts @@ -1,6 +1,8 @@ import { clearActionCreator as clear } from '../actions/clear' +import { thoughtspaceRuntime } from '../data-providers/thoughtspace' import store from '../stores/app' import { resetStores } from '../stores/ministore' +import waitForThoughtspaceIdle from './waitForThoughtspaceIdle' interface Params { /** @@ -17,13 +19,16 @@ interface Params { /** * Initializes the store. Defaults to clearing the store and skipping the tutorial. */ -const initStore = ({ persist, allowTutorial }: Params = {}) => { +const initStore = async ({ persist, allowTutorial }: Params = {}) => { // Use fake timers so throttled/debounced side effects (e.g., url/history updates, storage writes) // don't execute after the test completes and the environment is torn down. // This makes tests deterministic and prevents post-teardown access to window/localStorage. vi.useFakeTimers() if (!persist) { + await waitForThoughtspaceIdle() + await thoughtspaceRuntime.drop() + await thoughtspaceRuntime.init({ storage: 'memory' }) store.dispatch(clear()) // Ministores are module-level singletons that vitest only isolates per test file, so reset them diff --git a/src/test-helpers/moveThoughtAtFirstMatch.ts b/src/test-helpers/moveThoughtAtFirstMatch.ts index 6505ad46c9a..405ce1db76e 100644 --- a/src/test-helpers/moveThoughtAtFirstMatch.ts +++ b/src/test-helpers/moveThoughtAtFirstMatch.ts @@ -8,7 +8,10 @@ import rootedParentOf from '../selectors/rootedParentOf' import appendToPath from '../util/appendToPath' import head from '../util/head' -type Payload = Omit & { from: string[]; to: string[] } +type Payload = Omit & { + from: string[] + to: string[] +} /** * Get ranked old and new paths for the unranked paths. diff --git a/src/test-helpers/treecrdt/createTestSystemThoughtIndexes.ts b/src/test-helpers/treecrdt/createTestSystemThoughtIndexes.ts new file mode 100644 index 00000000000..92e111d9403 --- /dev/null +++ b/src/test-helpers/treecrdt/createTestSystemThoughtIndexes.ts @@ -0,0 +1,62 @@ +import type Index from '../../@types/IndexType' +import type Lexeme from '../../@types/Lexeme' +import type Thought from '../../@types/Thought' +import type Timestamp from '../../@types/Timestamp' +import { EM_TOKEN, ROOT_PARENT_ID, SETTINGS_TOKEN, SETTINGS_VALUE } from '../../constants' +import { SYSTEM_ROOT_THOUGHT_IDS } from '../../data-providers/treecrdt/systemThoughtIds' +import hashThought from '../../util/hashThought' + +/** Creates em-style indexes for the in-memory TreeCRDT unit-test provider. */ +export const createTestSystemThoughtIndexes = ( + created: Timestamp = 0 as Timestamp, +): { + thoughtIndex: Index + lexemeIndex: Index +} => { + const thoughtIndex: Index = {} + + for (const id of SYSTEM_ROOT_THOUGHT_IDS) { + thoughtIndex[id] = { + id, + value: id, + rank: 0, + created, + lastUpdated: created, + updatedBy: '', + parentId: ROOT_PARENT_ID, + childrenMap: {}, + } + } + + thoughtIndex[EM_TOKEN] = { + ...thoughtIndex[EM_TOKEN], + childrenMap: { + [SETTINGS_TOKEN]: SETTINGS_TOKEN, + }, + } + + thoughtIndex[SETTINGS_TOKEN] = { + id: SETTINGS_TOKEN, + value: SETTINGS_VALUE, + rank: 0, + created, + lastUpdated: created, + updatedBy: '', + parentId: EM_TOKEN, + childrenMap: {}, + } + + return { + thoughtIndex, + lexemeIndex: { + [hashThought(SETTINGS_VALUE)]: { + contexts: [SETTINGS_TOKEN], + created, + lastUpdated: created, + updatedBy: '', + }, + }, + } +} + +export default createTestSystemThoughtIndexes diff --git a/src/test-helpers/waitForThoughtspaceIdle.ts b/src/test-helpers/waitForThoughtspaceIdle.ts new file mode 100644 index 00000000000..07212ecd48a --- /dev/null +++ b/src/test-helpers/waitForThoughtspaceIdle.ts @@ -0,0 +1,23 @@ +import { thoughtspaceRuntime } from '../data-providers/thoughtspace' +import syncStatusStore from '../stores/syncStatus' + +/** True when all thoughtspace pulls visible to app tests have drained. */ +const isThoughtspacePullingSettled = (): boolean => { + const { isPulling } = syncStatusStore.getState() + return !isPulling +} + +/** Waits for real TreeCRDT persistence and pull queue work used by unit tests. */ +const waitForThoughtspaceIdle = async (): Promise => { + for (let i = 0; i < 3; i++) { + await thoughtspaceRuntime.waitForIdle() + + if (!isThoughtspacePullingSettled()) { + await syncStatusStore.once(isThoughtspacePullingSettled) + } + + await thoughtspaceRuntime.waitForIdle() + } +} + +export default waitForThoughtspaceIdle diff --git a/src/util/__tests__/initEvents.lifecycle.ts b/src/util/__tests__/initEvents.lifecycle.ts index 94e42a9843f..eb4ff7f3a23 100644 --- a/src/util/__tests__/initEvents.lifecycle.ts +++ b/src/util/__tests__/initEvents.lifecycle.ts @@ -31,8 +31,8 @@ vi.mock('page-lifecycle', () => ({ }, })) -beforeEach(() => { - initStore() +beforeEach(async () => { + await initStore() }) afterEach(() => { diff --git a/src/util/__tests__/removeHome.ts b/src/util/__tests__/removeHome.ts index 7c2f935495a..db744b50bf2 100644 --- a/src/util/__tests__/removeHome.ts +++ b/src/util/__tests__/removeHome.ts @@ -1,4 +1,4 @@ -import { HOME_TOKEN } from '../../constants' +import { HOME_DISPLAY_VALUE, HOME_TOKEN } from '../../constants' import removeHome from '../removeHome' it('remove home thought', () => { @@ -24,3 +24,11 @@ it('do not remove first thought if it is not a home root', () => { expect(removeHome(exported)).toBe(exported) }) + +it('shows the display label for a lone home root', () => { + expect(removeHome(`- ${HOME_TOKEN}`)).toBe(`- ${HOME_DISPLAY_VALUE}`) +}) + +it('shows the display label for a lone home root without a bullet', () => { + expect(removeHome(HOME_TOKEN)).toBe(HOME_DISPLAY_VALUE) +}) diff --git a/src/util/addContext.ts b/src/util/addContext.ts index fabbe89e5ee..8fdd391f959 100644 --- a/src/util/addContext.ts +++ b/src/util/addContext.ts @@ -1,7 +1,7 @@ import Lexeme from '../@types/Lexeme' import ThoughtId from '../@types/ThoughtId' import Timestamp from '../@types/Timestamp' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import timestamp from './timestamp' /** Returns a new thought plus the given context. Does not add duplicates. */ diff --git a/src/util/createChildrenMap.ts b/src/util/createChildrenMap.ts index e47c67244bb..8697b97713b 100644 --- a/src/util/createChildrenMap.ts +++ b/src/util/createChildrenMap.ts @@ -7,7 +7,7 @@ import isAttribute from '../util/isAttribute' import keyValueBy from '../util/keyValueBy' /** Creates a childrenMapKey based on the value of meta thoughts and the id for non-meta thoughts. Always use the id as key if there is a duplicate meta value. */ -export const childrenMapKey = (thoughtIndex: Index, child: Thought) => +export const childrenMapKey = (thoughtIndex: Index, child: Pick) => child && isAttribute(child.value) && !thoughtIndex[child.value] ? child.value : child.id /** Generates an object for O(1) lookup of a thought's children. Meta attributes are keyed by value and normal are keyed by id. Missing thoughts are excluded. */ diff --git a/src/util/createId.ts b/src/util/createId.ts index 0fe46e9a24b..d0184fc7b4d 100644 --- a/src/util/createId.ts +++ b/src/util/createId.ts @@ -1,11 +1,11 @@ -import { nanoid } from 'nanoid' import ThoughtId from '../@types/ThoughtId' -/** Creates a universally unique identifier. */ -// Should be safe to use a nanoid of length 13 rather than the default 21 since thoughts only need to be unique per thoughtspace. -// 100 IDs/hr @ length 13 for ~89 thousand years -> 1% probability of collision -// If thoughts are stored globally, we should increase this length. -// See: https://zelark.github.io/nano-id-cc/ -const createId: (length?: number) => ThoughtId = (length = 13) => nanoid(length) as ThoughtId +/** Creates a 128-bit random hex identifier compatible with treecrdt NodeId (32 lowercase hex chars, 16 bytes). */ +const createId = (): ThoughtId => { + const bytes = crypto.getRandomValues(new Uint8Array(16)) + let hex = '' + for (const b of bytes) hex += b.toString(16).padStart(2, '0') + return hex as ThoughtId +} export default createId diff --git a/src/util/importJson.ts b/src/util/importJson.ts index e578927b781..7d33cd473d0 100644 --- a/src/util/importJson.ts +++ b/src/util/importJson.ts @@ -10,7 +10,7 @@ import ThoughtIndices from '../@types/ThoughtIndices' import Timestamp from '../@types/Timestamp' import { deleteThought } from '../actions' import { EM_TOKEN, HOME_TOKEN } from '../constants' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import { anyChild } from '../selectors/getChildren' import getLexeme from '../selectors/getLexeme' import getNextRank from '../selectors/getNextRank' diff --git a/src/util/initialState.ts b/src/util/initialState.ts index f7d8c96b69d..45f33819270 100644 --- a/src/util/initialState.ts +++ b/src/util/initialState.ts @@ -5,7 +5,7 @@ import Thought from '../@types/Thought' import ThoughtIndices from '../@types/ThoughtIndices' import Timestamp from '../@types/Timestamp' import { ABSOLUTE_TOKEN, EM_TOKEN, HOME_TOKEN, LongPressState, ROOT_PARENT_ID, SCHEMA_LATEST } from '../constants' -import { clientId, tsidShared } from '../data-providers/yjs' +import { clientId, tsidShared } from '../data-providers/thoughtspaceSession' import storageModel from '../stores/storageModel' import hashThought from '../util/hashThought' import never from '../util/never' diff --git a/src/util/mergeBatch.ts b/src/util/mergeBatch.ts index dc2de1c4cb3..8d83b8d5f83 100644 --- a/src/util/mergeBatch.ts +++ b/src/util/mergeBatch.ts @@ -28,6 +28,7 @@ const mergeBatch = (accum: PushBatch, batch: Partial): PushBatch => ( ...batch.recentlyEdited, }, pendingDeletes: [...(accum.pendingDeletes || []), ...(batch.pendingDeletes || [])], + movePlacements: { ...(accum.movePlacements || {}), ...(batch.movePlacements || {}) }, updates: { ...accum.updates, ...batch.updates, diff --git a/src/util/moveLexemeThought.ts b/src/util/moveLexemeThought.ts index b5f40678998..1ccf12af8df 100644 --- a/src/util/moveLexemeThought.ts +++ b/src/util/moveLexemeThought.ts @@ -1,6 +1,6 @@ import Lexeme from '../@types/Lexeme' import State from '../@types/State' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import getThoughtById from '../selectors/getThoughtById' import concatOne from '../util/concatOne' import timestamp from '../util/timestamp' diff --git a/src/util/newLexeme.ts b/src/util/newLexeme.ts index 89bdaa5fec9..89c4b157ee8 100644 --- a/src/util/newLexeme.ts +++ b/src/util/newLexeme.ts @@ -1,7 +1,7 @@ import Lexeme from '../@types/Lexeme' import ThoughtId from '../@types/ThoughtId' import Timestamp from '../@types/Timestamp' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import timestamp from './timestamp' /** Creates a new Lexeme with a single context. Use addContext to add a context to an existing Lexeme. */ diff --git a/src/util/removeContext.ts b/src/util/removeContext.ts index 22565f5d4cb..063b01cafa6 100644 --- a/src/util/removeContext.ts +++ b/src/util/removeContext.ts @@ -1,7 +1,7 @@ import Index from '../@types/IndexType' import Lexeme from '../@types/Lexeme' import Timestamp from '../@types/Timestamp' -import { clientId } from '../data-providers/yjs' +import { clientId } from '../data-providers/thoughtspaceSession' import timestamp from './timestamp' /** Returns a shallow copy of an object with all keys that do not have a value of null. */ diff --git a/src/util/removeHome.ts b/src/util/removeHome.ts index ad100984dd5..e4bce8d69a7 100644 --- a/src/util/removeHome.ts +++ b/src/util/removeHome.ts @@ -1,18 +1,24 @@ +import { HOME_DISPLAY_VALUE } from '../constants' import isHome from './isHome' -/** - * Remove home token, de-indent (trim), and append newline to make tests more readable. - */ +/** Remove the Home wrapper from exports; for a bare root export, keep a readable placeholder instead of HOME_TOKEN. */ const removeHome = (exported: string) => { const firstLineBreakIndex = exported.indexOf('\n') - const firstThought = exported.slice(0, firstLineBreakIndex).slice(1).trim() + const hasChildren = firstLineBreakIndex !== -1 + const firstLine = hasChildren ? exported.slice(0, firstLineBreakIndex) : exported + const hasBullet = firstLine.startsWith('- ') + const firstThought = (hasBullet ? firstLine.slice(2) : firstLine).trim() return isHome([firstThought]) - ? exported - .slice(firstLineBreakIndex) - .split('\n') - .map(line => line.slice(2)) - .join('\n') + '\n' + ? hasChildren + ? exported + .slice(firstLineBreakIndex) + .split('\n') + .map(line => line.slice(2)) + .join('\n') + '\n' + : hasBullet + ? `- ${HOME_DISPLAY_VALUE}` + : HOME_DISPLAY_VALUE : exported } diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index 0aec2fb8518..8722d3c8657 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -5,6 +5,7 @@ interface ImportMetaEnv { readonly VITE_WEBSOCKET_HOST: string readonly VITE_WEBSOCKET_PORT: number readonly VITE_AI_URL: string + readonly VITE_TREECRDT_SYNC_BASE_URL?: string /** When truthy, enables the console proxy in src/util/consoleProxy.ts. Set for BrowserStack CI runs and AI agents that use the WDIO MCP; unset for production. The flag must be set at both build-time and run-time. */ readonly VITE_BROWSER_CONSOLE_CAPTURE?: string } diff --git a/tsconfig.json b/tsconfig.json index e0beb5fbd02..559b42f352a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -27,8 +27,10 @@ // DefaultRootState was removed in v8 and thus cannot be augmented anymore. // https://stackoverflow.com/a/78550104/480608 "react-redux-default": ["./node_modules/react-redux"], - "react-redux": ["./src/@types/react-redux.d.ts"] + "react-redux": ["./src/@types/react-redux.d.ts"], + "@treecrdt/wa-sqlite/client": ["./node_modules/@treecrdt/wa-sqlite/dist/client.d.ts"], + "@treecrdt/wa-sqlite/vite-plugin": ["./node_modules/@treecrdt/wa-sqlite/dist/vite-plugin.d.ts"] }, - "types": ["vitest/globals", "@wdio/globals/types", "@wdio/mocha-framework", "@wdio/browserstack-service"] + "types": ["node", "vitest/globals", "@wdio/globals/types", "@wdio/mocha-framework", "@wdio/browserstack-service"] } } diff --git a/vite.config.ts b/vite.config.ts index bc31586a25a..546db07ad9d 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,3 +1,4 @@ +import { treecrdt } from '@treecrdt/wa-sqlite/vite-plugin' import basicSsl from '@vitejs/plugin-basic-ssl' import react from '@vitejs/plugin-react' import { execSync } from 'child_process' @@ -81,11 +82,19 @@ export default defineConfig({ build: { outDir: 'build', }, + worker: { + format: 'es', + }, + optimizeDeps: { + // Avoid crawling stale local checkout directories left behind after removing the TreeCRDT submodule. + entries: ['index.html'], + }, define: { __COMMIT_HASH__: JSON.stringify(commitHash), }, plugins: [ react(), + treecrdt({ outDir: 'public/wa-sqlite' }), // Do not run vite-plugin-checker during tests, as it will clear the test output. // The dev server is usually running anyway, and tsc is run in lint:tsc which is triggered prepush. ...[!process.env.VITEST && !process.env.PUPPETEER ? checker({ typescript: true }) : undefined], @@ -96,7 +105,7 @@ export default defineConfig({ filename: 'service-worker.ts', injectManifest: { maximumFileSizeToCacheInBytes: 4 * 1024 * 1024, // Increase limit to 4 MiB - globPatterns: ['**/*.{js,css,html,webp,woff2}'], + globPatterns: ['**/*.{js,mjs,wasm,css,html,webp,woff2}'], }, manifest: { name: 'em', diff --git a/vitest.config.ts b/vitest.config.ts index b009f869553..3bb73f1c3ba 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,6 +2,7 @@ import { createRequire } from 'node:module' import Terminal from 'vite-plugin-terminal' import { defineConfig } from 'vitest/config' +const puppeteerMaxWorkers = Number(process.env.PUPPETEER_MAX_WORKERS || 2) const require = createRequire(import.meta.url) export default defineConfig({ @@ -9,6 +10,7 @@ export default defineConfig({ projects: [ { extends: './vite.config.ts', + plugins: [], test: { name: 'unit', globals: true, @@ -37,6 +39,9 @@ export default defineConfig({ exclude: ['node_modules/**'], environment: './src/e2e/puppeteer-environment.ts', setupFiles: ['./src/e2e/puppeteer/setup.ts'], + // Browserless runs all Puppeteer files in one Chrome service. Unbounded file parallelism overloads + // touch/focus handling and OPFS cleanup, so keep bounded parallelism instead of serializing the suite. + maxWorkers: puppeteerMaxWorkers, }, plugins: [ Terminal({ diff --git a/yarn.lock b/yarn.lock index ea7cb238ee2..7f55941cfce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2292,6 +2292,13 @@ __metadata: languageName: node linkType: hard +"@bufbuild/protobuf@npm:^2.10.2": + version: 2.12.0 + resolution: "@bufbuild/protobuf@npm:2.12.0" + checksum: 10c0/28959e371fd2e7c8d67d52af9e3c8f8e6770e2e116dcc5508f469ca736afa6905e216bc76183835898a5f3deb8aa2e7db47d4a0e706fba76115e9a1996893191 + languageName: node + linkType: hard + "@bufbuild/protobuf@npm:^2.5.2": version: 2.10.2 resolution: "@bufbuild/protobuf@npm:2.10.2" @@ -4575,6 +4582,20 @@ __metadata: languageName: node linkType: hard +"@noble/ed25519@npm:^3.0.0": + version: 3.1.0 + resolution: "@noble/ed25519@npm:3.1.0" + checksum: 10c0/6317722130649cf5884eff57d540b36ec1006cf68145b09ff427d88df359a3b12baaaf420aad87b747f49d30c5ae56065bf31e5ebde1a6d59c3ce9d0b6a68fdb + languageName: node + linkType: hard + +"@noble/hashes@npm:^1.8.0": + version: 1.8.0 + resolution: "@noble/hashes@npm:1.8.0" + checksum: 10c0/06a0b52c81a6fa7f04d67762e08b2c476a00285858150caeaaff4037356dd5e119f45b2a530f638b77a5eeca013168ec1b655db41bae3236cb2e9d511484fc77 + languageName: node + linkType: hard + "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5" @@ -6853,6 +6874,94 @@ __metadata: languageName: node linkType: hard +"@treecrdt/auth@npm:0.1.2": + version: 0.1.2 + resolution: "@treecrdt/auth@npm:0.1.2" + dependencies: + "@noble/ed25519": "npm:^3.0.0" + "@noble/hashes": "npm:^1.8.0" + "@treecrdt/interface": "npm:0.2.0" + "@treecrdt/sync-protocol": "npm:0.1.2" + cborg: "npm:^4.3.2" + checksum: 10c0/1de0bbbc2b93ae9d3803ca41710787b64af4c5acf35cac832c24d8ef1df8c053ac68a0f1810cb8bcfbebf41285813339442681e7bd96853a79554e71057ead4c + languageName: node + linkType: hard + +"@treecrdt/discovery@npm:0.1.0": + version: 0.1.0 + resolution: "@treecrdt/discovery@npm:0.1.0" + checksum: 10c0/3190e765222fa0e72a1427fa445311c8e4051208065fde410c4118f0ad2b37c50f91b475f6eec8c4d527b3fcf7b28b4dfa9174983b9b7f1809fcb98735862ffc + languageName: node + linkType: hard + +"@treecrdt/interface@npm:0.2.0": + version: 0.2.0 + resolution: "@treecrdt/interface@npm:0.2.0" + checksum: 10c0/50ff3dd6e32be057288150fbf4bf1906fb9aa97bb19aab04607187731dd8cf8517f2ba7265d604466acdab3ead2bd38cc3c5e2c5aef4240eb21d5102aa2f67b8 + languageName: node + linkType: hard + +"@treecrdt/riblt-wasm@npm:0.1.0": + version: 0.1.0 + resolution: "@treecrdt/riblt-wasm@npm:0.1.0" + checksum: 10c0/278908546bcc6a0fb020541e4718662baab59f9fd2999a6ee5725e0a3cfc4dad9bd4d27bed11b57def4b21b08e5715171aff0846d499102a9fedafe6786c1079 + languageName: node + linkType: hard + +"@treecrdt/sync-protocol@npm:0.1.2": + version: 0.1.2 + resolution: "@treecrdt/sync-protocol@npm:0.1.2" + dependencies: + "@bufbuild/protobuf": "npm:^2.10.2" + "@noble/hashes": "npm:^1.8.0" + "@treecrdt/interface": "npm:0.2.0" + "@treecrdt/riblt-wasm": "npm:0.1.0" + checksum: 10c0/4e6b9527890f849193f353a333103ea4004833baf084b2f55ed7160ca6542cd0765b1fb668f68aa947c8e65c6a3b367aed6ac83ae4519c6f9f68cee646057c0f + languageName: node + linkType: hard + +"@treecrdt/sync-server-core@npm:0.1.2": + version: 0.1.2 + resolution: "@treecrdt/sync-server-core@npm:0.1.2" + dependencies: + "@treecrdt/sync-protocol": "npm:0.1.2" + ws: "npm:^8.18.3" + checksum: 10c0/4abd0ac72a5c5741312e8aabcf4a22a379c370e85de02416548529f4efc728b62c726b89b35c6f4a1b2c55888541899584de7ea0fa301f08cf27a65dd4a343b0 + languageName: node + linkType: hard + +"@treecrdt/sync-sqlite@npm:0.1.2": + version: 0.1.2 + resolution: "@treecrdt/sync-sqlite@npm:0.1.2" + dependencies: + "@treecrdt/auth": "npm:0.1.2" + "@treecrdt/interface": "npm:0.2.0" + "@treecrdt/sync-protocol": "npm:0.1.2" + checksum: 10c0/8d9cf8af570aea8de188a89acf94c5ddccd968185e86ef0f7116211f502a625c482752f07765e3160af5574099c324c33fd1eff15ecd0c4935a1e33ecd10320e + languageName: node + linkType: hard + +"@treecrdt/sync@npm:0.1.2": + version: 0.1.2 + resolution: "@treecrdt/sync@npm:0.1.2" + dependencies: + "@treecrdt/discovery": "npm:0.1.0" + "@treecrdt/interface": "npm:0.2.0" + "@treecrdt/sync-protocol": "npm:0.1.2" + "@treecrdt/sync-sqlite": "npm:0.1.2" + checksum: 10c0/bf43f9e52dfd59055b811cd677be7603c1718a0e1bfbd4ef2b04178a5b06f468e81edc044038ddb05d4d1d8252f8e55fb70872107efe3363b43ca74973d7577f + languageName: node + linkType: hard + +"@treecrdt/wa-sqlite@npm:0.4.2": + version: 0.4.2 + resolution: "@treecrdt/wa-sqlite@npm:0.4.2" + dependencies: + "@treecrdt/interface": "npm:0.2.0" + checksum: 10c0/4031ea77dd862b4b1988887090792f32c2ecbe1c951e0770c4a0567f3fc29edfdc1d5b24a8805eba187889d490739dd6a4a168bee0adf52e83f16d397bc43d3c + languageName: node + linkType: hard + "@trickfilm400/rollup-plugin-off-main-thread@npm:^3.0.0-pre1": version: 3.0.0-pre1 resolution: "@trickfilm400/rollup-plugin-off-main-thread@npm:3.0.0-pre1" @@ -10111,6 +10220,15 @@ __metadata: languageName: node linkType: hard +"cborg@npm:^4.3.2": + version: 4.5.8 + resolution: "cborg@npm:4.5.8" + bin: + cborg: lib/bin.js + checksum: 10c0/9703fc3201aeb9814c6f27358f079f63bd2361bb946aae4fc82db3ea389418516251a2ea85a728839af887ff43fd67f9900353aae6daa0c0d8b1bc6f20addeb4 + languageName: node + linkType: hard + "chai@npm:^6.2.1": version: 6.2.1 resolution: "chai@npm:6.2.1" @@ -11683,6 +11801,15 @@ __metadata: "@testing-library/react": "npm:^16.3.2" "@testing-library/user-event": "npm:^14.6.4" "@total-typescript/ts-reset": "npm:^0.6.1" + "@treecrdt/auth": "npm:0.1.2" + "@treecrdt/discovery": "npm:0.1.0" + "@treecrdt/interface": "npm:0.2.0" + "@treecrdt/riblt-wasm": "npm:0.1.0" + "@treecrdt/sync": "npm:0.1.2" + "@treecrdt/sync-protocol": "npm:0.1.2" + "@treecrdt/sync-server-core": "npm:0.1.2" + "@treecrdt/sync-sqlite": "npm:0.1.2" + "@treecrdt/wa-sqlite": "npm:0.4.2" "@trivago/prettier-plugin-sort-imports": "npm:^6.0.2" "@types/clipboard": "npm:^2.0.10" "@types/dompurify": "npm:^3.2.0" @@ -11816,10 +11943,7 @@ __metadata: workbox-strategies: "npm:^7.4.1" workbox-window: "npm:^7.4.1" xhtml-purifier: "npm:^0.4.3" - y-indexeddb: "⚠️ OVERRIDDEN BY 'resolutions' - Source: https://github.com/raineorshine/y-indexeddb#y-indexeddb-multiplex | Tarball: https://codeload.github.com/raineorshine/y-indexeddb/tar.gz/60b960009085b1a988b5064ee35703229231531f | Configured in: package.json#resolutions" - y-protocols: "npm:^1.0.7" yallist: "npm:^5.0.0" - yjs: "npm:^13.6.32" languageName: unknown linkType: soft @@ -15894,13 +16018,6 @@ __metadata: languageName: node linkType: hard -"isomorphic.js@npm:^0.2.4": - version: 0.2.5 - resolution: "isomorphic.js@npm:0.2.5" - checksum: 10c0/7cd268c8e58146a8160c8cd16596291fd1fbf3e8799a325f269accda9dc1238806e371ccef0b66fe2ad957209230c55997248d8b6d02cf2d7c575ffeb759c789 - languageName: node - linkType: hard - "istanbul-lib-coverage@npm:^3.0.0, istanbul-lib-coverage@npm:^3.2.0": version: 3.2.2 resolution: "istanbul-lib-coverage@npm:3.2.2" @@ -17177,32 +17294,6 @@ __metadata: languageName: node linkType: hard -"lib0@npm:^0.2.74, lib0@npm:^0.2.85": - version: 0.2.98 - resolution: "lib0@npm:0.2.98" - dependencies: - isomorphic.js: "npm:^0.2.4" - bin: - 0ecdsa-generate-keypair: bin/0ecdsa-generate-keypair.js - 0gentesthtml: bin/gentesthtml.js - 0serve: bin/0serve.js - checksum: 10c0/074097bcb90a002449cfddaa11552f11d088e38233f0a20a3358b08d7d1952b2f5a491fb63e63ccf66a8ec037e16d82d40a27601bbacf5ded0d2ec810bdf550e - languageName: node - linkType: hard - -"lib0@npm:^0.2.99": - version: 0.2.114 - resolution: "lib0@npm:0.2.114" - dependencies: - isomorphic.js: "npm:^0.2.4" - bin: - 0ecdsa-generate-keypair: bin/0ecdsa-generate-keypair.js - 0gentesthtml: bin/gentesthtml.js - 0serve: bin/0serve.js - checksum: 10c0/3edaebb4ac80da164dfc097720a610bba4d08f44dc60ecb186aa42924d8e67679c12a6b7415eaffc07c47f1fbddccb15fd925bee7745c7aef16ab1a1e0a3cdc0 - languageName: node - linkType: hard - "lie@npm:~3.3.0": version: 3.3.0 resolution: "lie@npm:3.3.0" @@ -25473,6 +25564,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:^8.18.3": + version: 8.18.3 + resolution: "ws@npm:8.18.3" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 10c0/eac918213de265ef7cb3d4ca348b891a51a520d839aa51cdb8ca93d4fa7ff9f6ccb339ccee89e4075324097f0a55157c89fa3f7147bde9d8d7e90335dc087b53 + languageName: node + linkType: hard + "ws@npm:^8.21.0": version: 8.21.0 resolution: "ws@npm:8.21.0" @@ -25580,28 +25686,6 @@ __metadata: languageName: node linkType: hard -"y-indexeddb@https://codeload.github.com/raineorshine/y-indexeddb/tar.gz/60b960009085b1a988b5064ee35703229231531f": - version: 9.0.11-multiplex.0 - resolution: "y-indexeddb@https://codeload.github.com/raineorshine/y-indexeddb/tar.gz/60b960009085b1a988b5064ee35703229231531f" - dependencies: - lib0: "npm:^0.2.74" - peerDependencies: - yjs: ^13.0.0 - checksum: 10c0/fa4ba1b81f2c5e1fcee7445a1ff7a108947f83b1fea75a6bbf2efba925279726f0acc6b9c9c9f1e7f40642a1e34c9712e71ff3a528ca2890d42297ac82d92e43 - languageName: node - linkType: hard - -"y-protocols@npm:^1.0.7": - version: 1.0.7 - resolution: "y-protocols@npm:1.0.7" - dependencies: - lib0: "npm:^0.2.85" - peerDependencies: - yjs: ^13.0.0 - checksum: 10c0/70e14ab50b9200e97602c5effc3e14ae4ea0f51479a1628569af7cc12a6739fbc02484335828f5c508a4bf6fa0c3ef5ae1c17e13346b7a6093abeb9627f59021 - languageName: node - linkType: hard - "y18n@npm:^5.0.5": version: 5.0.8 resolution: "y18n@npm:5.0.8" @@ -25733,15 +25817,6 @@ __metadata: languageName: node linkType: hard -"yjs@npm:^13.6.32": - version: 13.6.32 - resolution: "yjs@npm:13.6.32" - dependencies: - lib0: "npm:^0.2.99" - checksum: 10c0/07b50f475d3749a6b135e959fdc805eea8e7fdad4d6177b151c9d30463399577586252e34b3949fdd03260ee7b8526fa01a8b10c64e2506fc96ca6e22ae58032 - languageName: node - linkType: hard - "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"