diff --git a/.dependency-cruiser.cjs b/.dependency-cruiser.cjs index 5606ed7c..252321dd 100644 --- a/.dependency-cruiser.cjs +++ b/.dependency-cruiser.cjs @@ -15,7 +15,7 @@ module.exports = { name: 'not-to-unresolvable', comment: 'A broken/typo/moved import must fail the build, not surface at runtime.', severity: 'error', - from: {}, + from: { path: '^src/' }, to: { couldNotResolve: true } }, { @@ -71,7 +71,8 @@ module.exports = { 'the boundary. A stray core->pro import ships paid source in the public repo.', severity: 'error', from: { - pathNot: '(loadProFeaturesMain|loadProFeaturesRenderer|main\\.tsx|bootstrap/proStub)' + pathNot: + '(loadProFeaturesMain|loadProFeaturesRenderer|main\\.tsx|bootstrap/proStub|\\.(test|spec)\\.[tj]sx?$|/__tests__/)' }, to: { path: 'bootstrap/proStub\\.ts$|(^|/)pro/(main|renderer)/' } }, @@ -107,7 +108,9 @@ module.exports = { } ], options: { - doNotFollow: { path: 'node_modules' }, + // Pro is a sibling checkout in CI. Inspect the core -> Pro edge, but do not + // traverse the private graph and apply open-core rules inside Pro itself. + doNotFollow: { path: 'node_modules|^pro/' }, tsConfig: { fileName: 'tsconfig.web.json' }, exclude: { path: 'node_modules|e2e/' }, tsPreCompilationDeps: true, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94b5ad78..d9496bff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ on: pull_request: push: branches: [main] + workflow_dispatch: jobs: # ONE job per repo, matching mobile and mobile-pro: typecheck, tests + coverage, boundaries, lint and the # Playwright e2e all report as a single `ci` check. @@ -25,6 +26,10 @@ jobs: timeout-minutes: 50 # backstop: a hung step fails fast instead of running for hours (unit gates + e2e tour) steps: - uses: actions/checkout@v4 + with: + # The authoritative coverage gate compares the checked-out change with + # origin/main. A depth-1 checkout has no merge base to measure. + fetch-depth: 0 # Check out pro on the branch that MATCHES this PR/push (so a coordinated # core+pro change is tested together). If pro has no such branch (most PRs), # this step's checkout fails quietly and the fallback below pulls pro `main`. @@ -40,6 +45,7 @@ jobs: token: ${{ secrets.CI_CROSS_REPO_TOKEN }} path: pro ref: ${{ github.head_ref || github.ref_name }} + fetch-depth: 0 # Don't leave the cross-repo PAT in ./pro/.git/config where a later # PR-controlled step could reuse it — we only need it for the clone. persist-credentials: false @@ -51,6 +57,7 @@ jobs: repository: off-grid-ai/desktop-pro token: ${{ secrets.CI_CROSS_REPO_TOKEN }} path: pro + fetch-depth: 0 persist-credentials: false # pro MUST be present: without it the guarded typecheck/coverage path is skipped and # CI would pass core-only with cratered/misleading coverage (the failure the @@ -63,7 +70,10 @@ jobs: fi - uses: actions/setup-node@v4 with: - node-version: '24' # node:sqlite (used by integration tests) is available unflagged + # Node 22 supports node:sqlite and both native SQLite dependencies. Node + # 24 aborts while Vitest unloads the cipher module; Node 26 cannot build + # better-sqlite3 12.6. + node-version: '22' # Shared packages are file: dependencies on the SIBLING shared monorepo. CI must provision it before # installing Desktop. actions/checkout refuses a path outside the workspace, so it lands inside and is # moved up one level - exactly where the file: specifiers point. @@ -107,6 +117,30 @@ jobs: npm --prefix ../shared ci npm --prefix ../shared/packages/sync run build npm --prefix ../shared/packages/models run build + npm --prefix ../shared/packages/speech run build + npm --prefix ../shared/packages/ui run build + - name: Check out the private Desktop speech runtime + uses: actions/checkout@v4 + with: + repository: off-grid-ai/executorch-speech + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + path: _executorch_speech + lfs: false + # Keep the checkout credential only until the selective LFS pull in + # the next step. That step removes it before any project code runs. + persist-credentials: true + - name: Put the speech runtime beside this checkout + run: | + if [ ! -f _executorch_speech/package.json ]; then + echo "::error::off-grid-ai/executorch-speech was not checked out. Check CI_CROSS_REPO_TOKEN." + exit 1 + fi + # Tests exercise the real native boundary, but ordinary CI does not need + # the 350 MB bundled English asset set used only by the Mac release. + git -C _executorch_speech lfs pull --include='native/bin/executorch-speech' --exclude='' + git -C _executorch_speech config --unset-all http.https://github.com/.extraheader || true + rm -rf ../executorch-speech + mv _executorch_speech ../executorch-speech - run: npm ci # Hard gates: types + the full test suite. - name: Typecheck (core) @@ -116,12 +150,23 @@ jobs: if: ${{ hashFiles('pro/tsconfig.json') != '' }} timeout-minutes: 6 run: cd pro && npx tsc --noEmit -p tsconfig.json - # Runs the full vitest suite AND enforces the coverage ratchet floor - # (vitest.config.ts `thresholds`) — the same gate as the pre-push hook, so a - # coverage regression fails CI, not just local pushes. The pro/** threshold - # group applies only when pro was checked out above (guarded in the config). - - name: Test + coverage thresholds - timeout-minutes: 10 + # npm's Electron install hooks leave sqlite compiled for Electron's ABI. The + # Vitest process is plain Node, and some default integration tests open the + # real database, so put the native module on Node's ABI before that gate. + # The later DB journey script restores the Electron ABI in its EXIT trap. + - name: Rebuild SQLite for the Node test runner + run: npm rebuild better-sqlite3-multiple-ciphers + # This is the fast report, not the complete coverage measurement. DB-only + # journeys run next and e2e can add coarse coverage later. Test failures still + # block here; the aggregate new-code gate below owns the coverage decision. + - name: Test + fast coverage report + # The tests finish in about four minutes. V8 then maps coverage across the + # complete core + Pro source set, including files no test imported. That + # aggregation can take another six minutes on a busy runner, so give report + # generation enough time to finish. + timeout-minutes: 20 + env: + OFFGRID_AGGREGATE_COVERAGE: '1' run: npm run test:coverage # The DB journeys - 74 files, 255 cases - which CI has NEVER run. # @@ -148,7 +193,7 @@ jobs: # reason is recorded in vitest.db.ci.config.ts with the evidence from the run that found it. 243 of the # 248 cases still run here. OFFGRID_DB_VITEST_CONFIG: vitest.db.ci.config.ts - run: npm run test:db + run: npm run test:db -- --coverage # Build/native/port integration tests (packaging, whisper build-staging, the # model-port + System Health seams that own :8439). These need a packaged # build / native toolchain / a live engine port the pure `verify` runner @@ -233,6 +278,27 @@ jobs: else echo "no e2e coverage captured (the suite may not have launched)" fi + # Match the pre-push authority: merge every report that can cover this change, + # then gate only executable lines added on this branch. The floors are the same + # ratchets as scripts/hooks/pre-push; neither whole-tree debt nor a partial suite + # can decide the result. + - name: Coverage gate (new code across all suites) + run: | + reports="coverage/coverage-final.json coverage-db/coverage-final.json" + for report in $reports; do + if [ ! -f "$report" ]; then + echo "::error::required coverage report is missing: $report" + exit 1 + fi + done + coarse="" + if [ -f coverage-e2e/coverage-final.json ]; then + coarse="--coarse=coverage-e2e/coverage-final.json" + fi + node ../shared/scripts/new-code-coverage.mjs . $reports $coarse \ + --min-statements=78 --min-branches=57 --min-functions=52 --min-lines=78 + node ../shared/scripts/new-code-coverage.mjs ./pro $reports $coarse \ + --min-statements=72 --min-branches=45 --min-functions=46 --min-lines=72 - name: Upload e2e coverage if: ${{ always() && vars.OFFGRID_CI_E2E == '1' }} uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d361fb64..fe62118c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,15 @@ on: type: string required: false default: '' + artifact_only: + # Internal demo build: the SAME signed + notarized DMG, but uploaded as a run + # artifact (repo-scoped, auto-expiring) instead of a published Release. Skips the + # release-create/upload, the pro-mac.yml + latest/nightly aliases, Slack, and the + # whole Windows job - so nothing lands on the releases page or the update feed. + description: 'Build the signed+notarized DMG as a run artifact only (no Release, no Windows, no Slack)' + type: boolean + required: false + default: false permissions: contents: write @@ -149,6 +158,12 @@ jobs: chmod +x resources/bin/meeting-recorder file resources/bin/meeting-recorder otool -l resources/bin/meeting-recorder | awk '/LC_BUILD_VERSION/{f=1} f&&/minos/{print "[ci] meeting-recorder minos="$2; exit}' + - name: Build Computer Use capture helper (exclude supervisor window) + run: | + MACOS_DEPLOYMENT_TARGET=13.0 bash scripts/build-computer-use-capture.sh resources/bin + chmod +x resources/bin/computer-use-capture + file resources/bin/computer-use-capture + otool -l resources/bin/computer-use-capture | awk '/LC_BUILD_VERSION/{f=1} f&&/minos/{print "[ci] computer-use-capture minos="$2; exit}' # Compile the dictation push-to-talk helper (CGEventTap) and stage it into # resources/bin so extraResources bundles it at Contents/Resources/bin — the # path PushToTalkHotkey resolves. Self-contained: no committed binary, built @@ -160,6 +175,17 @@ jobs: mkdir -p resources/bin cp scripts/dictation-hotkey/dictation-hotkey resources/bin/dictation-hotkey chmod +x resources/bin/dictation-hotkey + # Compile the native actions helper (EventKit) and stage it into resources/bin so + # extraResources bundles it at Contents/Resources/bin — the path runNativeAction + # resolves. Self-contained: no committed binary, built fresh against the pinned + # target. If it ever fails to ship, the calendar tools report "not available" and + # the rest of the app is unaffected, so it can't break a release. + - name: Build native actions helper (computer use, semantic rail) + run: | + bash scripts/build-actions-helper.sh + mkdir -p resources/bin + cp scripts/actions-helper/actions-helper resources/bin/actions-helper + chmod +x resources/bin/actions-helper # Stage the Parakeet STT runtime (sherpa-onnx CLI + ONNX model) into # resources/bin/parakeet. Additive: with SHERPA_ONNX_URL / PARAKEET_MODEL_URL # unset this is a no-op and transcription stays on whisper, so it can't break a @@ -228,7 +254,27 @@ jobs: # drifted lock stops the release instead of resolving a graph nobody committed. npm --prefix ../shared ci npm --prefix ../shared/packages/sync run build + # @offgrid/use (the durable action engine) is consumed the same way as sync + # and needs the same explicit build - file-dep prepare alone does not emit + # its dist/ types before the typecheck runs. + npm --prefix ../shared/packages/use run build npm --prefix ../shared/packages/models run build + - name: Check out the private Desktop speech runtime + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: off-grid-ai/executorch-speech + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + path: _executorch_speech + lfs: true # Mac releases bundle the default English voices for offline first use. + persist-credentials: false + - name: Put the speech runtime beside this checkout + run: | + if [ ! -f _executorch_speech/package.json ]; then + echo "::error::off-grid-ai/executorch-speech was not checked out. Check CI_CROSS_REPO_TOKEN." + exit 1 + fi + rm -rf ../executorch-speech + mv _executorch_speech ../executorch-speech - name: Install dependencies run: npm ci # Stamp the resolved version into package.json so electron-builder picks the @@ -268,6 +314,7 @@ jobs: # notarytool needs the API key as a file; write it from the secret. printf '%s' "$APPLE_API_KEY_CONTENT" > "$RUNNER_TEMP/AuthKey.p8" export APPLE_API_KEY="$RUNNER_TEMP/AuthKey.p8" + npm run prepare:speech-defaults # Build every artifact with publication disabled. The artifact hook verifies # both the updater ZIP and DMG before this command can complete. npx electron-builder --mac \ @@ -278,7 +325,18 @@ jobs: APP="$(find dist -mindepth 2 -maxdepth 2 -type d -name 'Off Grid AI Desktop.app' -print -quit)" test -n "$APP" node scripts/probe-packaged-tts.mjs "$APP" --synthesize + # Internal demo build: hand the signed+notarized DMG back as a run artifact and stop + # (the publish steps below are all skipped). Repo-scoped, auto-expiring, no Release. + - name: Upload signed DMG as a build artifact + if: ${{ inputs.artifact_only }} + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4 + with: + name: OffGrid-macOS-${{ needs.version.outputs.version }} + path: dist/OffGrid-*.dmg + if-no-files-found: error + retention-days: 14 - name: Stage verified update assets and publish the release + if: ${{ !inputs.artifact_only }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} @@ -310,6 +368,7 @@ jobs: gh release edit "$TAG" --draft=false --prerelease fi - name: Migrate legacy Pro update channel (publish pro-mac.yml) + if: ${{ !inputs.artifact_only }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} @@ -335,7 +394,7 @@ jobs: # --clobber replaces the copy from the prior run. The versioned DMG + updater # feed (latest-mac.yml) are untouched, so auto-update is unaffected. - name: Publish stable latest.dmg alias - if: needs.version.outputs.channel == 'stable' + if: ${{ !inputs.artifact_only && needs.version.outputs.channel == 'stable' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} @@ -349,7 +408,7 @@ jobs: # /releases/latest or the stable OffGrid-latest.dmg: # https://github.com/off-grid-ai/off-grid-ai-desktop/releases/download/nightly/OffGrid-nightly.dmg - name: Publish constant nightly.dmg link - if: needs.version.outputs.channel == 'beta' + if: ${{ !inputs.artifact_only && needs.version.outputs.channel == 'beta' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} @@ -367,7 +426,7 @@ jobs: # v$VERSION tag exist for beta too, so beta/nightly releases get their changelog on # the release page, not just stable. - name: Attach release notes - if: success() + if: ${{ !inputs.artifact_only && success() }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ needs.version.outputs.version }} @@ -378,7 +437,7 @@ jobs: # Slack outage never fails a release. Needs org/repo secret SLACK_WEBHOOK_URL (an # Incoming Webhook, channel-bound). No secret => the step is a logged no-op. - name: Announce release in Slack - if: success() + if: ${{ !inputs.artifact_only && success() }} env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} PRODUCT: Off Grid AI Desktop @@ -398,6 +457,8 @@ jobs: # (the repo's LFS binaries are macOS-only). build-win: needs: [version, build-mac] + # Skipped for an artifact-only demo build (macOS DMG is all the lead needs). + if: ${{ !inputs.artifact_only }} # windows-2022 = VS 2022 toolchain. windows-latest ships VS 2026, which node-gyp 11 # can't parse, breaking native-module (better-sqlite3) compiles. Pin until it catches up. runs-on: windows-2022 @@ -469,7 +530,28 @@ jobs: # drifted lock stops the release instead of resolving a graph nobody committed. npm --prefix ../shared ci npm --prefix ../shared/packages/sync run build + # @offgrid/use (the durable action engine) is consumed the same way as sync + # and needs the same explicit build - file-dep prepare alone does not emit + # its dist/ types before the typecheck runs. + npm --prefix ../shared/packages/use run build npm --prefix ../shared/packages/models run build + - name: Check out the private Desktop speech package + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: off-grid-ai/executorch-speech + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + path: _executorch_speech + lfs: false # The current native speech engine is macOS-only. + persist-credentials: false + - name: Put the speech package beside this checkout + shell: bash + run: | + if [ ! -f _executorch_speech/package.json ]; then + echo "::error::off-grid-ai/executorch-speech was not checked out. Check CI_CROSS_REPO_TOKEN." + exit 1 + fi + rm -rf ../executorch-speech + mv _executorch_speech ../executorch-speech - name: Install dependencies run: npm ci - name: Fetch Windows native binaries (llama/whisper/sd/ffmpeg) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index b0f0bc4b..5796b652 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -20,6 +20,15 @@ on: type: string required: false default: '' + shared_ref: + # off-grid-ai/shared carries @offgrid/models|sync|use. This core branch depends + # on shared feature work that is NOT on shared main, and the auto match below is + # by BRANCH NAME - which differs across repos here - so it would fall back to main + # and miss effectId/undo/computer_task. Pin it on dispatch (e.g. feat/use-approval-tiers). + description: 'off-grid-ai/shared ref for @offgrid/* (empty = match this branch name, else shared main)' + type: string + required: false + default: '' permissions: contents: read @@ -75,9 +84,81 @@ jobs: with: python-version: '3.12' + - name: Check out the private Desktop speech package + uses: actions/checkout@v4 + with: + repository: off-grid-ai/executorch-speech + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + path: _executorch_speech + lfs: false # The current native speech engine is macOS-only. + persist-credentials: false + - name: Put the speech package beside this checkout + shell: bash + run: | + if [ ! -f _executorch_speech/package.json ]; then + echo "::error::off-grid-ai/executorch-speech was not checked out. Check CI_CROSS_REPO_TOKEN." + exit 1 + fi + rm -rf ../executorch-speech + mv _executorch_speech ../executorch-speech + + # `@offgrid/sync` and `@offgrid/use` are file: dependencies on the shared + # monorepo, so it must sit BESIDE this checkout before `npm ci` runs - + # mirrors release.yml's build-win. The branch ref decides the shared ref: + # a build from an integration branch takes the shared branch of the same + # name when one exists, else shared main. + - name: Checkout shared at the matching ref + id: shared_branch + continue-on-error: true + uses: actions/checkout@v4 + with: + repository: off-grid-ai/shared + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + ref: ${{ inputs.shared_ref || github.ref_name }} + path: _shared + persist-credentials: false + - name: Fall back to shared main + if: ${{ steps.shared_branch.outcome != 'success' }} + continue-on-error: true + uses: actions/checkout@v4 + with: + repository: off-grid-ai/shared + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + path: _shared + persist-credentials: false + - name: Put shared beside this checkout + shell: bash + run: | + if [ ! -d _shared ]; then + echo "::error::off-grid-ai/shared was not checked out - @offgrid/sync and @offgrid/use cannot resolve. Check CI_CROSS_REPO_TOKEN." + exit 1 + fi + rm -rf ../shared + mv _shared ../shared + npm --prefix ../shared ci + npm --prefix ../shared/packages/models run build + npm --prefix ../shared/packages/sync run build + npm --prefix ../shared/packages/use run build + - name: Install dependencies run: npm ci + - name: Verify the computer-use input addon (nut.js) shipped for Windows + shell: pwsh + run: | + # The vision computer-use rail drives the cursor/keyboard through + # @nut-tree-fork/nut-js, whose Windows binding is the prebuilt (N-API) + # libnut-win32 addon - an OPTIONAL dependency. If npm skips it or the + # prebuild fails to land, the rail loads null and refuses every task + # silently. Fail the build loudly instead, the same way + # fetch-win-binaries.ps1 fails on a missing llama-server.exe. + $addon = "node_modules/@nut-tree-fork/libnut-win32/build/Release/libnut.node" + if (-not (Test-Path $addon)) { + Write-Error "Missing $addon - the Windows computer-use rail would refuse every task. Confirm @nut-tree-fork/libnut-win32 installed (optional dep)." + exit 1 + } + Write-Host "OK: computer-use addon present ($((Get-Item $addon).Length) bytes)" + - name: Fetch Windows native binaries (llama/whisper/sd/ffmpeg) shell: pwsh env: diff --git a/.gitignore b/.gitignore index 6eab2653..cc6a9aae 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ component-library-animations/ # Bundled MLX (mflux) Python env — large, rebuilt via scripts/build-mflux-env.sh resources/bin/mflux/ -# Off Grid SD-GGUF conversion build dir +# Off Grid AI SD-GGUF conversion build dir build-sd-gguf/ # Swift build artifacts (embedded repos) @@ -55,6 +55,7 @@ test-results/ # Marketing material (dev.to drafts, emails, assets) — publishing, not app code /marketing/ scripts/dictation-hotkey/dictation-hotkey +scripts/actions-helper/actions-helper # Local demo profile — synthetic-data run target for `npm run demo` (never real userData) .demo-profile/ diff --git a/AGENTS.md b/AGENTS.md index 148553d1..eea13e38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,8 @@ # Off Grid AI Desktop — agent guide -This is **Off Grid AI Desktop** — an Electron (macOS) desktop app. The product name is always **"Off Grid AI Desktop"** (never "Off Grid Desktop", "My Memories", etc.) — in window titles, OAuth client names, about screens, everywhere. +This is **Off Grid AI Desktop** — an Electron (macOS) desktop app. The product name is always **"Off Grid AI Desktop"** (never "Off Grid AI Desktop", "My Memories", etc.) — in window titles, OAuth client names, about screens, everywhere. -## Design — DESKTOP-FIRST, Off Grid brand +## Design — DESKTOP-FIRST, Off Grid AI brand Full design doc: **`docs/DESIGN.md`**. The essentials, which OVERRIDE any mobile-first or monochrome assumptions: @@ -100,7 +100,7 @@ When iterating (a request, a fix, a tweak the user just confirmed), add a test t - **Coverage bar — 85% is the standard, enforced by a rising floor.** The goal is **85% on every metric — statements, branches (which subsume conditions), functions, lines** — across the testable surface. The repo is not there yet (much of it is Electron/native I/O shell that requires a running integration harness), so `npm run test:coverage` gates a **ratchet floor** set just under current coverage: the pre-push hook blocks any REGRESSION and the floor only rises. Every change that adds logic adds tests, nudges the number up, and the floor follows it toward 85 (raise the `thresholds` in `vitest.config.ts` as coverage climbs; never lower them). New code never ships uncovered — add a branch or an error path, add the user-visible case. - **Never add unit tests. User-behavior integration tests only.** A new test must enter through a real product boundary (rendered UI, IPC, HTTP/API, CLI, service interface, database, packaged app) and exercise the actual owning layers together. Do not test an isolated helper, class, hook, reducer, or source string. Existing unit tests may remain until their code is touched; when extending behavior they cover, replace or supersede them with integration coverage instead of adding another unit case. -- **Real collaborators, not mocks.** Use the actual implementation and real collaborators wherever execution is possible: a temp SQLite DB, a temp `OFFGRID_USER_DATA` dir, the real renderer/main IPC bridge, a local HTTP server, real crypto/WASM, or a packaged Electron app. Fakes are allowed only at an uncontrollable external boundary such as a third-party network, native OS dialog, or hardware ID, and a fake-boundary test is not sufficient acceptance evidence when a real end-to-end path can run. A mock that stands in for Off Grid code hides the behavior under test and lets it rot green. +- **Real collaborators, not mocks.** Use the actual implementation and real collaborators wherever execution is possible: a temp SQLite DB, a temp `OFFGRID_USER_DATA` dir, the real renderer/main IPC bridge, a local HTTP server, real crypto/WASM, or a packaged Electron app. Fakes are allowed only at an uncontrollable external boundary such as a third-party network, native OS dialog, or hardware ID, and a fake-boundary test is not sufficient acceptance evidence when a real end-to-end path can run. A mock that stands in for Off Grid AI code hides the behavior under test and lets it rot green. - **Prompt and contract fixes are behavior too.** Exercise the real prompt composer/consumer or contract owner through its public boundary and assert the resulting user-visible or downstream behavior. Do not read source files and regex-match implementation text as the primary regression test. - **Tests guard the architecture too (SOLID + DRY).** Prove the seam through a real second implementation or integration harness so a test fails the moment a caller starts branching on a concrete type (`kind === 'x'`, `instanceof`) instead of the interface. Guard DRY by driving the public owner rather than re-hardcoding its mapping or rule in the test. - **E2E** — Playwright Electron tour in `e2e/` (`npm run test:e2e`), DOM-driven, fresh temp profile, `OFFGRID_PRO=0`. Assert new surfaces render. Screenshot key states via `page.screenshot({ path: 'e2e/screenshots/.png' })`; include those screenshots in the PR body. diff --git a/CLAUDE.md b/CLAUDE.md index 167f8270..adf905de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,8 +1,8 @@ # Off Grid AI Desktop — agent guide -This is **Off Grid AI Desktop** — an Electron (macOS) desktop app. The product name is always **"Off Grid AI Desktop"** (never "Off Grid Desktop", "My Memories", etc.) — in window titles, OAuth client names, about screens, everywhere. +This is **Off Grid AI Desktop** — an Electron (macOS) desktop app. The product name is always **"Off Grid AI Desktop"** (never "Off Grid AI Desktop", "My Memories", etc.) — in window titles, OAuth client names, about screens, everywhere. -## Design — DESKTOP-FIRST, Off Grid brand +## Design — DESKTOP-FIRST, Off Grid AI brand Full design doc: **`docs/DESIGN.md`**. The essentials, which OVERRIDE any mobile-first or monochrome assumptions: @@ -21,6 +21,7 @@ The window is WIDE. A list of cards/rows stretched edge-to-edge in a single colu - **Tight, consistent spacing on a 4/8/12px scale.** Dense data UIs use narrow gutters (8-12px) and small padding, NOT the 16-24px editorial spacing. Body text ~12-14px, compact line-height. Flat and sharp, per the brand. - **Group, then separate.** Reduce gaps _within_ a group (rows in a section) but keep clear separation _between_ functional groups (filters vs data, "On this device" vs "Available"). Section headers over a wall of identical rows. - **Progressive disclosure.** Secondary info and rarely-used controls go behind a detail panel / "…" / hover affordance — don't lay everything flat. Master list stays scannable; depth lives in the side panel or slide-over. +- **Side panels, not desktop modals.** Open settings, editors, previews, and other multi-step detail flows in the shared `SidePanel`. It must close with Escape, an outside click, and its close control. Reserve a centered dialog only for a short confirmation that blocks one immediate action, such as confirming a destructive delete. - **Sticky context.** Fix headers, tabs, filter bars, and column labels while the body scrolls, so context never scrolls away. - **Finesse the interactions.** Every click gets a small micro-interaction — `transition-all duration-150`, `active:scale-95` on buttons, slide+fade (not abrupt mount) for panels/slide-overs. State changes animate; nothing pops in or out hard. - **Offer density where it matters**, but the default IS dense — this is a terminal/brutalist desktop app, not a spacious mobile-first card feed. @@ -153,6 +154,22 @@ The `pro/` directory is a **git submodule** pointing at the private `desktop-pro > Run `node scripts/mirror-doctrine.mjs` in `shared/` after changing the canonical copy. > `--check` fails the build when a mirror drifts, so these cannot silently disagree. +## Debugging — reason from first principles + +**Ask what the thing IS, before you ask what is happening to it.** Name what the code should be in +one sentence ("a side panel is fixed to the right edge, full height"), read what it actually says, +and fix the gap. Almost every hard-looking bug here dissolves at that step. + +The failure mode is reaching for the environment instead: measuring window geometry, blaming an OS +setting, inspecting global CSS, theorising about the platform. Those are ways of not reading the +component. A real example: a gap between a side panel and the window edge got attributed to a macOS +tiled-window margin. The actual cause was in the component's own class list — it declared two +competing heights (`h-dvh` on top of `top-0 bottom-0`) inside a clipping wrapper. The fix was to say +the simple thing directly. + +So, before any tooling: if the answer requires unusual measurement to explain, the implementation is +probably wrong, and it is complicated where it should be plain. Simplify it and the symptom goes. + ## Debugging — start with the source of truth **Most bugs here are source-of-truth bugs, and the fix is almost always to collapse two sources into diff --git a/README.md b/README.md index 79d9ccde..88fb348e 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Three things in one app: Claude/LM-Studio/Ollama with everything on-device. 2. **A gateway** — one local OpenAI-compatible API (`http://127.0.0.1:7878/v1`, no key) for chat, vision, image, audio, and embeddings. Run it headless as just the gateway. -3. **Off Grid Pro** — an always-on private layer that _sees_ your work (screen → OCR), +3. **Off Grid AI Pro** — an always-on private layer that _sees_ your work (screen → OCR), _remembers_ it, helps you _reflect_, and _acts_ with your approval. On-device, opt-in. ## A look inside @@ -81,7 +81,7 @@ Three things in one app: Artifacts — HTML, React, SVG & Mermaid in a local sandbox
Artifacts -Off Grid Pro — the sees/remembers/reflects/acts layer
Pro +Off Grid AI Pro — the sees/remembers/reflects/acts layer
Pro Private by default — runs on your machine, no account
Private by default @@ -174,7 +174,7 @@ curl -X POST http://127.0.0.1:7878/v1/models/activate \ -H 'Content-Type: application/json' -d '{"id":"unsloth/gemma-4-E4B-it-GGUF"}' ``` -## Off Grid Pro — available now +## Off Grid AI Pro — available now The free app **runs** models. **Pro** adds the always-on layer that turns your own work into private, on-device memory — and an assistant that helps you act on it. Everything is diff --git a/ROADMAP_DESKTOP.md b/ROADMAP_DESKTOP.md index d44e2727..1d7669f5 100644 --- a/ROADMAP_DESKTOP.md +++ b/ROADMAP_DESKTOP.md @@ -1,4 +1,4 @@ -# Off Grid Desktop — Roadmap +# Off Grid AI Desktop — Roadmap The desktop product view of the plan. The shared, package-oriented plan lives in `../shared/ROADMAP.md`; this is the same arc re-cut around the **desktop app**, with honest current status. Sources folded in: `shared/ROADMAP.md`, root `CLAUDE.md`, `website/vision.md`. @@ -10,7 +10,7 @@ Legend: ✅ done · 🟡 partial / in progress · ⬜ not started ## Phase 0 — Foundation ✅ -- ✅ App shell, Off Grid design (Menlo / black / emerald), unified `userData` path, dock + tray icons +- ✅ App shell, Off Grid AI design (Menlo / black / emerald), unified `userData` path, dock + tray icons - ✅ Bundled local runtimes: `llama-server` (gemma-4 vision), `whisper.cpp`, `ffmpeg`, `sharp` - ✅ Local LLM plumbing: grammar-constrained JSON, `enable_thinking:false`, single-flight init, port 8439 - 🟡 Full design pass on every screen (ongoing polish) @@ -64,7 +64,7 @@ Legend: ✅ done · 🟡 partial / in progress · ⬜ not started - ⬜ **Google (Gmail/Calendar) via SCREEN CAPTURE, not an OAuth client** (decided June 2026, fully-offline). Google MCP has no DCR → would need a registered GCP client = anti-offline; rejected. We already OCR Gmail/Calendar on screen — zero setup, nothing leaves the device. - ⬜ **Capture ↔ connector intelligence (source-of-truth priority).** Be smart: capture is the universal SIGNAL of interest ("you're on a Notion page / a Linear issue / company XYZ"); when a connector exists for what you're looking at, **pull the authoritative data from the connector (source of truth) instead of leaning on OCR/AX.** Connector data **takes priority** over captured/OCR data in synthesis + dedup, and lets us **throttle/skip OCR** for those apps (cheaper, cleaner). Capture tells us _what you care about_; the connector gives the _correct_ version. - ⬜ **Cross-source synthesis** — join email ↔ calendar event ↔ person ↔ project ↔ ticket into one entity, across capture + connectors -- ⬜ **File system access (local file catalogue + auto-retrieval).** Index opt-in, **scoped folders** on-device — file metadata (path / name / type / size / mtime) + content embeddings into the RAG store (Phase 5) — so Off Grid knows _what files exist_ and _what's in them_. Then **fetch the right file at the right time instead of making the user attach it**: capture/context signals _what you're working on_ → the catalogue surfaces or auto-attaches the relevant document (meeting prep pulls the deck; a chat about project X pulls its docs; "the contract we discussed" resolves to the file). Local-only, per-folder enable/revoke under the consent model, incremental re-index via an fs watcher. The local-first peer of the source-of-truth-priority rule above: **capture tells us what you care about; the file system gives the actual document — no manual attach.** +- ⬜ **File system access (local file catalogue + auto-retrieval).** Index opt-in, **scoped folders** on-device — file metadata (path / name / type / size / mtime) + content embeddings into the RAG store (Phase 5) — so Off Grid AI knows _what files exist_ and _what's in them_. Then **fetch the right file at the right time instead of making the user attach it**: capture/context signals _what you're working on_ → the catalogue surfaces or auto-attaches the relevant document (meeting prep pulls the deck; a chat about project X pulls its docs; "the contract we discussed" resolves to the file). Local-only, per-folder enable/revoke under the consent model, incremental re-index via an fs watcher. The local-first peer of the source-of-truth-priority rule above: **capture tells us what you care about; the file system gives the actual document — no manual attach.** **Skills & action:** @@ -83,11 +83,11 @@ Legend: ✅ done · 🟡 partial / in progress · ⬜ not started - ✅ **Projects + RAG + chat** over all memory + ingested docs — file upload (txt/md/PDF/DOCX/image/audio/video) → MiniLM embeddings + better-sqlite3 vector store; cited sources; "include captured memory" toggle spans uploads + everything captured (`@offgrid/rag`, `rag/index.ts`, `ProjectsScreen.tsx`) - ⬜ Unified search - ✅ **Models** — HF browser / provider abstraction / download manager (`@offgrid/models`, `ModelsScreen.tsx`) -- ⬜ **Expose a local model server** — surface the on-device runtimes (the bundled `llama-server` on 8439, whisper, embeddings) as a **local, OpenAI-compatible API endpoint** so other apps on the machine — and, over the mesh, other paired devices — can use Off Grid AI Desktop as their private inference backend. Off Grid Desktop becomes the household's on-device model server (no cloud, no API keys). Auth-gated + opt-in (off by default); ties into the consent model and the cross-device mesh (Phase 6). +- ⬜ **Expose a local model server** — surface the on-device runtimes (the bundled `llama-server` on 8439, whisper, embeddings) as a **local, OpenAI-compatible API endpoint** so other apps on the machine — and, over the mesh, other paired devices — can use Off Grid AI Desktop as their private inference backend. Off Grid AI Desktop becomes the household's on-device model server (no cloud, no API keys). Auth-gated + opt-in (off by default); ties into the consent model and the cross-device mesh (Phase 6). -### Phase 5b — The Off Grid chat as a local AI Studio (in progress, June 2026) +### Phase 5b — The Off Grid AI chat as a local AI Studio (in progress, June 2026) -The Off Grid chat (`MemoryChat.tsx`) is becoming a full local-first studio — like Claude/LM Studio/Ollama, but everything on-device. Brand: brutalist/terminal (Menlo, emerald, flat). Done + planned: +The Off Grid AI chat (`MemoryChat.tsx`) is becoming a full local-first studio — like Claude/LM Studio/Ollama, but everything on-device. Brand: brutalist/terminal (Menlo, emerald, flat). Done + planned: - ✅ **Chat redesign** — brutalist composer, clickable example prompts, mode segmented control; light + dark. - ✅ **On-device image generation** — `stable-diffusion.cpp` (`sd-cli`, Metal) in `resources/bin/sd`; txt2img + img2img; per-model size/steps/seed/negative-prompt; **live per-step preview + progress bar + ETA + Stop/cancel**; **lightbox** (zoom/download/delete); **artifacts gallery** of every generated image (`main/imagegen.ts`, `MemoryChat.tsx`). @@ -157,7 +157,7 @@ Phases 0–2 largely done — the full **see → remember → reflect → act(de ## Later phase — UI standardization audit (design philosophy) -**Decided 2026-06-23.** Off Grid Desktop adopts the Wednesday Solutions standards-kit: **no custom UI components** — every element comes from the approved libraries (**shadcn/ui** foundation, **Aceternity** effects, **Magic UI** text/buttons, **Motion Primitives** transitions). Branding stays Off Grid (Menlo mono, emerald, brutalist); shadcn semantic tokens are mapped to `--og-*` in `main.css` `@theme` so library components inherit the brand automatically. +**Decided 2026-06-23.** Off Grid AI Desktop adopts the Wednesday Solutions standards-kit: **no custom UI components** — every element comes from the approved libraries (**shadcn/ui** foundation, **Aceternity** effects, **Magic UI** text/buttons, **Motion Primitives** transitions). Branding stays Off Grid AI (Menlo mono, emerald, brutalist); shadcn semantic tokens are mapped to `--og-*` in `main.css` `@theme` so library components inherit the brand automatically. - [ ] Audit every screen/component and replace hand-rolled markup with approved-library components (start: MemoryChat composer/messages, then Settings, Onboarding, Day/Replay/Reflect/Actions/Connectors/Meetings/Models/Entities screens). - [ ] Pull primitives via `npx shadcn add` (+ `@aceternity`/`@magicui` registries); pick from `component-library-animations/skills/component-library-index.md`. diff --git a/docs/API.md b/docs/API.md index 96b576ba..3bc9d3bf 100644 --- a/docs/API.md +++ b/docs/API.md @@ -241,7 +241,7 @@ onnxruntime. **Returns raw `audio/wav` bytes** by default (like OpenAI). # Raw WAV to a file curl http://127.0.0.1:7878/v1/audio/speech \ -H "Content-Type: application/json" \ - -d '{ "input": "Hello from Off Grid.", "voice": "af_heart" }' \ + -d '{ "input": "Hello from Off Grid AI.", "voice": "af_heart" }' \ --output speech.wav ``` diff --git a/docs/ASSISTANT_ARCHITECTURE.md b/docs/ASSISTANT_ARCHITECTURE.md new file mode 100644 index 00000000..82eba1e6 --- /dev/null +++ b/docs/ASSISTANT_ARCHITECTURE.md @@ -0,0 +1,320 @@ +# The assistant - system architecture (the act pipeline) + +**Status:** high-level design, August 13, 2026, from the architecture discussion. For team review. +Companion to `COMPUTER_USE.md` (the product model), `COMPUTER_USE_PLAN.md` (the build doc + schedule), and `PORTING_MAP.md` (port-vs-bespoke). This is the *design reference* for the system that executes actions - how it stays reliable on a weak local model and identical across desktop and mobile. The build order and timeline live in `COMPUTER_USE_PLAN.md`; follow that to build. + +--- + +## 1. The problem this design solves + +Two hard constraints shape everything: + +1. **Local models are unreliable at tool-calling.** A bundled small model malforms calls, hallucinates arguments, or answers in prose instead of calling the tool. We cannot couple "decide" and "do" in a single model turn, or a bad turn means a lost or half-done action. +2. **The core must be identical on desktop and mobile.** We do not want two implementations of the thing that decides and guarantees actions. + +The design below answers both: a durable action pipeline where the model is the least-trusted component, wrapped by deterministic machinery that guarantees execution. + +## 2. The core reframe: the model proposes, the pipeline guarantees + +Most agent systems fail because the model both decides and executes in one turn. We invert it: + +**The model only ever proposes a structured Action. A durable, deterministic pipeline guarantees it happens - exactly once, gated, verified.** + +The model does the smallest, most-constrained job (produce a valid Action), and everything downstream is deterministic. A bad proposal is caught at a validation boundary and discarded (fail closed); a good proposal is executed with exactly-once guarantees and effect-verification. This is the tenet the whole system rests on: + +> **Reliability lives in the system, not the model.** + +A capable model makes the *proposals* better (fewer rejections, better resolution). It never changes whether an approved action actually executes. That is what lets us swap in a smaller or fine-tuned model later with no change to the guarantee. + +## 3. The Action: a durable record and a state machine + +Everything - a proactive come-up, a tool the chat model called, a routine step, a scheduled trigger - normalizes into one durable **Action** record in local SQLite. That store is the queue. + +An Action carries: `id`, `type` (message / email / calendar / open / file-share / web-task / ...), `source` (reasoning / chat / routine / schedule), `intent` (the natural-language ask), `args` (resolved slots), `payloadHash` (the immutable contract of exactly what will run), `risk` (read / navigate / mutate / irreversible), `rail`, `idempotencyKey`, `attempts`, `verification`, `state`, `triggerAt`, and audit references. + +It moves through a persisted state machine: + +```mermaid +stateDiagram-v2 + [*] --> proposed + proposed --> rejected: invalid (grammar / schema) + proposed --> scheduled: has a trigger + proposed --> resolving: valid, run now + scheduled --> resolving: trigger fires + resolving --> awaiting_approval: mutate / irreversible + resolving --> ready: read / low-risk + awaiting_approval --> ready: approved + awaiting_approval --> rejected: rejected + ready --> executing + executing --> verifying + verifying --> done: effect confirmed + verifying --> executing: failed, retry once + verifying --> needs_help: still failed +``` + +Because the record is persisted, not a transient turn: a crash resumes it, a scheduled action waits durably, a retry does not double-send (idempotency), and an action is not `done` until its effect is verified. This durability is also exactly why the pattern fits mobile - a queue drained by a background worker survives the OS killing the app, which mobile does aggressively. + +## 4. The reliability stack (how it survives a weak model) + +Layered, weakest-model-work first: + +1. **Constrain the output.** Grammar-constrained decoding (GBNF) so the model can only emit a valid Action on valid arguments. For GUI steps, generate the grammar per step so it can only pick elements that exist right now. +2. **Validate at the boundary, fail closed.** A malformed or off-schema proposal never becomes an Action. Keep the action schema small and closed (fewer types = far better local accuracy); rank and prune available tools to the token budget. +3. **Decouple decision from execution.** The durable queue means a bad turn is a no-op, not a lost or half-done action. +4. **Bind the executed payload to the approved one.** The `payloadHash` the gate showed is exactly what runs - no re-resolution between confirm and act. +5. **Prefer determinism over the model.** Route to semantic rails and recorded traces first; the model does the least, most-constrained work, least often. Vision/GUI is the last resort. +6. **Verify, then retry once, then ask.** Observe the effect. If it did not happen, retry once; if it still did not, mark `needs_help` and surface it rather than looping. +7. **A cross-rail escalation is a re-fire, under the same policy.** Falling back from one rail to another (semantic timed out -> try the browser) is another execution attempt on the same Action, so it is governed by the same retry rules: only a retryable action (reversible, reliably verifiable, retry budget left) may escalate, and only after verification confirms the effect did NOT happen. A timed-out irreversible action goes to `needs_help`, never to another rail - that is how a double-send is made impossible even across rails. The durable Action record is the effect journal: every attempt records the rail it ran on. + +## 5. Focused, not general: a registry of typed action handlers + +Per the lead's steer, the assistant is not a general "call any tool" agent - it is a **curated set of first-class action types**, each with its own schema, grammar, resolver, rail, and verification. Adding a capability = adding a handler, not retraining anything. + +The v1 scope (things you do on your own machine), grouped by type and honest about reliability tier: + +| Action type | Examples | Rail | Reliability in v1 | +| --- | --- | --- | --- | +| Message | send a text | semantic (AppleScript / iMessage) | high | +| Email | send / compose | semantic (Mail, or Gmail connector) | high | +| Calendar and reminders | create event / reminder | semantic (EventKit) | high | +| Open / launch | open tabs, a URL, a YouTube video, an app | semantic (deep link / open) | high | +| Look up | contacts, "what's on my calendar" | semantic (read, inline) | high | +| File share | share a file over WhatsApp | GUI vision (Catalyst, dead AX tree) | best-effort, supervised | +| Web task | flight check-in, book a hotel, order | agent browser + takeover | best-effort, supervised | +| Proactive notice | "flight tonight, not checked in", "you promised the deck" | reasoning engine -> feeds the above | new, memory-driven | + +The pipeline is identical across all of them; only the rail and the reliability differ. Two tiers to set expectations honestly: **semantic actions (text, email, reminders, open) are solid; GUI and web tasks (WhatsApp file share, check-in, booking) are supervised and improving.** Same product, honestly tiered. + +## 6. One core, two platforms + +The pipeline is the `@offgrid/use` engine in `shared` (consumed as `file:../shared/packages/use`). The reliable parts are pure logic, so they are shared; only the platform-specific edges are adapters. + +**Naming (canonical).** Two layers: **the assistant** (the brain - reasoning, resolve, the queue, the router, the gate, verify) and **the rails** (the actuation layer - the executors that actually perform actions, behind the `DeviceController` interface). Each concrete path is a rail: the **semantic rail**, the **browser rail**, the **accessibility rail**, and the **vision rail**. "Computer use" means the vision rail specifically, not the whole layer - most actions never touch it. + +**The shape, at a glance.** This is a component diagram in the **ports-and-adapters (hexagonal)** pattern: the assistant is the core, the `DeviceController` is the port, and the rails are the swappable adapters implemented per platform. + +```mermaid +flowchart TB + RE[Reasoning engine] --> IN + CH[Chat / routine] --> IN + SC[Scheduler / trigger] --> IN + MEM[(Memory:
Replay, entities, RAG)] -.-> RE + MEM -.-> RS + + subgraph BRAIN["THE ASSISTANT · brain · @offgrid/use (shared, platform-free)"] + direction TB + IN[Intake + validate
grammar · schema · fail closed] + Q[(Durable queue · state machine)] + RS[Resolver · slots from memory + confidence] + GT{Gate · evidence + confidence} + RO[Router · cheapest reliable rail] + VF[Verify · retry once · else ask] + IN --> Q --> RS --> GT --> RO + VF -.re-queue on fail.-> Q + end + + RO ==>|"execute(action)"| DC{{DeviceController · the port}} + DC -.result.-> VF + + subgraph RAILS["THE RAILS · actuation · platform adapter"] + direction LR + R1[Semantic rail] + R2[Browser rail] + R3[Accessibility rail] + R4[Vision rail
= computer use] + end + DC --> R1 + DC --> R2 + DC --> R3 + DC --> R4 + + RAILS -.implemented per platform.-> PLAT["macOS · Windows · Android · iOS"] +``` + +**Shared core (platform-free):** the Action model + durable queue + state machine; the reasoning engine (commitment / gap detection); the resolver (slot-filling over memory, with confidence); the router (cheapest reliable rail); verification + retry / idempotency policy; the action-handler registry; the gate seam (a callback the host implements). + +**Per-platform adapters (behind interfaces the core calls):** +- **The rails (behind the `DeviceController` interface)** - how to actually run a thing. **Desktop v1 is macOS + Windows, in scope from day 1.** macOS: the Swift helper (EventKit / AppleScript), the agent browser, AX + CGEvent, vision. Windows: **local Outlook automation (COM / PowerShell) first** where Outlook exists - like the mac rail, a local write that syncs when the network returns - with Microsoft Graph as the fallback for setups without a local Outlook, and online-only actions labeled honestly; the shell for open / launch; the agent browser (shared, Electron); UI Automation + SendInput; vision. Android: intents + content providers + an accessibility-service portal. iOS: App Intents / Shortcuts only (no GUI or vision rail - the platform forbids reading or driving other apps). +- **Accessibility is primarily the eyes, not a fourth pair of hands.** The AX / UIA tree is the observation and verification layer serving every rail: anchors for recorded traces, read-back for verification, drift checks. Actuation through it stays deliberately capped - macOS keeps `AXPress` / set-value only (the Swift helper already has them; set-value beats replaying keystrokes), and Windows acts through SendInput at UIA-located targets rather than growing a second actuation surface. One maintenance surface less, per platform. +- **Offline scope, stated precisely:** the brain - detection, resolution, gating, the queue, verification logic - runs with zero network on every platform. An action whose effect lives on an external service (send an email, book a flight) needs that service reachable at execution time on any OS; the design preference is local-app rails whose writes land locally and sync later, which is exactly why local Outlook beats Graph as the Windows default. +- **MemoryStore** - read observations / entities / RAG. +- **Scheduler** - fire time and event triggers. +- **Approval and feed UI** - render the gate and the come-up feed (desktop renderer; mobile React Native). +- **Model client** - both call the local model through the OpenAI-compatible gateway. + +So "the core stays the same" is concrete: the queue, resolver, router, reasoning, and verification are one codebase; only the executor, store, scheduler, and UI are swapped per platform. Mobile is an adapter project on the same engine, not a rewrite. + +--- + +## 7. Decisions locked (present these as answered) + +1. **The model proposes, the durable queue guarantees.** Decision-and-execution are separated. The model produces a validated Action; the pipeline executes it. This is what makes the system reliable on a weak model. +2. **Reliability lives in the system, not the model.** The execution guarantee comes from the pipeline (constrain, validate, queue, deterministic rails, verify), never from the model being good. +3. **Model choice: capable now, model-agnostic pipeline, fine-tuning deferred.** We start with a good, capable model to prove the experience feels right. The pipeline is built to hold with a smaller model, so the bundled model (or a future LoRA fine-tuned on our action schema) slots in with zero change to the guarantee. Fine-tuning is an optional later reliability boost, not a v1 dependency. +4. **The queue lives in `shared` (`@offgrid/use`).** The queue engine and state machine are platform-free core; the storage and UI are platform adapters. This keeps the execution guarantee identical on desktop and mobile. +5. **Mutations go through the queue and gate; reads run inline.** Anything that changes the world (send, create, delete) - even when asked in chat - flows through the durable pipeline. Pure reads ("what is on my calendar") can run inline for latency, since there is nothing to guarantee. To the user this is invisible; chat can still act, it is just durable and gated underneath. +6. **Retry once, then ask.** On a verified failure, retry a single time; if it still fails, stop and surface `needs_help` rather than looping. +7. **The gate shows resolved values, evidence, and confidence, bound to the approved payload.** The approval card shows what was inferred and why ("Send Q3.pptx to Ali because ..."), and the exact payload approved is the exact payload that runs. +8. **Cheapest reliable rail first, vision last.** The router prefers a deterministic surface (deep link / API / AppleScript) over the agent browser over the accessibility tree over the vision-grounding model. +9. **Scope: a curated set of typed action handlers** (Section 5), spanning two honest reliability tiers - semantic actions are solid, GUI / web tasks are supervised. + +## 8. Open questions (for the team) - explained + +Each is a real decision with a tradeoff. Where we have a lean, it is stated so the team reacts to a proposal rather than a blank. + +### 8.1 Exactly-once per rail +**What it is.** The guarantee that an action runs one time and only one time, even across a retry or a crash. Example: the executor sends an iMessage, then the app crashes before recording success; on restart it must not send a second copy. +**Why it matters.** Double-sending a message, or creating two calendar events, is a visible, trust-damaging failure - worse than a clean failure. +**Options.** (a) *Idempotency key* - tell the target "this is operation X, ignore a duplicate" (works only if the target supports it). (b) *Check-before-act* - before creating, ask "does this already exist?" (c) *Verify-after* - after the attempt, look for the effect and only retry if it is missing. Feasibility is per-rail: calendar / reminders / mail are verifiable and roughly idempotent; iMessage / WhatsApp / a website form are fuzzy (no key, and "did it send?" is hard to answer cleanly). +**The decision.** Do we require every action handler to declare a verification or existence-check capability? And for the fuzzy rails, is the policy single-attempt-behind-the-gate, or verify-then-accept-a-small-residual-risk? +**Our lean.** Handlers declare how they verify; reversible actions retry-once with verify; irreversible fuzzy actions (an outbound send) are single-attempt behind the gate, so a wrong verify can never double-fire. Escalating to another rail is a re-fire under the same rule (Section 4, item 7) - a non-retryable action never escalates. + +### 8.2 Scheduling and triggers +**What it is.** How a routine fires at 09:00, or an event trigger fires ("when I open Slack", "20 minutes before a meeting"). +**Why it matters.** Proactive delivery and routines depend on triggers, and they must work when the app is backgrounded or killed - especially on mobile, where the OS controls wakeups. +**Options.** (a) *Core-owned trigger model* - the shared core holds the trigger definitions and a durable schedule table, and a thin platform adapter wakes the worker (launchd / a timer on Mac, WorkManager / BackgroundTasks on mobile). (b) *Platform-native scheduling wrapped* - each OS's scheduler owns the timing, the core just registers callbacks. +**The decision.** How much scheduling logic lives in the core vs the OS, and how we survive the app being closed. +**Our lean.** Core owns the trigger model and the durable schedule; a thin per-platform adapter is responsible only for waking the worker at the right time. + +### 8.3 Trust graduation (Suggest to Auto) +**What it is.** When an action or routine moves from Suggest (ask before each run) to Auto (runs unattended). +**Why it matters.** This is the whole "proactive but safe" arc. Too eager feels invasive or dangerous; too timid and it never saves time. +**Options.** (a) *Per action type* - reads auto, sends always ask. (b) *User-set per routine* - a manual Suggest/Auto toggle. (c) *Confidence threshold* - auto when confidence is high and the action is reversible. (d) *Learned* - auto after N successful approvals of the same shape. +**The decision.** What is the default, who controls the dial, and do irreversible actions ever run Auto. +**Our lean.** Default Suggest; the user promotes a routine to Auto; irreversible actions always gate even inside an Auto routine; reversible high-confidence actions may auto after a few confirmations. + +### 8.4 Mobile v1 target +**What it is.** What actually ships on mobile first, given the same core but very different rails. +**Why it matters.** The rail capabilities differ enormously by platform, and this sets expectations. Android can host the full stack (an accessibility-service portal plus intents and content providers). iOS is intents-only - Apple forbids an app from reading or driving other apps, so there is no GUI or vision rail there. Also, the mobile app does not consume the shared monorepo yet, which is a prerequisite regardless. +**The decision.** Is mobile v1 Android-first (full experience), iOS-first (intents-only, limited), or desktop-only for v1 with mobile as a fast-follow - and on what timeline. +**Our lean.** Desktop v1; mobile as an adapter project afterward, Android-first for the full experience, iOS shipped as intents-only with honest scope. + +### 8.5 Open-core placement +**What it is.** Which parts of the pipeline are open core (AGPL, in `shared` / the public repo) vs pro (in `desktop-pro`). +**Why it matters.** Open-core is a hard rule - pro business logic must not live in core. The reasoning engine, the resolver policy, the approval-queue UI, and routines are the "act pillar" and follow the existing pro spine; the rail primitives and the queue engine are closer to infrastructure. +**The decision.** Draw the line: what is the inert core shell vs the pro business logic. +**Our lean.** The queue engine, the action-handler interfaces, and the rail primitives live in `shared` / core (infrastructure); the reasoning engine, the resolver's policy, the approval and feed UI, and routines live in `desktop-pro`. + +### 8.6 Verification depth per rail +**What it is.** How thoroughly we confirm an action's effect actually happened before marking it `done`. +**Why it matters.** Verification is what makes retry-once safe and catches silent failures and false confirmations (the field's number-one trust failure is an agent saying "done" when the backend failed). +**Options.** (a) *None* - trust the executor's return. (b) *Light* - parse the return / status. (c) *Full re-observe* - query the world (is the event in the calendar, is the mail in Sent, re-read the AX tree or screenshot). Cost vs safety, and it differs per rail. +**The decision.** The minimum verification bar per rail, and whether irreversible or GUI actions require full effect-verification. +**Our lean.** At least "executor reported success and the effect is observable" for every mutation; full re-observe for irreversible actions and for the GUI / vision rail, where drift is most likely. + +--- + +## 9. How to present this + +The narrative for the team: the vision (the demo) is validated; the scope is a curated set of action types across two honest reliability tiers; the system is a durable action pipeline where the model only proposes and the pipeline guarantees, so it survives a weak local model and stays identical on desktop and mobile; the decisions in Section 7 are locked; and Section 8 is the six open questions we want the team to weigh in on. The natural next step after alignment is the detailed `@offgrid/use` spec - the Action schema, the handler interfaces, and the reliability policy in code form. + +--- + +## 10. System architecture diagrams (C4, swimlane, user flows) + +The TRD / PRD deliverables, in the standard house style. The component diagram in Section 6 is the C4 **component** level (Level 3); the two views below add the **context** (Level 1) and **container** (Level 2) levels above it, then a runtime swimlane and the product user flows. + +### 10.1 System context (C4 - Level 1) + +Who uses the system and what it touches. The assistant is on-device; the only external things are the user and the apps and services it acts on. + +```mermaid +C4Context + title System Context - Off Grid AI assistant + Person(user, "User", "Knowledge worker, on their Mac or phone") + System(oga, "Off Grid AI", "Private on-device assistant that notices what you need and acts, with approval") + System_Ext(apps, "The user's apps and services", "Calendar, Mail, Messages, WhatsApp, the browser, connectors") + Rel(user, oga, "Asks in chat, approves actions") + Rel(oga, user, "Surfaces come-ups, asks to confirm") + Rel(oga, apps, "Acts on the user's behalf, with approval") +``` + +### 10.2 Containers (C4 - Level 2) + +The parts inside Off Grid AI and how they talk. The assistant engine is the brain; the rails are the hands; everything runs on-device. + +```mermaid +C4Container + title Container view - Off Grid AI assistant (all on-device) + Person(user, "User", "") + System_Boundary(oga, "Off Grid AI (on-device)") { + Container(ui, "Approval and feed UI", "React / React Native", "Day feed, approval card, routines") + Container(assistant, "Assistant engine", "@offgrid/use, shared TypeScript", "Reasoning, resolve, durable queue, router, gate, verify") + Container(rails, "The rails", "DeviceController adapters, native per platform", "Semantic, browser, accessibility, vision") + ContainerDb(memory, "Memory", "SQLite plus LanceDB", "Replay observations, entities, RAG") + Container(model, "Local model gateway", "llama.cpp, OpenAI-compatible", "On-device LLM, grammar-constrained") + } + System_Ext(apps, "The user's apps and services", "Calendar, Mail, Messages, WhatsApp, web, connectors") + Rel(user, ui, "Sees come-ups, approves") + Rel(ui, assistant, "Proposes and approves actions") + Rel(assistant, model, "Proposes a validated action") + Rel(assistant, memory, "Detects patterns, resolves slots") + Rel(assistant, rails, "execute(action)") + Rel(rails, apps, "Deep links, EventKit, AppleScript, GUI") +``` + +### 10.3 Sequence / swimlane + +Swimlane by actor: it makes clear who is responsible at each step, and which part of the system assists the user. Example flow: the user acts on a proactive come-up ("send the deck I promised Ali"). The lanes are the actors: User, Assistant, Memory, Gate, Rails, and the target app. + +```mermaid +sequenceDiagram + actor U as User + participant A as Assistant (brain) + participant M as Memory + participant G as Gate / Approval + participant R as Rails (DeviceController) + participant T as Target app (Mail) + + Note over A: Reasoning engine notices a commitment + A->>U: Come-up "you promised Ali the deck" + U->>A: "Send it" + A->>A: Validate and enqueue a durable Action + A->>M: Resolve "the deck" and "Ali" + M-->>A: Q3-strategy.pptx, Ali Chherawalla (with confidence) + A->>G: Propose (mutate) with the evidence + G->>U: Approval card - resolved values plus evidence + U->>G: Approve and send + G-->>A: Approved, payload locked + A->>R: execute(action) on the cheapest reliable rail + R->>T: Send via Mail (semantic rail) + T-->>R: Sent + R-->>A: Result + A->>A: Verify the effect (retry once if needed) + A-->>U: "Sent to Ali" (real confirmation, not a guess) +``` + +For a GUI action (say a WhatsApp file share) the same lanes hold; only the rail changes to vision, and the target app is driven step by step with a pause at the send. + +### 10.4 User flows + +The paths a user can take through the product: the two entry points (the proactive Day feed, or asking in Chat) through the gate to a verified result, plus the two ways a routine is born. + +```mermaid +flowchart TD + S([Open Off Grid AI]) --> DAY[Day - the Needs you feed] + ASK([Ask in Chat]) --> REV[Review the action] + + DAY -->|reasoned come-up| REV + DAY -->|routine proposal| TR[Turn into routine] + DAY -->|record a routine| DEMO[Demonstrate it once] + + REV --> CARD[Approval card:
resolved values + evidence + confidence] + CARD -->|low confidence| PICK[Pick the right one] + PICK --> CARD + CARD -->|edit| CARD + CARD -->|dismiss| DAY + CARD -->|approve| EXE[Assistant runs it on a rail] + + EXE --> VER{Verified?} + VER -->|yes| DONE([Done - toast confirms]) + VER -->|no, retry once| EXE + VER -->|still no| HELP([Needs help - asks you]) + + TR --> CONF[Confirm the learned steps
and set a trigger] + DEMO --> CONF + CONF --> SAVE([Saved - starts as Suggest]) + SAVE -.runs on its trigger.-> REV +``` + +Two entry points - the proactive Day feed and Chat. Both land on the approval card, which shows the resolved values with their evidence and confidence; low confidence branches to a quick "which one did you mean" pick. Approve runs it on a rail, then verify decides done, retry-once, or ask you. A routine is born two ways - the assistant proposes a detected pattern, or you record one by demonstrating it - both converge on confirming the learned steps and setting a trigger, and a saved routine starts as Suggest until you trust it. diff --git a/docs/CHAT_UX_SPEC.md b/docs/CHAT_UX_SPEC.md index bd0c0b2e..da62e09d 100644 --- a/docs/CHAT_UX_SPEC.md +++ b/docs/CHAT_UX_SPEC.md @@ -1,4 +1,4 @@ -# Off Grid Desktop — Conversational UX Spec +# Off Grid AI Desktop — Conversational UX Spec > **This is one application of the app-wide design philosophy in > `docs/DESIGN_PHILOSOPHY.md` (read that first — it is binding for the whole app).** @@ -61,7 +61,7 @@ a choice materially changes the answer — so the conversation never feels like **Aceternity** (effects), **Magic UI** (text/buttons), **Motion Primitives** (transitions). Pull via `npx shadcn add` / `@aceternity` / `@magicui`; pick from `component-library-animations/skills/component-library-index.md`. -- **Brand stays Off Grid:** Menlo mono, emerald (`#34D399`/`#059669`), flat/brutalist, +- **Brand stays Off Grid AI:** Menlo mono, emerald (`#34D399`/`#059669`), flat/brutalist, dense. shadcn semantic tokens are mapped to `--og-*` in `main.css @theme`, so library components inherit the brand with zero per-component styling. - **Code standard (standards-kit):** cyclomatic complexity < 8; PascalCase UI/Types, @@ -73,6 +73,6 @@ a choice materially changes the answer — so the conversation never feels like - [ ] Model generates images **and** artifacts **and** clarifying questions inline, chosen automatically, each rendered with calm motion. - [ ] Responses **stream** (text, and artifacts build live in the canvas). -- [ ] Composer/messages/menus use approved-library components, themed to Off Grid. +- [ ] Composer/messages/menus use approved-library components, themed to Off Grid AI. - [ ] Conversation list searchable + grouped; artifact canvas docks right. - [ ] Verified on screen (not just typecheck) — feels smooth, easy, breathtaking. diff --git a/docs/COMPETITIVE_RESEARCH.md b/docs/COMPETITIVE_RESEARCH.md new file mode 100644 index 00000000..8670b5f1 --- /dev/null +++ b/docs/COMPETITIVE_RESEARCH.md @@ -0,0 +1,115 @@ +# Competitive and prior-art research - the proactive assistant + +Researched August 2026. Every facet of what we are building has prior art; none of the incumbents ship the whole loop, and two big pieces are open whitespace. This is reference material for product and design. Sources are linked inline. + +## Three strategic findings (read first) + +1. **Local-first is open whitespace, and the market just proved why it matters.** The two flagship local screen/audio-memory products both got acquired by Meta in Dec 2025 and effectively ended as local products (Rewind capture disabled Dec 19 2025; Limitless pendant pulled). Dot (New Computer), a beloved memory-driven companion, shut down Oct 2025 and users "grieved" lost months of context. The lesson every Rewind-alternative now leads with: **local means your memory survives the vendor and never leaves the device.** Screenpipe (local SQLite + OCR + on-device model, MIT) is the architecture to benchmark against. This is our moat, validated the hard way. + - https://the-gadgeteer.com/2026/05/05/best-ai-wearables-2026/ · https://techcrunch.com/2025/09/05/personalized-ai-companion-app-dot-is-shutting-down · https://github.com/screenpipe/screenpipe + +2. **"Resolve the reference, show the evidence, confirm before acting" is essentially unshipped.** Every assistant resolves a vague reference the same way (hybrid retrieval -> rerank -> LLM answer) and shows provenance as *post-hoc citations*. None show ranked candidates with the evidence for each, surface a confidence, and ask you to confirm the pick *before* acting. Shortwave computes per-feature confidence and discards it. Gmail's forgotten-attachment detector is the only shipping confirm-before-send gate, and it cannot even name the file. **The thing our approval card does - "Send Q3.pptx to Ali because you called it 'the deck' in Tuesday's call and it is the only deck shared with Ali" - is the exact whitespace.** + - https://arxiv.org/abs/2503.15739 (ECLAIR) · https://arxiv.org/abs/2206.07836 (PEL/CREL) · https://patents.google.com/patent/US10812427 + +3. **The GUI-automation reliability ceiling is real, and everyone hit it in 2026.** Google killed Project Mariner (May 2026) - screenshot-per-step vision was too slow, costly, and error-prone at scale. OpenAI quietly killed ChatGPT travel checkout (~Mar 2026) - "travel was too hard." Perplexity Comet's agentic mode is "wildly inconsistent" ("faster to do it yourself"). This validates our whole architecture: **route to the cheapest reliable rail, prefer demonstrated traces over novel automation, and gate everything.** Do not bet the product on pixel-level autonomy. + - https://en.wikipedia.org/wiki/Project_Mariner · https://www.tourismtribe.com/chatgpt-instant-checkout-travel-operators/ · https://www.eesel.ai/blog/perplexity-comet-reviews + +--- + +## 1. Proactive surfacing (the "come-up") + +**Who does it:** Rewind/Limitless and Microsoft Recall (recall, not proactive push), Screenpipe (local infra), Apple Siri Suggestions / Call Context, Google **Magic Cue** + **Daily Hub** (Pixel), Microsoft Copilot ("Your Day at a Glance"), **ChatGPT Pulse** (the reference morning-briefing), Martin / Ohai (act + reach you in your channel). + +**The recurring patterns (what to copy):** +- **The morning card feed** - a once-daily, scannable set that owns "the first five minutes of your day." Pulse, Daily Hub, Copilot, OpenClaw's briefing all converge here. Value is *density and relevance per card, not volume*. +- **Inline point-of-need chip (the best pattern)** - Magic Cue surfaces the thing *where you are already acting* (a chip in the message box, a confirmation code on the call screen), single tap to use, no feed to visit. Preferred over a feed for actionable items. +- **Notification -> answer-ready, never a dead alert** - Copilot's push opens straight into the pre-run answer and next action. Never surface "you have items waiting" with a blank prompt behind it. +- **Feedback + forward-preview** - Pulse ends each briefing previewing tomorrow's topics with a "curate" control, so the feed feels steerable. +- **Recall is a separate surface** - the scrubbable DVR timeline (Recall, Screenpipe) is for "find what I saw," kept distinct from the proactive push. + +**The hard constraint - the notification budget.** Independent research and Pulse's own complaints converge: **~3-5 unsolicited notifications/day total is the ceiling**; exceeding it means users mute by Friday. "Notifications sent is a vanity metric; dismissals look like engagement but predict churn." An interruption costs ~23 minutes of recovery. Prescription: a hard daily cap the surfacing engine must respect, value-vs-attention scoring per candidate, learned per-user dismiss thresholds, and displacement logic (a new item must out-rank the queued one to fire). Treat each notification as a withdrawal from a finite account. + - https://tianpan.co/blog/2026-05-13-background-agents-notification-budget-attention-economy · https://www.platformer.news/chatgpt-pulse-proactive-ai/ + +**Avoid:** a high-frequency engagement-optimized feed (Pulse's worst reviews: fatigue, "creepy," "my calendar does this free"); the come-up that only restates what the calendar/email already shows (the bar is *net-new synthesis*); over-automation without control (Motion's complaint); always-on capture without visible opt-in + encryption + per-app exclusions (Recall's 2024 near-death). Google's **Magic Cue** is the single best pattern to study. + - https://store.google.com/us/magazine/magic-cue · https://9to5google.com/2025/08/20/pixel-10-magic-cue-launch/ + +## 2. Context resolution ("which deck did they mean") + +**Who does it, and how (all the same shape):** ChatGPT memory + connectors (RAG over an index, live source sidebar), Gemini Workspace ("Sources" list, admits it "can make up a source"), **Glean** (the most sophisticated - a per-company entity knowledge graph that collapses variant names to one canonical identity, auditable traversal path), Microsoft 365 Copilot ("/" typeahead picker - the closest shipping "pick which one you meant", but only on explicit "/", not vague prose), Notion Q&A, Dropbox Dash, Slack AI (auto-extracts filters from a NL reference: author=Sarah, type=slides, last week), **Shortwave** (the best-documented pipeline: coref query-reformulation -> parallel feature extraction *with confidence* -> hybrid retrieval -> two-stage cross-encoder rerank). + +**The whitespace (finding #2 above):** every product shows provenance as *post-hoc citation*, never a pre-action evidence panel with candidates + confidence + a confirm/correct control. Confidence is computed and thrown away. The research blueprint exists (ECLAIR interactive disambiguation; PEL/CREL personal-entity linking = coref to trace "the deck" back to its first mention + bind to the file entity - a two-step our on-device entity graph is well-suited to) but is unshipped in consumer products. **Caveat:** entity-reference ambiguity is only ~23% of real ambiguity - the rest is which *version*, which *date*, a missing constraint - so a resolver must handle more than the noun. + +**The universal failure story:** confident wrong-source grounding. The Tow Center found >60% citation errors across AI search tools (ChatGPT ~67%); Google admits Gemini cites unused docs; Notion cannot reconcile duplicate/stale pages. The trust gap is precisely that these systems act on an unconfirmed pick and back-fill a citation users have learned not to trust. **Our answer:** show the evidence and confidence *before* acting, gate on it. + - https://www.glean.com/perspectives/what-role-does-a-knowledge-graph-play-inside-modern-enterprise-ai-software · https://www.zenml.io/llmops-database/building-a-production-grade-email-ai-assistant-using-rag-and-multi-stage-retrieval · https://support.microsoft.com/en-us/microsoft-365-copilot/refer-to-specific-files-and-more-in-microsoft-365-copilot + +## 3. Commitment / reasoned detection + +**Email tools mostly do NOT do semantic "I promised X" detection - they detect the structural proxy "you sent mail, got no reply in N days":** Gmail/Gemini **Nudges** (the canonical *cautionary tale* - right idea, but on-by-default, breaks inbox order, induces guilt, fires on already-closed threads; the textbook example of resurfacing done annoyingly), Superhuman Auto Reminders (with the key anti-nag lever: scope to "external recipients only"), Spark, Boomerang, SaneBox (the quieter "no-replies folder" vs Gmail's loud inbox-bump - a useful design axis). **Mailbutler** does real semantic commitment extraction with urgency tiers; **Shortwave deliberately keeps task-creation manual** (human-confirm to avoid false-positive spam). + +**Meeting-notes tools are where real "who owes what" extraction happens** (LLM over the transcript, owner by speaker, deadline from prose): Otter (cross-meeting dashboard, links to the transcript moment, weekly digest), Fireflies (cue-phrase extraction, ~90% after 2 weeks of correction, but speaker attribution "hit-or-miss"), Fathom (strong attribution, but **ownership is understood then lost at handoff** to task tools), Granola (uses your sparse notes as anchors to cut hallucination), Zoom (best-practice format "Owner + verb + deliverable + date"). **Failure mode to design against:** hallucinated action items and invented commitments ("assigned stories they didn't agree to write") - so link every extracted commitment to its exact source utterance and keep a confirm step. + +**The durable formal model** (Microsoft Research, HP Labs): a commitment is a **commissive speech act with a debtor (who owes), a creditor (who is owed), and an optional deadline**, detected at the *sentence* level. That cleanly gives our two lists: "you owe" (user is debtor) and "waiting on" (user is creditor). Commitment vocabulary generalizes across domains (so a bundled local model is plausible) but models overfit, and precision tops out ~80-90% = **1 in 5-10 flags is wrong** - which is exactly why every shipping product hedges ("suggested"), batches into a digest, or requires a human confirm. + - https://www.microsoft.com/en-us/research/blog/email-overload-using-machine-learning-to-manage-messages-commitments/ · https://techcrunch.com/2018/06/15/gmail-proves-that-some-people-hate-smart-suggestions/ · https://www.careful.industries/blog/2025-11-nine-risks-caused-by-ai-notetakers + +**Anti-nag levers actually used:** granular independent opt-outs; scope narrowing ("external only"); batching over real-time; hedged framing ("suggested," not "your tasks"); human-confirm-before-commit; urgency tiers as a soft confidence gate; link every item to its source. The louder the surface, the more a false positive hurts. + +## 4. Routines / teach-by-demonstration + +**The two failed ends of the spectrum:** coordinate/pixel replay (Apple Automator **"Watch Me Do"** - it *observed* via the accessibility tree then *replayed* via absolute coordinates, "playback continues regardless" of drift; that one choice is the entire failure mode) and pure-vision replay (Mariner - "learn the plan not the pixels" was the right idea but cloud vision every step was too slow/costly/error-prone to ship). **Our AX-anchored trace + memory-filled slots + local model sits in the gap both missed.** + +**Best authoring patterns (Apple Shortcuts, Keyboard Maestro, BetterTouchTool):** +- **Magic Variables** (Shortcuts) - every action's output is automatically a droppable, icon-tagged token you click to reinterpret. Best data-flow UX in the field. +- **Ask Each Time** (Shortcuts) - the simplest run-time slot; prompt when the value is not known. Pair with memory-fill: *resolve the slot from memory if known, fall back to Ask Each Time.* +- **Named Triggers with passed variables** (BTT) - the routine as a function with named arguments, invocable by many triggers; **Conditional Activation Groups** = context predicates gating when it may fire. +- **Use Model as one action in the stack** (Shortcuts, iOS 26) - Apple's own "an LLM step inside a deterministic routine," not "the model runs everything." Mirror this for slot-filling. +- **The reliability spectrum shown to the author** (KM: AX/semantic > found-image > coordinates, with "not found -> empty string -> branch"). + +**The RPA recorders are the gold standard for element anchoring and self-healing** (UiPath, Power Automate Desktop, Automation Anywhere): +- **Descriptor = target + anchors, not a bare selector.** UiPath's Unified Target captures the element *plus* 1-3 stable neighbor elements, with type-aware anchor selection (input -> label to the left/above via aria-labelledby; checkbox -> right). For an AX trace, record the target AX node **plus its labeling neighbor(s)**. +- **A redundant stack of targeting methods that race, first-match-wins** - strict path, fuzzy/Levenshtein match, visual/CV fallback - never a single point of failure, never raw coordinates except last resort. Critical refinement (Selenium's lesson): make the fallbacks *different in kind* (semantic + text + structural + visual), so one redesign cannot kill all at once. +- **Self-healing fires at the failure boundary, not the happy path.** UiPath **Healing Agent** and PAD **self-healing** (GA/preview 2025-26) run only after the element times out, give the model the **screenshot of the missing element + parent-window title + full-screen image**, and regenerate a fresh selector preserving intent. PAD runs this with GPT-4.1-mini + Claude Sonnet 4.5 - **a local model doing the same visual-grounding + AX-tree reasoning is a direct fit for our on-device design.** Two modes (auto-fix vs propose-for-approval), and the healed descriptor is *persisted* so the routine self-improves. Cascade cheap heuristics (close overlays, adaptive waits, semantic relabel match) before the LLM. +- **Record-with-narration** (PAD "Record with Copilot") - the user demonstrates while narrating; video + audio + UI metadata -> a flow with conditions and loops. The closest analog to us; voice narration disambiguates intent and variable slots that pure action capture cannot infer. + +**The one-line macro-vs-smart test:** if changing a button's CSS class, moving it in the DOM, or swapping its tag breaks the routine, it is a macro. If it still finds the control a user would call "Submit" and can re-derive it from accessibility semantics, it is smart. + +**The research is the actual build blueprint for slot induction + self-healing (this is what phase 4 implements):** +- **Agent Workflow Memory** (AWM, ICML 2025, arXiv:2409.07429) - the canonical "trace -> parameterized routine" mechanism: an LM extracts reusable workflows from trajectories and **represents the non-fixed parts with descriptive variable names** (literal "dry cat food" -> `{product-name}`). Works online (induce a workflow after each success, add to memory immediately - self-improving, no training). Proves **an LLM can induce named, described slots from as little as one successful trace** - directly how our local model turns a recorded AX trace into a parameterized routine. WebArena 23.5% -> 35.5%. +- **Alloy** (arXiv:2510.10049) - single demo -> a task-level graph (nodes with conditionals/loops); an Identifier agent replaces literals with **semantic placeholders** carrying a documented meaning, a Filter agent fills them from the user's stated intent. Two-level review UX to copy: **structural** editing (nodes/edges) + **behavioral** (edit a node's prompt or **re-record just that one step**). Re-record-one-step is the killer repair affordance. +- **SUGILITE (CHI 2017) / APPINITE (2018)** - our exact primitive from 2017: capture via the **accessibility API**, generalize a **single demo into a parameterized script** by combining **verbal command + demonstrated procedure + UI hierarchy**; APPINITE targets elements by **semantic "data descriptions" (property queries), not coordinates** - the canonical answer to semantic-grounding-vs-pixels. PLOW (2007): NL identifies which demonstrated values are the parameters. +- **LUMOS** (arXiv:2606.30697) - the closest published articulation of *our* thesis: ground actions to **OS accessibility-tree elements (role, label, state, hierarchy) not pixels**, because "when applications update visual styling or layout, the accessibility tree typically remains stable, preserving action validity"; it names the **macOS Accessibility API** as the surface. Cite as the robustness rationale for AX anchoring. Contrast: frontier GUI models (UI-TARS-2) are pixel-and-coordinate grounded, drift-fragile, and expose **no editable parameterized routine artifact** - our inspectable AX routine is a different, more robust design point. +- **Segment into subtasks, never a flat event log** (arXiv:2606.20978) - hierarchy "separates what to do from how to do it," which is what makes a routine reusable and parameterizable. The flat event list is exactly the Automator mistake. +- **Verify each step's effect** ("Don't Act Blindly", ACL 2026; VeriSafe pre-action logic checks) - the expected effect at step t becomes the verification hypothesis at t+1; the dominant *silent* failure is that "agents don't recognize they've failed, leading to cascading errors," so re-snapshot after a consequential action and replan on `NO_CHANGE`. **Morae** (arXiv:2508.21456) - confirm only at *consequential or ambiguous* steps (a critical-vs-non-critical classifier + ambiguity-gated pause), not every step. A clean tiered permission model from the 2026 survey: **Silent (read) -> Logged (writes shown) -> Confirmed (shell/network) -> Blocked (credentials)**. And graduated trust exactly like Shortcuts: default a new routine to **Run After Confirmation**, let the user promote it to **Run Immediately**. +- **trycua/cua** (MIT) already does the **screenshot + AX-tree hybrid** and, in 2026, drives macOS apps **in the background without stealing the cursor** - directly relevant to a local-first assistant that must not hijack the session. + +The four properties that separate smart from brittle, converged across the literature: **semantic anchoring** (AX role/label + vision fallback, not coordinates), **described slots induced from the trace + intent** (sourced from memory / ask-each-time / a data loop), **verify-and-self-heal** (check each effect, regenerate the anchor or replan on drift, persist the fix), and **confirm at the right moments** (graduated trust, consequential-step gating). A literal macro has none; a smart routine has all four. None of the systems that produce an editable parameterized artifact (Alloy, SUGILITE, AWM, Mirage-1) is a local-first, on-device macOS product with AX anchoring + memory-sourced slots - that combination is ours. + - https://www.dssw.co.uk/blog/2014-11-10-automator-watch-me-do/ · https://support.apple.com/guide/shortcuts-mac/variable-types-apdd2b316022/mac · https://www.uipath.com/blog/product-and-updates/technical-tuesday-how-healing-agent-solves-ui-automation-challenges · https://learn.microsoft.com/en-us/power-automate/desktop-flows/self-healing · https://learn.microsoft.com/en-us/power-automate/desktop-flows/create-flow-using-ai-recorder · https://arxiv.org/abs/2409.07429 (AWM) · https://arxiv.org/html/2510.10049 (Alloy) · https://toby.li/publications/c4/ (SUGILITE) · https://arxiv.org/pdf/2606.30697 (LUMOS) · https://arxiv.org/html/2508.21456 (Morae) · https://github.com/trycua/cua + +## 5. Confirm-before-acting (the gate) + +**Two distinct designs exist:** +- **Inline pause + human takeover** (OpenAI Operator, Gemini Auto Browse, Comet) - the human re-enters the surface to type sensitive data or press the final button; the "edit" is "do it yourself." +- **Structured resolved-action card** (Manus Plan Mode, OpenAI Agents SDK / LangChain approval interrupts, mrmr, NN/g "Intent Preview") - shows resolved parameters (To / Subject / Body, amount, file, date) with Proceed / **Edit** / Cancel. **Manus is the standout**: "click into the plan and rewrite anything; when you Confirm, that plan becomes the source of truth." **Editing the resolved value is the differentiator** - most agents make you take over instead. Our card maps to this pattern. + +**Converged rules across everyone:** +- **Handoff for sensitive steps is universal** - payments, logins, CAPTCHAs -> human takeover; do not screenshot what the user types in takeover; use stored credentials only with permission; route payment through a tokenized intermediary; decline some categories (banking) outright. +- **Calibrate friction by reversibility, not uniformly** - auto-do the reversible long tail, confirm the sensitive, hard-gate the irreversible. "Confirm everything" measurably degrades into rubber-stamping (Anthropic's own data: full auto-approve drifts from ~20% of new-user sessions to >40% for experienced users). A user-set autonomy dial (Suggest / Confirm / Auto) is the emerging control. +- **Enforce the confirm deterministically, below the model.** Every real incident (Replit deleting a prod DB despite an approval rule; Comet's OTP exfiltration; Manus SilentBridge) proves a prompt-level "ask first" instruction is not an enforcement boundary. The card must gate the actual side-effecting call and match that exact action and its exact arguments, so injection or model drift cannot act on values the user never saw. + +**The #1 trust killer - false confirmations.** It appears in every task-doer: Ohai "tells you it completed tasks it hasn't," Comet "booked a hotel for the wrong dates," ChatGPT's "invented confirmations when the backend fails," Alexa+ got both Uber addresses wrong. **An agent must return a real backend confirmation record, never a model-generated "done."** Bake this in: our post-action toast must reflect the actual result of the executor call, never the model's claim. + - https://manus.im/blog/manus-plan-mode · https://getmrmr.com/blog/approval-fatigue · https://www.anthropic.com/research/measuring-agent-autonomy · https://brave.com/blog/comet-prompt-injection/ · https://www.nngroup.com/articles/impressions-chatgpt-agent/ + +--- + +## What this means for us + +Our design holds up remarkably well against the field; several of our choices are the exact documented best-practice (route to cheapest rail, demonstrated traces over novel automation, gate everything, memory as the moat). Concrete things to fold in: + +1. **Own the two whitespaces**: local-first (memory survives the vendor) and **context-resolution-with-evidence-and-confidence-shown-before-acting**. The approval card that shows *why* it resolved a value is the single most differentiated thing we can ship, and nobody has it. +2. **Make the notification budget a real module** (hard 3-5/day cap, value-vs-attention scoring, learned dismiss thresholds, displacement) - test it as a pure ranking unit. This is the difference between "proactive" and "muted by Friday." +3. **The gate must show a real confirmation, never a model "done."** Wire the post-action toast to the executor's actual result. This is the field's #1 trust failure and it is cheap to get right. +4. **Self-healing = AX-anchor (target + neighbor anchors) + racing heterogeneous fallbacks + LLM recovery at the failure boundary, with the healed descriptor persisted.** The local model does what PAD does with GPT-4.1-mini + Claude. Two modes: auto-fix vs propose-in-review. +5. **Anti-nag levers**: scope ("external only"), quiet folder vs loud bump, batching, hedged "suggested" framing, and link every commitment to its source utterance. Commitment precision is ~80-90%, so 1 in 5-10 is wrong - never auto-act on a detected commitment without the gate. +6. **Recorder**: consider narrate-while-demonstrating (PAD "Record with Copilot") to disambiguate slots; present the trace as an editable draft of semantic cards (not an event log); Magic-Variable-style tokens + Ask-Each-Time slots that resolve from memory. +7. **Detect completion and auto-retire the commitment - the single biggest anti-nag move, and one only we can make.** Every commitment tool flags "you said you'd send the deck" but none notice that you *sent* it, so they keep nagging. Because Replay watches the whole day on-device, we can see the fulfilling action (the email went out, the file was shared) and retire the item automatically. That is the difference between a tracker and a scold, and a generic cloud assistant cannot do it because it never saw you do the thing. +8. **Bind the confirmation to the exact payload that executes.** The Alexa+ lesson: a read-back is worthless if the value shown is not provably the value acted on (it read back the right address then used the wrong one). The values on the approval card must be the literal values the executor runs - no re-resolution between confirm and act; confirm returns an immutable action object. + +**For the demo brief specifically:** Screen 2 (the approval card) is where our differentiator lives - it must show the *evidence and confidence* for each resolved value, not just the resolved value. Add a low-confidence/disambiguation state (the ECLAIR "did you mean A or B" with evidence per candidate). That is the screen no competitor can show. diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md new file mode 100644 index 00000000..0c8aaf83 --- /dev/null +++ b/docs/COMPUTER_USE.md @@ -0,0 +1,290 @@ +# Off Grid AI - the proactive assistant (the act pillar) + +**Status:** product model agreed August 12, 2026. This supersedes the earlier "replicate the mobile-use stack" framing: that described one rail (GUI automation), not the product. The product is a proactive, context-grounded local assistant. Computer use is the last rail it reaches for, not the point. +**Standing constraints:** local models only, nothing leaves the device; all UI and copy follow `off-grid-ai/brand` (see 11). + +--- + +## Current operator experience + +Computer Use and Web Use now run as supervised tasks in Off Grid AI Desktop. + +1. Start the task from Chat and approve it. +2. When a local Web Use attempt first starts, its task details open once. Off Grid AI closes the left + navigation drawer and the Chat workspace so the browser task has the full app window. Back, or a + terminal task state, restores the earlier layout. Later progress updates do not reopen details + after you close them. +3. Open **Tasks** to see the durable history on the left. +4. Select a task record to inspect its execution plan and ordered trace. Plan stages group the work + into outcomes and show the current, completed, and failed stage. The **Live task** pane remains + clearly labelled and keeps the current run visible. +5. Use **Pause**, **Stop**, or **Take Over** for Computer Use. Escape stops the active task when the + global shortcut is available. +6. While Web Use or Computer Use is running, use the guidance box to change its next decision. + Enter sends and Shift+Enter adds a line. Attach a local document or image when it gives useful + context. The trace shows when guidance is accepted and when the next decision uses it. +7. Use **Return to originating Chat** in task details to reopen the Chat that started the run. + +Web Use controls an embedded browser surface. A link clicked in Chat opens as a normal manual browser +tab and does not start automation. Computer Use controls the execution device's current screen and +shows a separate always-on-top supervisor while it runs. Mouse movement does not pause or stop a +task. + +Stopping Chat first sends the Stop command to the active Web Use or Computer Use owner. Off Grid AI +cancels the Chat turn only after that owner accepts Stop. A stop failure stays visible, and Chat does +not pretend the task ended. + +Computer Use checks the focused macOS Accessibility element before it types. A secure or unknown +private field stops before actuation and asks you to enter the value, then resume. The helper returns +only `safe`, `secure`, or `unknown`; it does not read the field value. Typed action content is also +redacted before task history, SQLite, or sync. Exact live guidance and attachment content stay in +the active task's memory. History and sync keep only safe accepted/applied markers. + +Task records and text traces sync through the Personal Mesh. Screen images and their filesystem paths +stay on the execution device. Another device shows which Mac owns the missing image. + +Every Computer Use, Web Use, and accessibility run stops at 200 planning steps. The durable detailed +trace keeps the newest 250 steps. These limits are shared constants, not separate UI defaults. + +The real Electron QA journey lives in `scripts/qa-agentic-studio.mjs`, with light/dark evidence in +`e2e/screenshots/agentic-studio`. The August 25 run proved the docked task history, execution-plan +detail, retry, native browser ownership, route hide/restore, keyboard resize, settings, and +device-local evidence copy. It used the earlier semantic Web Use decision fixture. It is not proof +of the current strict visual decision contract. The current vision-first Web Use rerun and a real +Computer Use control run remain open as CU-004. + +The Computer Use catalog follows the same filters, model cards, download state, and active-model +rules as the other model tabs. It lists only model packages with a shipped policy adapter, pinned +GGUF, and matching projector. + +In Task settings, **Same as Chat** keeps the resident Chat model. **Separate specialist** loads the +selected ready Computer Use model for the task and restores Chat after it ends. The strategy and the +selected specialist sync through the Personal Mesh. The task screen size, detail, checkpoint, and +panel layout remain device-local where hardware or screen geometry makes a shared value unsafe. + +### Vision-first pipeline status - August 26, 2026 + +| Requirement | Code and wiring | Verification state | +| --- | --- | --- | +| Fixed Web Use evidence | Web Use captures only the page viewport from its main-owned `WebContentsView`. App chrome, Chat, and task controls are not in the model image. | Focused capture tests pass. Current real Electron proof is open in CU-004. | +| One model decision | UI-Mate, UI-TARS, and general vision models use one strict direction, milestone, and zero-or-one-action response for each screenshot. The request includes the current Task brief, accepted guidance, milestone, verified actions, recent events, older facts, coordinate bounds, and the screenshot. | Adapter and graph tests pass. Remote paths still have a thinking and privacy gap in CU-015. | +| Response validation | The model boundary rejects missing or extra fields, invalid enum values, malformed JSON, a mismatched verdict, and more than one action. The request has one attempt. | Focused adapter tests pass. | +| Model authority | After a valid model decision approves an action, Web Use does not use DOM text, element counts, labels, or UI phrases to reject it. The browser boundary checks only document freshness, screenshot pixels, coordinate structure, safety policy, and execution results. | A canvas-only page regression proves the approved visual click executes without DOM target resolution. | +| Coordinates and action | Web Use maps inference pixels to the current page viewport by proportion, executes the one approved action through CDP, then returns to a fresh capture. | Mapping, resize, browser-driver, and graph tests pass. | +| Milestones | The Web Use graph advances only on the model's validated `milestone_complete` signal and advances one milestone once. | Focused graph tests pass. Desktop Computer Use still has a separate loop owner; see CU-013. | +| Evidence and model identity | Live details show the run-bound model name, current phase, milestone, operation, final decision, visible evidence, updates, screenshots, mapped actions, and errors. The model identity is stored with the task, so a later global model change cannot relabel it. | Focused main and renderer tests pass. Live visual proof is open in CU-004. | +| Stop and immersive start | Stop in Chat reaches the task owner before Chat cancellation. The first running local Web Use attempt opens its details and closes the left navigation and Chat workspace once. | Focused lifecycle and App navigation tests pass. Live visual proof is open in CU-004. | + +The August 26 pipeline sweep passed 15 files and 200 tests, and its node typecheck passed. The later +CU-014 gate passes four files and 58 tests. Its changed browser files have zero scoped lint errors. +The current node typecheck passes. The expanded run has 202 passing tests and one unrelated Chat UI +failure. The standalone desktop `vision-agent.test.ts` run still does not finish. These remaining +test items are not CU-014 regressions. A green focused gate is code evidence, not live Electron or +real-device proof. + +Web Use treats an empty or single-colour compositor frame as failed evidence. One capture waits for +up to six seconds and asks Chromium to repaint every 250 ms. The graph then allows two fresh +recoverable observations before it shows the clear blank- or empty-screenshot error. A renderer +reload does not replace the main-owned browser view. An SPA route change, visibility change, or +native-view resize can still leave that view without a painted compositor frame for a short time. A +main-process restart is different: it disposes the browser host, stops the run, and destroys the +view; it cannot continue the same capture. + +--- + +## 1. What we are building + +An assistant that **notices what you need and acts on it**, grounded in what OGAD already remembers about your day. Not a chatbot you command, and not a pixel-clicking robot - an assistant that: + +- **knows you** - Replay already captures your day (screen -> OCR -> observations -> entities). That memory is the raw material. +- **is proactive** - it surfaces the flight you have not checked in for, the presentation you promised, the routine you run every morning - before you ask. +- **is private** - all of it is on-device. That is the only reason a person would let something watch their whole day, and it is the moat. +- **routes to the cheapest reliable rail** - a deep link or a connector before a scripted action before driving a GUI before pixels. It acts through the app and content you actually mean, resolved from your context - "put on the show we were just talking about, in the app you use" - not a UI-clicking gamble. + +The differentiator is that combination, not raw GUI prowess. Local models will not beat frontier cloud agents at clicking arbitrary pixels this year, and chasing that is a trap. Knowing you, noticing, staying private, and routing well is the product. + +## 2. Two ways a task is born (the generators) + +Every task the assistant acts on comes from one of two generators. Both emit the same thing: **a proposed action with open slots** (the structure is known; the content is filled later, see 4). + +### 2.1 Routine proactivity - repetition + +The same flow, done again. Two authoring paths, one artifact (a routine = a trigger + an ordered, AX-anchored action trace): + +- **Auto-detected** - mined from the Replay observation log: "every weekday ~9am you open Mail then Slack and scan unread." Low fidelity (we know the sequence, not every exact target), so it is used to *propose*, then confirmed by a recording. +- **Demonstrated** - you hit record and do it once. High fidelity: the exact trace, directly replayable. See 5. + +Detection *proposes*; demonstration *records the reliable version*. "I noticed you do this every morning - show me once so I can do it exactly." They are one loop, not two features. + +### 2.2 Reasoned proactivity - situation + +No repetition at all. Given your situation, something *should* have happened and has not. The flight case: + +1. **Detect the commitment/event** - "flight tonight" from a conversation Replay captured, or a confirmation email. +2. **Know what it implies** - world knowledge the LLM already has: a flight means check-in, a boarding pass, a gate. Nobody programs "a flight entails check-in." +3. **Gap-check the actual state** - the agent goes and looks, read-only: is there a boarding pass in Gmail? any sign of check-in? +4. **Surface the gap** - "You fly tonight and haven't checked in. Want me to?" +5. **Act, then gate** - check in or open the check-in page; anything with identity or payment confirms first. + +Steps 1-4 - the *smart* part - are pure memory + LLM + read-only connectors. No vision, no risky automation. That is the most magical and the most reliable part; it lands early - R2 in the build plan, right after the chat action tool is released (R1). See `COMPUTER_USE_PLAN.md` for the order. + +**The routine engine gives reliable *doing*; the reasoning engine gives an assistant that *notices*.** Same spine underneath. + +## 3. One gated spine + +Both generators feed one path: + +```mermaid +flowchart TD + RG["routine generator\n(detected + demonstrated)"] --> P[proposed action + open slots] + XG["reasoning generator\n(commitment + world-knowledge + gap-check)"] --> P + P --> R["resolve slots\n(RAG over Replay + conversation + entities + files)"] + R --> C{gate} + C -->|read / reversible / high-confidence| X + C -->|sensitive OR low-confidence| A["approval card\nshows the RESOLVED values"] + A --> X[execute via the rails] + X --> V[verify: AX diff / screenshot / connector result] + V --> P +``` + +- **Resolve** - the slots ("the presentation", "the person I promised") are filled from memory at run time, each with a confidence. This is the "which presentation" intelligence (see 6). +- **Gate** - the approval card shows the *resolved* values: "Send `Q3-strategy.pptx` to Ali Chherawalla." One glance confirms the AI inferred correctly *and* that the action is safe. The gate is where inference and safety are confirmed together - it is the guard against a confident-but-wrong resolution, and the same mechanism handles "is it right" and "is it allowed." +- **Trust graduation** - suggest -> approve-each-run -> auto-run trusted routines. Irreversible steps (send, pay, delete, account-create) gate by default even inside a trusted routine. + +## 4. The rail hierarchy - cheapest reliable first + +The router picks the cheapest rail that will reliably do the step. Vision is the last resort, not the engine. + +| Rail | What it is | Reliability | Status | +| --- | --- | --- | --- | +| 0. Perception | Replay OCR + the accessibility tree - structured "sight", no ML grounding model | n/a | capture ships; AX reader exists | +| 1. Semantic | deep links / URL schemes, AppleScript / Apple Events, EventKit, Shortcuts, MCP connectors | ~100%, deterministic | **built** (calendar, reminders, contacts, messages, mail, open_url) | +| 2. Agent browser | embedded browser pane driven in-process, for novel web tasks (check-in, ordering) | good; no OS permissions | built; live device proof remains open | +| 3. AX-tree GUI | structured native control (AXPress / set-value) + replay of a demonstrated trace | good on well-behaved apps | supervised task rail built; recorder remains open | +| 4. Vision grounding | a downloadable model mapping pixels -> coordinates | the frontier ceiling (~35-45% novel, local) | UI-TARS and UI-Mate adapters built; live device proof remains open | + +**Three different things get called "seeing", and only rail 4 is the heavy one:** Replay OCR (rail 0, ships) powers detection and context; the AX tree (rail 0/3, exists) powers precise recording and reliable replay with no ML model; the grounding vision model (rail 4) only earns its place when the AX tree is dead (WhatsApp-class apps) or a recorded step drifted. So the assistant can do a great deal - and ship real value - before rail 4 exists. + +Your examples, mapped to rails: + +- **Open Maps** - rail 1, `open_url` (`maps://`). Built. Flawless. +- **Call a cab** - rail 1, deep link (`uber://?action=setPickup&dropoff=...`) opens the ride pre-filled; you confirm. Reliable. +- **Put on a movie** - rail 1 if the app has a title deep link (many do); rail 2/4 if it means driving the streaming UI. Mixed. +- **Order from Amazon** - rail 2, the agent browser driving the real site (no consumer API), ideally a pre-authored recipe for the reorder flow, payment behind the gate. Best-effort, improving. + +The rule is always: does the service expose a clean surface (deep link / API / connector / AppleScript)? If yes, reliable and cheap. If it is GUI-only, it is the hard long tail - the same ceiling every agent hits, worse with local models. "Does everything" is honest as a direction, delivered as: the clean-surface majority done flawlessly, the GUI long tail done assistively and improving, always honest about confidence. + +## 5. The demonstration recorder + +Record-by-showing turns "novel GUI automation is ~40% reliable" into "replay a known trace", because replaying a *known* path is a far easier task than figuring out a UI from scratch. + +- **Capture the action trace, not raw input** - for each meaningful step: the app, the AX element (fallback coordinate), the action (click/type/scroll/navigate), any typed text. The AX context is what turns a raw click into "clicked Send in Slack" and what makes replay survive window moves and resizes. +- **Primitives we already have** - the CGEvent tap (we ship the *listening* half in dictation-hotkey), the AX reader, and Replay frames for context and step verification. The recorder is Replay-with-intent plus AX-tagging, a new mode, not a new system. +- **Review + edit** - after recording we show the steps in plain language ("Open Slack", "Click Send", "Type: ..."); you delete, reorder, or **mark a step as a variable slot** (see 6). +- **Store** - as a skill with a trigger (manual / schedule / event), reusing the existing skills format. +- **Never record secrets** - secure-input detection (`IsSecureEventInputEnabled()`) hard-skips keystrokes into password fields. Recording credentials would be a serious mistake. + +On replay, deterministic trace execution runs through the rails; the LLM/vision comes in only as **recovery** when a step's AX target is gone or a verification fails. Deterministic automation with model fallback is strictly more reliable than model-drives-everything. + +## 6. Memory-grounded resolution (the recording gives the *how*, memory gives the *what*) + +A demonstrated trace stores the reliable UI path but leaves the content open. The slots - "the presentation I mentioned", "the person I promised" - resolve at run time by RAG over the memory spine: Replay observations + recent conversation + entity graph + files you touched, scoped by temporal and entity proximity, returning a value **plus a confidence**. + +- A generic assistant cannot do "send the deck I promised" - it has no record of your day. OGAD can, because it has both halves (the memory and the action). +- **Confidence drives the gate**: high + non-sensitive -> preview-and-go; sensitive -> gate with the resolved preview; ambiguous ("which of three decks?") -> disambiguate or show the top candidate for one-tap confirm. +- **Honest edges**: recency window needs temporal decay (grab *this* deck, not last month's); resolution quality rises and falls with what Replay captured (a healthy incentive to invest in memory); the dangerous case is confident-and-wrong, which the preview-at-gate catches for sensitive actions and a higher confidence bar catches for auto-run. + +**Slot resolution (data, from memory) is a different intelligence from UI-drift recovery (elements, from AX + vision).** Keep them separate: one finds content, one finds buttons. + +## 7. What is already built (the reliable foundation) + +The semantic rail and the shared gate exist on this branch (11 commits), and they are exactly the reliable execution layer this assistant needs: + +- **Transport-agnostic approval seam** - `actions:proposeApproval` with a read/navigate/mutate/irreversible risk taxonomy; the single gate every rail routes through. Backward-compatible with the current pro build. +- **Native actions helper (macOS)** - one Swift one-shot backend behind `runNativeAction`, covering calendar (create/list), reminders (create/list), contacts (search), Messages send, Mail send, and `open_url`. Mutations gate; reads run free; lenient date parsing; AppleScript values escaped against injection. +- **Wired into the chat tool loop** macOS-only, and shipped in CI. Fully unit-tested through an injected boundary. +- **TCC packaging** - the Info.plist usage strings and apple-events entitlement a signed build needs, brand-clean, guarded by a test. + +None of this is wasted by the reframe. It is rail 1, and rail 1 carries most of the value. + +## 8. What is genuinely new to build + +On top of the existing foundation. **The build order and schedule live in `COMPUTER_USE_PLAN.md` (the build doc), which sequences these as releases R1-R4** - the chat action tool ships first (R1, the lead's steer), then the reasoning/resolve layer, then routines, then the hard rails. Mapped to the releases: + +- **R1 - the chat action tool + the durable spine.** Turn the existing semantic rail (7) into a released, gated, verified tool the chat model calls, on a durable Action queue + state machine. This is the released foundation the rest layers on. +- **R2 - the reasoning engine + the slot/resolve layer.** Commitment/event detection + world-knowledge of required steps + read-only gap-checking + surfacing (the magic, and the safest - no risky automation); and RAG over the memory spine to fill "the presentation" with a confidence. +- **R3 - the demonstration recorder + the routine store.** Record-by-showing (recorder + AX tagging + review UI, see 5) and skills with schedule/event triggers; auto-detection feeds the "record this?" proposal. +- **R4 - the agent browser (rail 2) + the AX act-primitives and grounding vision model (rails 3-4).** The reasoned novel web tasks (check-in, ordering) and the dead-AX / drift-recovery fallback. Last. + +## 9. What we reuse (do not reinvent) + +This is the curated shortlist. The deep, component-by-component port map for the whole system (durable queue, brain, memory, routines, rails, models) with a port-vs-bespoke verdict per component lives in `PORTING_MAP.md`. + +| Source | License | What we take | +| --- | --- | --- | +| `@ui-tars/sdk` + the UI-TARS desktop app | Apache-2.0 | Operator seam + action parser; the ScreenMarker overlay trio (animated border, content-protected control widget, pre-action markers); desktopCapturer scaling; the macOS permission gate | +| nanobrowser | Apache-2.0 | TypeScript DOM-to-indexed-elements serialization for the agent browser | +| `@computer-use/nut-js` (or the community fork) | Apache-2.0 | input synthesis on the native rail | +| macos-automator-mcp | MIT | wrapped AppleScript/JXA intents plus its recipe knowledge base | +| bytebot (archived) | Apache-2.0 | takeover-as-recorded-actions (the human demonstration lands in the same action log) and the needs_help state - directly relevant to the recorder | +| Peekaboo (OpenClaw org) | MIT | reference for the AX-tree + vision hybrid on the native rail | +| OpenAdapt (MLDSAI) | MIT | the recorder / routines rail (R3): record once -> deterministic, self-healing local replay; each step carries a template crop, an OCR label, geometry, a structural locator, and postconditions (our per-step verify), and the model touches the script only to repair on drift. Port the trace format + self-heal rather than build one. | +| FlaUI / pywinauto | MIT / BSD-3 | the Windows UI Automation act-primitives reference for the accessibility rail (R3 Windows fast-follow) - UIA2/UIA3 element find + invoke / set-value, the analogue of the macOS AX act-primitives | +| Agent-S2 (Simular) | Apache-2.0 | open computer-use agent loop + router structure as a reference for the brain | +| OpenClaw | AGPL (patterns only) | the proactive cron/skills pattern and the killer briefing workflow; equally its incident record as the avoid-list (exposed gateways, sandbox-off, weak auth, unvetted skills) - we ship none of those surfaces | + +Everything adopted as code is Apache-2.0, MIT, or BSD - clean for the AGPL core + proprietary pro split (verify each license at the point of adoption; minitap/mobile-use asks for attribution). + +### Mobile (the adapter after v1) + +Mobile is a `DeviceController` adapter on the same engine (Section 6 and `ASSISTANT_ARCHITECTURE.md`), not a rewrite - and the actuation layer already exists to port rather than build: + +| Source | License | What we take | +| --- | --- | --- | +| Mobilerun (droidrun) | MIT | the mobile actuation rail for Android + iOS: inspect UI state, screenshot, tap / swipe / type, model-agnostic and local-model-capable (Ollama / OpenAI-compatible). The mobile `DeviceController` wraps this instead of writing driver glue. | +| minitap/mobile-use | Apache-2.0 (credit Minitap) | the mobile agent loop reference (first to 100% on AndroidWorld), a LangGraph multi-agent over low-level control | +| AppAgent / AppAgent-v2 (Tencent) | MIT | learn-by-demonstration + tagged-element perception (numeric tags over the Android view hierarchy) - mobile routines by showing | +| Mobile-Agent-v3 / GUI-Owl (X-PLUG) | MIT | full mobile agent reference + GUI-Owl as the shared grounding model (desktop + mobile trained) | +| Appium + appium-webdriveragent (iOS) + UiAutomator2 (Android) | Apache-2.0 | the low-level device drivers under the mobile rail (iOS via WebDriverAgent / XCTest, Android via UiAutomator2) | +| Maestro (mobile.dev) | Apache-2.0 | the YAML flow format as inspiration for the mobile routine trace | + +iOS stays intents-only for driving other apps (App Intents / Shortcuts) - Apple forbids reading or driving other apps, so the mobile GUI / vision rails are Android-first, exactly as the plan states. + +## 10. Safety + +- The **gate** is the confirmation of inference and safety together (3): irreversible classes (send, pay, delete, account-create) confirm even inside trusted routines; the card shows resolved values so a confident-but-wrong resolution is caught before it acts. +- **Screen content is untrusted input** - published studies show 86% attack success from adversarial pop-ups against GUI agents; prompt-level defenses fail, so the gate and an app allowlist are system-level. +- **Never see or type credentials** - secure-input detection hands password fields to the user; the recorder hard-skips them. +- **Takeover with a guarantee** - at logins and payments the agent pauses and frame capture stops while the user controls the surface. +- **Kill switch** - visible Stop and Escape halt execution. Pause and Take Over park it until an + explicit Resume. Mouse movement does not change task state. +- **Everything executed lands in the approvals audit log.** +- **Trust graduates** - suggest -> approve-each -> auto-run; never jump to autonomous (the OpenClaw MoltMatch lesson). + +## 11. Build guidelines (binding, all surfaces) + +- **Design** - `off-grid-ai/brand` `DESIGN_PHILOSOPHY.md`: brutalist/terminal, Menlo, emerald as the only accent, black/white base, hierarchy by size and weight not color, no gradients, no emojis. All values from `@offgrid/design` tokens (`off-grid-ai/shared`) - no hardcoded hex. Desktop density per this repo's `docs/DESIGN.md`. +- **Copy** - `off-grid-ai/brand` `brand_tone_voice.md` + the outcomes-first rule: lead with what the user gets, mechanism as proof; no em dashes, no curly quotes, no exclamation marks, banned-word list applies. Applies to every approval card, suggestion, and notification. + +## 12. Open-core placement + +Rail-1 helper primitives and adapter plumbing are core infrastructure (like OCR). The reasoning engine, the recorder, the routine store, the resolve layer, and the approvals integration follow the existing pro spine. `pro/` changes land in `desktop-pro` first, submodule bump after. The engine (agent loop / router / resolver) lives in `off-grid-ai/shared` as `@offgrid/use`, consumed via `file:../shared/packages/use`. + +## 13. Decisions and open questions + +1. **Decided** - the model above: two generators (routine + reasoned) on one gated spine, rails cheapest-first, memory-grounded resolution, vision last. +2. **Decided** - `@offgrid/use` package name and `file:../shared/packages/use` consumption; the sibling `../shared` checkout is a build requirement (main already adopted it). +3. **Decided** - the semantic rail (rail 1) is the foundation and is built. +4. **Open** - parameterization depth: start faithful-with-marked-slots resolved by memory, layer richer LLM generalization on top. Confirmed lean: start faithful. +5. **Decided** - build order is release-led (`COMPUTER_USE_PLAN.md`): the chat action tool + durable spine ships first (R1, the lead's steer), then the reasoning + resolve layer (R2), then routines (R3), then the hard rails (R4). +6. **Decided** (per `PORTING_MAP.md` Section 6) - the default grounding model is UI-TARS-1.5-7B on desktop (Apache-2.0, GGUF + mmproj already published and mainline-runnable); GUI-Owl-1.5 / Qwen3-VL for mobile. Still not on the critical path (R4). + +## 14. Sources + +- Mobile-use agent loop: minitap/mobile-use (100% AndroidWorld) https://github.com/minitap-ai/mobile-use ; Mobile-Agent-v3 / GUI-Owl https://github.com/X-PLUG/MobileAgent +- Product UX: Claude Desktop browser pane https://code.claude.com/docs/en/desktop ; Codex embedded browser https://chierhu.medium.com/openai-codexs-browser-use-feature-b7dffa761d45 ; browser-use raw CDP https://browser-use.com/posts/playwright-to-cdp ; nanobrowser https://github.com/nanobrowser/nanobrowser ; UI-TARS desktop https://github.com/bytedance/UI-TARS-desktop ; bytebot takeover https://github.com/bytebot-ai/bytebot +- OpenClaw teardown: https://github.com/openclaw/openclaw ; Peekaboo https://github.com/openclaw/Peekaboo ; exposed gateways https://www.bitsight.com/blog/openclaw-ai-security-risks-exposed-instances +- Reliability calibration: OSWorld https://os-world.github.io/ ; pop-up injection (86%) arXiv:2411.02391 +- Grounding models: GUI-Owl-1.5-8B https://huggingface.co/mPLUG/GUI-Owl-1.5-8B-Instruct ; Qwen3-VL grounding arXiv:2511.21631 ; Holo3.1 https://huggingface.co/blog/Hcompany/holo31 +- macOS surface: Electron `AXManualAccessibility` https://www.electronjs.org/docs/latest/tutorial/accessibility/ ; secure input TN2150 https://developer.apple.com/library/mac/technotes/tn2150/_index.html ; Electron debugger https://www.electronjs.org/docs/latest/api/debugger ; node-mac-permissions https://github.com/codebytere/node-mac-permissions +- Brand: https://github.com/off-grid-ai/brand ; `@offgrid/design` in https://github.com/off-grid-ai/shared diff --git a/docs/COMPUTER_USE_PLAN.md b/docs/COMPUTER_USE_PLAN.md new file mode 100644 index 00000000..cb2080d9 --- /dev/null +++ b/docs/COMPUTER_USE_PLAN.md @@ -0,0 +1,131 @@ +# The proactive assistant - build plan and timeline + +Companion to `COMPUTER_USE.md` (the product model), `ASSISTANT_ARCHITECTURE.md` (the system design), and `PORTING_MAP.md` (the port-vs-bespoke research). + +> **This is the doc to build from.** Work release by release, top to bottom: a release is not done until its checkpoint passes, and the next release does not start until it does. The other three docs are references. Adjust the plan here at each checkpoint; never fork a second plan. + +**Re-cut (August 14, 2026 - the lead's steer + R1 field feedback).** The release after R1 is **all four rails, chat-driven, on both platforms**, plus the approval UX rebuild the R1 pro-path test demanded. The reasoning engine (proactive) and routines move after it. R1 itself is done: 17/19 checklist boxes, both PRs open and green (OGAD #81, shared #4). + +**Standing assumptions** + +- Solo developer, AI authoring the code end to end. +- Release-led: each release is a real, demoable, shippable increment. Desktop = macOS + Windows. +- **Port the plumbing, build the product** - each release names its ports (all MIT / Apache-2.0 / BSD, all in-process); the full map is `PORTING_MAP.md`. +- **Offline scope, stated precisely.** The brain runs with zero network on every platform. An action whose effect lives on an external service needs that service reachable at execution time - so the rails prefer local apps whose writes land locally and sync later (EventKit / Mail on macOS, local Outlook on Windows), and online-only actions are labeled honestly. +- **Reliability rules the router must honor** (architecture doc, Section 4): effect-verification lives in the engine; a cross-rail escalation is a re-fire under the same retry policy - a non-retryable action never escalates. The DeviceController routing is a thin layer over these. +- Checkpoint discipline: a checkpoint is a verifiable, demoable milestone. + +## Build guidelines (standing, all releases) + +- **Design** - `off-grid-ai/brand` `DESIGN_PHILOSOPHY.md`: brutalist/terminal, Menlo, emerald-only accent, black/white base, hierarchy by size and weight not color, no gradients, no emojis. All values from `@offgrid/design` tokens. Desktop density per `docs/DESIGN.md`. +- **Copy** - `off-grid-ai/brand` `brand_tone_voice.md` + outcomes-first: no em dashes, no curly quotes, no exclamation marks, banned-word list applies. +- **Cross-platform from the seam.** Callers depend on the `DeviceController` port and the shared engine, never on a concrete OS. +- **Port before writing.** Check `PORTING_MAP.md` / `COMPUTER_USE.md` Section 9 first; verify the license at the point of adoption; honor the AGPL / source-available avoid-list. + +## Releases + +| Release | What ships | Status | +| --- | --- | --- | +| **R1. Chat actions on the durable engine** | The semantic rail in chat on macOS (reminders, calendar, messages, mail, open, lookups) through the `@offgrid/use` engine: durable queue, payload-hash gate, retry-once-with-verify, read-back verification, effect journal. Windows toolchain green (installer artifact); the Windows semantic rail (local Outlook COM) built behind the port. | **Done.** PRs: OGAD #81, shared #4. Record: `R1_CHECKLIST.md` | +| **R2. Full rails in chat, both platforms + Approval UX v2** | Windows chat exposure; the browser rail (watched web tasks, takeover at login); the vision rail (supervised GUI actions, UI-TARS-1.5-7B); the approval experience rebuilt (inline in chat, outcome feedback, risk-tiered auto-run); the safety pass. ~5-6 working days. | **next** | +| **R3. Notices you** (was R2) | Reasoning + resolve + gate: commitment/gap detection over Replay, memory-resolved slots with confidence, the proactive Day surface. Cross-platform (memory + LLM). Pro-side code lands in desktop-pro (access in place). ~3 days. | after R2 | +| **R4. Routines** (was R3) | Record-by-showing + self-healing, per-step-verified replay (OpenAdapt design). macOS-first; the Windows UIA adapter as the fast-follow (napi-rs over the `uiautomation` crate + SendInput, Terminator head-start). ~2-3 days + fast-follow. | after R3 | +| **R5. Model-agnostic computer use** (the tiered rail) | Demote the vision grounder to a last-resort fallback so computer use runs on the user's NORMAL chat model for the common case. **Tier 1: the accessibility driving rail** - extend the shipped macOS AX helper to emit structured interactive elements; the model picks by label; act via AXPress/click. Any chat model, no grounder, covers most native + Electron apps. **Tier 2: set-of-marks** - a small OmniParser-class detector numbers elements on dead-AX apps for a general vision model. Router prefers AX -> set-of-marks -> vision. Tier 3 (the R2 UI-TARS grounder) stays as the on-demand fallback, deferred here. | **building (pulled forward, before R3/R4)** | + +The split: `shared` holds the durable cross-platform brain (`@offgrid/use`, reused by mobile later); this repo holds the rails, surfaces, and product integration; pro business logic lands in `desktop-pro`. + +## R1 - chat actions on the durable engine (DONE) + +Shipped scope, guarantees, and evidence live in `R1_CHECKLIST.md` and the PR bodies. Merge order: **shared #4 before OGAD #81** (main's CI resolves `@offgrid/use` from shared main). The release DISPATCH waits for R2 per the re-cut - one versioned release ships both. + +**R1 field verdicts driving R2** (from the pro-path smoke test): + +- Approving a card gives no completion feedback - the chat message says "pending" forever and nothing reports the run. (The engine path already reports verified outcomes; the legacy pro path is the old system.) +- Reversible simple actions (a reminder) should not need a human gate at all. +- Chat-originated approvals belong INLINE in the conversation, not on a separate screen; the Actions screen's job is unattended actions (proactive, scheduled) plus the audit log. + +## R2 - full rails in chat, both platforms + Approval UX v2 (~5-6 days) + +Everything chat-drivable on both OSes, honestly tiered, with an approval experience that reads like a conversation instead of a queue. + +### A. Windows chat exposure (~1 day) + +- Per-platform tool specs: win32 exposes the engine-routed set the Outlook rail supports (calendar_create_event, reminders_create, mail_send, open_url); reads stay macOS-only until the Outlook read verbs land. +- A win32 inline runner for open/navigate; the engine path handles mutations end to end (the rail shipped in R1). +- Outlook read-back verifiers (list verbs mirroring the mac ones) so Windows gets verified outcomes too. + +### B. Approval UX v2 (~1-1.5 days, core + desktop-pro) + +- **Inline approval card in chat**: resolved values + Approve / Edit / Reject in the conversation flow, driven by the engine gate (`resolveActionGate`). The Actions screen remains the queue for unattended actions plus the audit log. +- **Outcome feedback everywhere**: approve -> the engine executes -> the verified result lands back in the chat turn and on the card ("Created - verified", or the honest failure). This is the pro approval-executor migration: pro's queue resolves the engine gate instead of running its own executor, so payload binding and verification hold on the pro path too. +- **Risk-tiered gating** (decision 8.3's lean, now policy): reads/navigate free; reversible mutations (reminder, calendar) auto-run with a verified confirmation and an Undo affordance; sends and irreversible actions keep the gate. + +### C. The browser rail (~1.5-2 days) - cross-platform on arrival + +- Embedded pane over Electron's `webContents.debugger` (raw CDP): **nanobrowser's** TS dom module + overlay as starting code, **browser-use's** snapshot + AX-merge + numeric-index as the algorithm, **Stagehand's** act/observe/extract + Zod as the API. +- Chat-drivable web tasks (check-in, ordering) - watched live, takeover at any login/identity step, gated at the identity boundary. + +### D. The vision rail (~1.5-2 days) - the supervised tier, labeled so + +- **UI-TARS-1.5-7B** catalog entry (Apache-2.0, GGUF + mmproj published; a ~5GB download via the Models screen); **OmniParser v3** (MIT) set-of-marks fallback for the bundled model. +- The operator spine from **@ui-tars/sdk** (nut.js swapped for **@nut-tree-fork**/robotjs); mac input via CGEvent, Windows via SendInput. +- Supervised UX: the ScreenMarker-style overlay, pause-on-user-input, the kill switch (Esc halts with the keypress consumed). +- The WhatsApp file-share recipe as the showcase (behind the gate). + +### E. Safety pass + the release + +- Injection-resistance review (screen content is untrusted input), kill-switch e2e, per-rail verification depth honored, release-readiness checklist. +- **Checkpoint / release dispatch:** on macOS AND Windows - a semantic action, a watched web task with takeover, and a supervised vision action all run from chat, gated by tier, with verified outcomes reported inline. One versioned release: the signed/notarized .dmg + the Windows NSIS .exe (unsigned until the cert - decision open with the lead). + +**R2 risks:** the vision tier on a 7B local grounder is best-effort - ship it labeled supervised or not at all; Windows browser/vision needs a human on a real Windows machine (CI proves builds, not clicks); the model download adds a Models-screen surface; Approval UX v2 touches the live chat surface (the R1 lesson stands - behavior tests per branch, the plain path untouched for non-action turns). + +## R3 - notices you (was R2, ~3 days) + +Scope unchanged: commitment and gap detection over the Replay observation + entity spine; the resolve layer (RAG over memory returning value + confidence); proposals surfacing on the Day feed and executing through the same engine and inline approval UX. Ports: sqlite-vec (inside the app DB), LlamaIndex.TS memory blocks, Mem0's dedup loop, Orama hybrid ranking; techniques: HippoRAG PageRank, bi-temporal facts, the WSDM commitment rubric. Pro-side code (reasoning, resolve policy, feed UI) lands in desktop-pro. Checkpoint: on a seeded profile, on both OSes, an un-actioned commitment surfaces and "send the deck I promised" resolves from context and runs, gated by tier. + +## R4 - routines (was R3, ~2-3 days + the Windows fast-follow) + +Record-by-showing + faithful replay per the OpenAdapt design (compiled-step schema, resolution ladder, postconditions, repair-as-diff); Playwright codegen for the browser lane; memory-resolved variable slots; the plain-language review UI. macOS AX-as-eyes with actuation capped at press/set-value; the Windows UIA adapter (reader + SendInput) as the fast-follow. Checkpoint: record a routine once; it replays per-step-verified with a slot resolved from memory at run time. + +## R5 - model-agnostic computer use (the tiered rail, pulled forward) + +The R2 vision rail grounds every click through a specialized 7B model (UI-TARS): RAM-heavy, model-specific, and - as the field test showed - it makes the grounder the DEFAULT for interactive GUI tasks. R5 inverts that: the user's NORMAL chat model drives the common case, and the grounder is the last resort. This is architecture rule 8 ("cheapest reliable rail first, vision last") finally built for interactive execution, not just intent routing. Agenda: **computer use works on most chat models.** Detail + evidence: `R5_CHECKLIST.md`. + +- **Tier 1 - the accessibility driving rail (BUILD FIRST).** The shipped macOS AX helper (`scripts/text-extractor`, Swift - already walks the AX tree and owns the permission plumbing) is extended to emit STRUCTURED interactive elements: role, label, frame (x/y/w/h), actionable (has AXPress), value, enabled - not today's flat text blob. The rail runs the browser-rail loop verbatim over that element list: numbered elements -> the model picks by label (a TEXT task) -> act via AXPress or a click at the element frame. Works with ANY chat model, zero extra model RAM, covers most native + Electron apps. Wired as the `accessibility` rail the router prefers before vision. +- **Tier 2 - set-of-marks for the dead-AX tail.** Catalyst / WhatsApp-class apps expose no usable AX tree. A small OmniParser-class detector (a YOLO-ish icon detector + a tiny captioner - ONNX-class, not a 7B) finds and numbers the interactive elements on the screenshot; the user's general VISION model picks the number. One small detection-model runtime, not a grounder. Built after tier 1 ships and real coverage shows the tail is worth it. +- **Tier 3 - the vision grounder (existing R2 rail, DEFERRED in R5).** UI-TARS becomes the last-resort fallback for pixel-precision cases (drag a slider, a canvas) the tiers above cannot reach. Its separate on-demand loader (image-gen eviction pattern) + pluggable grounding-format adapters are a later item - NOT in R5. + +**Architecture evolution (note against `ASSISTANT_ARCHITECTURE.md`):** that doc said "accessibility is primarily the eyes, actuation capped." R5 promotes AX to a first-class DRIVING rail (element-picking + AXPress/click at frame), because that is precisely what lets a normal model do computer use. Vision stays genuinely last. The router's cheapest-first order becomes: semantic -> browser -> **accessibility (driving)** -> set-of-marks -> vision. + +**R5 checkpoint:** "send a file to a contact in Slack" runs end to end on a general chat model - app nav + a native file dialog + a verified irreversible send - with the vision grounder never loaded. + +## Dependencies + +| What | Needed by | Note | +| --- | --- | --- | +| shared #4 merged before OGAD #81 | now | main's CI resolves `@offgrid/use` from shared main | +| Windows signing cert | R2 release | wiring exists (WIN_CSC_LINK secrets); publishes unsigned until then | +| A human on a real Windows machine | R2 | browser/vision click-through + the model-load smoke (`WINDOWS_TEST_PLAN.md`) | +| UI-TARS-1.5-7B GGUF + mmproj catalog entry | R2-D | the vision model install | +| desktop-pro access | R2-B, R3 | in place (cloned at pro/) | +| Seeded memory fixtures | R3 | detection + resolution tests without a live profile | +| OpenAdapt trace/replay port + the `axuielement` napi addon | R4 | the recorder + the mac AX read | + +## Risks + +| Risk | Mitigation | +| --- | --- | +| Vision reliability (the frontier ceiling) on a local 7B | supervised tier, labeled; cheapest-rail-first routing; set-of-marks fallback; the gate on everything consequential | +| Approval UX v2 touches the live chat surface | behavior tests per branch; the plain path stays untouched for non-action turns | +| The Windows human-testing gap | recorded dependency; release notes honest about machine-verified vs human-verified | +| Solo schedule | releases independently valuable; scope trims at the tail (the vision showcase, Windows polish), never the shipped core | + +## Out of scope (unchanged) + +The mobile adapter (post-v1: Appium/WebdriverIO + DroidRun Portal + GUI-Owl-1.5/Qwen3-VL), background/headless autonomous runs, store distribution. + +## Tracking + +- R1 record: `R1_CHECKLIST.md`. R2 gets its own checklist when it starts. +- Small commits per verified unit, merge not squash. PR evidence rules apply. +- Checkpoint review against this doc at each release; plan changes are edits here. diff --git a/docs/CONSOLE_PLAN.md b/docs/CONSOLE_PLAN.md index ec1143d8..86446dd7 100644 --- a/docs/CONSOLE_PLAN.md +++ b/docs/CONSOLE_PLAN.md @@ -1,10 +1,10 @@ -# Off Grid Console — build plan (the SaaS for the 4 planes) +# Off Grid AI Console — build plan (the SaaS for the 4 planes) The **org-side web application** — the UI and backend for the Control / Data / AI / Regulatory planes. This is **Fleet Control's console + the Gateway/Brain admin surface**: -the "app that connects to all the nodes" (Off Grid Desktop/Mobile). Next.js. +the "app that connects to all the nodes" (Off Grid AI Desktop/Mobile). Next.js. -This is a **new, separate product** from Off Grid Desktop. The nodes already carry the +This is a **new, separate product** from Off Grid AI Desktop. The nodes already carry the gateway and enforce policy locally (see `ENTERPRISE_BUILD_PLAN.md`). The Console does **not** run the intelligence or enforce policy — it **defines and observes**: provisions policy + knowledge + config _down_ to the fleet, aggregates audit + telemetry + distilled learnings @@ -15,13 +15,13 @@ _up_. ## Where it fits ``` - ┌───────────────────────── OFF GRID CONSOLE (Next.js, this plan) ─────────────────────────┐ + ┌───────────────────────── Off Grid AI CONSOLE (Next.js, this plan) ─────────────────────────┐ │ Control plane UI · Data plane UI · AI plane (Brain) UI · Regulatory (DPO) UI · Fleet mgmt │ │ + backend (API · DB · node protocol) │ └───────────────┬───────────────────────────────────────────────────────┬──────────────────┘ policy / config / SOPs ▼ (down) audit / telemetry / learnings ▲ (up) ┌─────────────────────────── FLEET OF NODES ───────────────────────────┐ - │ Off Grid Desktop / Mobile — gateway baked in, enforces policy locally │ + │ Off Grid AI Desktop / Mobile — gateway baked in, enforces policy locally │ └───────────────────────────────────────────────────────────────────────┘ Org systems (DBs · warehouses · SaaS) ──connectors──► Brain (org knowledge) ◄── Console manages ``` @@ -122,9 +122,9 @@ Navigation mirrors the planes (and the `ENTERPRISE_BUILD_PLAN.md` component map) ## Standards (decision locked) We follow the Wednesday **Standards Kit** for engineering and component sourcing, and the -**Off Grid brutalist brand** (`docs/DESIGN.md`) for visual identity. Where the kit's _visual_ -identity conflicts with Off Grid (it uses green→teal gradients, Instrument Serif, DM Sans, -shimmer, card-lift, rich animation), **Off Grid wins** — the Console is one product family +**Off Grid AI brutalist brand** (`docs/DESIGN.md`) for visual identity. Where the kit's _visual_ +identity conflicts with Off Grid AI (it uses green→teal gradients, Instrument Serif, DM Sans, +shimmer, card-lift, rich animation), **Off Grid AI wins** — the Console is one product family with the Desktop/Mobile nodes it manages, and a dense compliance/audit tool suits the flat, information-first look. @@ -137,7 +137,7 @@ information-first look. - Animate only `transform` / `opacity`; wrap motion in `prefers-reduced-motion`; mandatory `aria-label` on icons, `alt` on images; 4.5:1 contrast. -**Visual identity (Off Grid `docs/DESIGN.md` — overrides the kit):** +**Visual identity (Off Grid AI `docs/DESIGN.md` — overrides the kit):** - Menlo mono everywhere; single emerald accent (`#34D399`/`#059669`), **no gradients**. - Flat 8px radius, hairline borders, no shadow/lift; hierarchy via size+opacity, not color. @@ -187,7 +187,7 @@ gradients**, **single emerald accent**, Menlo mono, **no decorative animation**) where functional (loading), never decorative. Net: **zero custom components** — discover in the catalog, source from the real library, -re-theme to Off Grid. +re-theme to Off Grid AI. ## Tech stack @@ -277,7 +277,7 @@ diagrams + OSS map). All on real Postgres + LanceDB + SSO + the live `:7878` gat 2. **Multi-tenant Admin module + ABAC/RBAC** — tenants/orgs, provisioning (who the console is for + their access), ABAC (tenant/purpose/data-class) layered on existing RBAC. Do now to avoid a retrofit. Single interface (ours) — no white-labeling underlying tools. -3. **License & legal audit** — every integrated OSS tool's license vs Off Grid (AGPL-3.0, +3. **License & legal audit** — every integrated OSS tool's license vs Off Grid AI (AGPL-3.0, on-prem). Flag AGPL (Grafana/Loki), SSPL/ELv2 (Redis/Airbyte), commercial-only features; produce `LICENSES.md` + swap recommendations. Confirm no copyright infringement (diagrams are first-party Wednesday assets). diff --git a/docs/CORE_RELEASE_JOURNEY_AUDIT_0.0.40.md b/docs/CORE_RELEASE_JOURNEY_AUDIT_0.0.40.md index 8176c083..4a65e0e2 100644 --- a/docs/CORE_RELEASE_JOURNEY_AUDIT_0.0.40.md +++ b/docs/CORE_RELEASE_JOURNEY_AUDIT_0.0.40.md @@ -11,11 +11,11 @@ This report cross-checks Core-applicable rows in - Excluded as Pro-only: #2, #83-104, #107-128, and #138-140. - Rows that combine Core and Pro outcomes remain in scope, but this report names only the Core portion. Pro-only clauses must be marked N/A during a Core pass. -- `High` means the decisive product seam is exercised without replacing Off Grid code. The manual +- `High` means the decisive product seam is exercised without replacing Off Grid AI code. The manual pass still checks the installed artifact, operating-system boundary, or pixels. - `Medium` means material production owners run, but a native, network, installer, renderer, or relaunch boundary remains split or controlled. -- `Low` means the decisive boundary is still manual, the current UI test substitutes an Off Grid +- `Low` means the decisive boundary is still manual, the current UI test substitutes an Off Grid AI preload/service, or the manual checklist does not explicitly represent the CSV journey. ## Manual-checklist gaps @@ -95,7 +95,7 @@ This report cross-checks Core-applicable rows in | ID | Pri | Journey and manual representation | Exact automation evidence | Remaining Core manual boundary | Confidence | | --- | --- | ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | -| #32 | P0 | First local message replies - exact: Send the first local message | `src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx` | Send through installed preload/IPC to a real local model, inspect one streaming bubble, switch screens, and relaunch | Low - rendered test substitutes Off Grid preload/model ownership | +| #32 | P0 | First local message replies - exact: Send the first local message | `src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx` | Send through installed preload/IPC to a real local model, inspect one streaming bubble, switch screens, and relaunch | Low - rendered test substitutes Off Grid AI preload/model ownership | | #33 | P0 | No memory scope works - grouped: Verify memory scopes | `src/renderer/src/components/__tests__/MemoryChat.chat-lifecycle.test.tsx` | Send a real local turn with No memory and inspect visible/persisted scope | Low - rendered test substitutes the preload path | | #34 | P0 | All memory scope works - grouped: Verify memory scopes | `src/main/__tests__/rag-empty-memory.dbtest.ts` | Seed synthetic memory through product UI/capture, send from rendered chat, and inspect answer/citation | Medium - real IPC/RAG/SQLite run; model and rendered entry are split | | #35 | P1 | Empty memory degrades safely - grouped: Verify memory scopes | `src/main/__tests__/rag-empty-memory.dbtest.ts` | Use a truly empty installed profile and send a second turn after the empty-memory result | Medium - real IPC/RAG/controller release run; rendered/local-model boundary remains | diff --git a/docs/DEMO_DESIGN_BRIEF.md b/docs/DEMO_DESIGN_BRIEF.md new file mode 100644 index 00000000..894899a4 --- /dev/null +++ b/docs/DEMO_DESIGN_BRIEF.md @@ -0,0 +1,126 @@ +# Design brief - Off Grid AI proactive assistant demo + +**For:** whoever is generating the design artifacts (Claude, or a designer). +**Deliverable:** high-fidelity mockups of the "ideal outcome" demo as an **interactive HTML artifact** (desktop), that **looks like the real Off Grid AI Desktop app** - same shell, same components, same feel. Real content throughout, never lorem. Five screens tied into one story (Section 6). +**Most important instruction:** match the actual app in Section 4. The app already exists; do not invent a new visual language. If a screen would not sit comfortably next to the real Models or Chat screen, it is wrong. +**Self-contained:** the app's look and tokens are inlined below; you do not need the repo. + +--- + +## 1. What the product is + +Off Grid AI Desktop is a **private, on-device assistant that notices what you need and acts on it.** It already watches your day locally (screen capture -> on-device OCR -> a private memory of what you saw and did). We are adding the ability to **act**. This demo shows that. + +Four things make it different, and the design must make all four feel true: +1. **It knows you** - it acts on *your* context ("the deck I promised" resolves to the actual file from memory). +2. **It is proactive** - it surfaces the flight you have not checked in for, the promise you made, the routine you run every morning, before you ask. +3. **It is private** - everything runs on your Mac, nothing leaves the device. Reinforce it quietly (a small "on-device" cue), never a banner. +4. **It is one general engine, not a pile of features.** The flight nudge, the promised deck, a renewal, a reply you owe - all the *same* machinery. There is no "Flights" tab, no per-situation section. These are transient items the assistant generates, here when relevant, gone when handled. + +The interaction principle to convey: **it routes to the cheapest reliable path and acts through the app and content you actually mean**, and **always shows what it will do before it does it.** + +## 2. Who it is for and the tone + +A sharp knowledge worker (beachhead: engineers) who lives in many apps. The feeling: **calm, dense, immediate, trustworthy** - a terminal/developer tool, not a friendly consumer chat app. + +## 3. The look (from the real app - copy this exactly) + +The app is **monospace, flat, outlined, and quietly technical.** It is NOT razor-sharp brutalism and it is NOT airy editorial SaaS - it sits between: flat surfaces with **1px borders and moderate corner radius (about 6-8px)**, **no drop shadows**, a very subtle **dotted-grid background texture**, and **Menlo monospace for every character on screen**. + +- **Typeface:** **Menlo** (or `ui-monospace, "SF Mono", Menlo, monospace`) everywhere - labels, headings, body, numbers. Weights stay light-to-regular. Hierarchy from size, weight, spacing, uppercase - never a second font. +- **Uppercase, letter-spaced labels** for section headers, tabs, and status tags (e.g. `MODELS`, `AVAILABLE TO DOWNLOAD`, `TEXT` / `IMAGE` / `VOICE`, `VISION`). Body and buttons are normal case. +- **Accent: emerald, only emerald.** The single accent - active nav, the one primary action per screen, focus, links, success, status tags. Everything else is a monochrome gray hierarchy. Do not add a second accent or color-code categories. +- **Semantic colors exist only for their exact job:** a muted **amber** for a single caution label (the app uses it for a `CHALLENGER` tag), a **red** only for error/health ("Model stopped"). Used rarely. +- **Exact tokens (dark mode - the primary theme for the demo):** background `#0A0A0A`, surface `#141414`, surface-light `#1E1E1E`, surface-hover `#252525`, border `#1E1E1E`, border-light `#2A2A2A`, text near-white, muted text mid-gray, accent `#34D399`. +- **Exact tokens (light mode - also ship it):** background `#FFFFFF`, surface `#F5F5F5`, surface-light `#EBEBEB`, text `#0A0A0A`, border `#E5E5E5`, accent `#059669`. +- **Component vocabulary (reuse these shapes, do not invent new ones):** + - **Outlined button** - 1px border, ~6px radius, icon + label, flat (the app's `Import .gguf`, `Download`, `+ New chat`, `Back`). Hover lightens the surface. + - **Solid emerald button** - the one primary CTA per surface, emerald fill with dark text (the app's `Configure`). Circular emerald send button with an up-arrow in the composer. + - **Outlined pill toggle** - small, rounded, icon + label, emerald when active (the composer's `All memory`, `Thinking`, `Image`). + - **Status tag** - tiny uppercase, emerald 1px outline + emerald text + a small icon (the app's `VISION` tag). Use this shape for risk/confidence tags. + - **Metadata line** - gray, dot-separated: `Qwen · 4B · 3.4GB · Mar 2026`. + - **Bottom CTA card / toast** - a flat outlined card pinned near the bottom with an icon, a title + one gray subtitle line, a solid emerald action, and an X (the app's "Set up your local AI - Configure"). **This is the exact shape to reuse for a come-up and for a toast.** +- **Density:** comfortable-dense. Rows and cards have real breathing room (this is not a cramped table); 2-column card grids where it fits; sticky headers. Design at 1440px+ wide. +- **Motion:** restrained - 150ms transitions, slide+fade for panels, subtle active-press. Nothing pops in hard. + +## 4. The actual app shell (render this frame around every screen) + +**Left sidebar** (expanded, about 240-260px; the app can also collapse to an icon-only ~64px rail - show the expanded one): +- Top: the emerald chip logo + wordmark **`Off Grid AI`**, and a small panel-collapse icon. +- A full-width outlined **`< Back`** control. +- The nav list, each row = **monochrome icon + label**, generous row height: **Search, Day, Replay, Reflect, Meetings, Actions, Entities, Projects, Chat, Voice, Vault, Clipboard, Devices, Integrations, Models, Gateway.** For the demo, **add one new item after Actions: `Routines`.** +- **Active item styling (important):** emerald icon + emerald label + a subtle emerald-tinted row background + a thin emerald bar on the row's left edge. Inactive: gray icon, near-black/near-white label. +- A divider, then quiet utility rows: a health line with a red pulse icon (e.g. `Model running`), `Theme: System`, `Settings`, `Mobile app` (with an external-link glyph). + +**Main area:** a header row with a small icon, a **title + one gray subtitle** (e.g. Chat shows `Off Grid AI` / `Private, on-device - chat, generate, and build`), and a cluster of square outlined icon-buttons top-right. Below it, the screen's content. A faint dotted-grid texture bleeds in at the top and bottom edges. + +Every demo screen must sit inside this shell (sidebar + header), so it reads unmistakably as Off Grid AI. + +## 5. Where the assistant lives (real nav, minimal additions) + +- **Come-ups live in `Day`** - the ambient home. The proactive items surface as a **"Needs you" section pinned at the top of Day**, above the retrospective day timeline. Ephemeral rows, never tabs. Day *is* the assistant, forward-looking on top. +- **The gate lives inline + in `Actions`** - a come-up expands *in place* into the approval card (fast path); `Actions` (which already exists, with a checkbox icon) is the full queue and audit. +- **`Routines` is the one new tab** - the library of saved automations; recording opens as a modal from it. +- **When you are away:** a toast (the bottom-CTA-card shape) and a menu-bar count. + +## 6. The five screens (one day, one story) + +Each screen must be **self-understandable** - legible without a caption (the come-up says what it is; the card shows exactly what it will do). Keep the one-line "proves:" note as an annotation. + +**Screen 1 - Day, with "Needs you" on top. Proves: proactive, knows you, one general engine.** +The hero, inside the real shell with **Day** active in the sidebar. Header: a calendar icon + `Day` + a gray subtitle + the date. The main column opens with an uppercase gray section label **`NEEDS YOU`**, then a short list of come-ups as flat outlined rows (reuse the bottom-CTA-card shape, one per row). Show a **mix of situations** so the generality is obvious: +- `You fly to SFO tonight, 21:40. Not checked in, no boarding pass found.` -> emerald `Check me in` + quiet `Later`. +- `You told Ali you'd send the Q3 deck by tonight.` with a gray context line `from your 10:15 call` -> `Send it` + `Later`. +- `getoffgridai.co renews tomorrow. The card on file expired.` -> `Update card` + `Dismiss`. +- A detected routine that already ran: `Morning brief - 09:02 · 12 unread, 3 need you` with a two-line synthesis from Mail and Slack. +Below `NEEDS YOU`, an uppercase `EARLIER TODAY` section with a dense retrospective timeline of what you did (a few rows), so it reads as an evolution of the existing Day view. Quiet "on-device" cue somewhere unobtrusive. + +**Screen 2 - The approval card, expanded inline from a Day row. Proves: it acts on your real context, shows the evidence and its confidence, and you confirm before it acts.** +The single most important screen, and the one no competitor ships. The user hit `Send it`; the row **expands in place** into a flat outlined card. It shows the **resolved action, each slot with its evidence and a confidence tag** (not a vague action, and not just the value - the *proof* it picked right): +- Title line: **`Send Q3-strategy.pptx to Ali Chherawalla`**. +- **Resolved slots, each a row:** a label, the resolved value as an editable pill, a gray provenance line (the evidence), and a small confidence tag using the status-tag shape: + - `File` -> `Q3-strategy.pptx` · gray: `you called it "the deck" in your 10:15 call · last edited 20m ago` · emerald tag `HIGH`. + - `To` -> `Ali Chherawalla ` · gray: `the "Ali" you promised · only deck shared with him` · emerald tag `HIGH`. + - `Via` -> `Mail` (the rail it will use). +- A risk tag near the actions in the status-tag shape but amber: `SEND · NEEDS APPROVAL`. +- Actions: solid emerald **`Approve and send`**, quiet outlined **`Edit`**, text **`Dismiss`**. +- Then show the **post-action toast** (bottom-CTA-card shape): `Sent to Ali - Q3-strategy.pptx`. (Annotate: the toast reflects the real send result, never a guess; the full queue lives in `Actions`.) +- **Also design the low-confidence variant of one slot** (a second small card state): instead of a pre-filled value, the slot becomes a picker - `Which deck did you mean?` with two candidate rows, each showing its own evidence (`Q3-strategy.pptx - shared with Ali, edited 20m ago` vs `Q3-final.pptx - edited last week`) and a select control. Low confidence disambiguates *before* the confirm, it never guesses. + +**Screen 3 - The reasoned nudge in action (the flight). Proves: it notices what should happen and helps, handing off safely.** +The flight come-up expanded into a short flow. State one: `Check me in` / `Remind me at 20:00` / `Dismiss`. State two: it opened the airline check-in and filled the known fields (confirmation number, name from memory), then **handed off** at the identity/seat step - `Your turn - confirm your seat` (capture paused, shown as a small note). End state toast: `Boarding pass saved`. + +**Screen 4 - Record a routine (modal from Routines). Proves: the user can author automations by demonstrating.** +A modal/slide-over in the app's style. State one - **recording:** a calm indicator (a thin emerald border around the app, or a small emerald status pill `Recording routine - do it once, I'll learn it`), NOT a big red dot. State two - **review the captured steps:** an editable list of semantic step cards in plain language (`Open Slack`, `Go to #standup`, `Post: Standup - {date}`), one step showing a **variable slot** as an emerald pill (`{date}`, or `the deck`) that resolves from memory each run. Controls to reorder/delete a step, an inline hint to mark a value as a variable, and a **trigger** row (`Manual` / `Schedule` / `When I ...`). Primary solid emerald `Save routine`. + +**Screen 5 - Routines tab. Proves: detected and demonstrated routines live together on one spine.** +`Routines` active in the sidebar. Header: `Routines` + subtitle. A dense list/table: a mix of **detected** (`Morning brief`, auto-found) and **recorded** (`Standup note`, `Send weekly report`). Columns: name, trigger (`09:00 weekdays` / `manual` / `event`), last run, and a trust tag in the status-tag shape (`SUGGEST` / `AUTO`). A run control per row, an outlined `Record routine` button top-right, sticky header. + +## 7. Copy voice (every string) + +- **Lead with the outcome, in the user's language:** "Send the Q3 deck to Ali", not "Execute mail.send". +- Plain and direct; proof over adjectives. +- **No em dashes** (use " - "), no curly quotes, no exclamation marks, no emojis. +- Banned words: revolutionary, seamless, empower, leverage, robust, comprehensive, crucial, delve, tapestry, testament, foster, showcase, enhance; and AI-slop ("it's not X, it's Y", "serves as"). +- A control says exactly what it does; the toast says it happened. +- Real names and content (Ali Chherawalla, `Q3-strategy.pptx`, SFO 21:40, getoffgridai.co). + +## 8. Deliverable format + +- **One interactive HTML artifact** rendering the real app shell (sidebar + header) with the five screens; the sidebar switches Day / Actions / Routines, numbered steps handle the flight/record sub-states and the low-confidence card variant. Self-contained (inline CSS, monospace stack, no external fonts/CDNs). Designed for 1440px+. +- **Dark mode primary (tokens above); include a working light-mode toggle.** Both properly styled. +- Each screen annotated with its "proves:" line, but the screen must read on its own without it. +- If one artifact is too much, deliver **Screen 1 (Day) and Screen 2 (approval card)** first - they carry the demo. + +## 9. Do not + +- **Do not invent a new app shell or visual language.** Match Section 4. No "Assistant" tab, no "Flights"/"Bills"/"Travel" tabs - come-ups are transient content in Day. +- **Do not over-round or over-soften into consumer SaaS** (big rounded cards, drop shadows, gradients, pastel fills) - the app is flat, outlined, ~6px radius, monospace. +- **Do not over-sharpen into hard brutalism either** (zero-radius, heavy black rules, cramped rows) - the real app is calmer than that. Match the screenshots' feel. +- Do not use a second accent or color-code categories; emerald only, amber/red only for caution/error. +- Do not use a non-monospace font anywhere. +- Do not design mobile-first; wide desktop only. +- Do not make the assistant a chat-bubble feed; it speaks through the Day rows and approval cards. +- Do not over-explain privacy with a banner; a quiet, constant cue. + +The north star: **it looks like it shipped inside Off Grid AI** - monospace, flat, outlined, emerald-on-dark, dotted-grid - and every screen makes it obvious the assistant knows you, acts on your real context, shows the evidence and its confidence, and always confirms before it acts. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 2159bc64..8c885c0f 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -1,6 +1,6 @@ # Off Grid AI Desktop — Design -The desktop adaptation of the Off Grid design philosophy. The brand canon is the mobile docs (`../../mobile/docs/design/DESIGN_PHILOSOPHY_SYSTEM.md` + `VISUAL_HIERARCHY_STANDARD.md`); **this doc keeps the same soul and adapts it for a desktop app.** Where this conflicts with the mobile docs on _layout/interaction_, desktop wins; where it conflicts on _brand_ (font, color, voice), the brand wins. +The desktop adaptation of the Off Grid AI design philosophy. The brand canon is the mobile docs (`../../mobile/docs/design/DESIGN_PHILOSOPHY_SYSTEM.md` + `VISUAL_HIERARCHY_STANDARD.md`); **this doc keeps the same soul and adapts it for a desktop app.** Where this conflicts with the mobile docs on _layout/interaction_, desktop wins; where it conflicts on _brand_ (font, color, voice), the brand wins. --- diff --git a/docs/DESIGN_PHILOSOPHY.md b/docs/DESIGN_PHILOSOPHY.md index e8a305ab..bb22d7d5 100644 --- a/docs/DESIGN_PHILOSOPHY.md +++ b/docs/DESIGN_PHILOSOPHY.md @@ -27,7 +27,7 @@ on-device, and fast — the UI should feel that way too: calm, dense, immediate. (the repo's component files are demo stubs — pull the real one from the registry). - `components.json` is configured with the `@aceternity` + `@magicui` registries + shadcn. -## 3. Brand — Off Grid identity (binding) +## 3. Brand — Off Grid AI identity (binding) - **Typeface:** Menlo (monospace), everywhere. Terminal/brutalist. - **Accent:** emerald — `#34D399` (dark) / `#059669` (light). The only accent. diff --git a/docs/ENTERPRISE_BUILD_PLAN.md b/docs/ENTERPRISE_BUILD_PLAN.md index c15697ae..fffc4409 100644 --- a/docs/ENTERPRISE_BUILD_PLAN.md +++ b/docs/ENTERPRISE_BUILD_PLAN.md @@ -1,11 +1,11 @@ -# Off Grid for Organizations — full build plan +# Off Grid AI for Organizations — full build plan This plan **inherits the entire 5-plane agentic architecture** from the stack navigator (`wednesdayai/knowledge-base/agentic-ai-stack-navigator.md` = the canonical text of `cro/proposals/final/agentic-ai-stack-navigator.html`). Every component A1→E6, plus the -Physical plane, is accounted for below and mapped to Off Grid. +Physical plane, is accounted for below and mapped to Off Grid AI. -The navigator was written for a multi-tenant regulated bank on cloud/on-prem. Off Grid is +The navigator was written for a multi-tenant regulated bank on cloud/on-prem. Off Grid AI is **local-first, on-device, single-org-per-deployment, on-prem.** That changes _how_ each layer is realized, not _whether_ it exists. Six rules drive every mapping: @@ -110,10 +110,10 @@ _and_ the org's real data** — not observation alone: ## Phase A — DATA PLANE -> Bank version: get data out of source systems, prep, govern, land. Off Grid version: the +> Bank version: get data out of source systems, prep, govern, land. Off Grid AI version: the > work itself is the source; it lands on-device and (distilled) in Brain. -| # | Navigator component | Off Grid realization | Owner | Status | +| # | Navigator component | Off Grid AI realization | Owner | Status | | ---- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------- | ----------------------------------------------------------------------------------------------------- | | A1 | Source systems | **Two first-class sources.** (1) **Capture** (screen→OCR, messages, calls) — the work itself, on the node. (2) **Org digital data** — databases, warehouses, SaaS, document stores — via connectors on the org-side ingest service. | Personal AI capture · Brain-side connectors | ✅ capture (`watcher.ts`, `vision.ts`, `ocr.swift`, meeting recorder); ❌ enterprise connectors build | | A3 | CDC / ingestion | Node: continuous on-device ingest of capture. Org: CDC/connector pulls from DBs & warehouses → ETL → mask → Brain. | Personal AI ingest · Brain ingest service | ✅ `watcher.ts`, `ingest.ts`; ❌ org ingest service | @@ -132,7 +132,7 @@ _and_ the org's real data** — not observation alone: > The engine. This is largely **already built** — it's what `model-server.ts` is. -| # | Navigator component | Off Grid realization | Owner | Status | +| # | Navigator component | Off Grid AI realization | Owner | Status | | --- | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | --------------------------------------------- | | B1 | Doc parsing + chunking | OCR + audio transcription + doc extractors. | Gateway (AI plane) | ✅ `rag/extractors`, whisper, vision | | B2a | Reranking + hybrid search | Retrieval over captured + Brain content; BM25 + vector + rerank. | Gateway | ⚠️ embeddings exist; hybrid + rerank build | @@ -140,7 +140,7 @@ _and_ the org's real data** — not observation alone: | B5a | Provenance + citation | **Every SOP/answer traces to the captured source** (which screen, which call, when). The auditability beam. _(This is the "SANN-equivalent.")_ | Brain · Gateway output policy | ❌ build | | B7 | Tool layer (MCP) | `/mcp` — on-device models + actions + org connectors as scoped, audited tools. | Gateway | ✅ `mcp-server.ts`, extend with scope+audit | | B9 | Sandboxed code execution | Agents run untrusted code in isolation (microVM), never the host. | Gateway | ❌ build (later) | -| B11 | Memory (sidecar, 4 flavors) | Exactly Off Grid's memory: short-term, long-term vector, entity graph, file-based markdown. **Personal memory** + **org memory (Brain)**. | Personal AI · Brain | ✅ strong (`crm/*`, observations, memory) | +| B11 | Memory (sidecar, 4 flavors) | Exactly Off Grid AI's memory: short-term, long-term vector, entity graph, file-based markdown. **Personal memory** + **org memory (Brain)**. | Personal AI · Brain | ✅ strong (`crm/*`, observations, memory) | | B15 | Model serving / inference | Bundled llama-server, whisper, TTS, diffusion — unified at `:7878`. | Gateway | ✅ strong (`model-server.ts`) | | B16 | Fine-tuning + privacy ML | Adapt the local SLM to the org's domain & SOPs (LoRA), on-device or in Brain. | Brain | ❌ build (later) | @@ -151,7 +151,7 @@ _and_ the org's real data** — not observation alone: > The gateway spine. Wraps the AI plane we already have. **This is the bulk of the new > build**, and where Fleet Control plugs in. -| # | Navigator component | Off Grid realization | Owner | Status | +| # | Navigator component | Off Grid AI realization | Owner | Status | | --- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------- | ----------------------- | -------------------------------------------- | | C1 | AI gateway | `:7878` becomes a real chokepoint: routing local + **leashed cloud** (rule 5). | Gateway | ⚠️ started — single local model, add routing | | C2 | Input policy / guardrails | Injection scan — **especially of captured content** (hostile indirect input). | Gateway pre-hook | ❌ build | @@ -173,7 +173,7 @@ _and_ the org's real data** — not observation alone: > Where humans meet it. Mostly **already built** on the Personal AI side. -| # | Navigator component | Off Grid realization | Owner | Status | +| # | Navigator component | Off Grid AI realization | Owner | Status | | --- | ------------------------------ | ----------------------------------------------------------------------------- | --------------------------- | --------------------------------------- | | D1 | Agent runtime / orchestration | The agents that do the work + the **learning-loop batch jobs** (observe→SOP). | Personal AI · Brain | ⚠️ partial (`crm/agent.ts`); loop build | | D2 | Human-in-the-loop | Approval-gated actions on anything consequential. | Personal AI | ✅ `crm/approvals.ts` | @@ -188,7 +188,7 @@ _and_ the org's real data** — not observation alone: > Functions, not just tools. Realized mostly inside **Fleet Control** (the DPO's product). -| # | Navigator component | Off Grid realization | Owner | Status | +| # | Navigator component | Off Grid AI realization | Owner | Status | | --- | --------------------- | -------------------------------------------------------------------------------------- | ------------------------ | -------- | | E1 | Framework mapping | Map controls → DPDP/etc clauses. **The DPO single compliant view + one-click export.** | Fleet Control | ❌ build | | E2 | AI use policy | What staff may do; authored and **pushed to every device** by Fleet Control. | Fleet Control | ❌ build | @@ -199,7 +199,7 @@ _and_ the org's real data** — not observation alone: ## Phase 0 — PHYSICAL PLANE -| Navigator | Off Grid realization | Status | +| Navigator | Off Grid AI realization | Status | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | | GPUs / nodes / power / K8s | The **devices themselves** (laptops, phones) run inference on-device. Optional **on-prem org server** hosts Fleet Control + Brain. No hyperscaler in the path. | deployment | diff --git a/docs/FEATURES.md b/docs/FEATURES.md index a6348cca..15e58a7b 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -24,7 +24,7 @@ device** — no cloud inference, no account, no API key. Each feature has its ow | ---------------------------------------------------- | --------------------------------------------------------------- | | [Privacy & data](features/privacy.md) | 100% local inference, encryption at rest, fully offline. | | [Architecture (open core)](features/architecture.md) | The AGPL core, the `pro/` submodule, and the `activate()` seam. | -| [Off Grid Pro](features/pro.md) | The sees / remembers / acts layer — **coming July 2026**. | +| [Off Grid AI Pro](features/pro.md) | The sees / remembers / acts layer — **coming July 2026**. | --- diff --git a/docs/GAPS_BACKLOG.md b/docs/GAPS_BACKLOG.md index b051752c..eb6c790a 100644 --- a/docs/GAPS_BACKLOG.md +++ b/docs/GAPS_BACKLOG.md @@ -7,6 +7,69 @@ how to reproduce, and the fix direction. Close with evidence; never hide. ## OPEN +### CU-004 (P1) - The current vision-first paths lack complete live control proof + +**Earlier Web Use evidence (2026-08-25):** `scripts/qa-agentic-studio.mjs` ran a deterministic Web +Use task through production Electron, SQLite, IPC, CDP, and `WebContentsView`. Inspected captures +proved task history, the execution plan, route ownership, resize, settings, local evidence, pointer +display, takeover, resume, failure, and dark-page composition. That run used the earlier semantic +decision fixture. It does not close the current strict screenshot, judge, and action pipeline. + +**Current code evidence (2026-08-26):** the pipeline sweep passed 15 focused files and 200 tests. +They cover the strict model contract, Web Use graph, page-only capture, proportional mapping, browser +actuation, task ownership, live evidence, run-bound model identity, Chat Stop, and the immersive +first-start layout. Its node typecheck passed. The later CU-014 gate passes four files and 58 tests. +The current expanded run has 202 passing tests and one unrelated Chat-planner UI failure. This is not +live Electron or real-device evidence. + +**Remaining evidence:** run the current strict visual-decision fixture through the real Electron +Web Use journey and inspect its light/dark screenshots. Also run one safe Computer Use task on a real +Mac. Prove that screenshots, mapped pointer, fresh evidence, milestone progress, and the active model +are factual. For Computer Use, prove that Pause, Resume, Take Over, Stop, Chat Stop, and Esc stop or +park the real native actuator as shown. + +### CU-013 (P1) - Computer Use and Web Use have different visual workflow owners + +**Evidence (2026-08-26):** Web Use runs `runVisionTaskGraph` through `browser-visual-task.ts`. +Desktop Computer Use still runs the older `runVisionTask` loop through `vision-host.ts`. The old loop +adds a model answer to `policyHistory` before it knows whether actuation succeeds. A rejected action, +handoff, rethink, or terminal decision can therefore remain in the next model request as a prior +validated decision. The graph commits that history only after successful actuation and discards it +for rejected or non-action decisions. + +The standalone `vision-agent.test.ts` command does not finish and had to be stopped after 30 seconds. +Scoped ESLint also reports a hard error in `vision-policy-runner.ts`, plus size and complexity +warnings in both workflow owners. The superseded semantic `web-task-agent.ts` runtime and its tests +have been removed; Web Use now has one production workflow owner. + +**Impact:** the same valid model response can produce different history and recovery behavior on the +desktop and browser surfaces. Two owners can drift on Stop, evidence, milestone, and action-commit +rules. + +**Fix direction:** route both surfaces through one workflow owner with injected capture and action +boundaries. Commit model history only after successful action execution. Remove superseded runtime +loops when the shared path is wired. Make the focused desktop test finish, then clear the scoped lint +gate and rerun both surface journeys. + +### CU-015 (P0) - Selectable remote vision paths violate the local-only contract + +**Evidence (2026-08-26):** the standing product constraint says local models only and nothing leaves +the device. The current model UI can select OpenRouter or a custom remote server. +`vision-policy-runner.ts` then sends the base64 screenshot and full task context to that server. +Only OpenRouter receives an explicit `reasoning_effort`; Ollama, LM Studio, OGAD, and custom +OpenAI-compatible endpoints do not get a transport-level thinking control. The application does not +prove that every selectable remote model is thinking-enabled. + +**Impact:** a cloud endpoint can receive private screen evidence and task context without a product +exception to the local-only rule. Other remote paths can also run the required contract without a +verified thinking mode. + +**Fix direction:** remove network model-server selection from Computer Use and Web Use, or make a +documented product decision that changes the local-only constraint. Any approved exception needs an +explicit per-server privacy boundary, clear local-network versus cloud disclosure, user opt-in, and +a capability gate that rejects a model unless the required visual, structured-output, and thinking +features are verified. + ### SYN-004 (P1) - Late-pair full graph is not verified between the real Desktop and Mobile apps **Evidence (2026-08-13):** the production send paths now backfill state records, generated images, @@ -222,7 +285,7 @@ sidebar destinations and verified that each reaches its matching upgrade screen. provide the same journey. `ProPlaceholder` deliberately renders a non-interactive `motion.div` with no button, link, navigation intent, or keyboard affordance (`src/renderer/src/components/SettingsCard.tsx:147-193`). The visible `Device sync`, `You`, and -`What Off Grid has learned` cards therefore show a Pro badge and description but cannot explain how +`What Off Grid AI has learned` cards therefore show a Pro badge and description but cannot explain how to unlock the feature or take the user to the existing purchase/activation journey. The same run also found that `Proactive delivery` has canonical placeholder copy in `proSettingsCatalog.ts:61-70`, but Settings filters that slot out unconditionally at @@ -287,7 +350,7 @@ position while `SettingsCard` animates from zero to auto height (`SettingsCard.t `visible` result therefore does not guarantee that the recovery action is in the user's viewport. **Impact:** Screen Recording is required for Replay and local vision capture. At the exact point a -denied user returns from macOS and must relaunch to apply the grant, Off Grid appears empty and hides +denied user returns from macOS and must relaunch to apply the grant, Off Grid AI appears empty and hides the only next action off-screen. This makes a recoverable permission state look like an application failure and can leave capture permanently blocked for a user who does not discover the stale scroll position. @@ -302,7 +365,7 @@ layout-aware reveal/focus contract instead of adding another timeout or per-butt - `Review permissions` opens `/settings/permissions`, expands Setup & health, and places the Screen Recording card fully inside the Settings viewport after the accordion layout has settled. -- After `Enable Screen Recording` opens System Settings and Off Grid becomes active again, every +- After `Enable Screen Recording` opens System Settings and Off Grid AI becomes active again, every status transition (permission needed, checking, restart required, granted, or error) preserves a useful anchored viewport. `Relaunch Off Grid AI Desktop` is immediately visible, focused or predictably reachable, and clickable without manual scrolling; the canvas never becomes blank. @@ -491,10 +554,10 @@ though that was argued from the failure mechanism rather than proved against the 2,246 of the delivery rows on this Mac point at a `device_id` that is not in `sync_paired_devices` at all. They can never succeed, and they are counted: -| device_id | in paired table | rows | -|---|---|---| -| `6e1c3b7150fb1f088b52a4c2e99eda78` | no | 2,077 (1,559 skipped · 367 failed · 150 queued · 95 sent · 2 rejected) | -| `8Lj2tUzzpBQ1zJEPHGfrSw` | no | 119 (78 sent · 18 dismissed · 16 rejected · 5 skipped · 2 queued) | +| device_id | in paired table | rows | +| ---------------------------------- | --------------- | ---------------------------------------------------------------------- | +| `6e1c3b7150fb1f088b52a4c2e99eda78` | no | 2,077 (1,559 skipped · 367 failed · 150 queued · 95 sent · 2 rejected) | +| `8Lj2tUzzpBQ1zJEPHGfrSw` | no | 119 (78 sent · 18 dismissed · 16 rejected · 5 skipped · 2 queued) | A device is forgotten and its deliveries stay. So "158 queued" and the failure total describe work aimed at machines this Mac no longer has any relationship with, and no retry can ever clear them. @@ -534,10 +597,120 @@ for the service to expose a settled point to wait on. Found while landing the peer-link adoption. Not caused by it: nothing in that change touches knowledge-document sync, and the service under test builds no orchestrator. + +--- + +### DEF-009 (P2) - ModelPicker re-implements the Computer Use model-resolution rule in the renderer + +**Evidence (2026-08-27):** `src/renderer/src/components/ModelPicker.tsx` (~lines 175-179) re-derives +which model Computer Use will run: it checks `computerUseStrategy === 'same_as_chat'` and matches +models by `id` / `primaryFile` itself. The main process already owns that rule - the grounder loader's +`modelStrategy`, `selectedGrounderModelId`, and `resolveActiveBrowserVisionSelection` are the source +of truth. Two layers now answer "which model grounds Computer Use", kept in step by hand; any change +to the main-process rule (a new strategy, a different identity key) silently strands the renderer copy +and the picker labels the wrong model. + +**Fix direction:** collapse to one source. Add a main-process IPC (e.g. +`getComputerUseModelSelection()`) that returns the resolved model identity, expose it through the +preload, and have ModelPicker render that answer instead of re-deriving it. + +**Why deferred:** needs a new IPC channel plus a preload surface change, out of scope for a +quality-only pass. Filed during the 2026-08-27 code-quality sweep. + --- ## RESOLVED +### CU-014 (P1) - A DOM heuristic could reject a valid visual model action - RESOLVED 2026-08-26 + +`BrowserDriver.pageState()` now reads only the committed URL and document lifecycle state. It no +longer scans body text or visible DOM elements. `browser-vision-screen.ts` no longer uses text or +element counts to override an approved visual action. It asks for a fresh screenshot only when the +URL changed or the document started loading after capture. Pixel evidence, coordinate validation, +credential safety, and actual execution errors remain at their owning boundaries. + +The canvas-only regression passes a committed page through readiness, executes the exact approved +click through the real `BrowserDriver`, and proves that the readiness probe contains no +`querySelectorAll` or `innerText` rule. Four focused files and 58 tests pass. Scoped lint for the +changed browser files has zero errors, the node typecheck passes, and the diff check passes. + +### CU-012 (P0) - Live guidance can sync private text without redaction + +**Resolved (2026-08-25):** exact guidance and attachment content now stay in the active task's +memory-only queue. Task history and Personal Mesh receive only safe `GUIDANCE ACCEPTED` and +`GUIDANCE APPLIED` lifecycle markers. The task-history write boundary also replaces legacy +`USER GUIDANCE` rows, and startup migration removes old private values from SQLite. Focused unit, +real SQLite migration, Web Use, Vision, AX, and renderer tests prove the original value is absent. + +### CU-005 (P1) - The Tasks surface had no maintainable component boundary - RESOLVED 2026-08-25 + +`WatchedBrowserPane.tsx` is now a 240-line composition owner. History, selected-record detail, live +task state, execution plans, guidance, browser chrome, controls, layout, and selection each have a +focused component or hook under `components/browser/tasks`. The largest file in that surface is 286 +lines. Scoped ESLint reports no issue in the task-workspace files. The 70-test focused journey, +typecheck, production build, and real Electron harness passed after the split. + +### CU-011 (P1) - Keyboard resizing was announced but did not resize - RESOLVED 2026-08-25 + +Both task separators now support bounded Arrow Left and Arrow Right changes, announce current values, +and keep their device-local layout. The real Electron harness focuses `Resize Chat and task`, sends +eight Arrow Left events, and asserts a visible width change. The production run changed the Chat width +from 651.56 px to 400.96 px and captured `08-task-keyboard-resized.png`. Focused renderer tests also +cover both directions and bounds. + +### CU-006 (P0) - Vision typing had no structural credential boundary - RESOLVED 2026-08-25 + +The screenshot-only vision executor now checks the focused macOS Accessibility element before any +`typeText` call. The native helper reports only `safe`, `secure`, or `unknown`; it never reads or +returns the field value. A secure target, or content identified from password, PIN, OTP, token, API +key, payment-card, or credential goal context, stops before actuation and asks: `Enter the private +value, then resume`. The handoff does not include the value. Normal text remains available when the +focused editable element is verified safe. + +The handoff observation clears the parsed action and does not write the typed value to progress, +task details, SQLite, or sync. The state-sync outbound projector now applies the same step-detail +sanitizer again before encryption. Focused actuation and agent integration tests prove that private +typing never reaches the actuator or durable task projection, while verified ordinary typing still +works. The real SQLite typed-secret test and state-sync wire test cover both durable exits. The +shipped macOS helper was rebuilt with the focused-element inspector. + +### CU-001 (P1) - Web Use history collapsed runs by Chat - RESOLVED 2026-08-25 + +The task projection now keeps one history row per `taskId` while `journeyId` owns the shared browser +workspace. The rendered same-Chat journey test proves both runs remain present and selectable. + +### CU-003 (P1) - Esc availability copy contradicted the host - RESOLVED 2026-08-25 + +The host notice now owns Esc availability on both the floating supervisor and docked task surface. +When registration fails, both surfaces say to use the visible task controls and do not claim Esc +works. Rendered registration-success and registration-failure tests passed in the focused run. + +### CU-007 (P1) - Remote synced tasks exposed dead local controls - RESOLVED 2026-08-25 + +The dock now derives local Computer Use ownership from the live vision task and local Web Use +ownership from its browser session. Remote rows identify the execution device and expose no local +Stop, Pause, or Take Over buttons. Local control failures produce a visible alert. The remote/local +renderer contracts and the 15-test real state-sync suite passed. + +### CU-008 (P1) - Task record, live task, and Chat context were ambiguous - RESOLVED 2026-08-25 + +The two panes now identify `Task record` and `Live task` explicitly, so inspecting historical +evidence cannot be mistaken for controlling that run. Task details include `Return to originating +Chat` from `journeyId`. The renderer contract proves the navigation intent, and the real Electron +harness proves Tasks and the native browser region hide outside Chat and restore when Chat returns. + +### CU-010 (P1) - Structured and legacy traces rendered twice - RESOLVED 2026-08-25 + +Structured Computer Use details now replace the legacy step list. Legacy text renders only when no +structured detail exists. The rendered acceptance case proves the legacy duplicate is absent while +the safe decision, model evidence, mapped action, result, and return-to-Chat action remain available. + +### CU-002 (P1) - Hidden model reasoning reached task traces - RESOLVED 2026-08-25 + +The task-detail sanitizer now removes tagged reasoning and UI-TARS `Thought:` prefaces from both +model output and persisted model input. The focused sanitizer tests include both forms and passed on +2026-08-25. User-visible decision summaries remain separate from the hidden model reasoning. + ### DEF-007 (P0) - Secure notes are masked until deliberate reveal/copy - CLOSED 2026-08-09 The Vault service now treats a Secure Note body as secret data: list, add, and update return only its @@ -681,7 +854,7 @@ two specs that should run. Neither needed a model. They had stale selectors that the `Capture & processing` section that contains the residency controls; 2 of its 4 switch names were also stale (`Chat model residency` vs the rendered `Chat and capture model residency`). - `meeting-transcription` matched an onboarding CTA of `Start using Off Grid AI Desktop` while the - button renders `Start using Off Grid`, so it never left onboarding, then looked for `Meetings` + button renders `Start using Off Grid AI`, so it never left onboarding, then looked for `Meetings` exact where a locked-Pro item computes `Meetings Pro`. - Both Settings tests in `tour.spec.ts` clicked a second section while the first was open — sections are single-open, so the target was exit-animating out ("element detached"). @@ -1286,3 +1459,84 @@ Follow up with an isolated physical-device benchmark on a strong 5 GHz or 6 GHz desktop-to-phone transfer at a time, separate checksum preparation from wire time, and compare both directions. Also persist verified file checksums so a repeat send does not hash the same multi-GB model again after an app restart. Keep the 4 MiB authenticated frame format and bounded memory. + +--- + +## Desktop voice modes need the final physical audio pass + +**Status:** automation-backed; manual device verification is open. Filed 2026-08-24. + +The rendered Chat journey now proves Manual start/stop and cancellation, Auto end-on-silence, +Hands-free speech detection, the generation and playback lock, the two-second speaker-drain wait, +automatic rearming, pause, and the transition back to text mode. The global dictation reducer also +proves Hold, Toggle, and Both, including auto-repeat protection. The focused suites pass 41 tests. + +The remaining boundary is the installed macOS app with real hardware and models. On the release Mac: + +1. Select a Whisper model and a Parakeet model in turn. Confirm Chat reports and uses the selected + transcription model without changing the chat model. +2. Speak one turn in Manual, Auto, and Hands-free. Confirm Auto does not cut off a normal pause and + Hands-free does not record its own Kokoro reply. +3. Interrupt and pause Hands-free, switch to text during an active recording, deny and restore + microphone permission, and cancel during transcription. Confirm the mic indicator and audio output + stop and no discarded transcript appears. +4. In TextEdit and one other app, verify the Pro Voice Hold, Toggle, and Both gestures and confirm one + transcript is pasted for each completed turn. + +Close this gap only with the exact packaged build, an audible reply, the real macOS microphone +indicator, and a saved diagnostic excerpt that identifies the active speech-to-text model. + +--- + +## Personal Mesh visibility and Google OAuth need installed passes + +**Status:** automation-backed; manual macOS and provider verification is open. Filed 2026-08-24. + +The release tests prove the Personal Mesh lifecycle through the Shared contract, the Electron bridge, +and the macOS helper. They also prove that a failed advertising stop keeps the last true state and +that a later stop can retry. Complete rows PR-14 through PR-16 in +`docs/RELEASE_READINESS_CHECKLIST_0.0.40.csv` on the exact +release Mac with a second physical device. Confirm Hidden at cold launch, separate Discoverable and +Find nearby controls, an active encrypted session during visibility changes, a private IP or machine +name endpoint, one custom Sync port on every device, and failed-stop recovery through a diagnostic +helper. + +The Google connector UI and credential paths have automated coverage, but the real provider boundary +still needs one installed pass. Complete row PR-13 with a real test account and a Web application OAuth +client. Confirm both APIs are enabled, the account has consent or test-user approval, the exact local +callback completes, Gmail and Google Calendar connect, and both connection tests succeed. Relaunch, +reconnect, and confirm that the protected credentials still work. + +Close this gap only with the device names, OS versions, exact build commits, Google project test +status, redacted provider evidence, and completed checklist rows. Do not put client secrets, tokens, +mail, calendar records, or other private data in release evidence. + +--- + +## Remote task approval and live-frame sync need lifecycle closure + +**Status:** code resolved 2026-08-28; physical cross-device verification pending. + +The Release 107 implementation now has one Desktop-owned task state machine for Web Use and Computer +Use. Commits `4fe7f3a4`, `78658e16`, `f2a13c2`, `aefe8fe`, `922eb382`, `62c350a`, and `d69e3ff4` +provide the following closure: + +1. `takeover` is canonical at the shared and Desktop runtime ports. Pause, Resume, Stop, and Take Over + are covered for both task kinds. +2. Chat tasks start directly. Task approvals no longer enter the general synced ActionApproval path. + Desktop Actions keep their Desktop-owned approval, then create one execution chat. +3. Each control intent has a correlation id. The Desktop runtime publishes the authoritative applied + or rejected result. Subscriber UIs do not infer state transitions. +4. Live-frame persistence keeps one current winner instead of appending every frame. Terminal tasks + evict their cached frame. +5. An inbound task must match authenticated execution-device provenance before it is rendered or + controlled. +6. Each paired Mobile has a persisted, default-on remote-task permission on this Desktop. An explicit + off value rejects execution. Both task schemas expose the optional `execution_device` routing field, + and the Desktop strips that field before the task extension runs. + +Local evidence: Desktop and Desktop Pro typecheck pass; permission and routing tests pass 107/107; +the public MCP client/server integration passes 3/3; remote-task DB journeys pass 9/9; the production +Electron build passes. Keep this entry open only for the final two-Desktop physical journey on the +release build: default routing, exact named routing, permission off, live frame, all four controls, +offline recovery, and terminal cleanup. diff --git a/docs/GATEWAY_SPINE.md b/docs/GATEWAY_SPINE.md index 4f7787dd..4c8c557e 100644 --- a/docs/GATEWAY_SPINE.md +++ b/docs/GATEWAY_SPINE.md @@ -1,4 +1,4 @@ -# Off Grid Gateway — the spine +# Off Grid AI Gateway — the spine How the 5-layer agentic stack (`wednesdayai/knowledge-base/architecture.md`) maps onto the local-first desktop gateway at `127.0.0.1:7878` (`src/main/model-server.ts`), and the @@ -91,7 +91,7 @@ of the multi-tenant/regulatory machinery **collapses** — but a surprising amou **load-bearing**, often in a new guise. The privacy stakes are _higher_, not lower: capture sees everything on screen. -| BFSI layer | In Off Grid | Verdict | Where it lives | +| BFSI layer | In Off Grid AI | Verdict | Where it lives | | ---------------------------------------- | ----------------------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | | **A · Data plane** (CDC, lake, PII mask) | capture → OCR → entities → SQLite | Exists | `watcher.ts`, `vision.ts`, `database.ts`, `crm/*` — gateway _consumes_, doesn't own | | **A · PII masking** | redact before anything leaves the device | **Survives, critical** | post-hook egress DLP (see C16) | diff --git a/docs/MANUAL_RELEASE_TESTS_0.0.40.md b/docs/MANUAL_RELEASE_TESTS_0.0.40.md index b54b97a0..f820e60e 100644 --- a/docs/MANUAL_RELEASE_TESTS_0.0.40.md +++ b/docs/MANUAL_RELEASE_TESTS_0.0.40.md @@ -11,7 +11,7 @@ device pass. For a shareable execution sheet, import [`RELEASE_READINESS_CHECKLIST_0.0.40.csv`](RELEASE_READINESS_CHECKLIST_0.0.40.csv) into Google -Sheets. It contains 210 Core and Pro journeys: the canonical 155 product journeys plus 55 +Sheets. It contains 214 Core and Pro journeys: the canonical 155 product journeys plus 59 manually-audited native, security, release, and omitted-surface cases. Every row includes exact steps and expected results, strict automation status and evidence, remaining manual boundaries, calibrated regression confidence, and blank tester/result/evidence/defect columns for the release @@ -113,7 +113,7 @@ Stop and mark the build failed if any of these occurs: `/usr/bin/xcrun stapler validate `, and `/usr/sbin/spctl --assess --type execute --verbose=4 `. - Confirm version `0.0.40`, bundle identifier `co.getoffgridai.desktop.pro`, hardened runtime, - Off Grid Apple Team ID `84V6KCAC49`, and both ASAR fuses: + Off Grid AI Apple Team ID `84V6KCAC49`, and both ASAR fuses: `EnableEmbeddedAsarIntegrityValidation` and `OnlyLoadAppFromAsar`. - Confirm the only ASAR roots are `/node_modules`, `/out`, and `/package.json`; the only `/out` children are `main`, `preload`, and `renderer`; and there is no nested `.app` or case-normalized diff --git a/docs/MARKETING_ANGLES.md b/docs/MARKETING_ANGLES.md index 07a16fac..679af680 100644 --- a/docs/MARKETING_ANGLES.md +++ b/docs/MARKETING_ANGLES.md @@ -21,37 +21,37 @@ Voice rules (from `mobile/docs/brand_tone_voice.md`): proof-first, privacy state ## Angles **1. The amnesia angle** -You don't remember what you did last Tuesday. Your laptop does. Off Grid is a quiet record of your work that you can actually ask. +You don't remember what you did last Tuesday. Your laptop does. Off Grid AI is a quiet record of your work that you can actually ask. **2. Private by architecture, not by policy** -Most "private" AI still uploads your screen to someone's cloud. Off Grid runs the model in your laptop's own memory. Nothing is sent anywhere because there is no server to send it to. +Most "private" AI still uploads your screen to someone's cloud. Off Grid AI runs the model in your laptop's own memory. Nothing is sent anywhere because there is no server to send it to. **3. Background, not another tab** -Every AI tool waits for you to open it and explain yourself. Off Grid already watched the meeting, read the thread, and saw the ticket. You don't brief it. It briefs you. +Every AI tool waits for you to open it and explain yourself. Off Grid AI already watched the meeting, read the thread, and saw the ticket. You don't brief it. It briefs you. **4. Where your attention actually went** -Not hours logged. Mind share. Off Grid shows what you spent the day thinking about, how many times you context-switched, and how little uninterrupted focus you actually got. +Not hours logged. Mind share. Off Grid AI shows what you spent the day thinking about, how many times you context-switched, and how little uninterrupted focus you actually got. **5. A movie of your day** Scrub through your whole day like a recording. Every screen, in order, on device. The thing you swear you saw three hours ago is right there. **6. The off-grid line vs the cloud line** -Littlebird stores your captured work on AWS. Off Grid stores it on your disk. Same idea, opposite trust model. Yours never leaves the building. +Littlebird stores your captured work on AWS. Off Grid AI stores it on your disk. Same idea, opposite trust model. Yours never leaves the building. **7. Read the code** -The memory apps ask you to trust them. Off Grid is open source under AGPL. You can read exactly what leaves your device. The answer is nothing, and you can check. +The memory apps ask you to trust them. Off Grid AI is open source under AGPL. You can read exactly what leaves your device. The answer is nothing, and you can check. **8. The exocortex that fills itself** A second brain only works if you feed it. This one feeds itself. Capture, meetings, email, Notion, Linear, Jira, all flowing into one local memory without you copy-pasting anything. **9. Source of truth, not screenshots** -When you open a Notion page or a Linear issue, Off Grid pulls the real thing from the source, on device, instead of guessing from pixels. It knows what you care about from what you look at, then gets the accurate version. +When you open a Notion page or a Linear issue, Off Grid AI pulls the real thing from the source, on device, instead of guessing from pixels. It knows what you care about from what you look at, then gets the accurate version. **10. Your to-dos, found not written** -You wrote "we need to ship X by Friday" in an email. Off Grid noticed and put it on a list. It detects the commitments you make and the asks aimed at you, across everything you touch. +You wrote "we need to ship X by Friday" in an email. Off Grid AI noticed and put it on a list. It detects the commitments you make and the asks aimed at you, across everything you touch. **11. It acts, you approve** -Off Grid can draft the reply, file the ticket, update the doc. It never does it on its own. Every action is a proposal you approve, and every approval is logged. +Off Grid AI can draft the reply, file the ticket, update the doc. It never does it on its own. Every action is a proposal you approve, and every approval is logged. **12. For people who build** Tuned for software, design, and product. It speaks issues, PRs, cycles, and tickets, not generic "tasks". Linear, Jira, GitHub, Sentry, Vercel connect in one click. @@ -69,7 +69,7 @@ No training on your data. No selling it. No "we value your privacy" page. The sy Your laptop knows your work. Your phone knows your life. They sync over your own network while you sleep, never through a cloud relay, into one model of your day. **17. The relationship graph builds itself** -Off Grid quietly maps the people, projects, and topics you work with, and what's open with each. A CRM for everything, that you never have to update. +Off Grid AI quietly maps the people, projects, and topics you work with, and what's open with each. A CRM for everything, that you never have to update. **18. Time tracking with nothing to track** You don't start a timer. You don't tag anything. At the end of the day the breakdown is just there: 4h on the rewrite, 50m on Slack, 30m you'll wish you had back. @@ -84,7 +84,7 @@ You pick up your device and the day is already laid out. The 9am is with someone It only sees what you let it see, per device, with a visible recording indicator. Pause it anytime from the menu bar. Off-grid means in your control, not just off the cloud. **22. Intelligence for everyone, not a few** -A personal chief of staff used to be a privilege. The compute to run one now sits in every laptop. Off Grid is that layer, private and in your hands, not rented from a company that reads your mail. +A personal chief of staff used to be a privilege. The compute to run one now sits in every laptop. Off Grid AI is that layer, private and in your hands, not rented from a company that reads your mail. --- @@ -92,9 +92,9 @@ A personal chief of staff used to be a privilege. The compute to run one now sit Three products, one brand, one loop. They started as three problems Mac solved for himself, and they're becoming one thing. -- **Off Grid Mobile** - on-device intelligence in your pocket: chat, image, vision, voice, documents, all local. Becomes an opt-in offline recorder for your life. An AI meeting recorder, but for everything, processed on device, never shared. Never forget anything again. -- **Off Grid Desktop** - what My Memories becomes: the intelligence layer for your laptop. Captures your day there - meetings, email, the work context that makes the rest useful. -- **Off Grid Sync** - what Easy Share becomes: the private backbone that moves all of it between your devices over your own network, no third party in the middle. +- **Off Grid AI Mobile** - on-device intelligence in your pocket: chat, image, vision, voice, documents, all local. Becomes an opt-in offline recorder for your life. An AI meeting recorder, but for everything, processed on device, never shared. Never forget anything again. +- **Off Grid AI Desktop** - what My Memories becomes: the intelligence layer for your laptop. Captures your day there - meetings, email, the work context that makes the rest useful. +- **Off Grid AI Sync** - what Easy Share becomes: the private backbone that moves all of it between your devices over your own network, no third party in the middle. Together they close the loop between your physical and digital life, and make an assistant that knows everything about you possible without giving everything away. @@ -105,19 +105,19 @@ Origin (the honest version): **My Memories** came first - a macOS app that watch ## More angles (from the vision) **23. The price of useful AI** -The most useful AI is the one that knows everything about you. Today, getting that means giving everything away. Off Grid is the version where you don't have to. +The most useful AI is the one that knows everything about you. Today, getting that means giving everything away. Off Grid AI is the version where you don't have to. **24. No lock-in for your own thinking** Your conversations across ChatGPT, Claude, and Gemini end up in one place that is yours. Switch tools whenever you want. Your memory stays. **25. A recorder for your life, not just your meetings** -AI meeting recorders capture the call. Off Grid Mobile captures the day, opt-in, on device, nothing shared. The thing your partner said about dinner is a calendar event, not a forgotten text. +AI meeting recorders capture the call. Off Grid AI Mobile captures the day, opt-in, on device, nothing shared. The thing your partner said about dinner is a calendar event, not a forgotten text. **26. Close the loop** -Your phone holds your life. Your laptop holds your work. Off Grid joins them into one model of you, synced over your own network, so the assistant finally has the full picture. +Your phone holds your life. Your laptop holds your work. Off Grid AI joins them into one model of you, synced over your own network, so the assistant finally has the full picture. **27. Structured, not hand-the-keys** -This is not an open-ended agent you give the keys to and hope. Even the local open-source ones work that way. Off Grid is structured, with guardrails you set, so you stay in control. That structure is what makes the upside real instead of reckless. +This is not an open-ended agent you give the keys to and hope. Even the local open-source ones work that way. Off Grid AI is structured, with guardrails you set, so you stay in control. That structure is what makes the upside real instead of reckless. **28. The deal, plainly** The core stays open. The integration that ties the three products together is open source and free. Some advanced, opt-in features sit in Pro. Pro keeps the lights on so the core keeps getting better. diff --git a/docs/MASTER_PLAN.md b/docs/MASTER_PLAN.md index d4963f1b..d89ecfc7 100644 --- a/docs/MASTER_PLAN.md +++ b/docs/MASTER_PLAN.md @@ -1,4 +1,4 @@ -# Off Grid — Master Plan +# Off Grid AI — Master Plan The single forward plan for the enterprise product. Ties together `CONSOLE_PLAN.md`, `ENTERPRISE_BUILD_PLAN.md`, `LICENSES.md` (in `off-grid-ai/console`), and everything decided @@ -10,7 +10,7 @@ so far. Read this first; the others hold detail. A **common control plane for organizational AI**: one governed chokepoint for every model call, agent action, and byte of data — running on the org's own infrastructure, fully -auditable, built on open source. It manages a **fleet of on-device nodes** (Off Grid +auditable, built on open source. It manages a **fleet of on-device nodes** (Off Grid AI Desktop/Mobile), grounds them in the **organizational brain**, and proves compliance to a regulator. @@ -178,9 +178,9 @@ f. **FinOps + token issuance**: virtual keys scoped to user/project, budgets, co Single interface (ours) · gateway is the chokepoint · findings normalized onto the audit record · OTel as the one observability wire · permissive-only shipped · out-of-process aggregation · worker-owns-raw / org-sees-distilled · embed don't rebuild rich UIs · no tool's -copyleft ever linked into a closed module · **from the Off Grid ecosystem the console reuses only +copyleft ever linked into a closed module · **from the Off Grid AI ecosystem the console reuses only two things — the UI (design system) and the single Off Grid AI Gateway; it pulls in NO other -Off Grid packages (no `@offgrid/rag`, no desktop/mobile code) and stands on its own stack.** · +Off Grid AI packages (no `@offgrid/rag`, no desktop/mobile code) and stands on its own stack.** · **all model/inference (embeddings, grounding/NLI, multimodal) routes through that one gateway — never a third-party LLM; everything else is the console's own third-party OSS behind ports.** · **every capability is sellable alone (grounding without the Brain, the Brain without agents, …); diff --git a/docs/P0_P2_INTEGRATION_COVERAGE.md b/docs/P0_P2_INTEGRATION_COVERAGE.md index bd178b80..4c56eee1 100644 --- a/docs/P0_P2_INTEGRATION_COVERAGE.md +++ b/docs/P0_P2_INTEGRATION_COVERAGE.md @@ -10,7 +10,7 @@ manual claim does not count as complete integration coverage. - Counting rule: - Complete means the decisive production collaborators run through the real application seam. - A packaged or native-sensitive behavior needs an exact packaged/native proof. - - A test that replaces Off Grid code, preload/IPC ownership, persistence, or the decisive runtime + - A test that replaces Off Grid AI code, preload/IPC ownership, persistence, or the decisive runtime is partial. Partial does not count as done. - A fake at a genuinely uncontrollable remote or OS boundary may support a seam test, but it does not by itself prove the complete release journey. @@ -128,7 +128,7 @@ manual claim does not count as complete integration coverage. - The journey closes and reopens its disposable SQLite profile, reloads all modules, consumes all five modalities again without another remote request, and verifies their persisted selections. - Only HTTP delivery and the native llama.cpp, stable-diffusion.cpp, whisper.cpp, and Kokoro - processes are controlled boundaries. No Off Grid service, store, router, or runtime owner is + processes are controlled boundaries. No Off Grid AI service, store, router, or runtime owner is mocked. - Complete - active-service privacy deletion with no late-write resurrection: - `capture-deletion-race.integration.dbtest.ts` starts the real focus scheduler, native capture @@ -139,7 +139,7 @@ manual claim does not count as complete integration coverage. the admitted tick, erases its late output, restores only its own pause, and stays empty after profile relaunch. - Electron/TCC, focused-window discovery, LanceDB, and llama-server are the only controlled - boundaries. No Off Grid module is mocked. + boundaries. No Off Grid AI module is mocked. - Complete - concurrent workload ownership, cancellation, crash recovery, and clean relaunch: - `concurrent-workload-recovery.integration.dbtest.ts` starts four production catalog downloads against controlled HTTP streams: three active transfers and one queued transfer. @@ -155,7 +155,7 @@ manual claim does not count as complete integration coverage. dropped with actionable production health state and no additional native work. - Restoring only the TCC boundary lets the same interval and real SQLite `SettingsStore` capture again without restarting the app or scheduler. - - Electron/TCC and `get-windows` are the only controlled boundaries. No Off Grid module is mocked. + - Electron/TCC and `get-windows` are the only controlled boundaries. No Off Grid AI module is mocked. - Complete - atomic cross-store recovery from disk and database failures: - `meeting-atomic-recovery.integration.dbtest.ts` exercises production meeting persistence and recovery over real SQLite, files, CRM observations/actions, and the local-model HTTP client. @@ -163,7 +163,7 @@ manual claim does not count as complete integration coverage. followed by SQLite insert failure remains recoverable, is adopted and enriched exactly once on relaunch, and stays idempotent across a second relaunch. - Electron, native media executables, the local model process, and the injected filesystem - capacity failure are the only controlled boundaries. No Off Grid service is mocked. + capacity failure are the only controlled boundaries. No Off Grid AI service is mocked. - Verification: - All five commissioned journey cases passed in focused runs. - Download queue and application-shutdown checks: 9 tests passed. @@ -326,7 +326,7 @@ historical labels, not strict completion claims. Use the strict snapshot above f - #32 - First local message replies. `MemoryChat.chat-lifecycle.test.tsx` sends through the real rendered composer, routes a streamed token through production ownership, resolves the local-model boundary, and proves one assistant bubble with the exact answer is persisted once. - `workspace-production-bridge.ui.integration.dbtest.tsx` removes the substituted Off Grid preload + `workspace-production-bridge.ui.integration.dbtest.tsx` removes the substituted Off Grid AI preload API: the rendered composer uses the production preload mapping and IPC registrations, streams from only a controlled native-model socket, and persists the exact visible user and assistant turns in real SQLite. @@ -488,7 +488,7 @@ historical labels, not strict completion claims. Use the strict snapshot above f - #132 - Settings survive relaunch. Existing journeys 129 and 131 cover model residency and resource settings across relaunch. Core and Pro `settings-persistence.dbtest.ts` tests add the other owning stores: they change software-update, capture-privacy, identity, and proactive - delivery settings over real encrypted SQLite, close the database, reload every Off Grid module, + delivery settings over real encrypted SQLite, close the database, reload every Off Grid AI module, rehydrate each owner, and verify every value restores. - #134 - Clear cache preserves user data. `cache-cleanup.integration.test.ts` and the rendered Storage journey exercise the production control through IPC and prove its allowlist can reach @@ -767,7 +767,7 @@ historical labels, not strict completion claims. Use the strict snapshot above f Checking, preserve the stable channel, and never call install without approval. - #143 - Update channel persists. `src/main/__tests__/settings-persistence.dbtest.ts` changes the channel through the production update IPC handler, closes the encrypted database, reloads every - Off Grid module, and verifies the fresh update-preferences handler restores the beta channel. + Off Grid AI module, and verifies the fresh update-preferences handler restores the beta channel. - #146 - Model ports are single-owner. `model-port-ownership.integration.test.ts` starts a real foreign parent with the only fake native llama process on production port 8439, then proves the production contender preserves that live owner, starts no second engine, reports its own Chat diff --git a/docs/PORTING_MAP.md b/docs/PORTING_MAP.md new file mode 100644 index 00000000..af69b5da --- /dev/null +++ b/docs/PORTING_MAP.md @@ -0,0 +1,182 @@ +# Porting map - what we port vs what we build (deep prior-art research) + +**Status:** August 13, 2026. Answering the lead: "people must have already built stuff like this - what can we port instead of building?" This is the deep sweep across every layer of the assistant, with a blunt verdict per component. Companion to `COMPUTER_USE.md` (Section 9 is the curated shortlist), `ASSISTANT_ARCHITECTURE.md`, and `COMPUTER_USE_PLAN.md`. + +## The answer in one paragraph + +The lead is right, and the honest split matters: **we port the plumbing and keep the product.** Almost every mechanism we need exists as a permissively licensed open project - a durable-queue pattern, a state machine, constrained decoding, a vector store, a record-and-replay engine, the rails, the grounding models. What does NOT exist off the shelf is the thing that makes this product: a single-process, offline, on-device pipeline that joins actions to a personal screen-memory, gates them behind human approval, and verifies their effect. So the plan is: **assemble the pipeline from small permissive libraries + documented blueprints, and write only the product-defining glue** (the Action contract, the approval policy, the resolve-with-confidence layer, the commitment-gap reasoner, effect-verification, and the DeviceController + rail-selection). That glue is bespoke by nature - no upstream targets a single-process offline device wired to a personal memory - not by choice. + +**Verdict legend:** `port-wholesale` (adopt/vendor the code), `port-components` (lift specific modules/algorithms), `port-design` (reimplement its architecture), `inspiration-only` (study, don't copy), `adopt-as-model` (ship the weights), `bespoke` (must build - explained why). + +**Method:** five parallel research tracks (durable execution/HITL, agent brain/tool-calling, memory/resolve/proactive, record-replay/routines, rails/grounding-models), licenses verified per project. + +--- + +## 1. The durable action queue + state machine + scheduling + approval gate + +The lead's exact example. Finding: **durable execution is heavily built - but every mature engine is a server backed by Postgres/Cassandra/Kafka**, which is a non-starter for a single-process offline app (the same "bundled sidecar" fragility as the llama-server saga). No embeddable engine bundles queue + state machine + approval + verify. So we assemble it. + +| Need | Port from | License | Local-first fit | Verdict | What we take | +| --- | --- | --- | --- | --- | --- | +| Action state machine | **XState v5** | MIT | Yes, zero-dep, runs on RN too | **port-wholesale** | Each Action's lifecycle as a persisted statechart; `getPersistedSnapshot()` -> SQLite, rehydrate on launch; same machine on future mobile core | +| (lighter alt) | robot3 | BSD-2 | Yes, 3kb | port-components | If XState feels heavy; you write the serialize/restore glue | +| Durable SQLite queue | **sqliteq** (TS port of **goqite**) | MIT | Yes, better-sqlite3 | **port-wholesale** (transport) | SQS-style leased-message + visibility-timeout + auto-extend + retry loop | +| Scheduling (cron/delay) | plainjob | MIT | Yes, better-sqlite3 | port-components | Cron + delayed jobs; worker-death -> re-queue | +| Idempotent enqueue | better-queue-sqlite | MIT | Yes | port-components | Task-merge/dedup by id | +| Retry-once / resilience | **cockatiel** or **p-retry** | MIT | Yes, zero-dep | **port-wholesale** | Retry-once is a one-line policy; circuit-breaker/timeout free for connector calls | +| Approval gate (HITL) | **LangGraph.js** `interrupt -> resume` | MIT | Yes (checkpoint-sqlite) | port-components | The pause-before-side-effect -> surface proposed Action -> resume-from-checkpoint contract; do it locally against our own UI | +| HITL outcome model | HumanLayer | Apache-2.0 | No (cloud broker) | inspiration-only | The typed approve/deny/respond contract; de-couple request from response | +| Design spec | **DBOS Transact** semantics + **Gunnar Morling's "durable execution on SQLite"** blueprint | MIT / blog | reference | port-design | `(action_id, step)` PK, status per step, replay COMPLETE steps, idempotency key forwarded to side effects; Morling's PoC is near copy-paste | + +**Not viable for local-first (server + external DB, or license):** Temporal (MIT, needs Cassandra/Postgres), DBOS-TS (MIT, Postgres-bound - Go build has SQLite, TS not yet), OpenWorkflow (Apache-2.0, in-process TS step-checkpointing - the right shape, but Postgres-only today with SQLite "coming soon", early and fast-moving, and no approval/HITL or risk-aware retry; its step.run ergonomics are a design reference for our engine facade), Restate (**BUSL-1.1** runtime), Inngest (**SSPL** server), Trigger.dev (Postgres+Redis+Docker), Windmill (**AGPL** + Postgres), LittleHorse (**AGPL** + Kafka), Hatchet/Cadence/Resonate (all server). Their *semantics* are the gift; their deployment model is the disqualifier. **Re-evaluate list:** DBOS-TS and OpenWorkflow, if either ships a solid SQLite backend. + +**Bespoke (build it, ~a few hundred lines):** +- The orchestration glue that wires queue -> state machine -> approval -> execute -> verify. No library combines all five on-device. +- **Effect-verification** - "did the email actually send / the file actually move" - has **zero prior-art library**; it's inherently per-connector (read-back, re-query). Ours. +- The crash-after-execute-before-record window, closed with idempotency keys on the outbound side effect (every engine concedes this and solves it the same way). +- The checkpointer/queue adapter against our existing better-sqlite3 handle (SSOT: one DB answers "what is this Action's state"). + +--- + +## 2. The agent brain: constrained output, reliability, loop, router, tools + +Finding: **we already ship the best-fit constrained-decoding engine.** llama.cpp does JSON-schema -> GBNF and `response_format` grammar-constraining today. Most of this track is thin layers on top, in TypeScript. + +| Need | Port from | License | Local fit | Verdict | What we take | +| --- | --- | --- | --- | --- | --- | +| Constrain Action shape | **llama.cpp GBNF / `response_format`** (already bundled) | MIT | Yes | **port-wholesale** | Send the Action schema as `json_schema`; the model cannot emit invalid-shaped Action JSON. Gotcha: schema is NOT injected into the prompt - still describe the tool enum in the system prompt | +| Native tool-calling | llama-server `--jinja` lazy grammars | MIT | Yes | port-wholesale | Optional OpenAI-style `tools` mode for models with a good native template; prefer our own single-Action grammar for determinism | +| Weak-model reliability | **Schema-Aligned Parsing (SAP), from BAML** | Apache-2.0 | Yes | **port-components** | The single biggest weak-model jump in the literature (e.g. 19.8% -> 92.4%): coerce sloppy-but-close output to schema post-hoc. Reimplement a focused TS coercer keyed to the Action schema | +| Validate + retry | Instructor-JS pattern | MIT | Yes | port-components | Zod validate -> feed the error back -> re-ask, bounded to N. ~50 lines | +| Wrapper patterns | node-llama-cpp | MIT | Yes | port-components | The ChatWrapper seam (per-model template behind one interface) + optional-param grammar handling | +| Faster CFG engine | llguidance | MIT | Yes (build flag) | reserve | `-DLLAMA_LLGUIDANCE=ON` only if native GBNF coverage/perf bites; adds a build-gate surface, defer | +| Agent loop | LangGraph.js pattern | MIT | Yes | port-components | Checkpointed LLM-node <-> tool-node <-> conditional-edge loop; reimplement, don't take the LangChain dep | +| Router seam | Mastra (Apache core) / VoltAgent (MIT) | Apache/MIT | Yes | port-components | One interface over interchangeable model backends (our DSP rule). VoltAgent is MIT+TS+MCP+Zod - copy concrete code | +| Cheap-first routing | semantic-router concept | MIT (Python) | reimplement | port-components | Embedding-similarity intent classifier as the router's fast lane; skip the LLM when confident. Reimplement in TS over our local-embedding path | +| Connectors / tools | **MCP TypeScript SDK** | MIT/Apache-2.0 | Yes | **port-wholesale** | The whole client/server tool transport; this is our act surface, don't reinvent | + +**Model choice (verify per-checkpoint license before bundling):** function-calling-tuned small models - Salesforce xLAM-2, Hammer 2.1, NousResearch Hermes (native XML tool parser in llama-server), MeetKai functionary - picked on the Berkeley Function-Calling Leaderboard, not vibes. + +**Inspiration-only (Python, or wrong runtime):** Outlines/outlines-core, guidance, LMQL, jsonformer (Python), XGrammar (MLC not llama.cpp), Agent-S (Python, feeds the vision rail). + +**Bespoke:** the Action schema + durable pipeline (the SSOT for "what the agent is doing"); approval-gated execution + the privacy boundary; the router *policy* tuned to our bundled model's real behavior; the SAP coercion rules + retry prompts wired to our Action contract; the engine-health/stderr-classification path (`llama-error.ts` has no upstream equivalent). + +--- + +## 3. Memory + resolve/RAG + commitment/proactive detection + +Finding: the vector layer is a clean port; the "memory frameworks" are mostly Python (algorithm inspiration, not code); commitment/proactive detection is **genuinely bespoke** over our Replay spine. + +| Need | Port from | License | Local fit | Verdict | What we take | +| --- | --- | --- | --- | --- | --- | +| Vector store | **sqlite-vec** | Apache-2.0 / MIT | Yes, inside better-sqlite3 | **port-wholesale** | Vector KNN in the SAME DB file we already ship - one file, one transaction, one backup, no new process. Highest-value, lowest-risk port | +| Scaling alt | LanceDB (`@lancedb/lancedb`) | Apache-2.0 | Yes, embedded Node | port-wholesale (alt) | Real ANN when the corpus outgrows brute-force KNN (a second store to keep in sync - an SSOT tax) | +| Embeddings | bundled **llama-server `/embedding`** first; **Transformers.js** fallback | MIT / Apache-2.0 | Yes | port-wholesale | Reuse the endpoint we ship; Transformers.js (ONNX MiniLM/bge) if we want embeddings off the LLM's critical path | +| Memory-tier skeleton | **LlamaIndex.TS Memory Blocks** | MIT | Yes, TS-native | port-components | Write-time fact-extraction + short-term -> long-term + read-optimized; the only mature MIT TS-native option | +| Consolidation loop | Mem0 (has a TS SDK) | Apache-2.0 | partial (TS) | port-components | The ADD/UPDATE/DELETE dedup-on-write loop so memory doesn't bloat | +| Hybrid ranking | Orama | Apache-2.0 | Yes, TS | port-components | BM25 + vector fusion - lexical recall matters for OCR'd names/filenames/errors | +| Entity dedup (deterministic) | talisman + fuzzball.js | MIT | Yes, TS | port-components | Phonetics, Jaro-Winkler, blocking - the deterministic side of entity resolution | +| Multi-hop resolve (technique) | HippoRAG Personalized PageRank | MIT (Python) | reimplement | inspiration | PPR over the entity graph for "the deck" -> project -> file, instead of flat top-k | +| Memory linking (technique) | A-MEM Zettelkasten | MIT (Python) | reimplement | inspiration | Atomic note + keywords + auto-link + "evolution" rewrite of neighbors | +| Commitment lifecycle (concept) | Zep/Graphiti bi-temporal facts | Apache-2.0 | concept | inspiration | valid-from/valid-to per fact; new facts invalidate old - the backbone the commitment tracker needs | +| Capture triggers (concept) | Screenpipe | **source-available now (flag)** | concept only | inspiration | Event-driven capture (app-switch/click/pause) + accessibility-first, OCR-fallback. Its current tree is off-limits; take the ideas | +| Commitment detection (technique) | Microsoft WSDM 2019 definition | paper (patented method - note IP) | reimplement | inspiration | "sender-obligated + specific + not-yet-complete" as the LLM extraction rubric; commitment language is domain-independent so a small local model generalizes (~0.75 F1 is the bar) | + +**License flags (study only, no code into our permissive pro tier):** Reor, Khoj, OpenRecall (**AGPL**); Screenpipe (**source-available/commercial** post-2026-06); Letta / Zep-platform (server / proprietary). + +**Bespoke:** the RESOLVE layer returning `{value, confidence}` for a slot (no library does retrieval + slot-value + calibrated confidence); the entity-resolution pipeline (assembled from primitives, not adopted); and above all **the commitment-gap reasoner** - detecting the *unmet* commitment by joining it against captured observations and entity timelines has no prior art because it's defined entirely over our data model. + +--- + +## 4. Routines: record-and-replay (programming-by-demonstration) + +Finding: **OpenAdapt is an almost-exact architectural twin of our routines rail** - MIT, local-first, the same loop (record -> compile to anchored self-healing trace -> zero model calls on healthy runs -> local model only to repair drift -> halt instead of guess -> verify against a system of record). It's Python, so this is a **port-design** (reimplement in TS), not a code lift. + +| Need | Port from | License | Verdict | What we take | +| --- | --- | --- | --- | --- | +| Recorder + replay spine | **OpenAdapt / openadapt-flow** | MIT | **port-design** | The compiled-step schema (template crop + OCR label + geometry + structural locator + **postconditions**), the resolution ladder, system-of-record verification (their data: screen-only verify accepted wrong effects 75% of the time -> 12.5% with a system-of-record oracle), halt-on-uncertainty, repair-as-reviewable-diff | +| Self-heal technique | OpenAdapt resolution ladder (+ Healenium DOM tree-similarity, SikuliX OpenCV+Tesseract) | MIT / Apache-2.0 / MIT | port-design | Resolve each step by trying anchors in strict order (structural tree -> local template -> global template -> OCR label -> landmark geometry -> optional local grounding model); healthy runs never leave rung 1; write successful lower-rung resolutions back as a diff | +| Browser recorder | Playwright codegen | Apache-2.0 | port-components | Native TS recorder + its locator-priority heuristic (role -> text/label -> testid -> CSS) as the browser-lane anchor order | +| Browser trace format | Chrome DevTools Recorder `steps[]` | Apache-2.0 | inspiration | Per-step *array of alternative selectors* - a standardized "multiple anchors per step" schema to align to | +| Browser variable slots | browser-use workflow-use | **AGPL (flag)** | inspiration-only | The typed variable-slot idea; do NOT vendor the code, especially into pro | +| Mobile format | Maestro YAML flows | Apache-2.0 | inspiration | Human-readable flow format for the plain-language review surface + resilient text/id/AX matching | +| macOS AX recorder ref | open-record-replay | MIT | inspiration | Clean `events.jsonl` + AX-diff schema, AX-tree-as-primary-anchor | +| Multi-anchor capture | record-and-replay-skill | MIT | port-components | Recording several selectors per action (testId -> role+name -> id -> text -> css) so replay degrades gracefully | + +**Bespoke:** memory-resolved variable slots (every project treats variables as literals or LLM-extracted or manual; binding a slot to a memory query at run time is ours); the plain-language review UI (reuse an existing viewer component, don't fork); the TS-native cross-substrate recorder/runtime (OpenAdapt is Python; we need the ladder across macOS AX, browser CDP, later mobile); the local-only postcondition oracle (verify via our memory/observation layer, not the screen). + +--- + +## 5. The rails (actuation) - desktop + mobile + +Finding: input is a solved permissive dependency; the browser rail is free via Electron's CDP; the accessibility-tree read is a build-our-own napi-rs (Rust) addon with head-starts; the semantic rail is bespoke OS glue. **Convergent insight:** every rail's agent-facing contract is the same - a serialized element list with stable IDs, act-by-ID (browser-use's numeric index = Playwright's `ref` = Agent-S's ACI = the vision model's box). Design ONE DeviceController vocabulary; the vision rail manufactures the same IDs from pixels when no tree exists. + +| Rail | Port from | License | Verdict | What we take | +| --- | --- | --- | --- | --- | +| Desktop spine | **`@ui-tars/sdk`** (UI-TARS-desktop) | Apache-2.0 | port-components | The GUIAgent loop + `Operator` interface + coordinate scaling, in Electron+TS, local-model-ready. **Swap its nut.js operator for `@nut-tree-fork`** | +| Spine design | Agent-S/S2 ACI + a11y/vision fusion; Anthropic computer-use tool-schema + coord-scaling | Apache-2.0 / MIT | port-design | Accessibility-tree + vision fusion (the reliability lever for a weak model); the action vocabulary + normalized-coordinate convention | +| Desktop input | **robotjs** (revived, prebuilds) or **`@nut-tree-fork/nut-js`** | MIT / Apache-2.0 | adopt-as-dependency | Synthetic mouse/keyboard + capture (+ template match on the fork). **Avoid official `@nut-tree/*` - paid EULA** | +| Input (longevity) | enigo via napi-rs | MIT | port-components | Self-owned Rust input layer if we build our own addon | +| Desktop a11y read | **napi-rs addon over `axuielement` (macOS) + `uiautomation` (Windows) crates**; **Terminator** (Windows) + MacosUseSDK head-starts | MIT / Apache-2.0 | port-components / bespoke | No pure-Node lib reads both trees; FlaUI (.NET) / pywinauto (Python) are API references only | +| Browser rail | **nanobrowser** `dom/` module + overlay (starting code) + **browser-use** CDP snapshot/AX-merge/numeric-index (algorithm) + **Stagehand** act/observe/extract + Zod (API) | Apache-2.0 / MIT / MIT | port-components | All over Electron `webContents.debugger` (raw CDP - no Playwright dependency needed) | +| Mobile substrate | **Appium via WebdriverIO** | Apache-2.0 / MIT | adopt-as-dependency | One W3C protocol over iOS (WDA/XCTest) + Android (UiAutomator2), TS client, local, model-independent | +| Android host-free | DroidRun AccessibilityService "Portal" | MIT | port-components | On-device a11y-tree read + gesture dispatch with no host attached | +| Mobile seam | minitap/mobile-use | Apache-2.0 | port-components | Provider-agnostic model layer + multi-transport (ADB/idb/Appium) behind one interface | + +**Not viable:** official nut.js (paid EULA), Open Interpreter OS mode (AGPL + abandoned), Skyvern (AGPL, browser-only), Sonic (AGPL), c/ua (Python + VM-first). + +**Bespoke:** the semantic rail entirely (AppleScript/JXA, App Intents/Shortcuts, Microsoft Graph, deep links, Android intents - OS SDK glue behind the interface); the unified **DeviceController + rail-selection/fallback policy** (semantic -> browser -> accessibility -> vision - no prior art has all four behind one interface); the macOS-AX + Windows-UIA napi addon; **iOS on-device actuation** (genuinely needs a Mac-signed WDA/XCTest helper reached over USB - a permanent Apple constraint, plan the product around it). + +--- + +## 6. Grounding vision models (the vision rail's model) + +Finding: llama.cpp multimodal is real but base-gated (Qwen2-VL / Qwen2.5-VL / Qwen3-VL / InternVL / SmolVLM / Gemma 3 / Pixtral). A grounder is GGUF-runnable iff its base is one of these AND someone converted it. + +| Use | Model | Weights license | GGUF today? | Verdict | +| --- | --- | --- | --- | --- | +| **Desktop default** | **UI-TARS-1.5-7B** (Qwen2.5-VL base) | **Apache-2.0** | **Yes, published + mainline** | **adopt-as-model** - the only turnkey pick, no conversion work; ScreenSpot-V2 ~94% | +| Desktop 2nd | Holo1.5-7B (Qwen2.5-VL) | Apache-2.0 (7B only) | convertible | adopt-with-conversion - strong on ScreenSpot-Pro; avoid the 72B (research license) | +| **Mobile best** | **GUI-Owl-1.5-8B/4B** (Qwen3-VL) | **MIT** | needs one-time conversion | adopt-with-conversion - best open mobile grounding, multi-platform; OSWorld-Verified 52.3, AndroidWorld 69.0 | +| Mobile zero-conversion | Qwen3-VL-8B-Instruct | Apache-2.0 | Yes, official GGUF | adopt-as-model - ship day one, prompt/finetune for grounding; also the natural finetune target if we train our own | +| Pure-vision fallback (set-of-marks for a non-grounding LLM) | OmniParser **v3** detector (YOLOv9) + Florence-2 captioner | **MIT** (v3) | ONNX (not llama.cpp) | adopt-components - lets our bundled gemma click via labeled boxes. **Avoid v1/v2 icon_detect (AGPL YOLOv8)** | + +**Avoid (license or no GGUF path):** Qwen2.5-VL 3B/72B (research), Holo 72B (research), CogAgent (GLM-4V, non-commercial, no GGUF), Ferret-UI (Apple, non-commercial), SeeClick (Qwen-VL research), Aria-UI (custom MoE, no GGUF), OS-Atlas-4B (InternVL2 base, no safe GGUF), the closed UI-TARS-1.5 flagship. + +**Bespoke:** the GGUF + mmproj conversion + a ScreenSpot re-eval after quantization for any grounder beyond the turnkey UI-TARS-1.5-7B / Qwen3-VL (routine, but ours to own). + +--- + +## 7. The whole system, at a glance + +**Port these (the plumbing):** + +- Queue/state: **XState** + **sqliteq/goqite** + **plainjob** + **cockatiel** + **LangGraph interrupt contract**, spec'd from **DBOS + Morling**. +- Brain: **llama.cpp GBNF** (shipped) + **SAP (BAML)** + **Instructor retry** + **MCP TS SDK** + router seam from **Mastra/VoltAgent**. +- Memory: **sqlite-vec** + **LlamaIndex.TS memory blocks** + **Mem0 loop** + **Orama** hybrid ranking; techniques from **HippoRAG / A-MEM / Graphiti**. +- Routines: **OpenAdapt** design (resolution ladder + postconditions + self-heal), **Playwright/DevTools** for the browser lane. +- Rails: **@ui-tars/sdk** + **robotjs/nut-fork** + **nanobrowser/browser-use/Stagehand** over Electron CDP + **Appium/WebdriverIO** + **DroidRun Portal**; a napi-rs a11y addon over **axuielement/uiautomation** with **Terminator** head-start. +- Models: **UI-TARS-1.5-7B** (desktop), **GUI-Owl-1.5 / Qwen3-VL-8B** (mobile), **OmniParser v3** (fallback). + +**Build these (the product - bespoke by nature):** + +1. The **Action contract + durable pipeline** (queue -> FSM -> gate -> execute -> verify glue). +2. **Effect-verification** per connector (zero prior art anywhere) - lands in the R1 spine (the machine's verifying state + per-handler verify); R4's router only escalates through it, never rebuilds it. +3. The **resolve layer** returning `{value, confidence}`. +4. The **commitment-gap reasoner** (join a commitment against Replay observations). +5. The unified **DeviceController + rail-selection/fallback** policy. +6. The **macOS-AX + Windows-UIA napi-rs addon**. +7. The **approval-gate UX + privacy boundary** (nothing leaves the device). +8. **iOS on-device actuation** (Mac-signed WDA constraint). + +None of the bespoke items is NIH - each is bespoke because no upstream targets a single-process, offline, on-device app wired to a personal screen-memory. That is exactly the product. + +## 8. License avoid-list (carry forward) + +- **AGPL** (no code into the permissive pro tier): browser-use workflow-use, Skyvern, Open Interpreter OS mode, Windmill, LittleHorse, Reor, Khoj, OpenRecall, Sonic, OmniParser v1/v2 icon_detect (YOLOv8). +- **Source-available / SSPL / BUSL** (avoid depending): Screenpipe (post-2026-06), Inngest server (SSPL), Restate runtime (BUSL-1.1). +- **Paid EULA:** official `@nut-tree/*` nut.js (use `@nut-tree-fork`). +- **Non-commercial model weights** (do not bundle): Qwen2.5-VL 3B/72B, Holo 72B, CogAgent, Ferret-UI, SeeClick, the UI-TARS-1.5 flagship, xLAM/Hammer (verify per checkpoint). +- **Mixed/enterprise:** Mastra (use Apache-2.0 core only), Zep platform, Letta. + +Everything in the "port" column is MIT / Apache-2.0 / BSD. Verify each license at the point of adoption; a couple ask for attribution (minitap/mobile-use). diff --git a/docs/R1_CHECKLIST.md b/docs/R1_CHECKLIST.md new file mode 100644 index 00000000..1003e76b --- /dev/null +++ b/docs/R1_CHECKLIST.md @@ -0,0 +1,96 @@ +# R1 checklist - chat actions on the durable spine (Days 1 - 4) + +Execution checklist for R1 of `COMPUTER_USE_PLAN.md` (the build doc). The plan stays the source of truth for schedule and scope; this file only tracks R1's execution. Tick a box when its unit is landed green. + +**Rules for every box (from CLAUDE.md):** +- One box = one commit-sized unit. Land it as soon as it is green (`npx tsc --noEmit -p tsconfig.node.json && npx tsc --noEmit -p tsconfig.web.json && npm test`), then move on. Spine work commits in `../shared`. +- Tests land in the same commit as the change - one case per branch, condition, and error path. Coverage ratchet holds. +- Port before writing: the sources per component are in `PORTING_MAP.md`. Verify the license at the point of adoption. +- Any UI string follows the brand copy rules. + +**Design references:** the Action record and state machine are `ASSISTANT_ARCHITECTURE.md` Section 3; the reliability stack is Section 4; the gate contract is decision 7 (payload binding). The spine is platform-free and lives in `../shared/packages/use` (`@offgrid/use`); OGAD consumes it via `file:../shared/packages/use`. + +--- + +## Day 1 - the spine package (`@offgrid/use`, in `../shared`) + +- [x] **1. Scaffold `packages/use`** in the shared repo: tsup + node --test (the shared-repo house pattern), mirroring the sync engine layout; consumed from OGAD as `file:../shared/packages/use`. + *Done when:* the package builds, an empty test runs, and OGAD's tsc still passes with the dependency declared. +- [x] **2. The Action contract** (`packages/use/src/action.ts`): Zod schema + types for `id, type, source, intent, args, payloadHash, risk, rail, idempotencyKey, attempts, verification, state, triggerAt`, audit refs. Closed `type` enum (message / email / calendar / reminder / open / lookup / file-share / web-use). + *Done when:* schema tests cover each risk class, each type, and reject malformed input (fail closed). +- [x] **3. The state machine** (`packages/use/src/machine.ts`, XState v5): `proposed -> rejected | scheduled | resolving -> awaiting_approval | ready -> executing -> verifying -> done | executing(retry) | needs_help`, exactly as the architecture doc draws it. Persist via `getPersistedSnapshot()`; rehydrate on start. + *Done when:* every transition has a test, plus a snapshot -> restore roundtrip test (the crash-resume guarantee). +- [x] **4. The durable queue** (`packages/use/src/queue.ts`): the goqite/sqliteq pattern - lease + visibility timeout + auto-extend + attempts + `UNIQUE(idempotencyKey)` dedup - behind a small `Storage` interface (the spine stays platform-free; hosts inject the DB). + *Done when:* tested against better-sqlite3 `:memory:` - lease expiry re-queues, a duplicate enqueue dedups, attempts increment, a held lease blocks a second worker. + +## Day 2 - the guarantees + +- [x] **5. Retry policy** (`packages/use/src/retry.ts`; pure policy - the machine owns the loop, so no promise-retry dep): retry-once-with-verify for reversible actions; single-attempt-behind-the-gate for irreversible ones (decision 8.1 lean). + *Done when:* both policies are tested, including that an irreversible action never fires twice even when verify errors. +- [x] **6. The gate seam** (`packages/use/src/gate.ts`): the interrupt -> approve/edit/reject -> resume contract as a host callback; `payloadHash` computed at propose time and re-checked at execute time so the approved payload is exactly what runs. + *Done when:* tests cover approve, reject, edit-then-approve (hash changes, re-gate), and a tampered payload refusing to execute. +- [x] **7. The DeviceController port + handler registry** (`packages/use/src/device.ts`, `registry.ts`): `execute(action)` port; each action handler declares its rail, risk default, and how it verifies (read-back / status / none-fuzzy). Every attempt records the rail it ran on (the Action record is the effect journal), and escalation across rails is a re-fire governed by box 5's policy - a non-retryable action never escalates. + *Done when:* a fake DeviceController proves the seam - registering a second fake handler needs zero caller changes (the DSP test), and routing picks by declared rail. +- [x] **8. The engine facade + worker** (`packages/use/src/engine.ts`): `propose()` validates and enqueues; a worker drains the queue through machine -> gate -> execute -> verify. + *Done when:* the fake-device suite is green end to end: a routed action, the gate flow, a verify-retry scenario, crash-resume (kill mid-execute, rehydrate, no double-fire thanks to the idempotency key), exactly-once under a duplicate enqueue. **This is the engine checkpoint.** + +## Day 3 - wire into the app (macOS end to end) + +- [x] **9. The storage adapter in OGAD** (`src/main/actions/use-driver.ts` + `src/main/__tests__/use-storage.integration.dbtest.ts`): the queue/state tables live in the app's existing better-sqlite3 DB (one DB is the SSOT), with a migration. + *Done when:* an integration test runs the real engine against a temp app DB (no mocks at the DB seam). +- [x] **10. The semantic rail adapter** (`src/main/actions/semantic-rail.ts`): wrap the existing `runNativeAction` helper behind the DeviceController port; map the Action types to the helper's verbs (calendar, reminders, contacts, messages, mail, open_url). + *Done when:* each mapped type has a test through an injected helper boundary; unknown types are refused, not guessed. +- [x] **11. The gate host**: wire the existing `actions:proposeApproval` seam as the engine's gate callback; the approval card shows the resolved values from the bound payload. + *Done when:* an integration test proves approve runs exactly the approved payload and reject lands the Action in `rejected`. +- [x] **12. Emission hardening** (`src/main/actions/emit.ts`): the action tool's schema goes to llama-server as grammar-constrained `response_format`; a TS SAP coercer (ported from BAML's schema-aligned parsing, keyed to the Action schema) repairs near-misses; bounded Zod validate-and-retry feeds the error back. + *Done when:* coercion tests per branch (markdown fence, trailing prose, unquoted keys, missing optional), and a test that an unrepairable emission is rejected, never guessed. +- [x] **13. Chat tool integration**: mutations from the chat tool loop enqueue durable Actions through the engine; pure reads stay inline (decision 7.5). Existing native-tool behavior is preserved. + *Done when:* the existing native-action tests still pass, plus new tests that a mutation goes through the queue and gate while a read does not. +- [x] **14. Verification per handler**: calendar and reminders verify by read-back (list after create); messages and mail declare fuzzy -> single-attempt; open_url verifies by launch result. + *Done when:* each handler's declared verification has a test, including a failed read-back triggering the retry policy correctly. +- [x] **15. macOS checkpoint evidence** (free-build engine path; the approval-card capture lands with the pro migration): on a seeded demo profile (`npm run demo` seeding rules), a chat ask ("remind me to send the deck at 6pm") produces gate -> execute -> verified -> confirmation. Capture screenshots into `e2e/screenshots/`. + *Done when:* the flow runs clean and the screenshots show the approval card and the verified confirmation (validate the images before counting this done). + +## Day 4 - Windows + release + +- [x] **16. Windows toolchain** (pre-existing on main - build-win job, fetch-win-binaries.ps1 with llama-server.exe pinned to the mac engine ref, NSIS + auto-update, optional signing secrets; our delta: windows-build.yml gained the shared checkout, both workflows now build @offgrid/use, and run 31779453356 built this branch green with a 414MB installer artifact. Remaining as release items: the signing cert, and the model-load smoke on a real Windows machine per WINDOWS_TEST_PLAN.md) (start this in parallel as early as Day 1 - it is the schedule floor and has CI latency): electron-builder Windows target, code-signing, and the `llama-server` Windows engine build in `release.yml` with the same gates the mac build learned (deployment target / staged deps / no foreign paths, adapted to Windows). + *Done when:* CI produces a signed Windows build whose bundled engine loads a model. +- [x] **17. The Windows semantic rail** (`src/main/actions/semantic-rail-win.ts`), **local-first**: mail + calendar via local Outlook automation (COM / PowerShell) where Outlook exists - a local write that syncs later, matching the mac rail - with Microsoft Graph as the fallback for setups without local Outlook (online-only, labeled honestly, user's own sign-in); open via the Windows shell. iMessage is macOS-only in R1 (documented tier difference). + *Done when:* handler tests through an injected Graph boundary; the registry proves macOS and Windows rails swap with zero caller changes. +- [x] **18. E2E + evidence** (APP-250 in the suite; evidence in merged PR #81): a Playwright spec driving chat ask -> approval card -> done state on a fresh temp profile (`OFFGRID_PRO=0`, synthetic seed only); screenshots per surface, a short video of the golden path. + *Done when:* `npm run test:e2e` includes the new spec and passes; evidence attached to the PR per the repo's PR rules. +- [x] **18b. Release UX notes** (recorded; superseded by R2-B Approval UX v2 in the plan): Tools defaults ON (fresh installs) with native actions under the Tools category - verify in the e2e that a fresh profile can act without touching any toggle. Flag to the lead: the free-build inline-confirm question for mutate/irreversible actions (open-core line), and the R2 router retiring the per-turn toggle. +- [ ] **19. Ship it** (merged to main 2026-08-14, PR #81; the release DISPATCH ships with R2 per the re-cut): version bump, release via CI, checkpoint sign-off against the plan ("on both macOS and Windows, a chat ask calls the action tool and the action runs gated and verified"). Update `COMPUTER_USE_PLAN.md` if any date moved. + *Done when:* the release is out and the plan reflects reality. + +--- + +## Field verdicts from the R1 pro-path smoke test (drive R2's Approval UX v2) + +- Approving a card gives no completion feedback - the chat message stays "pending" + and nothing reports the run. The engine path reports verified outcomes; the + legacy pro path is the old system. Fixed by the pro migration (approve resolves + the engine gate) in R2-B. +- Reversible simple actions (a reminder) should not gate at all: R2-B ships the + risk-tiered policy (reversible mutations auto-run + verified confirmation + + Undo; sends keep the gate). +- Chat-originated approvals belong INLINE in the conversation; the Actions screen + is the queue for unattended actions + audit. + +## Windows follow-ups (fast-follow, recorded during box 17) + +- Windows chat-tool exposure: registerNativeActionTools stays darwin-gated; enabling a + filtered spec subset on win32 (calendar/reminders/mail/open via the engine path) + needs per-platform specs and a win inline runner for reads. +- Outlook read-back verifiers: calendar/reminder verification still speaks the mac + helper's list verbs; on Windows read_back reports unverifiable (retry policy treats + it honestly) until Outlook COM list scripts land. +- Graph OAuth wiring: the port + fallback logic are boundary-tested; production + passes no Graph port until sign-in lands. + +## Watch-list (honest risks inside R1) + +- **Box 16 is the long pole.** Windows CI signing + the engine build is net-new infra with slow feedback loops; kick it off on Day 1 and let it bake while the spine lands. +- **Box 12's SAP coercer is new surface** - err toward more coercion-branch tests, not fewer; every repair rule gets a regression case. +- **Boxes 9 - 11 touch the running app** - main-process changes need an app restart; do not over-restart during capture hours. +- If a box slips, the plan's rule applies: scope trims at the tail (Windows rail detail, evidence polish), never the released core. diff --git a/docs/R2_CHECKLIST.md b/docs/R2_CHECKLIST.md new file mode 100644 index 00000000..b95b2d53 --- /dev/null +++ b/docs/R2_CHECKLIST.md @@ -0,0 +1,132 @@ +# R2 checklist - full rails in chat, both platforms + Approval UX v2 + +Execution checklist for R2 of `COMPUTER_USE_PLAN.md`. Same rules as R1: one box = one +commit-sized unit, landed green (`tsc` node+web+pro, `npm test`), tests in the same +commit, port before writing, brand copy rules on every UI string. + +## A. Windows chat exposure (~1 day) + +- [x] **A1. Per-platform tool specs**: `specsForPlatform(platform)` in the logic file - + darwin keeps all eight; win32 exposes the engine-routed set the Outlook rail supports + (calendar_create_event, reminders_create, mail_send, open_url); everything else none. + A win32 system hint that never mentions iMessage or contacts. + *Done when:* filtering + hints tested per platform; the extension's schemas/canHandle/ + systemHint follow the platform; registerNativeActionTools registers on win32. +- [x] **A2. The win32 inline runner**: open/navigate on Windows goes through the shell + (injected opener); every other inline verb refuses honestly. The production boundary + picks the runner by platform in one place. + *Done when:* runner tests through the injected opener; unknown verbs refuse. +- [x] **A3. Outlook read-back verifiers**: list scripts for tasks (olFolderTasks 13) + and calendar range (olFolderCalendar 9, Restrict on [Start]) speaking the same + {reminders|events:[{title}]} shape as the mac helper, exposed as a RunNative reader + so `buildRegistry` works unchanged; the runtime picks the reader by platform. + *Done when:* script content + reader mapping tested; the read-back verifiers pass over + a scripted PS boundary; unknown verbs refuse. + +## B. Approval UX v2 (~1-1.5 days, core + desktop-pro) + +- [x] **B1. Risk-tiered gating policy**: reversible mutations (reminder, calendar) + auto-run + verified confirmation; sends and irreversible actions keep the gate. + Policy defined once (engine-side risk + handler declaration), tested per tier. +- [x] **B2. Undo affordance** for auto-run reversibles (delete the created item), in + chat next to the confirmation. (Engine half DONE with B1: engine.undo, effectId + stamping, delete verbs on both platforms; remaining = the chat chip, lands with B3.) +- [x] **B3. Inline approval card in chat**: resolved values + Approve / Edit / Reject + driven by `resolveActionGate`; the Actions screen stays the unattended queue + audit. +- [x] **B4. The pro migration** (desktop-pro): pro's approval queue resolves the engine + gate instead of running its own executor - payload binding + verification hold on + pro; outcome feedback lands back in the chat turn and on the card. + (desktop-pro PR #42: rows carry action_id; approve/reject resolve the gate; the row + records only the outcome the queue observes - the engine journal stays the SSOT.) + +## C. The browser rail (~1.5-2 days) + +- [x] **C1. CDP snapshot + indexed elements** over `webContents.debugger` (nanobrowser + dom module as start code, browser-use algorithm). +- [x] **C2. The watched pane + takeover** (login/identity boundary pauses, user acts). +- [x] **C3. web_use through the engine** (act/observe/extract API, Zod-validated), + gated at identity, verified by page-state postconditions. + +## D. The vision rail (~1.5-2 days, supervised tier) + +The whole spine landed, screen-free and tested (parser, guard, loop, engine +adapter), wired into the engine. What remains is the native actuation dep + +entitlements + a real-machine pass - a packaging decision, not code. Until it +lands the rail refuses cleanly and computer_task is NOT offered to the model, +so the tier is honestly gated (see the watch-list). + +- [x] **D1a. The UI-TARS action parser** (ported from @ui-tars/sdk, closed to the + shipped verbs; 0-1000 -> pixel denormalization, fail-closed). `computer_task` + added to the shared ACTION_TYPES enum. +- [x] **D1b. UI-TARS-1.5-7B catalog entry** in **OGAD's own `packages/models`** + (OGAD ships `file:./packages/models`, NOT the shared copy - the first attempt + landed in shared, which OGAD does not consume; corrected). mradermacher/ + UI-TARS-1.5-7B-GGUF, Q4_K_M weights (4.68GB) + f16 mmproj (1.35GB), Apache-2.0 + base, both URLs HEAD-verified 200, dist rebuilt. Downloadable from the Models + screen; regression test against the real catalog. (OmniParser v3 set-of-marks + fallback still open - a second-source-of-marks nicety, not on the critical path.) +- [x] **D1c. Model-agnostic + a grounder notice** (the maintainer's call): the + vision rail runs on any loaded model, but warns (never blocks) when it is not + a grounder. `grounder` flag on catalogued models + isGrounderModel() (flag + authoritative, name heuristic for a user's own HF pick); the supervisor overlay + shows an amber notice ('may click the wrong place') that names the fix. +- [x] **D2a. The operator spine**: the guard (kill switch terminal + outranks all, + pause-on-user-input, step budget), the supervised loop (screenshot -> ground -> + actuate, handoff + resume, re-check-before-dispatch), and the engine adapter + (computer_task on the vision rail, no-retry). The host shell captures via + desktopCapturer, grounds via the vision LLM, Esc kill switch wired. +- [x] **D2b-UX. The supervised surface**: the overlay (live step feed, visible + Stop + Pause, the takeover promise on-screen) + the controller routing + Stop/Pause/Resume to the running task's guard (fail-closed, stale-safe) + + step/state broadcasts + the preload vision namespace. Tested; mounted in chat. +- [x] **D2b-native. Actuation wired**: the ActuationPort is backed by + @nut-tree-fork/nut-js (optionalDependency; CGEvent mac / SendInput win), + dynamic-required so an absent/unbuilt addon degrades to a clean refusal. The + macOS Accessibility grant is checked at run start (prompts once, stops with a + clear message if missing). `computer_task` exposed as an engine-only tool. The + pure hotkey map (vision-keys) is tested; the actuation ITSELF still needs a + real machine to VERIFY (a display + the Accessibility grant) - the code is on + the branch for local testing, not proven headlessly. +- [ ] **D3. file_share recipe** (the WhatsApp share-a-file flow) as a labeled + showcase - a nicety on top of the general computer_task, once the rail is + verified on a real machine. + +## E. Safety pass + the release + +- [x] **E1. Injection-resistance review** (screen content is untrusted) + + per-rail prompt guards. `docs/SAFETY_REVIEW.md` records the threat / defense / + test per rail; `rail-injection-stance.test.ts` guards the prompt contracts; + the structural defenses (driver refuses credential fields, the vision guard's + terminal kill switch, re-check-before-dispatch) are tested in + browser-driver / vision-guard / vision-agent. **Kill-switch e2e** is blocked on + actuation (D2b): nothing actuates until then, so nothing halts - it is part of + the real-machine pass, not the headless tour (see the review). +- [ ] **E2. Release** - BLOCKED on: D2b (vision actuation + entitlements) so the + supervised tier is real; D1b (the UI-TARS catalog entry); the real-machine + click-through for browser + vision on both platforms (WINDOWS_TEST_PLAN.md); + and the Windows signing-cert decision (lead). Then: one versioned dispatch - + signed/notarized .dmg + Windows NSIS .exe; release notes honest about the + supervised tier and what was human-verified. + +## Watch-list + +- Vision on a local 7B is best-effort: labeled supervised or not shipped. +- Windows browser/vision needs a human on a real Windows machine before E2. +- B touches the live chat surface: behavior tests per branch; non-action turns stay on + the plain path untouched. +- B4 landed (desktop-pro PR #42): the pro queue resolves the engine gate, so the + Windows PRO path runs Outlook actions through the semantic rail on approval. Verify + on the real-Windows pass with the rest of WINDOWS_TEST_PLAN.md. +- Pro flaky watch: model-transfer-service.test.ts leaks a FileHandle at GC (an + unhandled-error line in every full run) - stabilize with the other sync flakes. + ambient-file-watcher / meeting-persistence flake locally (LLM/timing) but pass + in isolation and on CI; retry a blocked coverage push rather than chasing them. +- Vision rail actuation is capability-gated OFF (D2b): the spine is wired and + tested, but the native input addon + Accessibility/Screen-Recording + entitlements are unshipped, so computer_task is not offered to the model and + the host refuses cleanly. The E2 checkpoint's "supervised vision action from + chat" needs D2b first - on both platforms, with a human on a real machine. +- Shared `@offgrid/use` change (computer_task type) rides shared branch + feat/r2-full-rails (mirrors the OGAD branch name so CI's matching-branch + checkout finds it) and feat/use-approval-tiers; both need merging to shared + main with the OGAD PR. diff --git a/docs/R5_CHECKLIST.md b/docs/R5_CHECKLIST.md new file mode 100644 index 00000000..0a95aa4a --- /dev/null +++ b/docs/R5_CHECKLIST.md @@ -0,0 +1,70 @@ +# R5 - model-agnostic computer use (the tiered rail) + +Goal: **computer use works on most chat models.** The vision grounder (R2, UI-TARS) +becomes a last-resort fallback; the user's normal model drives the common case via +the accessibility tree, and a small detector covers the dead-AX tail. Router order: +semantic -> browser -> **accessibility** -> set-of-marks -> vision. + +Scope now: **Tier 1 (AX driving rail) + Tier 2 (set-of-marks).** Tier 3 (the +grounder's separate loader + pluggable formats) is deferred. + +Build rule (as everywhere): pure logic in Electron-free modules, unit-tested; the +native helper + the on-screen actuation are the injected boundaries, verified on a +real machine. Reuse the browser rail's loop - do not fork a parallel one. + +## Tier 1 - the accessibility driving rail + +- [x] **T1a. The element contract + parser (pure).** `AxElement` (role, label, value, + frame -> center cx/cy, actionable, enabled) + `parseAxElements` + a + `formatAxElementsForModel` that numbers them like the browser collector. Fail-closed + on malformed lines. `ax-elements.ts` (+ 7 tests). +- [x] **T1b. The picking loop (pure).** `runElementTask(goal, deps)` - snapshot, + model picks `{action: click|press|type|key|done|give_up, index, text, keys}` + (grammar-constrained, fail-closed), act via the injected actuator. Same shape as the + web-task loop; SHARED with the set-of-marks tier. `ax-agent.ts` (+ 16 tests). +- [x] **T1c. The Swift helper: structured-elements mode.** `--elements ` walks the + AX tree and emits one JSON object per interactive element (role, label, value, + frame, AXPress, enabled). Hardened for real apps: triggers the Chromium/Electron web + tree (AXManualAccessibility + AXEnhancedUserInterface), retries until it populates, + resolves the app to a foreground process. `--apps` lists candidates (NSWorkspace, no + SR grant). Built + minos-gated via `scripts/build-text-extractor.sh` (pinned 13.0). +- [x] **T1d. The AX reader + host (shell).** `ax-host.ts`: resolve the target app + (ax-target.ts, pure + tested), read via the helper, activate it so clicks land, and + drive `runElementTask` with the local model + the shared nut.js actuation. Reuses the + vision guard (Esc) + controller (overlay Stop). Excluded from coverage like the other + rail hosts. Target-picker: `ax-target.ts` (+ 7 tests). +- [x] **T1e. Engine wiring + the router.** `computer_task` tries the accessibility rail + FIRST and falls to vision when the AX tree is too thin (`ax-router.axRailViable`, 7 + tests) or the goal names no running app. The tiering is a pure, tested function + (`ax-rail.ts`, 5 tests); `use-runtime` wires the live hosts into the 'vision' branch. +- [ ] **T1f. Verify + evidence.** Real-machine pass: "open the DM with X in Slack", + "click Send", a native file dialog navigated by AX. The grounder must NOT load for + these. Screenshots + the step feed in the PR. (Helper verified: Slack 90 elements, + Chrome 231 - the live end-to-end drive is the remaining hands-on step.) + +## Tier 2 - set-of-marks (the dead-AX tail) + +- [ ] **T2a. The marks model.** A small OmniParser-class detector (icon/text element + detection -> boxes). Catalog + a detection-model runtime (ONNX-class) separate from + llama.cpp. Sized so it is NOT a 7B grounder. +- [ ] **T2b. The marks composition (pure).** Detector boxes -> numbered overlay -> + `AxElement[]`-shaped list (a box is just an element with no AX role) so tier-1's + loop and formatter are reused unchanged; the general VISION model picks the number. +- [ ] **T2c. Router fallthrough.** When AX yields too few actionable elements, fall to + set-of-marks before vision. One decision function, tested. +- [ ] **T2d. Verify.** A Catalyst / WhatsApp-class app driven by a general vision + model via numbered marks, on a real machine. + +## Deferred (not R5) + +- Tier 3 hardening: the grounder's separate on-demand loader (image-gen eviction + pattern) + per-model grounding-format adapters (UI-TARS / Aguvis / OS-Atlas). +- Windows: the UIA reader as tier-1's Windows twin (mirrors T1c/T1d via UIAutomation). + +## Watch-list + +- The AX driving rail is an architecture change: AX becomes a first-class hands, not + just eyes (see the R5 note in COMPUTER_USE_PLAN.md / ASSISTANT_ARCHITECTURE.md). +- Reuse the browser-rail loop for T1b/T2b - a second copy is a defect. +- Never run the push gate while a dev app is live (the DB-ABI swap in `test:db` + breaks the running app - learned the hard way during R2 testing). diff --git a/docs/RELEASE_DESKTOP.md b/docs/RELEASE_DESKTOP.md index 6e2b533d..7b275a9e 100644 --- a/docs/RELEASE_DESKTOP.md +++ b/docs/RELEASE_DESKTOP.md @@ -51,7 +51,7 @@ These builds are **unsigned** (no cert/Apple-ID prompts) and never touch GitHub. "Activated. Restart to finish unlocking Pro." → Restart → pro tabs unlock. 3. Things that broke before — verify explicitly: local model server starts on :7878 (chat responds), whisper/ffmpeg present, no missing-binary errors in the - console (`Console.app` → filter "Off Grid"). + console (`Console.app` → filter "Off Grid AI"). > Both DMGs use distinct appIds (`…desktop` / `…desktop.pro`) so they can be > installed side by side. They share the canonical userData dir diff --git a/docs/RELEASE_READINESS_CHECKLIST_0.0.40.csv b/docs/RELEASE_READINESS_CHECKLIST_0.0.40.csv index 1d4014aa..e8368035 100644 --- a/docs/RELEASE_READINESS_CHECKLIST_0.0.40.csv +++ b/docs/RELEASE_READINESS_CHECKLIST_0.0.40.csv @@ -26,11 +26,11 @@ Release,Journey ID,Phase,Tier,Priority,Manual test,Exact manual steps,Expected r 0.0.40,25,2 Models and downloads,Both,P0,Interrupted download recovers,Quit the app during a download and reopen,The item resumes or becomes explicitly retryable; it never remains falsely ready,Yes,PARTIAL,Integration,model-integrity.integration.test.ts,"Interrupted download recovers. `model-integrity.integration.test.ts` interrupts a real streamed partial, reloads the manager, resumes with the correct HTTP range, verifies exact final bytes and installation, then reloads again to prove completed state stays cleared.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W26=""PASS"",100,0)","=IF(OR(W26=""FAIL"",W26=""BLOCKED""),0,ROUND(O26*0.7+P26*0.3,0))",=100-Q26,"=IF(OR(W26=""FAIL"",W26=""BLOCKED""),""BLOCKED"",IF(AND(J26=""COMPLETE"",W26=""PASS""),""DONE"",IF(J26=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J26=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,26,2 Models and downloads,Both,P0,Truncated GGUF is rejected,Interrupt or substitute a file below the integrity floor,The file is not promoted to installed or loadable,Yes,PARTIAL,Integration,model-integrity.integration.test.ts,"Truncated GGUF is rejected. `model-integrity.integration.test.ts` drives the real model manager against a temp filesystem, with only HTTP delivery faked, and proves truncated downloads and local imports are rejected before promotion, installation, copying, or registration.","ts` drives the real model manager against a temp filesystem, with only HTTP delivery faked, and proves truncated downloads and local imports are rejected before promotion, installation, copying, or registration.",65,"=IF(W27=""PASS"",100,0)","=IF(OR(W27=""FAIL"",W27=""BLOCKED""),0,ROUND(O27*0.7+P27*0.3,0))",=100-Q27,"=IF(OR(W27=""FAIL"",W27=""BLOCKED""),""BLOCKED"",IF(AND(J27=""COMPLETE"",W27=""PASS""),""DONE"",IF(J27=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J27=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: ts` drives the real model manager against a temp filesystem, with only HTTP delivery faked, and proves truncated downloads and local imports are rejected before promotion, installation, copying, or registration.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Adversarial release check 0.0.40,27,2 Models and downloads,Both,P0,Disk write failure does not crash,Use a nearly full disposable volume or unwritable test location and start a download,The download fails; Off Grid AI Desktop stays open and other features remain usable,Yes,PARTIAL,Integration,,"Disk write failure does not crash. The same real model-manager integration injects `ENOSPC` only at the OS write boundary and proves the error is contained, no partial model is installed, failed status is recorded, and an existing installed model remains readable.","Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",55,"=IF(W28=""PASS"",100,0)","=IF(OR(W28=""FAIL"",W28=""BLOCKED""),0,ROUND(O28*0.7+P28*0.3,0))",=100-Q28,"=IF(OR(W28=""FAIL"",W28=""BLOCKED""),""BLOCKED"",IF(AND(J28=""COMPLETE"",W28=""PASS""),""DONE"",IF(J28=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J28=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Never risk a real data volume -0.0.40,28,2 Models and downloads,Both,P0,Active text model survives relaunch,Activate a chat model; quit fully; reopen,The same model remains active and answers a new message,Yes,PARTIAL,Integration,model-integrity.integration.test.ts,"Active text model survives relaunch. `model-integrity.integration.test.ts` installs and activates a real catalog text fixture through the production model manager, reloads every module, and proves the same installed model remains the active chat selection.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W29=""PASS"",100,0)","=IF(OR(W29=""FAIL"",W29=""BLOCKED""),0,ROUND(O29*0.7+P29*0.3,0))",=100-Q29,"=IF(OR(W29=""FAIL"",W29=""BLOCKED""),""BLOCKED"",IF(AND(J29=""COMPLETE"",W29=""PASS""),""DONE"",IF(J29=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J29=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, +0.0.40,28,2 Models and downloads,Both,P0,Active text model survives relaunch,Activate a chat model; quit fully; reopen,The same model remains active and answers a new message,Yes,PARTIAL,Integration,model-integrity.integration.test.ts; model-switch-ownership.integration.test.ts,"Active text model survives relaunch. `model-integrity.integration.test.ts` installs and activates a real catalog text fixture through the production model manager, reloads every module, and proves the same installed model remains the active chat selection. `model-switch-ownership.integration.test.ts` additionally imports and activates two real local GGUF fixtures, starts the production native-process and SSE path, changes selection while Model A is streaming, proves that admitted turn completes on A, then proves the next turn starts Model B.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W29=""PASS"",100,0)","=IF(OR(W29=""FAIL"",W29=""BLOCKED""),0,ROUND(O29*0.7+P29*0.3,0))",=100-Q29,"=IF(OR(W29=""FAIL"",W29=""BLOCKED""),""BLOCKED"",IF(AND(J29=""COMPLETE"",W29=""PASS""),""DONE"",IF(J29=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J29=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,29,2 Models and downloads,Both,P1,Active modal models survive relaunch,Activate image STT and TTS choices; quit fully; reopen,Each modality restores its own selection without cross-over,Yes,COMPLETE,Integration,model-integrity.integration.test.ts,"Active modal models survive relaunch. `model-integrity.integration.test.ts` installs and activates real image, STT, and TTS catalog fixtures, reloads every manager module, and proves each persisted modality restores its own selection without crossing into another modality.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W30=""PASS"",100,0)","=IF(OR(W30=""FAIL"",W30=""BLOCKED""),0,ROUND(O30*0.7+P30*0.3,0))",=100-Q30,"=IF(OR(W30=""FAIL"",W30=""BLOCKED""),""BLOCKED"",IF(AND(J30=""COMPLETE"",W30=""PASS""),""DONE"",IF(J30=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J30=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,, 0.0.40,30,2 Models and downloads,Both,P1,Deleting active model clears selection,Activate then delete a model for each available modality,No dangling active pointer remains; the UI asks for or selects a valid replacement,Yes,COMPLETE,Integration + Contract/package gate,model-integrity.integration.test.ts,"Deleting an active model clears selection. `model-integrity.integration.test.ts` activates installed text, vision, image, speech, and transcription fixtures through the production model manager, deletes each one, and proves all runtime and persisted selections remain cleared after a fresh module load.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W31=""PASS"",100,0)","=IF(OR(W31=""FAIL"",W31=""BLOCKED""),0,ROUND(O31*0.7+P31*0.3,0))",=100-Q31,"=IF(OR(W31=""FAIL"",W31=""BLOCKED""),""BLOCKED"",IF(AND(J31=""COMPLETE"",W31=""PASS""),""DONE"",IF(J31=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J31=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,, 0.0.40,31,2 Models and downloads,Both,P2,Models use desktop density,Resize the window across normal desktop widths,Model cards form a dense multi-column grid; controls stay beside their card content,Yes,PARTIAL,E2E,e2e/desktop-polish.spec.ts,Models use desktop density. `e2e/desktop-polish.spec.ts` resizes the real Electron window from 1280 to 1800 pixels and proves the production model collection forms three then four computed columns while its controls remain reachable.,"Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",70,"=IF(W32=""PASS"",100,0)","=IF(OR(W32=""FAIL"",W32=""BLOCKED""),0,ROUND(O32*0.7+P32*0.3,0))",=100-Q32,"=IF(OR(W32=""FAIL"",W32=""BLOCKED""),""BLOCKED"",IF(AND(J32=""COMPLETE"",W32=""PASS""),""DONE"",IF(J32=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J32=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Visual check against docs/DESIGN.md -0.0.40,32,3 Chat and conversations,Both,P0,First local message replies,Select a local text model; create a chat; send a prompt,A response streams into one assistant bubble and is persisted,Yes,PARTIAL,Automated test,MemoryChat.chat-lifecycle.test.tsx,"First local message replies. `MemoryChat.chat-lifecycle.test.tsx` sends through the real rendered composer, routes a streamed token through production ownership, resolves the local-model boundary, and proves one assistant bubble with the exact answer is persisted once.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",40,"=IF(W33=""PASS"",100,0)","=IF(OR(W33=""FAIL"",W33=""BLOCKED""),0,ROUND(O33*0.7+P33*0.3,0))",=100-Q33,"=IF(OR(W33=""FAIL"",W33=""BLOCKED""),""BLOCKED"",IF(AND(J33=""COMPLETE"",W33=""PASS""),""DONE"",IF(J33=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J33=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, +0.0.40,32,3 Chat and conversations,Both,P0,First local message replies,Select a local text model; create a chat; send a prompt,A response streams into one assistant bubble and is persisted,Yes,PARTIAL,Integration,MemoryChat.chat-lifecycle.test.tsx; workspace-production-bridge.ui.integration.dbtest.tsx,"First local message replies. `MemoryChat.chat-lifecycle.test.tsx` sends through the real rendered composer, routes a streamed token through production ownership, resolves the local-model boundary, and proves one assistant bubble with the exact answer is persisted once. `workspace-production-bridge.ui.integration.dbtest.tsx` removes the substituted Off Grid AI preload API: the rendered composer uses the production preload mapping and IPC registrations, streams from only a controlled native-model socket, and persists the exact visible user and assistant turns in real SQLite.","tsx` removes the substituted Off Grid AI preload API: the rendered composer uses the production preload mapping and IPC registrations, streams from only a controlled native-model socket, and persists the exact visible user and assistant turns in real SQLite.",65,"=IF(W33=""PASS"",100,0)","=IF(OR(W33=""FAIL"",W33=""BLOCKED""),0,ROUND(O33*0.7+P33*0.3,0))",=100-Q33,"=IF(OR(W33=""FAIL"",W33=""BLOCKED""),""BLOCKED"",IF(AND(J33=""COMPLETE"",W33=""PASS""),""DONE"",IF(J33=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J33=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: tsx` removes the substituted Off Grid AI preload API: the rendered composer uses the production preload mapping and IPC registrations, streams from only a controlled native-model socket, and persists the exact visible user and assistant turns in real SQLite.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,33,3 Chat and conversations,Both,P0,No memory scope works,Choose No memory and send a prompt,The reply uses the conversation only and the selected scope remains visible,Yes,PARTIAL,Integration,,"No memory scope works. The same rendered integration keeps No memory visibly selected, sends a turn through the production chat path with retrieval disabled and no project scope, then renders the conversation-only answer normally.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W34=""PASS"",100,0)","=IF(OR(W34=""FAIL"",W34=""BLOCKED""),0,ROUND(O34*0.7+P34*0.3,0))",=100-Q34,"=IF(OR(W34=""FAIL"",W34=""BLOCKED""),""BLOCKED"",IF(AND(J34=""COMPLETE"",W34=""PASS""),""DONE"",IF(J34=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J34=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,34,3 Chat and conversations,Both,P0,All memory scope works,Seed or capture memory; choose All memory; ask about it,The answer uses retrieved local context and citations where applicable,Yes,PARTIAL,Integration + Contract/package gate,rag-empty-memory.dbtest.ts; memory-rag-chat-lifecycle.integration.dbtest.ts,"All memory scope works. `rag-empty-memory.dbtest.ts` seeds a synthetic capture into real SQLite/FTS storage, invokes the production `rag:chat` IPC handler in All memory mode, and proves the local-model prompt, answer, streamed retrieval count, and returned `[S1]` citation all carry the exact matching source. `memory-rag-chat-lifecycle.integration.dbtest.ts` additionally proves captured memory enters and leaves project-scoped retrieval through the persisted project policy.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",75,"=IF(W35=""PASS"",100,0)","=IF(OR(W35=""FAIL"",W35=""BLOCKED""),0,ROUND(O35*0.7+P35*0.3,0))",=100-Q35,"=IF(OR(W35=""FAIL"",W35=""BLOCKED""),""BLOCKED"",IF(AND(J35=""COMPLETE"",W35=""PASS""),""DONE"",IF(J35=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J35=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,A blank demo profile must be seeded before this check 0.0.40,35,3 Chat and conversations,Both,P1,Empty memory degrades safely,On a truly fresh profile choose All memory and send a prompt,The app answers without context or shows a clear empty-state message; no generic crash bubble,Yes,PARTIAL,Integration,rag-empty-memory.dbtest.ts,"Empty memory degrades safely. `rag-empty-memory.dbtest.ts` invokes the real `rag:chat` IPC handler on an empty SQLite/RAG corpus, verifies a normal answer, empty context and zero retrieval counts, then completes an immediate second turn to prove the queue and controller were released.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W36=""PASS"",100,0)","=IF(OR(W36=""FAIL"",W36=""BLOCKED""),0,ROUND(O36*0.7+P36*0.3,0))",=100-Q36,"=IF(OR(W36=""FAIL"",W36=""BLOCKED""),""BLOCKED"",IF(AND(J36=""COMPLETE"",W36=""PASS""),""DONE"",IF(J36=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J36=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, @@ -41,7 +41,7 @@ Release,Journey ID,Phase,Tier,Priority,Manual test,Exact manual steps,Expected r 0.0.40,40,3 Chat and conversations,Both,P1,Queued message order,Send a second message while the first reply is still streaming,Messages run in order without collision duplication or loss,Yes,PARTIAL,Automated test,MemoryChat.chat-lifecycle.test.tsx,"Queued message order. `MemoryChat.chat-lifecycle.test.tsx` sends a second message through the real composer while the first model-boundary promise is pending, then proves production queue draining preserves user/assistant order without collision, duplication, or loss.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",40,"=IF(W41=""PASS"",100,0)","=IF(OR(W41=""FAIL"",W41=""BLOCKED""),0,ROUND(O41*0.7+P41*0.3,0))",=100-Q41,"=IF(OR(W41=""FAIL"",W41=""BLOCKED""),""BLOCKED"",IF(AND(J41=""COMPLETE"",W41=""PASS""),""DONE"",IF(J41=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J41=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,41,3 Chat and conversations,Both,P0,Conversation switch isolation,Start work in conversation A then switch to B,B shows none of A's spinner progress or partial state; A completes against A's history,Yes,PARTIAL,Integration,,"Conversation switch isolation. The same integration starts and streams conversation A, switches the rendered screen to B, proves B receives none of A's partial or completed state, then reopens A and retrieves its correctly persisted result.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W42=""PASS"",100,0)","=IF(OR(W42=""FAIL"",W42=""BLOCKED""),0,ROUND(O42*0.7+P42*0.3,0))",=100-Q42,"=IF(OR(W42=""FAIL"",W42=""BLOCKED""),""BLOCKED"",IF(AND(J42=""COMPLETE"",W42=""PASS""),""DONE"",IF(J42=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J42=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,42,3 Chat and conversations,Both,P0,Project switch isolation,Start a project-scoped generation then change the project selection,The turn and resulting artifacts stay attributed to the project captured at send time,Yes,PARTIAL,Integration,,"Project switch isolation. The same integration sends from Project Alpha, changes the real project selector to Project Beta while the model boundary is pending, and proves the result and parsed HTML artifact retain the Alpha project and conversation captured at send time.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W43=""PASS"",100,0)","=IF(OR(W43=""FAIL"",W43=""BLOCKED""),0,ROUND(O43*0.7+P43*0.3,0))",=100-Q43,"=IF(OR(W43=""FAIL"",W43=""BLOCKED""),""BLOCKED"",IF(AND(J43=""COMPLETE"",W43=""PASS""),""DONE"",IF(J43=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J43=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, -0.0.40,43,3 Chat and conversations,Both,P0,Chat survives relaunch,Create several conversations and fully quit,All conversations messages scopes and project associations restore correctly,Yes,PARTIAL,Integration,chat-relaunch.dbtest.ts,"Chat survives relaunch. `chat-relaunch.dbtest.ts` creates a conversation and ordered user and assistant messages with exact scope, attachment, and finish context through the production repository, closes the real SQLite profile, reopens it, and verifies the conversation, message count, order, content, and context. The composed memory/RAG lifecycle also reloads the application modules and runs another scoped chat against the reopened profile. Reloading the visible conversation list remains manual.",Reloading the visible conversation list remains manual.,65,"=IF(W44=""PASS"",100,0)","=IF(OR(W44=""FAIL"",W44=""BLOCKED""),0,ROUND(O44*0.7+P44*0.3,0))",=100-Q44,"=IF(OR(W44=""FAIL"",W44=""BLOCKED""),""BLOCKED"",IF(AND(J44=""COMPLETE"",W44=""PASS""),""DONE"",IF(J44=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J44=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Reloading the visible conversation list remains manual.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, +0.0.40,43,3 Chat and conversations,Both,P0,Chat survives relaunch,Create several conversations and fully quit,All conversations messages scopes and project associations restore correctly,Yes,PARTIAL,Integration,chat-relaunch.dbtest.ts; workspace-production-bridge.ui.integration.dbtest.tsx,"Chat survives relaunch. `chat-relaunch.dbtest.ts` creates a conversation and ordered user and assistant messages with exact scope, attachment, and finish context through the production repository, closes the real SQLite profile, reopens it, and verifies the conversation, message count, order, content, and context. The composed memory/RAG lifecycle also reloads the application modules and runs another scoped chat against the reopened profile. `workspace-production-bridge.ui.integration.dbtest.tsx` closes and reopens that real database, then renders the production Projects and Chat surfaces through production preload/IPC. The project, chat count, ordered messages, and project-scoped artifact all reappear visibly from durable state.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W44=""PASS"",100,0)","=IF(OR(W44=""FAIL"",W44=""BLOCKED""),0,ROUND(O44*0.7+P44*0.3,0))",=100-Q44,"=IF(OR(W44=""FAIL"",W44=""BLOCKED""),""BLOCKED"",IF(AND(J44=""COMPLETE"",W44=""PASS""),""DONE"",IF(J44=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J44=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,44,3 Chat and conversations,Both,P2,Rename conversation,Rename a conversation and navigate away and back,The new name persists everywhere it is shown,Yes,COMPLETE,E2E,e2e/chat-actions.spec.ts,"Rename conversation. `e2e/chat-actions.spec.ts` drives the real Electron UI through rename, production preload/IPC and SQLite, verifies the old title disappears, navigates to another conversation and back, then fully relaunches on the same profile and restores the new title.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W45=""PASS"",100,0)","=IF(OR(W45=""FAIL"",W45=""BLOCKED""),0,ROUND(O45*0.7+P45*0.3,0))",=100-Q45,"=IF(OR(W45=""FAIL"",W45=""BLOCKED""),""BLOCKED"",IF(AND(J45=""COMPLETE"",W45=""PASS""),""DONE"",IF(J45=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J45=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,, 0.0.40,45,3 Chat and conversations,Both,P1,Delete conversation cascades,Create a chat with messages and an artifact then delete it,The chat messages and chat-owned artifact disappear with no orphaned sidebar item,Yes,PARTIAL,Integration,conversation-delete-cascade.dbtest.ts,Delete conversation cascades. `conversation-delete-cascade.dbtest.ts` proves real messages and artifacts do not survive conversation deletion.,"Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W46=""PASS"",100,0)","=IF(OR(W46=""FAIL"",W46=""BLOCKED""),0,ROUND(O46*0.7+P46*0.3,0))",=100-Q46,"=IF(OR(W46=""FAIL"",W46=""BLOCKED""),""BLOCKED"",IF(AND(J46=""COMPLETE"",W46=""PASS""),""DONE"",IF(J46=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J46=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,46,3 Chat and conversations,Both,P2,Copy assistant reply,Use the copy action on an assistant message,Pasting into another app yields the expected message text,Yes,COMPLETE,E2E,e2e/chat-actions.spec.ts,"Copy assistant reply. `e2e/chat-actions.spec.ts` clicks Copy on a rendered assistant reply, crosses production IPC, verifies visible success feedback, and reads the exact expected message from the real macOS clipboard.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W47=""PASS"",100,0)","=IF(OR(W47=""FAIL"",W47=""BLOCKED""),0,ROUND(O47*0.7+P47*0.3,0))",=100-Q47,"=IF(OR(W47=""FAIL"",W47=""BLOCKED""),""BLOCKED"",IF(AND(J47=""COMPLETE"",W47=""PASS""),""DONE"",IF(J47=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J47=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,,Real clipboard boundary @@ -130,7 +130,7 @@ Release,Journey ID,Phase,Tier,Priority,Manual test,Exact manual steps,Expected r 0.0.40,129,11 Settings privacy licensing and updates,Both,P1,Runtime residency toggles persist,Change image STT and TTS residency then relaunch,Each setting persists and actual loading behavior matches it,Yes,COMPLETE,E2E + Integration,e2e/settings-residency.spec.ts,"Runtime residency toggles persist. `e2e/settings-residency.spec.ts` changes image, STT, and TTS through the real Settings controls, verifies production IPC, fully relaunches Electron, and verifies all values reload. The SQLite and runtime-manager integrations prove that same map controls persistence and re-warm behavior.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W130=""PASS"",100,0)","=IF(OR(W130=""FAIL"",W130=""BLOCKED""),0,ROUND(O130*0.7+P130*0.3,0))",=100-Q130,"=IF(OR(W130=""FAIL"",W130=""BLOCKED""),""BLOCKED"",IF(AND(J130=""COMPLETE"",W130=""PASS""),""DONE"",IF(J130=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J130=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,, 0.0.40,130,11 Settings privacy licensing and updates,Both,P1,Chat residency stays required,Open runtime residency settings,Chat model displays in-memory required and cannot be toggled off,Yes,COMPLETE,E2E + Integration,e2e/settings-residency.spec.ts,"Chat residency stays required. `e2e/settings-residency.spec.ts` verifies the production switch stays checked and disabled before and after relaunch, while the real SQLite integration proves an on-demand write is normalized back to `resident`.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W131=""PASS"",100,0)","=IF(OR(W131=""FAIL"",W131=""BLOCKED""),0,ROUND(O131*0.7+P131*0.3,0))",=100-Q131,"=IF(OR(W131=""FAIL"",W131=""BLOCKED""),""BLOCKED"",IF(AND(J131=""COMPLETE"",W131=""PASS""),""DONE"",IF(J131=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J131=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,, 0.0.40,131,11 Settings privacy licensing and updates,Both,P1,Resource mode applies,Choose Conservative or another available resource preset,The selected limits and model behavior apply without freezing the UI,Yes,PARTIAL,E2E + Integration + Contract/package gate,resource-mode.integration.test.ts,"Resource mode applies. `resource-mode.integration.test.ts` drives all three presets through the real LLM settings owner, disk persistence, fresh-service launch arguments, recommendation, and setup planner with only host RAM controlled; the Electron tour proves selection stays responsive and the sizing guards enforce memory clamps.","ts` drives all three presets through the real LLM settings owner, disk persistence, fresh-service launch arguments, recommendation, and setup planner with only host RAM controlled; the Electron tour proves selection stays responsive and the sizing guards enforce memory clamps.",80,"=IF(W132=""PASS"",100,0)","=IF(OR(W132=""FAIL"",W132=""BLOCKED""),0,ROUND(O132*0.7+P132*0.3,0))",=100-Q132,"=IF(OR(W132=""FAIL"",W132=""BLOCKED""),""BLOCKED"",IF(AND(J132=""COMPLETE"",W132=""PASS""),""DONE"",IF(J132=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J132=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: ts` drives all three presets through the real LLM settings owner, disk persistence, fresh-service launch arguments, recommendation, and setup planner with only host RAM controlled; the Electron tour proves selection stays responsive and the sizing guards enforce memory clamps.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, -0.0.40,132,11 Settings privacy licensing and updates,Both,P0,Settings survive relaunch,Change several model capture privacy update and Pro settings then quit fully,Every changed value restores from the owning store,Yes,PARTIAL,Integration + Contract/package gate,settings-persistence.dbtest.ts,"Settings survive relaunch. Existing journeys 129 and 131 cover model residency and resource settings across relaunch. Core and Pro `settings-persistence.dbtest.ts` tests add the other owning stores: they change software-update, capture-privacy, identity, and proactive delivery settings over real encrypted SQLite, close the database, reload every Off Grid module, rehydrate each owner, and verify every value restores.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",75,"=IF(W133=""PASS"",100,0)","=IF(OR(W133=""FAIL"",W133=""BLOCKED""),0,ROUND(O133*0.7+P133*0.3,0))",=100-Q133,"=IF(OR(W133=""FAIL"",W133=""BLOCKED""),""BLOCKED"",IF(AND(J133=""COMPLETE"",W133=""PASS""),""DONE"",IF(J133=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J133=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, +0.0.40,132,11 Settings privacy licensing and updates,Both,P0,Settings survive relaunch,Change several model capture privacy update and Pro settings then quit fully,Every changed value restores from the owning store,Yes,PARTIAL,Integration + Contract/package gate,settings-persistence.dbtest.ts,"Settings survive relaunch. Existing journeys 129 and 131 cover model residency and resource settings across relaunch. Core and Pro `settings-persistence.dbtest.ts` tests add the other owning stores: they change software-update, capture-privacy, identity, and proactive delivery settings over real encrypted SQLite, close the database, reload every Off Grid AI module, rehydrate each owner, and verify every value restores.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",75,"=IF(W133=""PASS"",100,0)","=IF(OR(W133=""FAIL"",W133=""BLOCKED""),0,ROUND(O133*0.7+P133*0.3,0))",=100-Q133,"=IF(OR(W133=""FAIL"",W133=""BLOCKED""),""BLOCKED"",IF(AND(J133=""COMPLETE"",W133=""PASS""),""DONE"",IF(J133=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J133=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,133,11 Settings privacy licensing and updates,Both,P1,Storage usage is truthful,Open Storage after downloading models and creating artifacts,Reported totals and per-category sizes roughly match files on disk,Yes,PARTIAL,Integration,storage-usage.integration.dbtest.ts,"Storage usage is truthful. `storage-usage.integration.dbtest.ts` writes exact synthetic model, capture, meeting, image, artifact, and thumbnail byte counts to a temp profile, then proves production storage owners and the rendered Storage/Data Privacy panels report their real totals, categories, models, and orphaned partials.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W134=""PASS"",100,0)","=IF(OR(W134=""FAIL"",W134=""BLOCKED""),0,ROUND(O134*0.7+P134*0.3,0))",=100-Q134,"=IF(OR(W134=""FAIL"",W134=""BLOCKED""),""BLOCKED"",IF(AND(J134=""COMPLETE"",W134=""PASS""),""DONE"",IF(J134=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J134=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,134,11 Settings privacy licensing and updates,Both,P0,Clear cache preserves user data,Use the cache cleanup control then reopen the app,Ephemeral cache is removed while chats projects models and vault remain,Yes,PARTIAL,Integration,cache-cleanup.integration.test.ts,"Clear cache preserves user data. `cache-cleanup.integration.test.ts` and the rendered Storage journey exercise the production control through IPC and prove its allowlist can reach only Electron's `cache` data type. Chats, projects, models, vault, settings, entitlement, and unknown app files are unreachable by construction; success and failure states are both visible.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W135=""PASS"",100,0)","=IF(OR(W135=""FAIL"",W135=""BLOCKED""),0,ROUND(O135*0.7+P135*0.3,0))",=100-Q135,"=IF(OR(W135=""FAIL"",W135=""BLOCKED""),""BLOCKED"",IF(AND(J135=""COMPLETE"",W135=""PASS""),""DONE"",IF(J135=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J135=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,, 0.0.40,135,11 Settings privacy licensing and updates,Both,P0,Delete category is scoped,Delete one selected personal-data category,Only that category disappears; unrelated stores and credentials remain unless explicitly included,Yes,PARTIAL,Integration,,"Delete category is scoped. The real SQLite/filesystem integration deletes the Chats category through `clearCategory` and proves memory, projects, connectors, encrypted tokens, models, and unrelated personal files remain.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W136=""PASS"",100,0)","=IF(OR(W136=""FAIL"",W136=""BLOCKED""),0,ROUND(O136*0.7+P136*0.3,0))",=100-Q136,"=IF(OR(W136=""FAIL"",W136=""BLOCKED""),""BLOCKED"",IF(AND(J136=""COMPLETE"",W136=""PASS""),""DONE"",IF(J136=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J136=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Use synthetic data @@ -141,7 +141,7 @@ Release,Journey ID,Phase,Tier,Priority,Manual test,Exact manual steps,Expected r 0.0.40,140,11 Settings privacy licensing and updates,Pro,P0,Offline entitlement behavior,Activate online; quit; disable network; reopen,The signed cached entitlement follows policy without exposing Pro on an unentitled profile,Yes,PARTIAL,Integration,,"Offline entitlement behavior. The licensing integration activates a lifetime entitlement, reloads with its network boundary unavailable, and proves the OS-protected encrypted cache remains entitled while a fresh profile, a forged plaintext cache, and an expired entitlement stay locked.","Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",55,"=IF(W141=""PASS"",100,0)","=IF(OR(W141=""FAIL"",W141=""BLOCKED""),0,ROUND(O141*0.7+P141*0.3,0))",=100-Q141,"=IF(OR(W141=""FAIL"",W141=""BLOCKED""),""BLOCKED"",IF(AND(J141=""COMPLETE"",W141=""PASS""),""DONE"",IF(J141=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J141=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Pro artifact only 0.0.40,141,11 Settings privacy licensing and updates,Both,P1,Core and Pro override behavior,Launch dev builds with OFFGRID_PRO=0 and OFFGRID_PRO=1,Overrides force the documented free and Pro states without changing persisted entitlement,Yes,PARTIAL,Integration,,Core and Pro override behavior. The licensing integration applies both development overrides through the production bootstrap seam and proves neither mutates the encrypted persisted entitlement; a core build remains incapable of force-loading Pro implementation code.,"Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W142=""PASS"",100,0)","=IF(OR(W142=""FAIL"",W142=""BLOCKED""),0,ROUND(O142*0.7+P142*0.3,0))",=100-Q142,"=IF(OR(W142=""FAIL"",W142=""BLOCKED""),""BLOCKED"",IF(AND(J142=""COMPLETE"",W142=""PASS""),""DONE"",IF(J142=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J142=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Development check only 0.0.40,142,11 Settings privacy licensing and updates,Both,P1,Manual update check,Settings -> Check for updates on stable,The current or available version is reported without a stuck checking state,Yes,PARTIAL,Integration,update-check.integration.dbtest.ts,"Manual update check. `update-check.integration.dbtest.ts` drives the rendered Settings action through production updater IPC, real encrypted update preferences, and deterministic updater events, proving current `0.0.103`, available `0.0.104`, and offline error states all leave Checking, preserve the stable channel, and never call install without approval.","Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",55,"=IF(W143=""PASS"",100,0)","=IF(OR(W143=""FAIL"",W143=""BLOCKED""),0,ROUND(O143*0.7+P143*0.3,0))",=100-Q143,"=IF(OR(W143=""FAIL"",W143=""BLOCKED""),""BLOCKED"",IF(AND(J143=""COMPLETE"",W143=""PASS""),""DONE"",IF(J143=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J143=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Packaged build only -0.0.40,143,11 Settings privacy licensing and updates,Both,P1,Update channel persists,Toggle nightly builds then relaunch,The selected stable or beta channel remains selected and checks that channel,Yes,PARTIAL,Integration,src/main/__tests__/settings-persistence.dbtest.ts,"Update channel persists. `src/main/__tests__/settings-persistence.dbtest.ts` changes the channel through the production update IPC handler, closes the encrypted database, reloads every Off Grid module, and verifies the fresh update-preferences handler restores the beta channel.","Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",55,"=IF(W144=""PASS"",100,0)","=IF(OR(W144=""FAIL"",W144=""BLOCKED""),0,ROUND(O144*0.7+P144*0.3,0))",=100-Q144,"=IF(OR(W144=""FAIL"",W144=""BLOCKED""),""BLOCKED"",IF(AND(J144=""COMPLETE"",W144=""PASS""),""DONE"",IF(J144=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J144=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Packaged build only +0.0.40,143,11 Settings privacy licensing and updates,Both,P1,Update channel persists,Toggle nightly builds then relaunch,The selected stable or beta channel remains selected and checks that channel,Yes,PARTIAL,Integration,src/main/__tests__/settings-persistence.dbtest.ts,"Update channel persists. `src/main/__tests__/settings-persistence.dbtest.ts` changes the channel through the production update IPC handler, closes the encrypted database, reloads every Off Grid AI module, and verifies the fresh update-preferences handler restores the beta channel.","Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",55,"=IF(W144=""PASS"",100,0)","=IF(OR(W144=""FAIL"",W144=""BLOCKED""),0,ROUND(O144*0.7+P144*0.3,0))",=100-Q144,"=IF(OR(W144=""FAIL"",W144=""BLOCKED""),""BLOCKED"",IF(AND(J144=""COMPLETE"",W144=""PASS""),""DONE"",IF(J144=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J144=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Packaged build only 0.0.40,144,12 Resilience and desktop polish,Both,P0,Local use works offline,Disable network after required models are installed,Chat image OCR replay search dictation and vault remain local; network features fail clearly,Yes,PARTIAL,Integration,,"Local use works offline. A shared offline boundary rejects and records every outbound request while preserving real loopback transports. Connected Core and Pro integrations prove local chat, image generation, Vision OCR, replay, SQLite/FTS search, dictation, and KDBX/Argon2 vault operations remain usable with zero unexpected egress; the real model manager also proves a network-only download fails clearly and retries without corrupting installed state.","Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",55,"=IF(W145=""PASS"",100,0)","=IF(OR(W145=""FAIL"",W145=""BLOCKED""),0,ROUND(O145*0.7+P145*0.3,0))",=100-Q145,"=IF(OR(W145=""FAIL"",W145=""BLOCKED""),""BLOCKED"",IF(AND(J145=""COMPLETE"",W145=""PASS""),""DONE"",IF(J145=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J145=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the listed steps on the exact installed 0.0.40 artifact with the real macOS, hardware, network, or external-system boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,, 0.0.40,145,12 Resilience and desktop polish,Both,P0,Cold relaunch after forced quit,Force quit during non-destructive activity then reopen,The app boots without a white screen or permanently busy state and committed data remains,Yes,COMPLETE,E2E,e2e/chat-memory.spec.ts,"Cold relaunch after forced quit. `e2e/chat-memory.spec.ts` kills the real Electron main process during non-destructive chat activity, waits for process exit, reopens the same profile, and verifies clean boot, preload availability, usable input, and durable committed chat data.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",100,"=IF(W146=""PASS"",100,0)","=IF(OR(W146=""FAIL"",W146=""BLOCKED""),0,ROUND(O146*0.7+P146*0.3,0))",=100-Q146,"=IF(OR(W146=""FAIL"",W146=""BLOCKED""),""BLOCKED"",IF(AND(J146=""COMPLETE"",W146=""PASS""),""DONE"",IF(J146=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J146=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Execute and attach evidence for the remaining manual boundary: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",HIGH,The strict ledger confirms the decisive production collaborators run through the real application seam.,NOT RUN,,,,, 0.0.40,146,12 Resilience and desktop polish,Both,P1,Model ports are single-owner,Start one app instance then attempt a second development or capture instance,The conflict is diagnosed clearly and does not masquerade as model corruption,Yes,PARTIAL,Integration,model-port-ownership.integration.test.ts,"Model ports are single-owner. `model-port-ownership.integration.test.ts` starts a real foreign parent with the only fake native llama process on production port 8439, then proves the production contender preserves that live owner, starts no second engine, reports its own Chat health as Down rather than borrowing the other process's readiness, and exposes the actionable `port_in_use` reason while the first engine remains responsive.","Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",65,"=IF(W147=""PASS"",100,0)","=IF(OR(W147=""FAIL"",W147=""BLOCKED""),0,ROUND(O147*0.7+P147*0.3,0))",=100-Q147,"=IF(OR(W147=""FAIL"",W147=""BLOCKED""),""BLOCKED"",IF(AND(J147=""COMPLETE"",W147=""PASS""),""DONE"",IF(J147=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J147=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the listed visible journey on the exact installed 0.0.40 artifact and inspect state, persistence, errors, and pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Do not use this as a normal two-instance workflow @@ -189,7 +189,7 @@ Release,Journey ID,Phase,Tier,Priority,Manual test,Exact manual steps,Expected r 0.0.40,NR-33,18 P3 compatibility,Both,P3,Unicode and space-containing paths work,Install and run from paths containing spaces and use a disposable macOS account with a non-ASCII username; download/import models and create logs/artifacts.,"Models, helpers, logs, database, updates, exports, and native processes work without shell-escaping or path-encoding failures.",Yes,PARTIAL,Package gate,Packaged-helper tests cover an application path with spaces but not a Unicode home directory.,Space handling is partially covered.,Run under a real Unicode account/home path.,40,"=IF(W189=""PASS"",100,0)","=IF(OR(W189=""FAIL"",W189=""BLOCKED""),0,ROUND(O189*0.7+P189*0.3,0))",=100-Q189,"=IF(OR(W189=""FAIL"",W189=""BLOCKED""),""BLOCKED"",IF(AND(J189=""COMPLETE"",W189=""PASS""),""DONE"",IF(J189=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J189=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run under a real Unicode account/home path.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,First explicit P3 compatibility case. 0.0.40,NR-34,18 P3 compatibility,Both,P3,"Timezone, locale clock, and DST changes preserve chronology","Change timezone and 12/24-hour preference while running; use synthetic timestamps around a DST boundary; inspect meetings, capture, Day, retention, and filenames.","Absolute order and retention remain correct, labels update truthfully, and no event is duplicated, skipped, or assigned to the wrong day.",Yes,PARTIAL,Integration,Reflect/time-range and persistence tests cover narrower fixed-time cases.,"Time calculations have integration evidence, but live OS timezone/DST changes do not.",Exercise actual timezone and locale changes with the installed app.,55,"=IF(W190=""PASS"",100,0)","=IF(OR(W190=""FAIL"",W190=""BLOCKED""),0,ROUND(O190*0.7+P190*0.3,0))",=100-Q190,"=IF(OR(W190=""FAIL"",W190=""BLOCKED""),""BLOCKED"",IF(AND(J190=""COMPLETE"",W190=""PASS""),""DONE"",IF(J190=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J190=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Exercise actual timezone and locale changes with the installed app.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Use synthetic data only. 0.0.40,NR-35,18 P3 compatibility,Both,P3,Live light/dark appearance changes keep every layer legible,"Switch macOS appearance while dialogs, menus, lightboxes, side panels, native prompts, and popup windows are open.","Theme, title bar, focus, icons, contrast, overlays, and native integration update without restart, blank content, or unreadable combinations.",Yes,PARTIAL,Unit/theme contract,src/renderer/src/__tests__/theme.test.ts,Theme selection logic is covered; live multi-window pixels are not.,Inspect every window and transient layer during real macOS appearance changes.,40,"=IF(W191=""PASS"",100,0)","=IF(OR(W191=""FAIL"",W191=""BLOCKED""),0,ROUND(O191*0.7+P191*0.3,0))",=100-Q191,"=IF(OR(W191=""FAIL"",W191=""BLOCKED""),""BLOCKED"",IF(AND(J191=""COMPLETE"",W191=""PASS""),""DONE"",IF(J191=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J191=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Inspect every window and transient layer during real macOS appearance changes.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,P3 visual compatibility. -0.0.40,PR-01,"19 Pro entity, action, and service guardrails",Pro,P0,No Pro background service activates before entitlement,"Start the production app on a fresh unentitled profile and watch processes, global shortcuts, clipboard, notifications, capture, meetings, tray, media ports, and Pro database writes; then activate and restart.","Before entitlement no Pro service, shortcut, watcher, capture, or write activates; after entitlement each service starts once under its owning lifecycle.",Yes,PARTIAL,Production composition + real SQLite integration,pro/main/__tests__/service-activation.integration.dbtest.ts; src/main/__tests__/release-packaging.integration.test.ts,"A fresh packaged-mode profile starts no Pro service. A real Keygen-boundary activation persists an encrypted entitlement; after every Off Grid module reloads, the production Core loader invokes actual Pro activateMain, migrates real SQLite, starts the owned focus/shortcut/tray services, and application shutdown tears them down. Clearing entitlement and reloading stays locked with no service start.",Observe the exact installed locked and entitled artifact against real OS services.,65,"=IF(W192=""PASS"",100,0)","=IF(OR(W192=""FAIL"",W192=""BLOCKED""),0,ROUND(O192*0.7+P192*0.3,0))",=100-Q192,"=IF(OR(W192=""FAIL"",W192=""BLOCKED""),""BLOCKED"",IF(AND(J192=""COMPLETE"",W192=""PASS""),""DONE"",IF(J192=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J192=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Observe the exact installed locked and entitled artifact against real OS services.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Privacy and open-core release blocker. +0.0.40,PR-01,"19 Pro entity, action, and service guardrails",Pro,P0,No Pro background service activates before entitlement,"Start the production app on a fresh unentitled profile and watch processes, global shortcuts, clipboard, notifications, capture, meetings, tray, media ports, and Pro database writes; then activate and restart.","Before entitlement no Pro service, shortcut, watcher, capture, or write activates; after entitlement each service starts once under its owning lifecycle.",Yes,PARTIAL,Production composition + real SQLite integration,pro/main/__tests__/service-activation.integration.dbtest.ts; src/main/__tests__/release-packaging.integration.test.ts,"A fresh packaged-mode profile starts no Pro service. A real Keygen-boundary activation persists an encrypted entitlement; after every Off Grid AI module reloads, the production Core loader invokes actual Pro activateMain, migrates real SQLite, starts the owned focus/shortcut/tray services, and application shutdown tears them down. Clearing entitlement and reloading stays locked with no service start.",Observe the exact installed locked and entitled artifact against real OS services.,65,"=IF(W192=""PASS"",100,0)","=IF(OR(W192=""FAIL"",W192=""BLOCKED""),0,ROUND(O192*0.7+P192*0.3,0))",=100-Q192,"=IF(OR(W192=""FAIL"",W192=""BLOCKED""),""BLOCKED"",IF(AND(J192=""COMPLETE"",W192=""PASS""),""DONE"",IF(J192=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J192=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Observe the exact installed locked and entitled artifact against real OS services.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Privacy and open-core release blocker. 0.0.40,PR-02,"19 Pro entity, action, and service guardrails",Pro,P1,Manual to-do creation enriches context without pollution,"Jot a synthetic to-do naming a known entity, implied priority, and due date; wait for enrichment; quit and reopen.","The to-do appears immediately, enrichment terminates, preserves the original intent, adds justified entity/date/priority/source context, creates no duplicate entity, and survives relaunch.",Yes,PARTIAL,Rendered integration + real SQLite + local HTTP,pro/main/__tests__/manual-todo-journey.integration.dbtest.ts,"The real Actions screen, Jot service, action and entity services, and disposable SQLite profile run together. The journey proves immediate raw persistence, visible enrichment progress, success and malformed-model recovery, canonical entity linking, immutable source evidence, and close/reopen persistence; only the local model HTTP response is controlled.",Repeat through the packaged Electron IPC bridge and the selected production local model.,70,"=IF(W193=""PASS"",100,0)","=IF(OR(W193=""FAIL"",W193=""BLOCKED""),0,ROUND(O193*0.7+P193*0.3,0))",=100-Q193,"=IF(OR(W193=""FAIL"",W193=""BLOCKED""),""BLOCKED"",IF(AND(J193=""COMPLETE"",W193=""PASS""),""DONE"",IF(J193=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J193=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat through the packaged Electron IPC bridge and the selected production local model.",HIGH,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Guards entity and contextual to-do quality. 0.0.40,PR-03,"19 Pro entity, action, and service guardrails",Pro,P1,Contextual to-do lifecycle preserves source and entity links,"Move a sourced to-do through Open, Waiting, Done, Dismissed, Undo, entity filtering, entity detail, and a stale/deleted source target; relaunch between transitions.","Status/counts persist, Undo restores correctly, entity and provenance stay attached, filters show the same record, and stale targets fail closed without opening another item.",Yes,PARTIAL,Integration + rendered integration,pro/main/__tests__/action-lifecycle-renderer.integration.dbtest.ts; pro/main/__tests__/approval-relaunch.dbtest.ts; pro/main/__tests__/entity-action-guardrails.integration.dbtest.ts,"The real Actions UI, production action/entity services, and disposable SQLite profile now run together through Open, Waiting, Done, Dismissed, Restore, and toast Undo. Counts, entity filters, exact provenance, stale-target failure, delegated visibility, and close/reopen persistence follow the same canonical record.",Repeat the lifecycle through the packaged Electron IPC bridge and verify native notification entry points.,70,"=IF(W194=""PASS"",100,0)","=IF(OR(W194=""FAIL"",W194=""BLOCKED""),0,ROUND(O194*0.7+P194*0.3,0))",=100-Q194,"=IF(OR(W194=""FAIL"",W194=""BLOCKED""),""BLOCKED"",IF(AND(J194=""COMPLETE"",W194=""PASS""),""DONE"",IF(J194=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J194=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Repeat the lifecycle through the packaged Electron IPC bridge and verify native notification entry points.",HIGH,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Broader than canonical journey #113. 0.0.40,PR-04,"19 Pro entity, action, and service guardrails",Pro,P1,Entity correction and extension preserve the guarded record,"Using synthetic multi-source evidence, rename/retype, add/remove alias and fact, set photo, hide/unhide, merge, split/reassign an observation, re-synthesize, then relaunch and open every linked source/action.","One canonical entity owns the right evidence and relationships, rollback/correction is possible, no self/noise duplicate is introduced, and every linked surface follows the surviving ID.",Yes,PARTIAL,Rendered integration + real SQLite/filesystem,pro/main/__tests__/entity-context-pipeline.integration.dbtest.ts; pro/main/__tests__/entity-corrections-renderer.integration.dbtest.ts; pro/main/__tests__/entity-action-guardrails.integration.dbtest.ts; pro/main/crm/__tests__/resolve.integration.test.ts; pro/main/crm/__tests__/entity-photo.integration.test.ts,"The real screen extractor, dictation memory sink, meeting processor, manual-todo owner, Entities UI, correction services, disposable SQLite profile, and app-owned photo filesystem run together. Screen, voice, meeting, and manual evidence converges on one guarded entity; three contextual actions retain exact source provenance; rename updates every linked action projection atomically; and the corrected graph survives close/reopen. The suite also covers retype, aliases, photo import/replacement/rollback, hide/unhide, merge, observation reassign/unlink, stale-survivor rejection, and transaction rollback.",Add and integrate rendered fact add/remove and observation split controls; neither is invoked by the shipped Entity UI today. Then repeat through packaged Electron IPC.,70,"=IF(W195=""PASS"",100,0)","=IF(OR(W195=""FAIL"",W195=""BLOCKED""),0,ROUND(O195*0.7+P195*0.3,0))",=100-Q195,"=IF(OR(W195=""FAIL"",W195=""BLOCKED""),""BLOCKED"",IF(AND(J195=""COMPLETE"",W195=""PASS""),""DONE"",IF(J195=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J195=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Add and integrate rendered fact add/remove and observation split controls; neither is invoked by the shipped Entity UI today. Then repeat through packaged Electron IPC.",HIGH,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Entity is a protected cross-product unit; this is a release-significant guardrail. @@ -209,3 +209,7 @@ Release,Journey ID,Phase,Tier,Priority,Manual test,Exact manual steps,Expected r 0.0.40,CR-06,21 Core secondary surfaces and controls,Both,P2,"Model catalog search, filters, detail, and reset stay coherent","At 1280 and 1800 widths, combine modality tabs, text search, credibility, size, sort, RAM buckets, and use-case filters; open/close details; download/activate/delete; reset filters.","Counts and sections remain truthful, no stale/duplicate card appears, detail reflects the selected model, actions stay adjacent and reachable, and reset returns the complete dense grid.",Yes,PARTIAL,E2E + rendered integration,e2e/desktop-polish.spec.ts; src/renderer/src/components/__tests__/desktop-polish.integration.test.tsx,"Responsive density, focus, and representative model controls are automated.",Manually combine filters/actions against real catalog and installed models and inspect pixels.,80,"=IF(W209=""PASS"",100,0)","=IF(OR(W209=""FAIL"",W209=""BLOCKED""),0,ROUND(O209*0.7+P209*0.3,0))",=100-Q209,"=IF(OR(W209=""FAIL"",W209=""BLOCKED""),""BLOCKED"",IF(AND(J209=""COMPLETE"",W209=""PASS""),""DONE"",IF(J209=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J209=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Manually combine filters/actions against real catalog and installed models and inspect pixels.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Expands canonical density journey #31. 0.0.40,CR-07,21 Core secondary surfaces and controls,Both,P1,Connector lifecycle controls stay synchronized,"Add one stdio and one remote connector; test, enable/disable, sync with and without a query, inspect tools, force an error, reconnect, delete, and relaunch.","Status and tools match the real connector, disabled connectors cannot run, sync is bounded, errors recover without blocking others, deletion removes owned secrets, and state persists exactly once.",Yes,PARTIAL,Database/process integration,integration-tests/mcp-connector-setup.dbtest.ts; src/main/__tests__/mcp-connector-tool-extension.dbtest.ts; src/main/__tests__/mcp-timeout.dbtest.ts; src/main/__tests__/connector-delete-secrets.dbtest.ts,"Real connector repository, encryption, stdio child, discovery, tools, timeout, and deletion run with remote boundaries controlled.",Drive every rendered control and real remote/OAuth provider through relaunch.,65,"=IF(W210=""PASS"",100,0)","=IF(OR(W210=""FAIL"",W210=""BLOCKED""),0,ROUND(O210*0.7+P210*0.3,0))",=100-Q210,"=IF(OR(W210=""FAIL"",W210=""BLOCKED""),""BLOCKED"",IF(AND(J210=""COMPLETE"",W210=""PASS""),""DONE"",IF(J210=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J210=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Drive every rendered control and real remote/OAuth provider through relaunch.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Complements canonical connector tool journeys. 0.0.40,CR-08,21 Core secondary surfaces and controls,Both,P0,Normal quit leaves no owned helper or capture resource,"After using chat, image, TTS, STT, gateway, media, meeting, dictation, clipboard, and capture, quit normally and inspect Activity Monitor, microphone/screen indicators, ports 7878/7879/8439, and mounted images.","Every owned process, listener, timer, shortcut, audio/capture resource, and DMG mount is released; committed data reopens and no feature restarts before entitlement/settings allow.",Yes,PARTIAL,Production composition integration,integration-tests/application-shutdown.integration.test.ts; pro/main/__tests__/service-shutdown.integration.test.ts; pro/main/__tests__/system-lifecycle.integration.test.ts; pro/main/__tests__/service-activation.integration.dbtest.ts; pro/main/__tests__/clipboard-popup-journey.dbtest.ts,"Core now has one idempotent shutdown owner for runtime engines, TTS, media, gateway, listeners, and Pro activation. Pro has one disposer for capture, meetings, dictation, clipboard, shortcuts, timers, tray, and console resources. Integration coverage proves reverse-order cleanup and failure isolation without skipping later owners.","Inspect actual host processes, permission indicators, shortcuts, ports, and mounted images after one full signed-app session.",65,"=IF(W211=""PASS"",100,0)","=IF(OR(W211=""FAIL"",W211=""BLOCKED""),0,ROUND(O211*0.7+P211*0.3,0))",=100-Q211,"=IF(OR(W211=""FAIL"",W211=""BLOCKED""),""BLOCKED"",IF(AND(J211=""COMPLETE"",W211=""PASS""),""DONE"",IF(J211=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J211=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Inspect actual host processes, permission indicators, shortcuts, ports, and mounted images after one full signed-app session.",MEDIUM,"Useful real integration exists, but at least one rendered, persistence, runtime, or full-journey seam is still substituted or separate.",NOT RUN,,,,,Final release stop condition. +0.0.40,PR-13,20 Pro secondary surfaces and controls,Pro,P0,Google OAuth client connects Gmail and Calendar,Create a Web application OAuth client; enable the Gmail and Google Calendar APIs; add http://127.0.0.1:33418/callback; approve the test user; save the client; connect and test both connectors; relaunch and reconnect.,The setup card says Your Google client; both connectors show Connected; both tests succeed before and after relaunch.,Yes,PARTIAL,Integration + rendered test,pro/renderer/__tests__/GoogleClientSetup.integration.test.tsx; pro/main/__tests__/google-client.test.ts; pro/main/__tests__/google-ipc.test.ts; pro/main/__tests__/google-rest-connector.integration.dbtest.ts; src/main/__tests__/mcp-oauth-loopback.integration.test.ts,"The client setup, protected credential owner, Google REST connector, callback state, and connector IPC run through real application seams with the Google boundary controlled.","Use a real Google Cloud project, system browser, consent or test-user approval, Gmail account, and Calendar account in the exact installed app.",55,"=IF(W212=""PASS"",100,0)","=IF(OR(W212=""FAIL"",W212=""BLOCKED""),0,ROUND(O212*0.7+P212*0.3,0))",=100-Q212,"=IF(OR(W212=""FAIL"",W212=""BLOCKED""),""BLOCKED"",IF(AND(J212=""COMPLETE"",W212=""PASS""),""DONE"",IF(J212=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J212=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Use a real Google Cloud project, system browser, consent or test-user approval, Gmail account, and Calendar account in the exact installed app.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,"Redact the Client ID, secret, tokens, mail, and calendar data." +0.0.40,PR-14,20 Pro secondary surfaces and controls,Pro,P0,Discoverable and Find nearby stay independent,Pair a second device; cold-launch with Discoverable off; turn each visibility control off in turn; send a small record on the active session.,Hidden applies before startup advertising; Discoverable off does not stop browsing; Find nearby off does not stop advertising; the active encrypted session stays active.,Yes,PARTIAL,Integration + native helper contract,pro/main/sync/__tests__/macos-proximity.test.ts; pro/main/__tests__/identity-discovery.test.ts; pro/main/sync/__tests__/state-bridge.test.ts; pro/renderer/__tests__/sync-state.test.ts,"The owning state, Electron bridge, macOS helper contract, and rendered visibility state prove separate browse and advertise lifecycles and Hidden startup order.",Run both controls and a cold launch with the exact installed app and a second physical device on the real network and radio boundary.,55,"=IF(W213=""PASS"",100,0)","=IF(OR(W213=""FAIL"",W213=""BLOCKED""),0,ROUND(O213*0.7+P213*0.3,0))",=100-Q213,"=IF(OR(W213=""FAIL"",W213=""BLOCKED""),""BLOCKED"",IF(AND(J213=""COMPLETE"",W213=""PASS""),""DONE"",IF(J213=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J213=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run both controls and a cold launch with the exact installed app and a second physical device on the real network and radio boundary.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,"Record both device names, OS versions, and exact build commits." +0.0.40,PR-15,20 Pro secondary surfaces and controls,Pro,P0,Private endpoint and custom Sync port reconnect,Save the paired device private IP address or machine name; connect; set one disposable non-default Sync port on every device; restart all apps; restore 37878.,The same paired identity reconnects by the saved endpoint; the custom port works only when all devices match; the default port works after restore.,Yes,PARTIAL,Database + integration contract,pro/main/sync/__tests__/sync-prefs-rules.test.ts; pro/main/sync/__tests__/state-bridge-persistence.test.ts; pro/main/__tests__/production-sync.dbtest.ts,"Port and private-host parsing, persistence, runtime state, and production Sync composition run through automated application seams.",Reconnect real paired devices through a private IP or machine name and one matching non-default port on every installed app.,55,"=IF(W214=""PASS"",100,0)","=IF(OR(W214=""FAIL"",W214=""BLOCKED""),0,ROUND(O214*0.7+P214*0.3,0))",=100-Q214,"=IF(OR(W214=""FAIL"",W214=""BLOCKED""),""BLOCKED"",IF(AND(J214=""COMPLETE"",W214=""PASS""),""DONE"",IF(J214=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J214=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Reconnect real paired devices through a private IP or machine name and one matching non-default port on every installed app.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Use one disposable port from 1024 to 65535 and restore 37878. +0.0.40,PR-16,20 Pro secondary surfaces and controls,Pro,P0,Failed advertising stop keeps true state and retries,Use a diagnostic helper that rejects the first advertising stop; turn Discoverable off; remove the failure; turn it off again.,The first action reports failure and the runtime switch and saved value stay on; the retry stops advertising and all three states turn off.,Yes,PARTIAL,Integration + native helper contract,pro/main/sync/__tests__/macos-proximity.test.ts; pro/main/sync/__tests__/state-bridge.test.ts; pro/renderer/__tests__/sync-state.test.ts,"The helper contract, runtime owner, persistence boundary, and rendered state fail closed and permit a later retry.",Run the rejection and retry with the exact diagnostic helper in the installed app and inspect the real advertisement from a second device.,55,"=IF(W215=""PASS"",100,0)","=IF(OR(W215=""FAIL"",W215=""BLOCKED""),0,ROUND(O215*0.7+P215*0.3,0))",=100-Q215,"=IF(OR(W215=""FAIL"",W215=""BLOCKED""),""BLOCKED"",IF(AND(J215=""COMPLETE"",W215=""PASS""),""DONE"",IF(J215=""COMPLETE"",""MANUAL VERIFICATION LEFT"",IF(J215=""PARTIAL"",""AUTOMATION + MANUAL LEFT"",""NOT AUTOMATED""))))","Join the split automation seams into one production journey, then verify: Run the rejection and retry with the exact diagnostic helper in the installed app and inspect the real advertisement from a second device.",LOW,"Useful automation exists, but it does not prove the decisive installed, native, hardware, network, or external-system boundary.",NOT RUN,,,,,Do not replace this check with a mocked renderer result. diff --git a/docs/RELEASE_READINESS_WORKFLOW.md b/docs/RELEASE_READINESS_WORKFLOW.md index 8c989780..5d9bd0d3 100644 --- a/docs/RELEASE_READINESS_WORKFLOW.md +++ b/docs/RELEASE_READINESS_WORKFLOW.md @@ -30,7 +30,7 @@ node scripts/generate-release-readiness-checklist.mjs npx vitest run src/main/__tests__/p0-p2-coverage-ledger.test.ts ``` -The generator must produce 210 unique rows with one header row and 28 columns. The validation test +The generator must produce 214 unique rows with one header row and 28 columns. The validation test prevents canonical journey/status drift, duplicate IDs, invalid tiers/confidence, empty evidence explanations, and non-pending manual results in a newly generated sheet. @@ -40,12 +40,12 @@ as a new sheet. Do not enable automatic conversion of IDs such as `NR-01`, `PR-0 ## Status snapshot after the current integration hardening pass -- Total: 210 journeys - 21 `COMPLETE`, 179 `PARTIAL`, 10 `OPEN`. -- P0: 93 journeys - 59.7% automation coverage, 41.8% initial readiness before manual results. +- Total: 214 journeys - 21 `COMPLETE`, 183 `PARTIAL`, 10 `OPEN`. +- P0: 97 journeys - 60.2% automation coverage, 42.1% initial readiness before manual results. - P1: 93 journeys - 63.7% automation coverage, 44.6% initial readiness before manual results. - P2: 21 journeys - 71.2% automation coverage, 49.8% initial readiness before manual results. - P3: 3 journeys - 45.0% automation coverage, 31.5% initial readiness before manual results. -- Overall: 62.4% automation coverage, 43.7% initial readiness before manual results. +- Overall: 62.6% automation coverage, 43.8% initial readiness before manual results. - This pass closed the gateway and OAuth P0 automation gaps, added production composition coverage for TCC revocation and shutdown, replaced clipboard and chat tours with real persistence/runtime integrations, and added guarded entity graph, contextual Jot, and complete rendered action-state @@ -97,7 +97,7 @@ date, evidence, and the exact artifact/profile/device identity. 3. Record the exact test path and what it proves. Do not write only "tests pass". 4. Keep the remaining manual boundary specific enough that a tester can reproduce it. 5. Never promote a row to `COMPLETE` because a unit test, source assertion, rendered shell, fake - Off Grid service, or separate package probe exists. + Off Grid AI service, or separate package probe exists. 6. Regenerate the CSV and run the validation test in the same commit. 7. Update the relevant Core, Pro, or native audit when a finding is closed or a new gap is found. diff --git a/docs/SAFETY_REVIEW.md b/docs/SAFETY_REVIEW.md new file mode 100644 index 00000000..ca1af2c5 --- /dev/null +++ b/docs/SAFETY_REVIEW.md @@ -0,0 +1,90 @@ +# Safety review - the act pillar (R2-E1) + +The rails act on the user's behalf, and two of them (browser, vision) take +untrusted content as input: a web page or an on-screen app can display text +that tries to redirect the agent. This is the injection-resistance review for +the released rails. It records, per rail, what the threat is, what stops it, +and where that defense is tested - so a later change that weakens a defense +fails a test instead of shipping. + +The governing principle: **the model only proposes; the pipeline guarantees.** +Every mutation is a durable Action that gates for approval, binds its payload +by hash, executes once, and verifies. Injection cannot manufacture an approved +action out of nothing - it can only try to steer a task the user already +approved. So the defenses below are about bounding that steering, and about +never letting the agent cross an identity or payment boundary on its own. + +## The threats and the defenses, per rail + +### Semantic rail (calendar, reminders, mail, open) + +- **Threat:** low. The arguments come from the user's chat turn, not from + scraped content. The model fills a typed tool schema. +- **Defense:** the payload-hash gate - what the user approves is byte-for-byte + what runs; an edit re-binds and re-gates. Sends are `none_fuzzy` and single- + attempt, so a wrong verify can never double-send. +- **Tested:** `shared/packages/use` retry + machine tests (never-double-fire), + `use-runtime.integration.dbtest.ts` (real propose -> verify -> undo). + +### Browser rail (web_use) + +- **Threat:** high. The page is untrusted. Two attacks: (a) page text says + "ignore your task, do X"; (b) a page tries to get the agent to type + credentials or submit a payment. +- **Defenses:** + 1. **Page text is DATA, not instructions** - the shared visual prompt treats + page and app text as untrusted content and stays anchored to the user's task. + 2. **The identity boundary is enforced in the driver, not the prompt.** Typing + into a password / one-time-code field is _refused_ by `BrowserDriver.type` + with a takeover signal - no prompt injection can talk the agent past code + that refuses to run. Clicking a login field is allowed (that is how the + human takes over); credentials never enter the snapshot the model sees. + 3. **The step budget** bounds how far a fully-fooled model could be steered + before the task stops. + 4. **The watched pane** - the user sees every step and can take over or cancel. +- **Tested:** `browser-driver.test.ts` (the driver refuses identity fields and + dispatches nothing), `vision-task-graph.test.ts` (budget, Stop, and handoff), + `rail-injection-stance.test.ts` (the shared prompt contract), and + `page-script.test.ts` (the collector never includes a credential value). + +### Vision rail (computer_task) - supervised tier + +- **Threat:** highest. The model drives real synthetic input on the live + desktop from a screenshot, and the screenshot is untrusted (any app in view + can show adversarial text). +- **Defenses (layered; the structural ones are load-bearing):** + 1. **The user is watching and the guard is the override.** The kill switch + (Esc) is terminal and outranks everything; any user touch pauses until they + resume; a step budget halts a flailing model. `canActuate()` is re-checked + immediately before every dispatch, so an Esc mid-decision actuates nothing + more. + 2. **Credentials are a handoff, never typed.** The prompt makes any sign-in / + one-time-code / payment a `call_user`, and the agent is told on-screen text + is untrusted content. + 3. **Capability-gated OFF until it is real.** Actuation needs a native addon + + Accessibility/Screen-Recording entitlements; until those land the rail + refuses cleanly and `computer_task` is not offered to the model. The tier + ships labeled or not at all. +- **Tested:** `vision-guard.test.ts` (the kill switch is terminal and outranks a + pause; the budget halts), `vision-agent.test.ts` (re-check-before-dispatch: a + kill mid-decision actuates nothing), `rail-injection-stance.test.ts` (the + prompt contract). + +## Kill switch - the e2e note + +The kill switch is a global `Escape` shortcut wired in the vision host, and its +_logic_ (terminal halt, outranks pause, re-check before dispatch) is unit-tested +in `vision-guard`/`vision-agent`. The full end-to-end - a real keypress halting +a real actuation loop and being consumed - can only be exercised once actuation +is available (D2b) on a real machine, so it is part of the real-machine pass in +`WINDOWS_TEST_PLAN.md`, not the headless e2e tour. Until then there is nothing +to actuate, so there is nothing to halt. + +## Open items before the release (E2) + +- **Actuation + entitlements (D2b)** for the vision tier, then the kill-switch + e2e on a real machine, both platforms. +- **Real-machine click-through** for the browser and vision rails (CI proves + builds, not clicks) - `WINDOWS_TEST_PLAN.md`. +- **Release notes** honest about the supervised tier: what is verified, what is + best-effort, and that computer-use is off until actuation ships. diff --git a/docs/SESSION_HANDOFF_2026-08-11.md b/docs/SESSION_HANDOFF_2026-08-11.md index f4449cd2..4882a4bb 100644 --- a/docs/SESSION_HANDOFF_2026-08-11.md +++ b/docs/SESSION_HANDOFF_2026-08-11.md @@ -1,4 +1,4 @@ -# Off Grid — close the gaps in `desktop/docs/GAPS_BACKLOG.md` +# Off Grid AI — close the gaps in `desktop/docs/GAPS_BACKLOG.md` ## Branch rule — read first, do not get this wrong diff --git a/docs/SYNC_PLAN.md b/docs/SYNC_PLAN.md index 8120aa16..c801e5ce 100644 --- a/docs/SYNC_PLAN.md +++ b/docs/SYNC_PLAN.md @@ -1,6 +1,6 @@ -# Off Grid — Cross-Device Sync & Offload Plan +# Off Grid AI — Cross-Device Sync & Offload Plan -> **Vision:** every Off Grid device is a window onto _all_ of your information. +> **Vision:** every Off Grid AI device is a window onto _all_ of your information. > Open the phone, the laptop, the tablet — same chats, same projects, same > memory, same search — and when a more capable device is nearby, heavy work > (LLM inference, big search, media) transparently runs there. No cloud, no @@ -20,7 +20,7 @@ Status: **planning**. Nothing here is built yet except the pieces noted as ## 1. Principles (do not drift) - **Local-first, no cloud.** Devices talk **directly** over the LAN. Not a - single byte goes to a server we own. (Same posture as the rest of Off Grid.) + single byte goes to a server we own. (Same posture as the rest of Off Grid AI.) - **One encrypted session, reused for everything.** All cross-device traffic rides the existing `@offgrid/sync` NaCl-encrypted, paired channel. We do **not** open a second unauthenticated LAN port. @@ -113,7 +113,7 @@ gating for free. "In vicinity" is simply: the paired peer is visible on mDNS. ## 4. Cross-platform feasibility — four OSes, two codebases -Off Grid is **two products**: the Electron **desktop** (macOS + Windows) and the +Off Grid AI is **two products**: the Electron **desktop** (macOS + Windows) and the RN **mobile** app (iOS + Android). The mesh must span all four. Every sync primitive is already cross-platform, so this is two adapter implementations (one Node, one RN), not four. diff --git a/docs/WINDOWS_SUPPORT.md b/docs/WINDOWS_SUPPORT.md index c2ad8f87..12f8ca85 100644 --- a/docs/WINDOWS_SUPPORT.md +++ b/docs/WINDOWS_SUPPORT.md @@ -81,6 +81,28 @@ Do **not** expect it in GitHub Releases; this workflow deliberately doesn't publ --- +## Action rails & computer use — Windows status + +The three action rails live in **core** (`src/main/actions`, `src/main/vision`, `src/main/input`) +and are chosen per-platform through one seam (`use-runtime.ts` → `pickByPlatform`), so no caller +branches on the OS. Status reflects code inspection; the vision rail still needs a run on real +Windows hardware. + +| Rail | Status | Notes / evidence | +| --- | --- | --- | +| **Browser rail** (`web_use`) | 🟢 | Electron CDP — no platform-specific code; identical path to macOS. | +| **Semantic rail** (calendar / reminder / email / open) | 🟢 | `semantic-rail-win.ts` drives **local Outlook via PowerShell/COM**: create, `calendar.listEvents` / `reminders.list` read-back, and delete-by-EntryID undo — wired with a real `runPowerShell` + `shell.openExternal`. `message` (iMessage) is refused honestly (macOS-only). Covered by `semantic-rail-win.test.ts` + `platform-picks.test.ts`. | +| ↳ Semantic rail — **Microsoft Graph online fallback** | 🟡 | The `GraphPort` + fallback logic ship, but production passes no port — a PC without local Outlook gets an honest refusal until the device-code OAuth wiring + an Azure app registration land. Fast-follow. | +| **Vision / computer-use rail** (screenshot → grounder → cursor/keyboard) | 🟢 needs real-HW test | Capture (`desktopCapturer`) + grounder (local GGUF) are cross-platform. Actuation uses `@nut-tree-fork/nut-js` (prebuilt N-API `libnut-win32`), packaged via `postinstall` `install-app-deps` + electron-builder smartUnpack, with a **fail-loud presence gate** in `windows-build.yml`. The one Windows-specific fix — **DPI/scale coordinate mapping** so clicks land on 125%/150% displays — is `src/main/input/coordinate-mapping.ts` (12 unit tests), wired into `vision-host.ts`. Needs a real-Windows run to confirm actuation + fractional scaling. | + +**Already handled:** `accessibilityBlock()` (the macOS Accessibility-grant prompt) no-ops off `darwin` — +Windows needs no such grant for synthetic input. `permissions.ts` returns "granted" for every check +off `darwin`, so the setup flow does not block on Windows. + +**Follow-ups:** mixed-DPI multi-monitor coordinate mapping (needs a physical-bounds source Electron +does not expose directly); Graph OAuth wiring for the Outlook-less fallback. + + ## Pro layer (out of scope) The Pro "sees / remembers / reflects / acts" layer is **not part of core** and is **not diff --git a/docs/features/clipboard.md b/docs/features/clipboard.md index ca67f6f5..c0348f64 100644 --- a/docs/features/clipboard.md +++ b/docs/features/clipboard.md @@ -16,4 +16,4 @@ lives **on your device** - nothing is synced or uploaded. - **Retention controls** - cap the history size and auto-expire old clips. - **On-device** - the history is stored locally; capture can be paused any time. -→ Part of [Off Grid Pro](pro.md). +→ Part of [Off Grid AI Pro](pro.md). diff --git a/docs/features/connectors.md b/docs/features/connectors.md index d0bc47eb..b5a833f8 100644 --- a/docs/features/connectors.md +++ b/docs/features/connectors.md @@ -12,3 +12,50 @@ Use [Model Context Protocol](https://modelcontextprotocol.io) servers right insi - **In chat** — turn **Connectors** on in the composer; the model can call connector tools, reads run inline. - Transports: hosted **HTTP** and local **stdio** servers. + +## Connect Gmail and Google Calendar with your own Google client + +Your own Google OAuth client lets Off Grid AI connect directly to Gmail and Google Calendar. Off Grid AI +stores the client credentials with the operating system protected credential store. Google data moves +between this device and Google. + +### Before you start + +- You need a Google Cloud project that you can configure. +- You need permission to enable the Gmail API and Google Calendar API. +- If the consent screen is in test mode, add the Google account that you will connect as a test user. +- Create an OAuth client with the **Web application** type. A Desktop application client does not use + the callback that Off Grid AI requires. + +### Configure Google Cloud + +1. Open [Google Cloud credentials](https://console.cloud.google.com/apis/credentials) and select or + create the project that will own the client. +2. Enable the [Gmail API](https://console.cloud.google.com/apis/library/gmail.googleapis.com). +3. Enable the [Google Calendar API](https://console.cloud.google.com/apis/library/calendar-json.googleapis.com). +4. Configure the [OAuth consent screen](https://console.cloud.google.com/apis/credentials/consent). + Select the audience that your Google organization permits. For an External app in test mode, add + your Google account under **Test users**. Complete any approval that your organization requires. +5. Select **Create credentials > OAuth client ID**. +6. Select **Web application**. +7. Add this exact authorized redirect URI: + + `http://127.0.0.1:33418/callback` + + This is the callback shown in the Off Grid AI setup panel. The release source is + `src/shared/mcp-oauth-callback.ts`. Do not select another port or change the path. +8. Create the client. Copy its Client ID and Client secret. The Project ID is optional. + +### Save and connect + +1. Open **Settings > Connectors** in Off Grid AI. +2. Open the Google client setup panel. +3. Enter the Client ID and Client secret. Enter the Project ID if you use it for project records. +4. Select **Save client**. +5. Select Gmail or Google Calendar, then select **Connect**. +6. Sign in in the browser and approve the requested access. Return to Off Grid AI after Google sends the + browser to the local callback. + +When setup succeeds, the setup card says **Your Google client**. The connector moves to +**Connected**, and its connection test succeeds. If you change the Google client, consent audience, +or test users, save the client again and reconnect each Google connector. diff --git a/docs/features/dictation.md b/docs/features/dictation.md index 5592703a..f1ef9362 100644 --- a/docs/features/dictation.md +++ b/docs/features/dictation.md @@ -24,4 +24,4 @@ a full-screen window too. Nothing is uploaded; the audio never leaves the machin - **Folds into your timeline** - an on-device LLM pulls the people, projects, and to-dos out of what you dictated, through the same observation pipeline as meetings. -→ Part of [Off Grid Pro](pro.md). +→ Part of [Off Grid AI Pro](pro.md). diff --git a/docs/features/models.md b/docs/features/models.md index 05061d33..033adf76 100644 --- a/docs/features/models.md +++ b/docs/features/models.md @@ -6,10 +6,10 @@ - **Catalog** with curated, **size-bucketed** recommendations (≤2/4/6/8/16 GB) per modality (text, vision, image, voice, transcription) — compete-with-LM-Studio picks, with release - dates and credibility tiers (official / verified / community / Off Grid). + dates and credibility tiers (official / verified / community / Off Grid AI). - **Direct Hugging Face search**, scoped to the focused modality; auto-detects GGUF / GGML / ONNX variants. - **Download manager** — progress, cancel, and a per-modality **active model** that the gateway loads on demand. -- Off Grid publishes correctly-converted SDXL GGUFs under the +- Off Grid AI publishes correctly-converted SDXL GGUFs under the [`offgrid-ai`](https://huggingface.co/offgrid-ai) org. diff --git a/docs/features/pro.md b/docs/features/pro.md index a1154c68..e019cfe0 100644 --- a/docs/features/pro.md +++ b/docs/features/pro.md @@ -1,8 +1,8 @@ -# Off Grid Pro — coming July 2026 +# Off Grid AI Pro — coming July 2026 [← All features](../FEATURES.md) -![Off Grid Pro — coming July 2026](../screenshots/07-pro-upgrade.png) +![Off Grid AI Pro — coming July 2026](../screenshots/07-pro-upgrade.png) The free app **runs** models. **Pro** adds the always-on layer that **sees, remembers, and acts**, on-device: diff --git a/docs/features/voice.md b/docs/features/voice.md index 510f0b4a..69a7f913 100644 --- a/docs/features/voice.md +++ b/docs/features/voice.md @@ -2,6 +2,33 @@ [← All features](../FEATURES.md) -- **Speech → text** with `whisper.cpp` (tiny → large-v3-turbo). -- **Text → speech** with Kokoro (multiple voices) — tap **Speak** on any message. -- **Voice mode** turns chat into a hands-free, voice-note conversation. +Speak instead of typing, and hear a reply without sending your words to a cloud service. + +## In Chat + +Select the voice-mode button at the top of Chat. Then select how each turn works: + +- **Manual:** Select the microphone to start. Select it again to stop and send. +- **Auto:** Select the microphone to start. Off Grid AI stops and sends after you stop speaking. +- **Hands-free:** Off Grid AI listens for your voice, sends after you stop speaking, plays the reply, + and then listens again. Select the microphone to pause or resume. + +During transcription, the microphone shows the installed speech-to-text model that is in use. Select +the cancel control to discard that turn. When you return to text mode, Off Grid AI stops the active +recording and keeps the text composer ready. + +## Dictation in other apps (Pro) + +Open **Voice** and set the Option+Space gesture: + +- **Hold:** Hold Option+Space while you speak. Release it to stop. +- **Toggle:** Press Option+Space once to start. Press it again to stop. +- **Both:** A quick press toggles recording. A hold works as push-to-talk. + +Off Grid AI transcribes with the active Whisper or Parakeet model. Auto-send can paste the result at the +current cursor. Saved recordings stay searchable in Voice. + +## Hear a reply + +Select **Speak** on a text reply, or play a voice-mode reply. Off Grid AI uses the selected Kokoro +language and voice. Only one reply plays at a time. diff --git a/docs/release-readiness-supplemental-0.0.40.json b/docs/release-readiness-supplemental-0.0.40.json index cb8eaaaf..9d81d30c 100644 --- a/docs/release-readiness-supplemental-0.0.40.json +++ b/docs/release-readiness-supplemental-0.0.40.json @@ -570,7 +570,7 @@ "status": "PARTIAL", "layer": "Production composition + real SQLite integration", "evidence": "pro/main/__tests__/service-activation.integration.dbtest.ts; src/main/__tests__/release-packaging.integration.test.ts", - "proof": "A fresh packaged-mode profile starts no Pro service. A real Keygen-boundary activation persists an encrypted entitlement; after every Off Grid module reloads, the production Core loader invokes actual Pro activateMain, migrates real SQLite, starts the owned focus/shortcut/tray services, and application shutdown tears them down. Clearing entitlement and reloading stays locked with no service start.", + "proof": "A fresh packaged-mode profile starts no Pro service. A real Keygen-boundary activation persists an encrypted entitlement; after every Off Grid AI module reloads, the production Core loader invokes actual Pro activateMain, migrates real SQLite, starts the owned focus/shortcut/tray services, and application shutdown tears them down. Clearing entitlement and reloading stays locked with no service start.", "remaining": "Observe the exact installed locked and entitled artifact against real OS services.", "confidence": "MEDIUM", "notes": "Privacy and open-core release blocker." @@ -878,5 +878,69 @@ "remaining": "Inspect actual host processes, permission indicators, shortcuts, ports, and mounted images after one full signed-app session.", "confidence": "MEDIUM", "notes": "Final release stop condition." + }, + { + "id": "PR-13", + "phase": "20 Pro secondary surfaces and controls", + "tier": "Pro", + "priority": "P0", + "manualTest": "Google OAuth client connects Gmail and Calendar", + "steps": "Create a Web application OAuth client; enable the Gmail and Google Calendar APIs; add http://127.0.0.1:33418/callback; approve the test user; save the client; connect and test both connectors; relaunch and reconnect.", + "expected": "The setup card says Your Google client; both connectors show Connected; both tests succeed before and after relaunch.", + "status": "PARTIAL", + "layer": "Integration + rendered test", + "evidence": "pro/renderer/__tests__/GoogleClientSetup.integration.test.tsx; pro/main/__tests__/google-client.test.ts; pro/main/__tests__/google-ipc.test.ts; pro/main/__tests__/google-rest-connector.integration.dbtest.ts; src/main/__tests__/mcp-oauth-loopback.integration.test.ts", + "proof": "The client setup, protected credential owner, Google REST connector, callback state, and connector IPC run through real application seams with the Google boundary controlled.", + "remaining": "Use a real Google Cloud project, system browser, consent or test-user approval, Gmail account, and Calendar account in the exact installed app.", + "confidence": "LOW", + "notes": "Redact the Client ID, secret, tokens, mail, and calendar data." + }, + { + "id": "PR-14", + "phase": "20 Pro secondary surfaces and controls", + "tier": "Pro", + "priority": "P0", + "manualTest": "Discoverable and Find nearby stay independent", + "steps": "Pair a second device; cold-launch with Discoverable off; turn each visibility control off in turn; send a small record on the active session.", + "expected": "Hidden applies before startup advertising; Discoverable off does not stop browsing; Find nearby off does not stop advertising; the active encrypted session stays active.", + "status": "PARTIAL", + "layer": "Integration + native helper contract", + "evidence": "pro/main/sync/__tests__/macos-proximity.test.ts; pro/main/__tests__/identity-discovery.test.ts; pro/main/sync/__tests__/state-bridge.test.ts; pro/renderer/__tests__/sync-state.test.ts", + "proof": "The owning state, Electron bridge, macOS helper contract, and rendered visibility state prove separate browse and advertise lifecycles and Hidden startup order.", + "remaining": "Run both controls and a cold launch with the exact installed app and a second physical device on the real network and radio boundary.", + "confidence": "LOW", + "notes": "Record both device names, OS versions, and exact build commits." + }, + { + "id": "PR-15", + "phase": "20 Pro secondary surfaces and controls", + "tier": "Pro", + "priority": "P0", + "manualTest": "Private endpoint and custom Sync port reconnect", + "steps": "Save the paired device private IP address or machine name; connect; set one disposable non-default Sync port on every device; restart all apps; restore 37878.", + "expected": "The same paired identity reconnects by the saved endpoint; the custom port works only when all devices match; the default port works after restore.", + "status": "PARTIAL", + "layer": "Database + integration contract", + "evidence": "pro/main/sync/__tests__/sync-prefs-rules.test.ts; pro/main/sync/__tests__/state-bridge-persistence.test.ts; pro/main/__tests__/production-sync.dbtest.ts", + "proof": "Port and private-host parsing, persistence, runtime state, and production Sync composition run through automated application seams.", + "remaining": "Reconnect real paired devices through a private IP or machine name and one matching non-default port on every installed app.", + "confidence": "LOW", + "notes": "Use one disposable port from 1024 to 65535 and restore 37878." + }, + { + "id": "PR-16", + "phase": "20 Pro secondary surfaces and controls", + "tier": "Pro", + "priority": "P0", + "manualTest": "Failed advertising stop keeps true state and retries", + "steps": "Use a diagnostic helper that rejects the first advertising stop; turn Discoverable off; remove the failure; turn it off again.", + "expected": "The first action reports failure and the runtime switch and saved value stay on; the retry stops advertising and all three states turn off.", + "status": "PARTIAL", + "layer": "Integration + native helper contract", + "evidence": "pro/main/sync/__tests__/macos-proximity.test.ts; pro/main/sync/__tests__/state-bridge.test.ts; pro/renderer/__tests__/sync-state.test.ts", + "proof": "The helper contract, runtime owner, persistence boundary, and rendered state fail closed and permit a later retry.", + "remaining": "Run the rejection and retry with the exact diagnostic helper in the installed app and inspect the real advertisement from a second device.", + "confidence": "LOW", + "notes": "Do not replace this check with a mocked renderer result." } ] diff --git a/e2e/app025-model-download-verification.spec.ts b/e2e/app025-model-download-verification.spec.ts index 75d31534..9bd2e40c 100644 --- a/e2e/app025-model-download-verification.spec.ts +++ b/e2e/app025-model-download-verification.spec.ts @@ -2,7 +2,7 @@ * APP-025: a rendered model download is staged, verified, activated, used, and restored. * * Only Hugging Face delivery is controlled. The retry payload is a real Qwen GGUF and the app uses - * its actual llama-server; all Off Grid UI, IPC, integrity, filesystem, runtime selection, chat, + * its actual llama-server; all Off Grid AI UI, IPC, integrity, filesystem, runtime selection, chat, * persistence, and relaunch behavior stay production. */ import { diff --git a/e2e/app045-local-streaming.spec.ts b/e2e/app045-local-streaming.spec.ts index 08dd8b5d..f2e8f6a5 100644 --- a/e2e/app045-local-streaming.spec.ts +++ b/e2e/app045-local-streaming.spec.ts @@ -4,7 +4,7 @@ * The only fake is the native llama-server executable/HTTP boundary. MemoryChat, * navigation, preload, IPC, the streaming transport, SQLite, and both Electron * processes are production code. The fake binds loopback only and audits the actual - * request, making the local-only model path explicit without mocking Off Grid code. + * request, making the local-only model path explicit without mocking Off Grid AI code. */ import { expect, test, type ElectronApplication, type Page } from '@playwright/test' import type { ChildProcess } from 'node:child_process' diff --git a/e2e/app105-free-pro-isolation.spec.ts b/e2e/app105-free-pro-isolation.spec.ts index e7956f45..6674ec28 100644 --- a/e2e/app105-free-pro-isolation.spec.ts +++ b/e2e/app105-free-pro-isolation.spec.ts @@ -8,7 +8,7 @@ * * This launches the real Electron app, uses the real sidebar and Settings UI, * and probes only the public preload/IPC boundary plus durable profile output. - * No Off Grid module or store is mocked. + * No Off Grid AI module or store is mocked. */ import { expect, test, type ElectronApplication, type Page } from '@playwright/test' import fs from 'node:fs' @@ -50,10 +50,10 @@ const SETTINGS_PLACEHOLDERS = [ }, { title: 'You', - description: 'Tell Off Grid who you are' + description: 'Tell Off Grid AI who you are' }, { - title: 'What Off Grid has learned', + title: 'What Off Grid AI has learned', description: 'Preferences distilled from the suggestions you dismiss' } ] as const diff --git a/e2e/app143-approval-gate.spec.ts b/e2e/app143-approval-gate.spec.ts index 98f35b3b..2e1712b3 100644 --- a/e2e/app143-approval-gate.spec.ts +++ b/e2e/app143-approval-gate.spec.ts @@ -115,7 +115,7 @@ test('holds a connector write until approval, then executes it once with visible connectorId: id, tool: 'create_external_task', args: { title: externalTitle, project: externalProject }, - entityName: 'Off Grid Desktop', + entityName: 'Off Grid AI Desktop', source: 'APP-143 rendered E2E' }) as Promise }, diff --git a/e2e/app159-vault-secret-protection.spec.ts b/e2e/app159-vault-secret-protection.spec.ts index b9833482..4585dc5a 100644 --- a/e2e/app159-vault-secret-protection.spec.ts +++ b/e2e/app159-vault-secret-protection.spec.ts @@ -78,7 +78,7 @@ async function openVault(): Promise { await vault.click() await expect( page - .getByText('Off Grid Vault', { exact: true }) + .getByText('Off Grid AI Vault', { exact: true }) .or(page.getByRole('button', { name: 'Lock vault', exact: true })) ).toBeVisible() } diff --git a/e2e/app250-chat-action-engine.spec.ts b/e2e/app250-chat-action-engine.spec.ts new file mode 100644 index 00000000..9cfbce96 --- /dev/null +++ b/e2e/app250-chat-action-engine.spec.ts @@ -0,0 +1,139 @@ +/** + * APP-250 — the R1 golden path: a chat ask becomes a durable, verified action. + * + * The rendered app, tool loop, tool-call parsing, the @offgrid/use engine + * (queue, gate, semantic rail, read-back verification), IPC, and MemoryChat + * are production code. Two fakes stand at the true boundaries: a scripted + * llama-server (emits the tool call as text, the way small local models do) + * and a scripted actions helper (records creates, answers list read-backs). + * + * Proves, on a fresh profile with Tools enabled through the real composer + * menu (default-off until R2's per-turn router; see checklist 18b): + * chat ask -> tool call -> durable Action -> semantic rail create -> + * read-back verify -> confirmed in chat. The helper log pins the order: + * exactly one create, then a list (the read-back actually ran). + */ +import { expect, test, type ElectronApplication, type Page } from '@playwright/test' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { completeOnboarding } from './helpers/onboarding' +import { launchOffGrid, targetIsPackaged } from './helpers/launch' + +let app: ElectronApplication | null = null +let page: Page +let profileDir: string +let helperLog: string + +function stageWorld(): void { + // The model boundary: a stub gguf + the scripted server as llama-server. + const modelsDir = path.join(profileDir, 'models') + const llamaDir = path.join(profileDir, 'bin', 'llama') + fs.mkdirSync(modelsDir, { recursive: true }) + fs.mkdirSync(llamaDir, { recursive: true }) + const gguf = Buffer.alloc(2_048) + gguf.write('GGUF') + fs.writeFileSync(path.join(modelsDir, 'app250-local.gguf'), gguf) + fs.writeFileSync( + path.join(modelsDir, 'active-model.json'), + JSON.stringify({ id: 'app250-local-model', primary: 'app250-local.gguf', mmproj: null }) + ) + fs.copyFileSync( + path.join(process.cwd(), 'e2e', 'fixtures', 'app250-actions-llama-server.mjs'), + path.join(llamaDir, 'llama-server') + ) + fs.chmodSync(path.join(llamaDir, 'llama-server'), 0o755) + + // The OS boundary: the scripted helper where dev resolution looks first + // (cwd/scripts/actions-helper/actions-helper - the spec launches the app + // with cwd pointed at the profile dir). + const helperDir = path.join(profileDir, 'scripts', 'actions-helper') + fs.mkdirSync(helperDir, { recursive: true }) + fs.copyFileSync( + path.join(process.cwd(), 'e2e', 'fixtures', 'app250-actions-helper.mjs'), + path.join(helperDir, 'actions-helper') + ) + fs.chmodSync(path.join(helperDir, 'actions-helper'), 0o755) +} + +const helperCalls = (): Array<{ command: string; args: Record }> => { + if (!fs.existsSync(helperLog)) { + return [] + } + return fs + .readFileSync(helperLog, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) +} + +test.beforeEach(async () => { + test.skip(targetIsPackaged(), 'dev-target journey: the packaged app resolves its helper from Resources') + profileDir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-app250-')) + helperLog = path.join(profileDir, 'helper-log.jsonl') + stageWorld() + app = await launchOffGrid({ + cwd: profileDir, + env: { + ...process.env, + OFFGRID_USER_DATA: profileDir, + OFFGRID_BIN_DIR: path.join(profileDir, 'bin'), + OFFGRID_APP250_HELPER_LOG: helperLog, + OFFGRID_PRO: '0', + NODE_ENV: 'production' + } + }) + page = await app.firstWindow() + await page.waitForLoadState('domcontentloaded') + await completeOnboarding(page) +}) + +test.afterEach(async () => { + const running = app + app = null + if (running) { + await running.close() + } + fs.rmSync(profileDir, { recursive: true, force: true }) +}) + +test('a chat ask becomes a created, read-back-verified reminder', async () => { + await page.getByRole('button', { name: 'Chat', exact: true }).click() + const composer = page.getByPlaceholder(/ask anything/i) + await expect(composer).toBeVisible() + const captureDismiss = page.getByRole('button', { name: 'Dismiss', exact: true }) + if (await captureDismiss.isVisible().catch(() => false)) { + await captureDismiss.click() + } + + // Enable Tools the way a user does: the composer's + menu. + await page.getByRole('button', { name: 'Composer options' }).click() + await page.getByRole('menuitem', { name: /^Tools/ }).click() + await page.keyboard.press('Escape') + + await composer.fill('remind me to send the deck at 6pm today') + await composer.press('Enter') + + // The model's confirmation only streams on the SECOND turn - after the + // tool ran through the engine and reported its verified outcome. + // .last(): the conversation rail previews the same text; the transcript + // copy is the one that matters. + await expect(page.getByText('Done - the reminder is set for 6pm today.').last()).toBeVisible({ + timeout: 90_000 + }) + + // The tool activity row shows the engine's verified outcome, not a guess. + await expect(page.getByText('reminders_create → Created the reminder.')).toBeVisible() + + // The helper log pins the guarantee: exactly one create, and at least one + // list AFTER it - the read-back verification actually observed the world. + const calls = helperCalls() + const creates = calls.filter((c) => c.command === 'reminders.create') + expect(creates).toHaveLength(1) + expect(creates[0]?.args.title).toBe('Send the deck') + const createIndex = calls.findIndex((c) => c.command === 'reminders.create') + const listAfter = calls.slice(createIndex + 1).some((c) => c.command === 'reminders.list') + expect(listAfter).toBe(true) + + await page.screenshot({ path: 'e2e/screenshots/r1-chat-action-verified.png' }) +}) diff --git a/e2e/chat-actions.spec.ts b/e2e/chat-actions.spec.ts index 0424776b..822f0c1c 100644 --- a/e2e/chat-actions.spec.ts +++ b/e2e/chat-actions.spec.ts @@ -55,7 +55,7 @@ async function closeApp(): Promise { async function finishOnboarding(): Promise { for (let step = 0; step < 8; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() } diff --git a/e2e/chat-large-collection.spec.ts b/e2e/chat-large-collection.spec.ts index faaba28a..6adf7341 100644 --- a/e2e/chat-large-collection.spec.ts +++ b/e2e/chat-large-collection.spec.ts @@ -15,7 +15,7 @@ let userDataDir: string async function finishOnboarding(): Promise { for (let step = 0; step < 6; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() } diff --git a/e2e/chat-memory.spec.ts b/e2e/chat-memory.spec.ts index 0af3523c..d58e3e6e 100644 --- a/e2e/chat-memory.spec.ts +++ b/e2e/chat-memory.spec.ts @@ -55,7 +55,7 @@ const forceCloseApp = async (): Promise => { const enterChat = async (): Promise => { for (let i = 0; i < 8; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click() await page.waitForTimeout(300) @@ -205,7 +205,7 @@ test('streaming placeholder appears immediately after send', async () => { const assistantBubble = page .locator('div') .filter({ - hasText: /searching|working|sorry|error|off grid/i + hasText: /searching|working|sorry|error|Off Grid AI/i }) .first() await expect(assistantBubble) @@ -319,7 +319,7 @@ test('cancelling a tool-owned image keeps its text answer after a full relaunch' // Re-launch against faithful native-process boundaries. The production LLMService // spawns the fake llama executable and speaks real HTTP/SSE; imagegen spawns the // fake sd-cli and must kill it through the rendered Stop control. SQLite, IPC, - // toolChat, MemoryChat, and the process relaunch are all real Off Grid code. + // toolChat, MemoryChat, and the process relaunch are all real Off Grid AI code. await closeApp() const modelsDir = path.join(userDataDir, 'models') diff --git a/e2e/computer-use-model-strategy.spec.ts b/e2e/computer-use-model-strategy.spec.ts new file mode 100644 index 00000000..f9336a61 --- /dev/null +++ b/e2e/computer-use-model-strategy.spec.ts @@ -0,0 +1,210 @@ +import { expect, test, type ElectronApplication, type Page } from '@playwright/test' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import sharp from 'sharp' +import { completeOnboarding } from './helpers/onboarding' +import { launchOffGrid, targetIsPackaged } from './helpers/launch' +import { gotoSettings, openSettingsSection } from './helpers/settings' + +let app: ElectronApplication | null = null +let page: Page +let profileDir: string + +const CHAT_ID = 'unsloth/Qwen3.5-0.8B-GGUF' +const CHAT_NAME = 'Qwen 3.5 0.8B' +const SPECIALIST_ID = 'mradermacher/UI-TARS-1.5-7B-GGUF' + +// This journey starts the native model boundary, reopens Electron, and runs a complete visual +// task. The default 60-second budget can expire during model startup before a product assertion. +test.setTimeout(180_000) + +function executable(source: string, target: string): void { + fs.copyFileSync(source, target) + fs.chmodSync(target, 0o755) +} + +async function stageWorld(): Promise { + const models = path.join(profileDir, 'models') + const bin = path.join(profileDir, 'bin') + const llama = path.join(bin, 'llama') + fs.mkdirSync(models, { recursive: true }) + fs.mkdirSync(llama, { recursive: true }) + for (const file of [ + 'Qwen3.5-0.8B-Q4_K_M.gguf', + 'mmproj-Qwen3.5-0.8B-BF16.gguf', + 'UI-TARS-1.5-7B.Q4_K_M.gguf', + 'UI-TARS-1.5-7B.mmproj-f16.gguf' + ]) { + const bytes = Buffer.alloc(2_048) + bytes.write('GGUF') + fs.writeFileSync(path.join(models, file), bytes) + } + fs.writeFileSync( + path.join(models, 'active-model.json'), + JSON.stringify({ + id: CHAT_ID, + primary: 'Qwen3.5-0.8B-Q4_K_M.gguf', + mmproj: 'mmproj-Qwen3.5-0.8B-BF16.gguf' + }) + ) + fs.writeFileSync( + path.join(models, 'active-modalities.json'), + JSON.stringify({ computer_use: SPECIALIST_ID }) + ) + executable( + path.join(process.cwd(), 'e2e/fixtures/computer-use-hybrid-llama-server.mjs'), + path.join(llama, 'llama-server') + ) + executable( + path.join(process.cwd(), 'e2e/fixtures/computer-use-capture.mjs'), + path.join(bin, 'computer-use-capture') + ) + await sharp({ + create: { width: 640, height: 400, channels: 4, background: '#f5f5f5' } + }) + .png() + .toFile(path.join(profileDir, 'computer-use-frame.png')) + + const nut = path.join(profileDir, 'node_modules/@nut-tree-fork/nut-js') + fs.mkdirSync(nut, { recursive: true }) + fs.writeFileSync( + path.join(nut, 'index.js'), + `class Point { constructor(x, y) { this.x = x; this.y = y } } +const done = async () => undefined +module.exports = { + Point, + Button: { LEFT: 1, RIGHT: 2, MIDDLE: 3 }, + Key: {}, + mouse: { setPosition: done, leftClick: done, rightClick: done, click: done, doubleClick: done, drag: done, scrollUp: done, scrollDown: done, scrollLeft: done, scrollRight: done }, + keyboard: { type: done, pressKey: done, releaseKey: done } +}` + ) +} + +async function launch(): Promise { + app = await launchOffGrid({ + cwd: profileDir, + env: { + ...process.env, + OFFGRID_USER_DATA: profileDir, + OFFGRID_BIN_DIR: path.join(profileDir, 'bin'), + OFFGRID_PRO: '1', + OFFGRID_E2E_HEADLESS: '1', + OFFGRID_E2E_PRO_TASKS: '1', + OFFGRID_E2E_COMPUTER_USE_BOUNDARY: '1', + OFFGRID_E2E_COMPUTER_USE_FRAME: path.join(profileDir, 'computer-use-frame.png'), + NODE_PATH: path.join(profileDir, 'node_modules'), + NODE_ENV: 'production' + } + }) + page = await app.firstWindow() + page.on('pageerror', (error) => console.error(`[renderer error] ${error.stack ?? error.message}`)) + await page.waitForLoadState('domcontentloaded') + await completeOnboarding(page) + await expect(page.getByRole('button', { name: /Model server: model running/i })).toBeVisible({ + timeout: 60_000 + }) +} + +async function close(): Promise { + const running = app + app = null + if (running) await running.close() +} + +async function selectStrategy(label: string): Promise { + await gotoSettings(page) + await openSettingsSection(page, 'Computer use') + const strategy = page.getByRole('button', { name: 'Computer Use model strategy' }) + await strategy.click() + await page.getByRole('menuitemradio', { name: label, exact: true }).click() + await expect(strategy).toContainText(label) +} + +async function gotoChat(): Promise { + await page.keyboard.press('Meta+K') + const palette = page.getByRole('dialog', { name: 'Search Off Grid AI' }) + await expect(palette).toBeVisible() + await palette.getByPlaceholder(/^Search everything/).fill('Chat') + await page.getByTestId('palette-screen-memory-chat-root').click() + await expect(page.getByPlaceholder(/ask anything/i)).toBeVisible() +} + +async function computerUseSummary(): Promise { + await gotoChat() + await page.getByRole('button', { name: 'Active models' }).click() + const region = page.getByRole('region', { name: 'Computer Use' }) + await expect(region).toBeVisible() + await expect(region).not.toContainText('No Computer Use model is selected.', { timeout: 30_000 }) + const text = await region.innerText() + await page.getByRole('button', { name: 'Close' }).click() + return text +} + +test.beforeEach(async () => { + test.skip(targetIsPackaged(), 'dev-target journey uses scripted native boundaries') + profileDir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-computer-use-strategy-')) + await stageWorld() + await launch() +}) + +test.afterEach(async () => { + await close() + fs.rmSync(profileDir, { recursive: true, force: true }) +}) + +test('all strategies project their effective roles and Text + Specialist survives relaunch', async () => { + await selectStrategy('Same as Chat') + expect(await computerUseSummary()).toContain('Same as Chat') + expect(await computerUseSummary()).toContain(CHAT_NAME) + + await selectStrategy('Specialist') + const specialist = await computerUseSummary() + expect(specialist).toContain('Specialist') + expect(specialist.toLowerCase()).toContain('grounding specialist') + expect(specialist.toLowerCase()).not.toContain('reasoner') + + await selectStrategy('Text + Specialist') + const hybrid = await computerUseSummary() + expect(hybrid.toLowerCase()).toContain('reasoner') + expect(hybrid.toLowerCase()).toContain('grounding specialist') + + await close() + await launch() + expect(await computerUseSummary()).toContain('Text + Specialist') +}) + +test('a Chat request runs the hybrid reasoner and specialist through one Computer Use task', async () => { + await selectStrategy('Text + Specialist') + await gotoChat() + const composer = page.getByPlaceholder(/ask anything/i) + await page.getByRole('button', { name: 'Composer options' }).click() + await page.getByRole('menuitem', { name: /^Tools/ }).click() + await page.keyboard.press('Escape') + await composer.fill('Move the pointer to the center of this test window.') + await composer.press('Enter') + + await expect( + page.getByText(/(?:Done\. Task reference|is running now and will finish shortly)/i).last() + ).toBeVisible({ timeout: 60_000 }) + await expect( + page.getByRole('button', { + name: /Computer Use done Move the pointer to the center of the visible Off Grid AI test window/ + }) + ).toBeVisible({ timeout: 120_000 }) + expect( + fs.existsSync(path.join(profileDir, 'hybrid-model-state.json')), + 'the scripted native model boundary must record hybrid reasoning' + ).toBe(true) + const state = JSON.parse( + fs.readFileSync(path.join(profileDir, 'hybrid-model-state.json'), 'utf8') + ) as { reasonerCalls: number } + expect(state.reasonerCalls).toBeGreaterThanOrEqual(2) + const requests = fs + .readFileSync(path.join(profileDir, 'hybrid-model-requests.jsonl'), 'utf8') + .trim() + .split('\n') + expect(requests.some((request) => request.includes('delegate_grounded_action'))).toBe(true) + expect(requests.some((request) => request.includes('"max_tokens":200'))).toBe(true) +}) diff --git a/e2e/desktop-polish.spec.ts b/e2e/desktop-polish.spec.ts index 08aaa99a..2c9c9ddf 100644 --- a/e2e/desktop-polish.spec.ts +++ b/e2e/desktop-polish.spec.ts @@ -10,7 +10,7 @@ let userDataDir: string async function finishOnboarding(): Promise { for (let step = 0; step < 6; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() } @@ -190,7 +190,7 @@ test('keyboard focus follows navigation, form, dialog, and primary-action order // The production command palette is the app's real modal dialog. It must move // focus into its form, keep keyboard focus inside, and retain the visible ring. await page.keyboard.press('Meta+K') - const dialog = page.getByRole('dialog', { name: 'Search Off Grid' }) + const dialog = page.getByRole('dialog', { name: 'Search Off Grid AI' }) await expect(dialog).toBeVisible() // The REAL placeholder. This said 'Search everything…', which is not a substring of what the palette // renders ('Search everything, or jump to a screen…'), so the locator matched nothing and the focus diff --git a/e2e/devices-sync.spec.ts b/e2e/devices-sync.spec.ts index b882ba16..d976848e 100644 --- a/e2e/devices-sync.spec.ts +++ b/e2e/devices-sync.spec.ts @@ -31,10 +31,17 @@ import { type PendingMembershipRevocation } from '@offgrid/sync' import { NodeTcpTransport } from '@offgrid/sync/node' -import { createKnowledgeDocumentSource } from '../pro/main/sync/knowledge-document-transfer' import type { KnowledgeDocumentSnapshot } from '../src/main/sync-knowledge-document' const PRO_PRESENT = fs.existsSync(path.resolve('pro/package.json')) +// Pro implementation modules load lazily behind PRO_PRESENT: a static import +// fails spec COLLECTION in a core-only checkout, before the guard can skip. +const knowledgeDocumentTransfer = PRO_PRESENT + ? // eslint-disable-next-line @typescript-eslint/no-require-imports + (require('../pro/main/sync/knowledge-document-transfer') as { + createKnowledgeDocumentSource: (...args: never[]) => unknown + }) + : null const SYNCED_PROJECT_ID = '22222222-2222-4222-8222-222222222222' const SYNCED_CONVERSATION_ID = '33333333-3333-4333-8333-333333333333' const SYNCED_MESSAGE_ID = '44444444-4444-4444-8444-444444444444' @@ -586,7 +593,7 @@ test.describe('Devices surface — pro tier', () => { } await syntheticFiles.sendFile( desktop.localDevice.id, - createKnowledgeDocumentSource(knowledgeDocument) + knowledgeDocumentTransfer!.createKnowledgeDocumentSource(knowledgeDocument as never) ) const knowledgeOp = syntheticLog.record( 'knowledge_document', diff --git a/e2e/explore.spec.ts b/e2e/explore.spec.ts new file mode 100644 index 00000000..366e2da1 --- /dev/null +++ b/e2e/explore.spec.ts @@ -0,0 +1,93 @@ +/** + * Explore surface - the capability-panel catalog renders on both of its placements + * (the Explore screen and the chat empty state) and never leaks a preset's raw + * prompt onto a card: cards show the label + blurb only, the prompt stays behind + * the tap. Free build, fresh profile - the catalog needs no model and no seed. + * + * Screenshots land in e2e/screenshots/ for PR evidence. + */ +import { test, expect, type ElectronApplication, type Page } from '@playwright/test' +import { launchOffGrid } from './helpers/launch' +import { completeOnboarding } from './helpers/onboarding' +import os from 'os' +import path from 'path' +import fs from 'fs' + +let app: ElectronApplication +let page: Page +let userDataDir: string + +test.beforeAll(async () => { + userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-e2e-explore-')) + app = await launchOffGrid({ + env: { + ...process.env, + OFFGRID_USER_DATA: userDataDir, + OFFGRID_PRO: '0', + NODE_ENV: 'production' + } + }) + page = await app.firstWindow() + await page.waitForLoadState('domcontentloaded') + await completeOnboarding(page) + try { + await page.getByRole('button', { name: 'Expand sidebar' }).click({ timeout: 4000 }) + } catch { + /* already open */ + } +}) + +test.afterAll(async () => { + await app?.close() + try { + fs.rmSync(userDataDir, { recursive: true, force: true }) + } catch { + /* ignore */ + } +}) + +test('the Explore screen renders capability panels with labels, never the prompt', async () => { + await page.getByRole('button', { name: 'Explore', exact: true }).first().click() + await expect(page.getByRole('heading', { level: 1, name: 'Explore' })).toBeVisible() + + // Every capability panel is on screen. + for (const panel of [ + 'Browse the web for you', + 'Drive your Mac', + 'Build client-ready work', + "Remembers what you've seen", + "Your Mac's tools, from your phone" + ]) { + await expect(page.getByText(panel, { exact: true })).toBeVisible() + } + + // A card carries its label + blurb - the seeded prompt never appears on the surface. + await expect(page.getByTestId('explore-preset-find-flight')).toBeVisible() + await expect(page.getByText('Find me a flight to book', { exact: false })).toHaveCount(0) + + // A gated card says why it cannot just run. + await expect(page.getByTestId('explore-preset-phone-summarize')).toContainText(/paired phone/i) + + await page.screenshot({ path: 'e2e/screenshots/explore-screen.png' }) +}) + +test('proposal setup scopes the content, style, and output folders before chat', async () => { + await page.getByTestId('explore-preset-proposal-deck').click() + const setup = page.getByTestId('proposal-deck-setup') + await expect(setup).toBeVisible() + await expect(setup.getByRole('textbox', { name: 'Content folder' })).toHaveValue('') + await expect(setup.getByRole('textbox', { name: 'Save under' })).toHaveValue('') + await expect(setup.getByRole('textbox', { name: 'Style example (optional)' })).toHaveValue('') + await setup.getByRole('textbox', { name: 'Content folder' }).fill('/tmp/client-material') + await expect(setup.getByRole('button', { name: 'Start in chat' })).toBeDisabled() + await setup.getByRole('textbox', { name: 'Save under' }).fill('/tmp/client-output') + await expect(setup.getByRole('button', { name: 'Start in chat' })).toBeEnabled() + await page.screenshot({ path: 'e2e/screenshots/explore-proposal-setup.png' }) +}) + +test('the chat empty state reuses the same catalog with its compact intro', async () => { + await page.getByRole('button', { name: 'Chat', exact: true }).first().click() + await expect(page.getByText('Explore what Off Grid AI can do')).toBeVisible() + await expect(page.getByTestId('explore-preset-best-nearby')).toBeVisible() + await page.screenshot({ path: 'e2e/screenshots/explore-chat-empty.png' }) +}) diff --git a/e2e/feature-smoke.spec.ts b/e2e/feature-smoke.spec.ts index e937c1f0..54f56ac2 100644 --- a/e2e/feature-smoke.spec.ts +++ b/e2e/feature-smoke.spec.ts @@ -39,7 +39,7 @@ test.beforeAll(async () => { page = await app.firstWindow() await page.waitForLoadState('domcontentloaded') for (let i = 0; i < 8; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click().catch(() => {}) await page.waitForTimeout(400) diff --git a/e2e/fixtures/app025-model-download-network-boundary.cjs b/e2e/fixtures/app025-model-download-network-boundary.cjs index 84f56f01..f2df4744 100644 --- a/e2e/fixtures/app025-model-download-network-boundary.cjs +++ b/e2e/fixtures/app025-model-download-network-boundary.cjs @@ -2,7 +2,7 @@ * Hugging Face delivery boundary for APP-025. * * The first response is a complete-but-corrupt staged payload. The retry receives a real GGUF - * from the host's verified local fixture. Off Grid still owns catalog resolution, download queue, + * from the host's verified local fixture. Off Grid AI still owns catalog resolution, download queue, * progress, Range retry, integrity checks, filesystem promotion, activation, and the model runtime. */ /* eslint-disable @typescript-eslint/no-require-imports, @typescript-eslint/explicit-function-return-type -- Electron loads this CommonJS bootstrap before the production bundle. */ diff --git a/e2e/fixtures/app093-image-runtime-boundary.mjs b/e2e/fixtures/app093-image-runtime-boundary.mjs index 4eda0640..883bd541 100644 --- a/e2e/fixtures/app093-image-runtime-boundary.mjs +++ b/e2e/fixtures/app093-image-runtime-boundary.mjs @@ -15,7 +15,7 @@ if (!outputPath) { } // A real, checksum-valid PNG. The native diffusion executable is the sole controlled -// boundary in this test; all Off Grid ownership, persistence and rendering remains real. +// boundary in this test; all Off Grid AI ownership, persistence and rendering remains real. const png = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAGAAAABACAIAAABqVuVZAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAv0lEQVR4nO3ZsQnDQADFUDc3rBbKvOkDcRPCd/HgJhD68tm+zivnfIdwoXNuFQEogPplJQwKoBjU/x41JhZAMSgTa3Wh1aAAikGZWBrUM78riHQAxaBMLA1q3mORDqAYlIk1746LYgDFoEyseYC8rAZQDMrE0qDmMRbpAIpBmVjz6LgoBlAMysR6yPFnNYBiUCaWBjXvsUgHUAzKxJp3x0UxgGJQJtY8QF5WAygGZWJpUPMYi3QAxaBMrHl0Ps4bguphafFRYPAAAAAASUVORK5CYII=', 'base64' diff --git a/e2e/fixtures/app142-meeting-recorder-boundary.mjs b/e2e/fixtures/app142-meeting-recorder-boundary.mjs index 6e0120d6..27a687d8 100644 --- a/e2e/fixtures/app142-meeting-recorder-boundary.mjs +++ b/e2e/fixtures/app142-meeting-recorder-boundary.mjs @@ -6,7 +6,7 @@ * The production MeetingController owns when this process starts and stops. This * executable only replaces ScreenCaptureKit/AVFoundation, which cannot be driven * deterministically in CI. It emits a small valid video so production finalization, - * persistence, IPC, and UI state can run without mocking any Off Grid module. + * persistence, IPC, and UI state can run without mocking any Off Grid AI module. */ import { spawnSync } from 'node:child_process' import fs from 'node:fs' diff --git a/e2e/fixtures/app250-actions-helper.mjs b/e2e/fixtures/app250-actions-helper.mjs new file mode 100755 index 00000000..8284767d --- /dev/null +++ b/e2e/fixtures/app250-actions-helper.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +// APP-250's OS boundary: a scripted actions helper. Records every command it +// receives and answers reminders.list with what actually landed, so the +// engine's read-back verification runs for real against this fake world - +// and the log proves create-then-list ordering. + +import fs from 'node:fs' + +const logFile = process.env.OFFGRID_APP250_HELPER_LOG +const raw = process.argv[2] ?? '{}' +const cmd = JSON.parse(raw) + +const record = (entry) => { + if (logFile) { + fs.appendFileSync(logFile, `${JSON.stringify(entry)}\n`) + } +} + +const reply = (payload) => { + process.stdout.write(`${JSON.stringify(payload)}\n`) + process.exit(0) +} + +record({ command: cmd.command, args: cmd.args ?? {} }) + +if (cmd.command === 'reminders.create') { + reply({ ok: true, result: { id: `e2e-${Date.now()}` } }) +} +if (cmd.command === 'reminders.list') { + const lines = logFile && fs.existsSync(logFile) ? fs.readFileSync(logFile, 'utf8').split('\n').filter(Boolean) : [] + const reminders = lines + .map((line) => JSON.parse(line)) + .filter((entry) => entry.command === 'reminders.create') + .map((entry) => ({ id: 'e2e', title: String(entry.args.title ?? '') })) + reply({ ok: true, result: { reminders } }) +} +reply({ ok: true, result: {} }) diff --git a/e2e/fixtures/app250-actions-llama-server.mjs b/e2e/fixtures/app250-actions-llama-server.mjs new file mode 100755 index 00000000..b65cd213 --- /dev/null +++ b/e2e/fixtures/app250-actions-llama-server.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +// APP-250's model boundary: a scripted llama-server. The production app still +// owns model discovery, the tool loop, tool-call parsing, the @offgrid/use +// engine, the semantic rail, read-back verification, IPC, and rendering. +// Turn 1 (an agentic turn carrying the reminders_create schema): emit the +// tool call AS TEXT, exactly how small local models do. Turn 2 (the request +// carries the tool's result): confirm in plain text. + +import http from 'node:http' + +const args = process.argv.slice(2) +const portFlag = Math.max(args.indexOf('--port'), args.indexOf('-p')) +const port = portFlag >= 0 ? Number(args[portFlag + 1]) : 8439 + +const delta = (content, finishReason = null) => + `data: ${JSON.stringify({ choices: [{ delta: content ? { content } : {}, finish_reason: finishReason }] })}\n\n` + +const TOOL_CALL = + '{"name":"reminders_create","arguments":{"title":"Send the deck","due":"2026-08-14T18:00:00"}}' +const CONFIRMATION = 'Done - the reminder is set for 6pm today.' + +const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) + return + } + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'app250-local-model' }] })) + return + } + if (request.method !== 'POST' || !String(request.url).includes('/chat/completions')) { + response.writeHead(404) + response.end() + return + } + let body = '' + request.on('data', (chunk) => { + body += chunk + }) + request.on('end', () => { + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }) + const isAgenticFirstTurn = body.includes('reminders_create') && !body.includes('Created the reminder.') + const text = isAgenticFirstTurn ? TOOL_CALL : CONFIRMATION + for (const piece of text.match(/.{1,24}/gs) ?? []) { + response.write(delta(piece)) + } + response.write(delta(null, 'stop')) + response.write('data: [DONE]\n\n') + response.end() + }) +}) + +server.listen(port, '127.0.0.1', () => { + console.log(`app250 fake llama-server listening on ${port}`) +}) diff --git a/e2e/fixtures/computer-use-capture.mjs b/e2e/fixtures/computer-use-capture.mjs new file mode 100644 index 00000000..5c3b7845 --- /dev/null +++ b/e2e/fixtures/computer-use-capture.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node + +// macOS screen-capture boundary for the hybrid E2E. It copies a synthetic +// frame into the exact output path requested by the production capture port. +import fs from 'node:fs' + +fs.copyFileSync(process.env.OFFGRID_E2E_COMPUTER_USE_FRAME, process.argv[2]) diff --git a/e2e/fixtures/computer-use-hybrid-llama-server.mjs b/e2e/fixtures/computer-use-hybrid-llama-server.mjs new file mode 100644 index 00000000..0671dd67 --- /dev/null +++ b/e2e/fixtures/computer-use-hybrid-llama-server.mjs @@ -0,0 +1,134 @@ +#!/usr/bin/env node + +// The only model fake for the hybrid Computer Use E2E. The app still owns +// settings, IPC, model swaps, the action engine, task graph, capture evidence, +// actuation adapter, task history, supervisor, and rendered task state. +import fs from 'node:fs' +import http from 'node:http' +import path from 'node:path' + +const args = process.argv.slice(2) +const portFlag = Math.max(args.indexOf('--port'), args.indexOf('-p')) +const port = portFlag >= 0 ? Number(args[portFlag + 1]) : 8439 +const stateFile = path.join(process.env.OFFGRID_USER_DATA, 'hybrid-model-state.json') +const requestLog = path.join(process.env.OFFGRID_USER_DATA, 'hybrid-model-requests.jsonl') + +const readCount = () => { + try { + return JSON.parse(fs.readFileSync(stateFile, 'utf8')).reasonerCalls ?? 0 + } catch { + return 0 + } +} + +const writeCount = (reasonerCalls) => fs.writeFileSync(stateFile, JSON.stringify({ reasonerCalls })) + +const toolCall = (name, value) => ({ + index: 0, + id: `call_${name}`, + type: 'function', + function: { name, arguments: JSON.stringify(value) } +}) + +const responseFor = (body) => { + if (body.includes('delegate_grounded_action')) { + const count = readCount() + writeCount(count + 1) + return count === 0 + ? { + toolCalls: [ + toolCall('delegate_grounded_action', { + instruction: 'Move the pointer to the center of the visible test window.', + summary: 'Point at the visible test window.', + visible_evidence: 'The Off Grid AI test window is visible.' + }) + ] + } + : { + toolCalls: [ + toolCall('complete_milestone', { + summary: 'The pointer moved to the test window.', + visible_evidence: 'The current screen remains visible after the pointer move.' + }) + ] + } + } + // The production UI-TARS adapter owns the specialist request budget. Match that native + // boundary instead of a presentation string that is not required to include the model name. + if (body.includes('"max_tokens":200')) { + return { content: "mouse_move(point='500 500')" } + } + if (body.includes('execution plan') || body.includes('Execution plan')) { + return { + content: JSON.stringify({ + version: 1, + phases: [{ id: 'move-pointer', title: 'Move the pointer to the test window' }] + }) + } + } + if (body.includes('computer_task') && !body.includes('The pointer moved to the test window.')) { + return { + content: + '{"name":"computer_task","arguments":{"goal":"Move the pointer to the center of the visible Off Grid AI test window."}}' + } + } + return { content: 'Done - the hybrid Computer Use task completed.' } +} + +const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) + return + } + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'e2e-hybrid-model' }] })) + return + } + if (request.method !== 'POST' || !String(request.url).includes('/chat/completions')) { + response.writeHead(404) + response.end() + return + } + let raw = '' + request.on('data', (chunk) => { + raw += chunk + }) + request.on('end', () => { + const body = JSON.parse(raw) + fs.appendFileSync(requestLog, `${JSON.stringify(body)}\n`) + const turn = responseFor(raw) + if (body.stream !== true) { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [ + { + message: { + content: turn.content ?? '', + ...(turn.toolCalls ? { tool_calls: turn.toolCalls } : {}) + } + } + ], + usage: { total_tokens: 0 } + }) + ) + return + } + response.writeHead(200, { 'Content-Type': 'text/event-stream' }) + if (turn.toolCalls) { + response.write( + `data: ${JSON.stringify({ choices: [{ delta: { tool_calls: turn.toolCalls } }] })}\n\n` + ) + } else if (turn.content) { + response.write( + `data: ${JSON.stringify({ choices: [{ delta: { content: turn.content } }] })}\n\n` + ) + } + response.write('data: [DONE]\n\n') + response.end() + }) +}) + +server.listen(port, '127.0.0.1') diff --git a/e2e/fixtures/cu004-web-use-llama-server.mjs b/e2e/fixtures/cu004-web-use-llama-server.mjs new file mode 100644 index 00000000..f5fc9764 --- /dev/null +++ b/e2e/fixtures/cu004-web-use-llama-server.mjs @@ -0,0 +1,305 @@ +#!/usr/bin/env node +/* eslint-disable @typescript-eslint/explicit-function-return-type -- executable JavaScript boundary */ + +// CU-004 model boundary. The production Web Use host, screenshot capture, +// canonical vision adapter, coordinate mapping, CDP driver, pointer injection, +// takeover coordinator, IPC, task store, and renderer stay real. This process +// returns deterministic strict visual decisions for the local QA page. +import http from 'node:http' +import { writeFileSync } from 'node:fs' +import path from 'node:path' + +const MODEL_ID = 'mradermacher/UI-TARS-1.5-7B-GGUF' +const VISUAL_FIELDS = [ + 'direction', + 'milestone_complete', + 'action_verdict', + 'summary', + 'visible_evidence', + 'action', + 'action_reason' +] +const args = process.argv.slice(2) +const portFlag = Math.max(args.indexOf('--port'), args.indexOf('-p')) +const port = portFlag >= 0 ? Number(args[portFlag + 1]) : 8439 +const profile = process.env.OFFGRID_USER_DATA +if (profile) writeFileSync(path.join(profile, 'qa-model-port'), String(port)) +const pendingDecisions = [] +let lastDecision = null +let lastPrompt = '' +let lastAudit = null +let visualRequestCount = 0 + +const textParts = (payload) => + (payload.messages ?? []).flatMap((message) => { + if (typeof message?.content === 'string') return [message.content] + if (!Array.isArray(message?.content)) return [] + return message.content + .filter((part) => part?.type === 'text' && typeof part.text === 'string') + .map((part) => part.text) + }) + +const imageParts = (payload) => + (payload.messages ?? []).flatMap((message) => + Array.isArray(message?.content) + ? message.content.filter( + (part) => + part?.type === 'image_url' && + typeof part.image_url?.url === 'string' && + part.image_url.url.startsWith('data:image/') + ) + : [] + ) + +const auditVisualRequest = (payload, prompt) => { + const schema = payload.response_format?.json_schema + const required = [...(schema?.schema?.required ?? [])].sort() + const errors = [] + if (payload.stream !== true) errors.push('visual request was not streamed') + if (schema?.name !== 'visual_step_decision' || schema?.strict !== true) { + errors.push('visual_step_decision was not strict') + } + if (required.join(',') !== [...VISUAL_FIELDS].sort().join(',')) { + errors.push('visual decision fields did not match the canonical contract') + } + if (schema?.schema?.additionalProperties !== false) { + errors.push('visual decision allowed additional properties') + } + if (payload.chat_template_kwargs?.enable_thinking !== true) { + errors.push('visual decision was not thinking-enabled') + } + if (payload.reasoning_format !== 'deepseek') { + errors.push('visual reasoning was not separated') + } + if (imageParts(payload).length !== 1) errors.push('visual request did not contain one screenshot') + for (const section of [ + 'Task brief:', + 'Current milestone:', + 'Recent verified actions:', + 'Screenshot coordinate space:' + ]) { + if (!prompt.includes(section)) errors.push(`visual request omitted ${section}`) + } + return { valid: errors.length === 0, errors } +} + +const verifiedActions = (prompt) => + prompt.match( + /Recent verified actions:\n([\s\S]*?)(?:\n\nPrior validated judge decisions:|\n\nRecent task events:|\n\nScreenshot coordinate space:)/ + )?.[1] ?? '' + +const screenshotBounds = (prompt) => { + const match = prompt.match(/screenshot is (\d+) pixels wide and (\d+) pixels high/i) + return match ? { width: Number(match[1]), height: Number(match[2]) } : null +} + +const verdict = ({ summary, evidence, action = null, milestoneComplete = false }) => ({ + direction: 'aligned', + milestone_complete: milestoneComplete, + action_verdict: milestoneComplete ? 'none' : action ? 'approve' : 'rethink', + summary, + visible_evidence: evidence, + action, + action_reason: milestoneComplete + ? 'The visible result completes the current milestone.' + : action + ? 'This one visible action advances the current milestone.' + : 'A verified target is not available for the current milestone.' +}) + +const decisionFor = (prompt) => { + if (prompt.includes('Create a short execution plan for a web agent.')) { + return { + phases: ['Enter the requested text', 'Click the target', 'Confirm the protected account step'] + } + } + if (prompt.includes('resumed by the user')) return null + const actions = verifiedActions(prompt) + if (prompt.includes('Current milestone:\nEnter the requested text')) { + if (actions.includes('type text')) { + return verdict({ + summary: 'The requested text is present.', + evidence: 'The focused text field visibly contains the requested text.', + milestoneComplete: true + }) + } + if (actions.includes('click at (')) { + return verdict({ + summary: 'Enter the requested text in the focused field.', + evidence: 'The Type target field is visible and focused.', + action: "type(content='cursor stays visible')" + }) + } + const bounds = screenshotBounds(prompt) + if (!bounds) { + return verdict({ + summary: 'The screenshot bounds are unavailable.', + evidence: 'No exact screenshot coordinate space is available.' + }) + } + const x = Math.round(bounds.width * 0.26) + const y = Math.round(bounds.height * 0.32) + return verdict({ + summary: 'Focus the visible text field.', + evidence: 'The Type target field is visible at the specified point.', + action: `click(point='${x} ${y}')` + }) + } + if (prompt.includes('Current milestone:\nClick the target')) { + const clickCount = actions.match(/click at \(/g)?.length ?? 0 + if (clickCount >= 2) { + return verdict({ + summary: 'The click target was activated.', + evidence: 'The page visibly reports that the pointer click was recorded.', + milestoneComplete: true + }) + } + const bounds = screenshotBounds(prompt) + if (!bounds) { + return verdict({ + summary: 'The screenshot bounds are unavailable.', + evidence: 'No exact screenshot coordinate space is available.' + }) + } + const x = Math.round(bounds.width * 0.26) + const y = Math.round(bounds.height * 0.58) + return verdict({ + summary: 'Activate the visible click target.', + evidence: 'The Click target button is visible at the specified point.', + action: `click(point='${x} ${y}')` + }) + } + if (prompt.includes('Current milestone:\nConfirm the protected account step')) { + return verdict({ + summary: 'The protected account step requires the user.', + evidence: 'A password field is visible on the current page.', + action: "call_user(content='Confirm the protected account step yourself.')" + }) + } + return verdict({ + summary: 'The current milestone is not available.', + evidence: 'The prompt has no recognized current milestone.' + }) +} + +const sendDecision = (response, decision, stream, audit) => { + if (!audit.valid) { + response.writeHead(422, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + error: { message: `CU-004 canonical request mismatch: ${audit.errors.join('; ')}` } + }) + ) + return + } + if (!decision) { + response.writeHead(500, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ error: { message: 'CU-004 terminal model failure' } })) + return + } + const content = JSON.stringify(decision) + if (stream) { + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }) + response.write( + `data: ${JSON.stringify({ choices: [{ delta: { reasoning_content: 'Reviewed the fresh screenshot and current milestone.' } }] })}\n\n` + ) + response.write( + `data: ${JSON.stringify({ choices: [{ delta: { content }, finish_reason: 'stop' }] })}\n\n` + ) + response.end('data: [DONE]\n\n') + return + } + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [ + { + message: { role: 'assistant', content }, + finish_reason: 'stop' + } + ], + usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 } + }) + ) +} + +const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) + return + } + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: MODEL_ID }] })) + return + } + if (request.method === 'GET' && request.url === '/props') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ chat_template: '{% if enable_thinking %}{% endif %}' })) + return + } + if (request.method === 'GET' && request.url === '/qa/pending-decision') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ pending: pendingDecisions.length > 0 })) + return + } + if (request.method === 'GET' && request.url === '/qa/state') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ lastDecision, lastPrompt, lastAudit, visualRequestCount })) + return + } + if (request.method === 'POST' && request.url === '/qa/release-decision') { + pendingDecisions.shift()?.() + response.writeHead(204) + response.end() + return + } + if (request.method !== 'POST' || !String(request.url).includes('/chat/completions')) { + response.writeHead(404) + response.end() + return + } + let body = '' + request.on('data', (chunk) => { + body += chunk + }) + request.on('end', () => { + let payload = null + try { + payload = JSON.parse(body) + if (typeof payload === 'string') payload = JSON.parse(payload) + } catch { + response.writeHead(400, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ error: { message: 'CU-004 request JSON was malformed' } })) + return + } + const prompt = textParts(payload).join('\n\n') + const isPlan = prompt.includes('Create a short execution plan for a web agent.') + if (isPlan) { + const decision = decisionFor(prompt) + lastPrompt = prompt + lastDecision = decision + sendDecision(response, decision, false, { valid: true, errors: [] }) + return + } + const audit = auditVisualRequest(payload, prompt) + visualRequestCount += 1 + lastAudit = audit + pendingDecisions.push(() => { + const decision = decisionFor(prompt) + lastPrompt = prompt + lastDecision = decision + sendDecision(response, decision, payload.stream === true, audit) + }) + }) +}) + +server.listen(port, '127.0.0.1') +const close = () => server.close(() => process.exit(0)) +process.on('SIGTERM', close) +process.on('SIGINT', close) diff --git a/e2e/functional-real-engine.spec.ts b/e2e/functional-real-engine.spec.ts index 9552ff50..2c194fea 100644 --- a/e2e/functional-real-engine.spec.ts +++ b/e2e/functional-real-engine.spec.ts @@ -3,7 +3,7 @@ * "does it actually work" smoke: real llama-server generates chat, real kokoro synthesizes * speech, real whisper transcribes it back, real sd generates an image. No fakes. * - * Local-only by design: it needs the real model files (~/Library/Application Support/Off Grid + * Local-only by design: it needs the real model files (~/Library/Application Support/Off Grid AI * AI Desktop/models) and the real engine binaries (resources/bin). It SKIPS anywhere those are * absent (CI, a fresh checkout) so it never turns the suite red where the models can't exist. * Profile stays synthetic (seeded temp dir); only the model files are the real, shared ones. @@ -80,7 +80,7 @@ test.beforeAll(async () => { page = await app.firstWindow() await page.waitForLoadState('domcontentloaded') for (let i = 0; i < 8; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click().catch(() => {}) await page.waitForTimeout(300) diff --git a/e2e/helpers/launch.ts b/e2e/helpers/launch.ts index 67e157cf..0edb1859 100644 --- a/e2e/helpers/launch.ts +++ b/e2e/helpers/launch.ts @@ -114,10 +114,21 @@ export interface LaunchOptions { env?: Record /** Extra Chromium/Electron flags (e.g. fake media devices). Applied to both targets. */ extraArgs?: string[] + /** Working directory for the DEV target's app process. The native actions + * helper resolves dev candidates relative to cwd, so a spec can plant a + * fake helper in a temp dir and point the app at it. Ignored when + * packaged (resolution uses resourcesPath there). */ + cwd?: string } export const launchOffGrid = async (options: LaunchOptions = {}): Promise => { const env = withCoverage({ ...process.env, ...options.env } as Record) + // A runner that itself lives inside Electron (VS Code tasks, agent + // sandboxes) exports ELECTRON_RUN_AS_NODE=1; inherited, it turns the + // launched app into plain Node - electron.app is undefined and every spec + // dies with "Process failed to launch". The app under test must never + // run as node. + delete env.ELECTRON_RUN_AS_NODE const extraArgs = options.extraArgs ?? [] if (targetIsPackaged()) { @@ -134,5 +145,6 @@ export const launchOffGrid = async (options: LaunchOptions = {}): Promise => { const cta = page.getByRole('button', { name: ONBOARDING_CTA }).first() diff --git a/e2e/helpers/settings.ts b/e2e/helpers/settings.ts index 541ea240..77ba04d4 100644 --- a/e2e/helpers/settings.ts +++ b/e2e/helpers/settings.ts @@ -40,13 +40,14 @@ export const settingsSectionHeader = (page: Page, title: string): Locator => /** * Collapse whichever section is currently the open detail, returning to the card grid. - * Uses the group's own Cmd/Ctrl+] shortcut — the same seam the app ships — so this keeps - * working if the header markup changes. No-op when nothing is open. + * The open SettingsCard is the only button whose accessible name starts with "All settings". + * Scope to that product boundary so expanded sidebar navigation groups are not mistaken for + * Settings details. No-op when nothing is open. */ export const closeSettingsSection = async (page: Page): Promise => { - const open = page.locator('button[aria-expanded="true"]') + const open = page.getByRole('button', { name: /^All settings\s/ }) if ((await open.count()) === 0) return - await page.keyboard.press('Control+]') + await open.click() await expect(open).toHaveCount(0) } diff --git a/e2e/onboarding-resume.spec.ts b/e2e/onboarding-resume.spec.ts index efce20c0..96c662c6 100644 --- a/e2e/onboarding-resume.spec.ts +++ b/e2e/onboarding-resume.spec.ts @@ -43,7 +43,7 @@ async function launchApp(): Promise { async function finishOnboarding(): Promise { for (let step = 0; step < 6; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() } @@ -126,7 +126,7 @@ test('relaunch resumes onboarding progress and one interrupted transfer (#12)', await launchApp() await expect(page.getByRole('heading', { name: 'Models' })).toBeVisible() - await expect(page.getByRole('button', { name: /Continue|Start using Off Grid/i })).toHaveCount(0) + await expect(page.getByRole('button', { name: /Continue|Start using Off Grid AI/i })).toHaveCount(0) const downloads = await page.evaluate(async () => window.api.listDownloads()) expect(downloads).toEqual([ diff --git a/e2e/p0-capture-consent-stop.spec.ts b/e2e/p0-capture-consent-stop.spec.ts index 0c677cde..ea08566f 100644 --- a/e2e/p0-capture-consent-stop.spec.ts +++ b/e2e/p0-capture-consent-stop.spec.ts @@ -103,7 +103,7 @@ test('APP-119 and APP-121: capture requires opt-in, then pause stops it everywhe test.setTimeout(90_000) await gotoCaptureSettings() - // A new device is paused by Off Grid even though the controlled OS boundary says permission is + // A new device is paused by Off Grid AI even though the controlled OS boundary says permission is // granted. Waiting past a full scheduler interval proves this is not a first-paint race. const settingsStatus = page.getByRole('status').filter({ hasText: 'Paused' }).first() await expect(settingsStatus).toBeVisible() diff --git a/e2e/pro.spec.ts b/e2e/pro.spec.ts index 722ec40e..5213bbe9 100644 --- a/e2e/pro.spec.ts +++ b/e2e/pro.spec.ts @@ -148,7 +148,7 @@ test.beforeAll(async () => { await page.waitForLoadState('domcontentloaded') // Click through onboarding into the app shell. for (let i = 0; i < 8; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click() await page.waitForTimeout(400) @@ -174,7 +174,7 @@ test.afterAll(async () => { test('Replay is unlocked in the pro build (renders the manager, not the upgrade screen)', async () => { await nav('Replay') - await expect(page.getByText('Off Grid Pro · Available now')).toHaveCount(0) + await expect(page.getByText('Off Grid AI Pro · Available now')).toHaveCount(0) // The seeded day has frames, so the film + scrubber render. await expect(page.getByText(/frames?$/).first()).toBeVisible() }) @@ -365,7 +365,7 @@ test('Search opens Replay at the selected captured moment instead of a timeline expect(expected.hit.imagePath).toBe(expected.selected.path) await page.keyboard.press('Meta+K') - const search = page.getByRole('dialog', { name: 'Search Off Grid' }) + const search = page.getByRole('dialog', { name: 'Search Off Grid AI' }) await expect(search).toBeVisible() await search.getByPlaceholder('Search everything…').fill(expected.query) const result = search.getByText(expected.hit.snippet, { exact: true }) @@ -406,7 +406,7 @@ test('Capture and processing share one actionable Settings detail with keyboard test('Clipboard is unlocked in the pro build', async () => { await nav('Clipboard') - await expect(page.getByText('Off Grid Pro · Available now')).toHaveCount(0) + await expect(page.getByText('Off Grid AI Pro · Available now')).toHaveCount(0) await expect(page.getByPlaceholder('Search content or tags…')).toBeVisible() }) @@ -456,7 +456,7 @@ test('Clipboard quick-open renders populated content on the first native hotkey test('Voice is unlocked in the pro build (renders the dictation library)', async () => { await nav('Voice') - await expect(page.getByText('Off Grid Pro · Available now')).toHaveCount(0) + await expect(page.getByText('Off Grid AI Pro · Available now')).toHaveCount(0) // The real screen: a search box, the dictation CTA, and the file-transcribe entry. await expect(page.getByPlaceholder('Search transcripts')).toBeVisible() await expect(page.getByRole('button', { name: 'Start dictation' })).toBeVisible() diff --git a/e2e/projects-layout.spec.ts b/e2e/projects-layout.spec.ts index 4cf3b004..b319e471 100644 --- a/e2e/projects-layout.spec.ts +++ b/e2e/projects-layout.spec.ts @@ -16,7 +16,7 @@ let userDataDir: string async function finishOnboarding(): Promise { for (let step = 0; step < 6; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() } diff --git a/e2e/screenshots-quality-hardening.spec.ts b/e2e/screenshots-quality-hardening.spec.ts index ff392311..6a37f2f1 100644 --- a/e2e/screenshots-quality-hardening.spec.ts +++ b/e2e/screenshots-quality-hardening.spec.ts @@ -50,7 +50,7 @@ test.beforeAll(async () => { page = await app.firstWindow() await page.waitForLoadState('domcontentloaded') for (let i = 0; i < 8; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click().catch(() => {}) await page.waitForTimeout(400) diff --git a/e2e/screenshots/agentic-studio/01-chat-tasks-docked-dark.png b/e2e/screenshots/agentic-studio/01-chat-tasks-docked-dark.png new file mode 100644 index 00000000..57798eaf Binary files /dev/null and b/e2e/screenshots/agentic-studio/01-chat-tasks-docked-dark.png differ diff --git a/e2e/screenshots/agentic-studio/02-task-detail-step-expanded.png b/e2e/screenshots/agentic-studio/02-task-detail-step-expanded.png new file mode 100644 index 00000000..7b383527 Binary files /dev/null and b/e2e/screenshots/agentic-studio/02-task-detail-step-expanded.png differ diff --git a/e2e/screenshots/agentic-studio/03-task-history-after-back.png b/e2e/screenshots/agentic-studio/03-task-history-after-back.png new file mode 100644 index 00000000..79f90148 Binary files /dev/null and b/e2e/screenshots/agentic-studio/03-task-history-after-back.png differ diff --git a/e2e/screenshots/agentic-studio/04-retry-created-new-attempt.png b/e2e/screenshots/agentic-studio/04-retry-created-new-attempt.png new file mode 100644 index 00000000..50d82982 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04-retry-created-new-attempt.png differ diff --git a/e2e/screenshots/agentic-studio/04-retry-resumed-same-task.png b/e2e/screenshots/agentic-studio/04-retry-resumed-same-task.png new file mode 100644 index 00000000..0bc71007 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04-retry-resumed-same-task.png differ diff --git a/e2e/screenshots/agentic-studio/04-web-use-start-immersive-detail.png b/e2e/screenshots/agentic-studio/04-web-use-start-immersive-detail.png new file mode 100644 index 00000000..37800ed3 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04-web-use-start-immersive-detail.png differ diff --git a/e2e/screenshots/agentic-studio/04a-web-use-pointer-initial-native-window.png b/e2e/screenshots/agentic-studio/04a-web-use-pointer-initial-native-window.png new file mode 100644 index 00000000..8f1d2ba6 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04a-web-use-pointer-initial-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/04b-web-use-pointer-and-takeover.png b/e2e/screenshots/agentic-studio/04b-web-use-pointer-and-takeover.png new file mode 100644 index 00000000..f03a6117 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04b-web-use-pointer-and-takeover.png differ diff --git a/e2e/screenshots/agentic-studio/04b-web-use-pointer-between-actions-native-window.png b/e2e/screenshots/agentic-studio/04b-web-use-pointer-between-actions-native-window.png new file mode 100644 index 00000000..bcd80875 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04b-web-use-pointer-between-actions-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/04c-web-use-pointer-and-takeover-native-window.png b/e2e/screenshots/agentic-studio/04c-web-use-pointer-and-takeover-native-window.png new file mode 100644 index 00000000..8d8a3015 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04c-web-use-pointer-and-takeover-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/04d-web-use-resumed-and-complete.png b/e2e/screenshots/agentic-studio/04d-web-use-resumed-and-complete.png new file mode 100644 index 00000000..36aff7d0 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04d-web-use-resumed-and-complete.png differ diff --git a/e2e/screenshots/agentic-studio/04d-web-use-resumed-and-failed.png b/e2e/screenshots/agentic-studio/04d-web-use-resumed-and-failed.png new file mode 100644 index 00000000..cfd97507 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04d-web-use-resumed-and-failed.png differ diff --git a/e2e/screenshots/agentic-studio/04d-web-use-terminal-shell-restored.png b/e2e/screenshots/agentic-studio/04d-web-use-terminal-shell-restored.png new file mode 100644 index 00000000..d8b0d1b4 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04d-web-use-terminal-shell-restored.png differ diff --git a/e2e/screenshots/agentic-studio/04e-web-use-pointer-terminal-failure-native-window.png b/e2e/screenshots/agentic-studio/04e-web-use-pointer-terminal-failure-native-window.png new file mode 100644 index 00000000..e09d987f Binary files /dev/null and b/e2e/screenshots/agentic-studio/04e-web-use-pointer-terminal-failure-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/04f-web-use-pointer-terminal-dark-page-native-window.png b/e2e/screenshots/agentic-studio/04f-web-use-pointer-terminal-dark-page-native-window.png new file mode 100644 index 00000000..e09d987f Binary files /dev/null and b/e2e/screenshots/agentic-studio/04f-web-use-pointer-terminal-dark-page-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/04g-web-use-execution-plan-detail.png b/e2e/screenshots/agentic-studio/04g-web-use-execution-plan-detail.png new file mode 100644 index 00000000..d6b493f1 Binary files /dev/null and b/e2e/screenshots/agentic-studio/04g-web-use-execution-plan-detail.png differ diff --git a/e2e/screenshots/agentic-studio/05-live-browser-example.png b/e2e/screenshots/agentic-studio/05-live-browser-example.png new file mode 100644 index 00000000..a24441b5 Binary files /dev/null and b/e2e/screenshots/agentic-studio/05-live-browser-example.png differ diff --git a/e2e/screenshots/agentic-studio/05b-live-browser-native-window.png b/e2e/screenshots/agentic-studio/05b-live-browser-native-window.png new file mode 100644 index 00000000..812e234f Binary files /dev/null and b/e2e/screenshots/agentic-studio/05b-live-browser-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/06-models-task-hidden.png b/e2e/screenshots/agentic-studio/06-models-task-hidden.png new file mode 100644 index 00000000..b41dd03a Binary files /dev/null and b/e2e/screenshots/agentic-studio/06-models-task-hidden.png differ diff --git a/e2e/screenshots/agentic-studio/07-chat-task-restored.png b/e2e/screenshots/agentic-studio/07-chat-task-restored.png new file mode 100644 index 00000000..3ba84169 Binary files /dev/null and b/e2e/screenshots/agentic-studio/07-chat-task-restored.png differ diff --git a/e2e/screenshots/agentic-studio/07b-chat-task-restored-native-window.png b/e2e/screenshots/agentic-studio/07b-chat-task-restored-native-window.png new file mode 100644 index 00000000..e15ade6f Binary files /dev/null and b/e2e/screenshots/agentic-studio/07b-chat-task-restored-native-window.png differ diff --git a/e2e/screenshots/agentic-studio/08-task-full-width.png b/e2e/screenshots/agentic-studio/08-task-full-width.png new file mode 100644 index 00000000..6437b069 Binary files /dev/null and b/e2e/screenshots/agentic-studio/08-task-full-width.png differ diff --git a/e2e/screenshots/agentic-studio/08-task-keyboard-resized.png b/e2e/screenshots/agentic-studio/08-task-keyboard-resized.png new file mode 100644 index 00000000..732a6bdc Binary files /dev/null and b/e2e/screenshots/agentic-studio/08-task-keyboard-resized.png differ diff --git a/e2e/screenshots/agentic-studio/09-task-settings.png b/e2e/screenshots/agentic-studio/09-task-settings.png new file mode 100644 index 00000000..4906711a Binary files /dev/null and b/e2e/screenshots/agentic-studio/09-task-settings.png differ diff --git a/e2e/screenshots/agentic-studio/10-chat-tasks-docked-light.png b/e2e/screenshots/agentic-studio/10-chat-tasks-docked-light.png new file mode 100644 index 00000000..9f24b330 Binary files /dev/null and b/e2e/screenshots/agentic-studio/10-chat-tasks-docked-light.png differ diff --git a/e2e/screenshots/explore-chat-empty.png b/e2e/screenshots/explore-chat-empty.png new file mode 100644 index 00000000..a4d52862 Binary files /dev/null and b/e2e/screenshots/explore-chat-empty.png differ diff --git a/e2e/screenshots/explore-proposal-setup.png b/e2e/screenshots/explore-proposal-setup.png new file mode 100644 index 00000000..5697bcb3 Binary files /dev/null and b/e2e/screenshots/explore-proposal-setup.png differ diff --git a/e2e/screenshots/explore-screen.png b/e2e/screenshots/explore-screen.png new file mode 100644 index 00000000..d89044b5 Binary files /dev/null and b/e2e/screenshots/explore-screen.png differ diff --git a/e2e/screenshots/r1-chat-action-verified.png b/e2e/screenshots/r1-chat-action-verified.png new file mode 100644 index 00000000..3d29b47f Binary files /dev/null and b/e2e/screenshots/r1-chat-action-verified.png differ diff --git a/e2e/settings-section-motion.spec.ts b/e2e/settings-section-motion.spec.ts index 4dd2341c..4afe9a68 100644 --- a/e2e/settings-section-motion.spec.ts +++ b/e2e/settings-section-motion.spec.ts @@ -40,7 +40,7 @@ test.beforeAll(async () => { await page.waitForLoadState('domcontentloaded') // Dismiss onboarding if present. for (let i = 0; i < 8; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click().catch(() => {}) await page.waitForTimeout(300) diff --git a/e2e/smoke.spec.ts b/e2e/smoke.spec.ts index 99cea20d..8736c161 100644 --- a/e2e/smoke.spec.ts +++ b/e2e/smoke.spec.ts @@ -84,15 +84,15 @@ test('opens filling the screen, not in a small window', async () => { }) test('shows onboarding on a fresh install', async () => { - await expect(page.getByText(/Off Grid/i).first()).toBeVisible() - await expect(page.getByRole('button', { name: /Continue|Start using Off Grid/i })).toBeVisible() + await expect(page.getByText(/Off Grid AI/i).first()).toBeVisible() + await expect(page.getByRole('button', { name: /Continue|Start using Off Grid AI/i })).toBeVisible() }) test('onboarding surfaces the Pro capability grid', async () => { // Advance until the Pro step renders its capability cards, then assert a few // capabilities are shown by name (Replay, Meetings, Vault). Regression guard // for the onboarding redesign that showcases the Pro layer. - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) for (let i = 0; i < 6; i++) { if ( await page @@ -112,9 +112,9 @@ test('onboarding surfaces the Pro capability grid', async () => { }) test('completes onboarding and lands in the app shell', async () => { - // Click through every onboarding step (Continue × N, then "Start using Off Grid"). + // Click through every onboarding step (Continue × N, then "Start using Off Grid AI"). for (let i = 0; i < 6; i++) { - const btn = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const btn = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await btn.isVisible().catch(() => false))) break await btn.click() await page.waitForTimeout(400) diff --git a/e2e/tts-speak.spec.ts b/e2e/tts-speak.spec.ts index f34dbbcd..808033a2 100644 --- a/e2e/tts-speak.spec.ts +++ b/e2e/tts-speak.spec.ts @@ -17,7 +17,7 @@ let spokenTextPath: string async function finishOnboarding(): Promise { for (let step = 0; step < 8; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() } diff --git a/e2e/update-rollback-packaged.spec.ts b/e2e/update-rollback-packaged.spec.ts index c5444a44..d478d181 100644 --- a/e2e/update-rollback-packaged.spec.ts +++ b/e2e/update-rollback-packaged.spec.ts @@ -47,7 +47,7 @@ test.beforeAll(async () => { await page.waitForLoadState('domcontentloaded') for (let step = 0; step < 8; step++) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) break await button.click() } diff --git a/e2e/voice-real-audio.spec.ts b/e2e/voice-real-audio.spec.ts index 7bc90b83..3ceb45e2 100644 --- a/e2e/voice-real-audio.spec.ts +++ b/e2e/voice-real-audio.spec.ts @@ -54,7 +54,7 @@ function assertDisposableProfile(candidate: string): void { async function finishOnboarding(): Promise { for (let step = 0; step < 8; step += 1) { - const button = page.getByRole('button', { name: /Continue|Start using Off Grid/i }) + const button = page.getByRole('button', { name: /Continue|Start using Off Grid AI/i }) if (!(await button.isVisible().catch(() => false))) return await button.click() await page.waitForTimeout(250) diff --git a/electron-builder.yml b/electron-builder.yml index 424c846a..07527b82 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -60,13 +60,28 @@ mac: NSCameraUsageDescription: Off Grid AI may use the camera for on-device vision features. Frames are processed locally and never leave your device. NSDocumentsFolderUsageDescription: Off Grid AI reads documents you add to a project so it can answer questions about them — locally, on your device. NSDownloadsFolderUsageDescription: Off Grid AI reads files you add to a project so it can answer questions about them — locally, on your device. - NSLocalNetworkUsageDescription: Off Grid AI Desktop uses your local network to find and sync directly with your devices. Sync traffic is encrypted and no Off Grid server receives it. + NSLocalNetworkUsageDescription: Off Grid AI Desktop uses your local network to find and sync directly with your devices. Sync traffic is encrypted and no Off Grid AI server receives it. NSBonjourServices: - _offgrid._tcp - _offgrid-sync._tcp - _offgrid-sync._udp + # Computer use — the semantic action rail. Each key is the OS prompt shown the + # first time the agent uses that capability; without it a hardened-runtime build + # is refused access before any prompt. Copy follows off-grid-ai/brand: lead with + # what the user gets, privacy as the proof, no em dashes. + NSAppleEventsUsageDescription: Off Grid AI controls apps like Messages, Mail, and Notes to carry out actions you approve. Everything runs on your Mac and nothing leaves the device. + NSCalendarsFullAccessUsageDescription: Off Grid AI reads and creates calendar events when you ask, on your device. Your calendar never leaves your Mac. + NSCalendarsUsageDescription: Off Grid AI reads and creates calendar events when you ask, on your device. Your calendar never leaves your Mac. + NSRemindersFullAccessUsageDescription: Off Grid AI reads and creates reminders when you ask, on your device. Your reminders never leave your Mac. + NSContactsUsageDescription: Off Grid AI reads your contacts to complete actions you ask for, like sending a message. Your contacts never leave your Mac. + NSPhotoLibraryUsageDescription: Off Grid AI works with photos you ask it to, like picking one to send. Your photos never leave your Mac. notarize: true icon: resources/icon.png + extraResources: + - from: ../executorch-speech/native/bin/executorch-speech + to: bin/executorch-speech + - from: ../executorch-speech/generated/default-assets + to: speech-assets # NOTE: no afterSign re-sign hook. electron-builder already signs every nested # binary (resources/bin/*) with hardenedRuntime + these entitlements and then # notarizes + staples. A post-sign re-sign (the old resign.js) re-signed AFTER diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 0db0678d..cd287093 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -41,13 +41,13 @@ export default defineConfig({ define: proDefine, build: { sourcemap: coverageSourcemap, + // The embedding worker is a SECOND main-process entry, bundled beside index.js so + // embeddings.ts can spawn it by path. It must not be folded into the main chunk: the point + // is that its ONNX/WASM inference runs off the thread that owns the window. rollupOptions: { - // The TTS worker must live inside app.asar beside its JavaScript - // dependencies. Copying the raw source into Resources makes ESM resolve - // from that external directory, where kokoro-js does not exist. input: { index: resolve('src/main/index.ts'), - 'tts-worker': resolve('resources/tts-worker.mjs') + 'embeddings-worker': resolve('src/main/embeddings-worker.ts') } } }, @@ -59,11 +59,7 @@ export default defineConfig({ // vault recovery-phrase feature (pro/main/vault/vault-recovery.ts). plugins: [ externalizeDepsPlugin({ - // Kokoro owns Transformers v3 transitively. The worker imports its env - // directly so cache configuration and Kokoro share one module instance; - // keep that native Node package external instead of bundling browser shims. - include: ['@huggingface/transformers'], - exclude: ['@scure/bip39', '@noble/hashes'] + exclude: ['@scure/bip39', '@noble/hashes', '@offgrid/executorch-speech'] }) ], resolve: { diff --git a/integration-tests/memory-chat-tts.ui.integration.dbtest.ts b/integration-tests/memory-chat-tts.ui.integration.dbtest.ts index 815dbde2..c7c62fa0 100644 --- a/integration-tests/memory-chat-tts.ui.integration.dbtest.ts +++ b/integration-tests/memory-chat-tts.ui.integration.dbtest.ts @@ -3,12 +3,13 @@ * Release checklist #105 through the real rendered assistant action, TTS IPC handler, * persisted voice setting, synthesis service, subprocess protocol, and WAV validation. * The fake subprocess replaces only the heavyweight Kokoro/ONNX worker; Audio replaces - * Chromium's media boundary. All Off Grid code between those boundaries stays production. + * Chromium's media boundary. All Off Grid AI code between those boundaries stays production. */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import { cleanup, fireEvent, screen, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { Blob as NodeBlob } from 'node:buffer' +import { EventEmitter } from 'node:events' import fs from 'node:fs' import os from 'node:os' import path from 'node:path' @@ -23,10 +24,21 @@ import { type FakeLlamaServer } from '../src/main/__tests__/harness/fake-llama-server' -type IpcEvent = { sender: { send: (channel: string, payload: unknown) => void } } +type IpcEvent = { + sender: EventEmitter & { + id: number + isDestroyed: () => boolean + send: (channel: string, payload: unknown) => void + } +} type IpcHandler = (event: IpcEvent, ...args: unknown[]) => unknown const handlers = new Map() const listeners = new Map unknown>() +const rendererSender = Object.assign(new EventEmitter(), { + id: 1, + isDestroyed: (): boolean => false, + send: (_channel: string, _payload: unknown): void => undefined +}) const root = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-memory-chat-tts-')) const dataDir = path.join(root, 'data') const resourceDir = path.join(root, 'resources') @@ -87,6 +99,7 @@ interface AudioBoundary { } const audios: AudioBoundary[] = [] +const pendingSpeech = new Set>() let fakeLlama: FakeLlamaServer class RecorderBoundary { @@ -118,10 +131,21 @@ function executable(relativePath: string, source: string): void { fs.writeFileSync(target, source, { mode: 0o755 }) } +function resourceExecutable(relativePath: string, source: string): void { + const target = path.join(resourceDir, 'bin', relativePath) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, source, { mode: 0o755 }) +} + async function invoke(channel: string, ...args: unknown[]): Promise { const handler = handlers.get(channel) if (!handler) throw new Error(`IPC handler not registered: ${channel}`) - return (await handler({ sender: { send: () => undefined } }, ...args)) as T + const result = Promise.resolve(handler({ sender: rendererSender }, ...args)) + if (channel === 'tts:speak') { + pendingSpeech.add(result) + void result.finally(() => pendingSpeech.delete(result)).catch(() => undefined) + } + return (await result) as T } function installProductionVoiceBridge(boundary: ChatBoundary): void { @@ -137,8 +161,9 @@ function installProductionVoiceBridge(boundary: ChatBoundary): void { invoke('rag:truncate-messages', id, keepCount), getSettings: () => invoke('settings:get'), saveSetting: (key: string, value: unknown) => invoke('settings:save', key, value), - transcribeAudio: (audio: ArrayBuffer | Uint8Array, ext?: string) => - invoke('voice:transcribe', audio, ext), + transcribeAudio: (audio: ArrayBuffer | Uint8Array, ext: string, requestId: string) => + invoke('voice:transcribe', audio, ext, requestId), + cancelTranscription: (requestId: string) => invoke('voice:cancel-transcription', requestId), ragChat: (...args: unknown[]) => invoke('rag:chat', ...args), speak: (text: string, voice?: string) => invoke('tts:speak', text, voice) }) @@ -151,13 +176,11 @@ function setEnv(name: string, original: string | undefined): void { } function installRealSpeechBridge(boundary: ChatBoundary): void { - const handler = handlers.get('tts:speak') - if (!handler) throw new Error('TTS IPC handler was not registered') ;( boundary.api as unknown as { speak: (text: string, voice?: string) => Promise<{ dataUrl: string }> } - ).speak = (text, voice) => handler(undefined, text, voice) as Promise<{ dataUrl: string }> + ).speak = (text, voice) => invoke('tts:speak', text, voice) installBoundary(boundary) } @@ -168,11 +191,13 @@ beforeAll(async () => { fs.writeFileSync(path.join(dataDir, 'models', 'ggml-base.bin'), 'synthetic whisper model') executable('ffmpeg', ['#!/bin/sh', 'for last; do :; done', 'printf RIFF > "$last"'].join('\n')) executable('whisper/whisper-cli', '#!/bin/sh\nprintf "Schedule the stable release review\\n"') - fs.writeFileSync( - path.join(resourceDir, 'tts-worker.mjs'), + resourceExecutable( + 'executorch-speech', [ - "import fs from 'node:fs'", - 'const [, , command, output, voice] = process.argv', + '#!/usr/bin/env node', + "const fs = require('node:fs')", + 'const args = process.argv.slice(2)', + 'const value = flag => args[args.indexOf(flag) + 1]', "let input = ''", "process.stdin.setEncoding('utf8')", "process.stdin.on('data', chunk => { input += chunk })", @@ -180,15 +205,14 @@ beforeAll(async () => { " if (fs.existsSync(process.env.OFFGRID_TTS_TEST_FAILURE_MARKER || '')) {", ' fs.rmSync(process.env.OFFGRID_TTS_TEST_FAILURE_MARKER, { force: true })', " process.stderr.write('local speech model is unavailable')", + ' process.exitCode = 23', ' return', ' }', - " if (command !== 'speak' || !output) return", ' fs.writeFileSync(process.env.OFFGRID_TTS_TEST_INPUT_RECORD, input)', - " fs.writeFileSync(process.env.OFFGRID_TTS_TEST_VOICE_RECORD, voice || '')", - " fs.writeFileSync(output, Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(60, 1)]))", + " fs.writeFileSync(process.env.OFFGRID_TTS_TEST_VOICE_RECORD, value('--voice') || '')", + " fs.writeFileSync(value('--output'), Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(60, 1)]))", '})' - ].join('\n'), - { mode: 0o755 } + ].join('\n') ) setupIPC() saveSetting('ttsVoice', 'af_bella') @@ -201,6 +225,10 @@ beforeAll(async () => { }) beforeEach(() => { + // The DB suite reuses one process across files. Recreate this file's explicit profile boundary + // before every journey so another file's teardown cannot leave a closed database with no parent. + fs.mkdirSync(dataDir, { recursive: true }) + saveSetting('ttsVoice', 'af_bella') audios.length = 0 RecorderBoundary.instances = [] fs.rmSync(failureMarker, { force: true }) @@ -225,7 +253,8 @@ beforeEach(() => { ) }) -afterEach(() => { +afterEach(async () => { + await Promise.allSettled([...pendingSpeech]) cleanup() vi.unstubAllGlobals() }) @@ -266,8 +295,13 @@ describe('assistant reply speech integration (#105)', () => { const user = userEvent.setup() renderChat({ conversationId: 'conversation-b' }) + await waitFor(() => expect(inTranscript('Conversation B baseline')).toBeTruthy(), { + timeout: 10_000 + }) await user.click(await screen.findByRole('button', { name: 'Speak' })) - await waitFor(() => expect(screen.getByRole('button', { name: 'Stop' })).toBeTruthy()) + // This crosses a real child-process boundary. Shared Linux runners can take + // more than 10 seconds to schedule the native speech worker under DB-suite load. + expect(await screen.findByRole('button', { name: 'Stop' }, { timeout: 30_000 })).toBeTruthy() expect(audios).toHaveLength(1) expect(audios[0]!.play).toHaveBeenCalledOnce() @@ -275,8 +309,8 @@ describe('assistant reply speech integration (#105)', () => { expect(metadata).toBe('data:audio/wav;base64') expect(Buffer.from(encoded!, 'base64').subarray(0, 4).toString('ascii')).toBe('RIFF') expect(fs.readFileSync(inputRecord, 'utf8')).toBe('Conversation B baseline') - expect(fs.readFileSync(voiceRecord, 'utf8')).toBe('af_bella') - }) + expect(path.basename(fs.readFileSync(voiceRecord, 'utf8'))).toContain('af_heart.bin') + }, 40_000) it('surfaces a real synthesis failure as an actionable rendered error', async () => { fs.writeFileSync(failureMarker, 'fail') @@ -285,6 +319,9 @@ describe('assistant reply speech integration (#105)', () => { const user = userEvent.setup() renderChat({ conversationId: 'conversation-b' }) + await waitFor(() => expect(inTranscript('Conversation B baseline')).toBeTruthy(), { + timeout: 10_000 + }) await user.click(await screen.findByRole('button', { name: 'Speak' })) expect((await screen.findByRole('alert')).textContent).toMatch( @@ -292,7 +329,7 @@ describe('assistant reply speech integration (#105)', () => { ) expect(audios).toHaveLength(0) expect(screen.getByRole('button', { name: 'Speak' })).toBeTruthy() - }) + }, 15_000) it('records, transcribes, chats, speaks, stops, recovers, and reopens one voice turn', async () => { const conversationId = 'voice-conversation-lifecycle' @@ -326,20 +363,23 @@ describe('assistant reply speech integration (#105)', () => { const user = userEvent.setup() const view = renderChat({ conversationId }) - await user.click(await screen.findByTitle('Voice mode off')) + await user.click(await screen.findByRole('button', { name: 'Voice', pressed: false })) await waitFor(() => expect(database.getSetting('composerVoiceMode', false)).toBe(true)) - await user.click(screen.getByText('Tap to record a voice note')) + await user.click(screen.getByRole('button', { name: 'Start voice recording' })) expect(RecorderBoundary.instances).toHaveLength(1) expect(RecorderBoundary.instances[0]!.state).toBe('recording') - await user.click(screen.getByText('Recording — tap to send')) + await user.click(screen.getByRole('button', { name: 'Stop voice recording' })) - await waitFor(() => expect(screen.getAllByText('Show transcript')).toHaveLength(2), { + await waitFor(() => expect(screen.getAllByText('Show transcript')).toHaveLength(1), { timeout: 10_000 }) - for (const toggle of screen.getAllByText('Show transcript')) await user.click(toggle) + await user.click(screen.getByText('Show transcript')) expect(screen.getByText('Schedule the stable release review')).toBeTruthy() - expect(inTranscript('The release review is scheduled locally.')).toBeTruthy() + await waitFor( + () => expect(inTranscript('The release review is scheduled locally.')).toBeTruthy(), + { timeout: 10_000 } + ) expect((await screen.findByRole('alert')).textContent).toMatch( /speech could not be generated.*text-to-speech is installed in settings/i ) @@ -355,7 +395,7 @@ describe('assistant reply speech integration (#105)', () => { }) await user.click(screen.getAllByTitle('Play').at(-1)!) - await waitFor(() => expect(screen.getByTitle('Pause')).toBeTruthy()) + expect(await screen.findByTitle('Pause', {}, { timeout: 10_000 })).toBeTruthy() expect(audios).toHaveLength(1) expect(audios[0]!.play).toHaveBeenCalledOnce() await user.click(screen.getByTitle('Pause')) @@ -365,18 +405,23 @@ describe('assistant reply speech integration (#105)', () => { installProductionVoiceBridge(new ChatBoundary()) renderChat({ conversationId }) - expect(await screen.findByTitle('Voice mode on — speak and listen in voice notes')).toBeTruthy() + expect(await screen.findByRole('button', { name: 'Voice', pressed: true })).toBeTruthy() const reopenedTranscripts = await screen.findAllByText('Show transcript') - expect(reopenedTranscripts).toHaveLength(2) - await user.click(reopenedTranscripts[1]!) - expect(inTranscript('The release review is scheduled locally.')).toBeTruthy() + expect(reopenedTranscripts).toHaveLength(1) + await user.click(reopenedTranscripts[0]!) + await waitFor( + () => expect(inTranscript('The release review is scheduled locally.')).toBeTruthy(), + { timeout: 10_000 } + ) expect(database.getSetting('ttsVoice', '')).toBe('af_bella') expect(database.getRagMessages(conversationId)).toHaveLength(2) - await user.click(screen.getByTitle('Voice mode on — speak and listen in voice notes')) + await user.click(screen.getByRole('button', { name: 'Voice', pressed: true })) await waitFor(() => expect(database.getSetting('composerVoiceMode', true)).toBe(false)) const composer = await screen.findByPlaceholderText(/^ask /i) - fireEvent.change(composer, { target: { value: 'Typed chat remains usable after voice recovery' } }) + fireEvent.change(composer, { + target: { value: 'Typed chat remains usable after voice recovery' } + }) expect((screen.getByPlaceholderText(/^ask /i) as HTMLTextAreaElement).value).toBe( 'Typed chat remains usable after voice recovery' ) diff --git a/integration-tests/remote-task-sync.integration.dbtest.ts b/integration-tests/remote-task-sync.integration.dbtest.ts new file mode 100644 index 00000000..adaee9db --- /dev/null +++ b/integration-tests/remote-task-sync.integration.dbtest.ts @@ -0,0 +1,363 @@ +/** + * Release 107 remote-task journey through the real task-history owner, frame encoder and task + * guard. Electron window/screen objects are the only unavailable OS boundary. + */ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import sharp from 'sharp' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { parseSyncedTaskRun } from '@offgrid/sync' + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-remote-task-sync-')) +process.env.OFFGRID_DATA_DIR = root + +vi.mock('electron', () => ({ + app: { getPath: () => root, isPackaged: false }, + BrowserWindow: { getAllWindows: () => [] }, + WebContentsView: class {}, + ipcMain: { handle: () => undefined, on: () => undefined }, + safeStorage: { + isEncryptionAvailable: () => false, + encryptString: (value: string) => Buffer.from(value), + decryptString: (value: Buffer) => value.toString() + } +})) + +const taskHistory = await import('../src/main/tasks/task-history') +const { VisionGuard } = await import('../src/main/vision/vision-guard') +const { registerVisionSession } = await import('../src/main/vision/vision-controller') +const { + observeTaskRunFrame, + disposeTaskRunFrameProjection, + evictTerminalTaskRunFrame, + syncedTaskRunFromSnapshot +} = await import('../pro/main/sync/task-run-projection') +const { applySyncedTaskControl, configureTaskControlSync, disposeTaskControlSync } = + await import('../pro/main/sync/task-control-sync') + +const DEVICE_ID = 'desktop-release-107' + +beforeAll(() => { + taskHistory.configureTaskExecutionDevice({ id: DEVICE_ID, name: 'Studio Mac' }) + configureTaskControlSync(DEVICE_ID) +}) + +afterAll(() => { + disposeTaskRunFrameProjection() + disposeTaskControlSync() + taskHistory.resetTaskHistoryForTests() + fs.rmSync(root, { recursive: true, force: true }) +}) + +describe('a task controlled from its synced Mobile chat', () => { + it.each(['web_use', 'computer_use'] as const)( + 'pauses the existing %s guard and consumes the direction-safe intent once', + async (kind) => { + const taskId = `remote-${kind}` + const conversationId = `chat-${kind}` + taskHistory.recordTaskRun({ + taskId, + journeyId: conversationId, + kind, + title: `Run ${kind}`, + status: 'running', + executionDeviceId: DEVICE_ID, + executionDeviceName: 'Studio Mac' + }) + const guard = new VisionGuard() + const request = new AbortController() + const release = registerVisionSession(taskId, guard, request) + const consumed: string[] = [] + const settled: Array> = [] + const fields = { + version: 1 as const, + controlId: `control-${kind}`, + taskId, + conversationId, + executionDeviceId: DEVICE_ID, + requestingDeviceId: 'mobile-release-107', + sequence: 1, + kind: 'pause' as const, + requestedAt: Date.now() + } + const provenance = { + originDeviceId: 'mobile-release-107', + originDeviceName: 'Release phone' + } + + expect( + applySyncedTaskControl( + fields.controlId, + fields, + provenance, + (_taskId, result) => settled.push(result), + (id) => consumed.push(id) + ) + ).toBe(true) + expect(guard.isPaused).toBe(true) + expect(consumed).toEqual([fields.controlId]) + expect(settled).toEqual([ + expect.objectContaining({ + controlId: fields.controlId, + kind: 'pause', + outcome: 'applied' + }) + ]) + expect( + applySyncedTaskControl( + fields.controlId, + fields, + provenance, + (_taskId, result) => settled.push(result), + (id) => consumed.push(id) + ) + ).toBe(false) + expect(consumed).toEqual([fields.controlId]) + expect(settled).toHaveLength(1) + release() + } + ) + + it.each(['web_use', 'computer_use'] as const)( + 'resumes and then stops the existing %s runtime with correlated receipts', + (kind) => { + const taskId = `resume-stop-${kind}` + const conversationId = `resume-stop-chat-${kind}` + taskHistory.recordTaskRun({ + taskId, + journeyId: conversationId, + kind, + title: `Resume and stop ${kind}`, + status: 'paused', + executionDeviceId: DEVICE_ID, + executionDeviceName: 'Studio Mac' + }) + const guard = new VisionGuard() + guard.pauseForUser('paused for the integration journey') + const request = new AbortController() + const release = registerVisionSession(taskId, guard, request) + const receipts: Array> = [] + const control = (controlKind: 'resume' | 'stop', sequence: number): boolean => + applySyncedTaskControl( + `${controlKind}-${kind}`, + { + version: 1, + controlId: `${controlKind}-${kind}`, + taskId, + conversationId, + executionDeviceId: DEVICE_ID, + requestingDeviceId: 'mobile-release-107', + sequence, + kind: controlKind, + requestedAt: Date.now() + }, + { originDeviceId: 'mobile-release-107', originDeviceName: 'Release phone' }, + (_settledTaskId, result) => receipts.push(result), + () => undefined + ) + + expect(control('resume', 1)).toBe(true) + expect(guard.snapshot().state).toBe('running') + expect(control('stop', 2)).toBe(true) + if (kind === 'web_use') { + expect(taskHistory.getTaskRun(taskId)?.status).toBe('stopped') + } else { + expect(guard.snapshot().state).toBe('halted') + expect(request.signal.aborted).toBe(true) + } + expect(receipts).toEqual([ + expect.objectContaining({ controlId: `resume-${kind}`, outcome: 'applied' }), + expect.objectContaining({ controlId: `stop-${kind}`, outcome: 'applied' }) + ]) + release() + } + ) + + it.each(['web_use', 'computer_use'] as const)( + 'hands the existing %s guard to the Mobile user with Take Over', + (kind) => { + const taskId = `takeover-${kind}` + const conversationId = `takeover-chat-${kind}` + taskHistory.recordTaskRun({ + taskId, + journeyId: conversationId, + kind, + title: `Take over ${kind}`, + status: 'running', + executionDeviceId: DEVICE_ID, + executionDeviceName: 'Studio Mac' + }) + const guard = new VisionGuard() + const request = new AbortController() + const release = registerVisionSession(taskId, guard, request) + const fields = { + version: 1 as const, + controlId: `takeover-control-${kind}`, + taskId, + conversationId, + executionDeviceId: DEVICE_ID, + requestingDeviceId: 'mobile-release-107', + sequence: 1, + kind: 'takeover' as const, + requestedAt: Date.now() + } + + expect( + applySyncedTaskControl( + fields.controlId, + fields, + { originDeviceId: 'mobile-release-107', originDeviceName: 'Release phone' }, + () => undefined, + () => undefined + ) + ).toBe(true) + expect(guard.snapshot()).toMatchObject({ + state: 'paused', + reason: 'you took over from the supervisor' + }) + release() + } + ) + + it('rejects a control whose authenticated peer does not match its requesting device', () => { + const consumed: string[] = [] + expect( + applySyncedTaskControl( + 'forged-control', + { + version: 1, + controlId: 'forged-control', + taskId: 'remote-web_use', + conversationId: 'chat-web_use', + executionDeviceId: DEVICE_ID, + requestingDeviceId: 'another-phone', + sequence: 2, + kind: 'stop', + requestedAt: Date.now() + }, + { originDeviceId: 'mobile-release-107', originDeviceName: 'Release phone' }, + () => undefined, + (id) => consumed.push(id) + ) + ).toBe(false) + expect(consumed).toEqual([]) + }) + + it('answers a valid late control once instead of letting the UI infer success', () => { + taskHistory.recordTaskRun({ + taskId: 'finished-task', + journeyId: 'finished-chat', + kind: 'computer_use', + title: 'Finished task', + status: 'done', + executionDeviceId: DEVICE_ID, + executionDeviceName: 'Studio Mac' + }) + const results: Array> = [] + const consumed: string[] = [] + const fields = { + version: 1 as const, + controlId: 'late-pause', + taskId: 'finished-task', + conversationId: 'finished-chat', + executionDeviceId: DEVICE_ID, + requestingDeviceId: 'mobile-release-107', + sequence: 1, + kind: 'pause' as const, + requestedAt: Date.now() + } + const apply = (): boolean => + applySyncedTaskControl( + fields.controlId, + fields, + { originDeviceId: 'mobile-release-107', originDeviceName: 'Release phone' }, + (_taskId, result) => results.push(result), + (controlId) => consumed.push(controlId) + ) + + expect(apply()).toBe(true) + expect(results).toEqual([ + expect.objectContaining({ + controlId: fields.controlId, + outcome: 'rejected', + message: 'The task cannot apply this control in its current state.' + }) + ]) + expect(consumed).toEqual([fields.controlId]) + expect(apply()).toBe(false) + expect(results).toHaveLength(1) + }) +}) + +describe('the live Mobile frame projection', () => { + it('uses the Desktop PiP screenshot and pointer evidence within the wire limits', async () => { + const taskId = 'computer-live-frame' + const screenshotPath = path.join(root, 'current-screen.png') + await sharp({ + create: { width: 1_440, height: 900, channels: 3, background: '#16352c' } + }) + .png() + .toFile(screenshotPath) + taskHistory.recordTaskRun({ + taskId, + journeyId: 'mobile-chat-live-frame', + kind: 'computer_use', + title: 'Send the release message', + status: 'running', + executionDeviceId: DEVICE_ID, + executionDeviceName: 'Studio Mac', + screenshotPath, + screenshotDeviceId: DEVICE_ID, + stepDetails: [ + { + stepId: 'step-1', + at: Date.now(), + phase: 'acting', + mappedAction: JSON.stringify({ type: 'click', point: { x: 720, y: 450 } }), + actionCoordinateSpace: 'inference', + screenshot: { + path: screenshotPath, + availability: 'device_local', + originalWidth: 1_440, + originalHeight: 900, + inferenceWidth: 1_440, + inferenceHeight: 900 + } + } + ] + }) + let fields: Record | undefined + observeTaskRunFrame(taskId, (value) => { + fields = value + }) + + await vi.waitFor(() => expect(fields).toBeDefined()) + const portable = parseSyncedTaskRun(fields) + expect(portable).toMatchObject({ + taskId, + conversationId: 'mobile-chat-live-frame', + kind: 'computer_use', + executionDevice: { id: DEVICE_ID, name: 'Studio Mac' }, + status: 'running', + frame: { mimeType: 'image/jpeg' }, + cursor: { x: 480, y: 300 } + }) + expect(Buffer.from(portable!.frame!.payloadBase64, 'base64').byteLength).toBeLessThanOrEqual( + 512 * 1024 + ) + expect(portable!.frame).toMatchObject({ width: 960, height: 600 }) + + const finished = taskHistory.recordTaskRun({ + taskId, + journeyId: 'mobile-chat-live-frame', + kind: 'computer_use', + title: 'Send the release message', + status: 'done', + executionDeviceId: DEVICE_ID, + executionDeviceName: 'Studio Mac' + }) + expect(evictTerminalTaskRunFrame(taskId)).toBe(true) + expect(syncedTaskRunFromSnapshot(finished).frame).toBeUndefined() + }) +}) diff --git a/integration-tests/workspace-production-bridge.ui.integration.dbtest.tsx b/integration-tests/workspace-production-bridge.ui.integration.dbtest.tsx index 80c2d430..5db784cd 100644 --- a/integration-tests/workspace-production-bridge.ui.integration.dbtest.tsx +++ b/integration-tests/workspace-production-bridge.ui.integration.dbtest.tsx @@ -129,10 +129,11 @@ let TooltipProvider: typeof import('../src/renderer/src/components/ui/tooltip'). async function bootProductionMain(): Promise { bridge.handlers.clear() bridge.mainListeners.clear() - const [{ setupIPC }, { setupRagIPC }, { llm }] = await Promise.all([ + const [{ setupIPC }, { setupRagIPC }, { llm }, { registerTaskHistoryIpc }] = await Promise.all([ import('../src/main/ipc'), import('../src/main/rag-ipc'), - import('../src/main/llm') + import('../src/main/llm'), + import('../src/main/tasks/task-history-ipc') ]) const service = llm as unknown as { port: number; initialized: boolean; paused: boolean } service.port = fake.port @@ -140,6 +141,7 @@ async function bootProductionMain(): Promise { service.paused = false setupIPC() setupRagIPC() + registerTaskHistoryIpc() } function renderChat(target?: { conversationId?: string; projectId?: string }): void { diff --git a/outputs/ios-macos-sync-manual-gate-20260729/Off-Grid-iOS-macOS-Sync-Manual-Test-Matrix.xlsx.inspect.ndjson b/outputs/ios-macos-sync-manual-gate-20260729/Off-Grid-iOS-macOS-Sync-Manual-Test-Matrix.xlsx.inspect.ndjson index 75f9f226..ea29a387 100644 --- a/outputs/ios-macos-sync-manual-gate-20260729/Off-Grid-iOS-macOS-Sync-Manual-Test-Matrix.xlsx.inspect.ndjson +++ b/outputs/ios-macos-sync-manual-gate-20260729/Off-Grid-iOS-macOS-Sync-Manual-Test-Matrix.xlsx.inspect.ndjson @@ -1,6 +1,6 @@ {"kind":"workbook","id":"wb/8xkkif","sheets":4,"tables":4} {"kind":"sheet","id":"ws/kynbkt","name":"Summary","index":0,"range":"A1:F24","address":"A1:F24","tables":1,"formulas":34} -{"kind":"table","sheet":"Summary","address":"A1:F24","rows":24,"cols":6,"values":[["Off Grid iOS ↔ macOS Sync Manual Gate",null,null,null,null,null],["Release scope: iOS and macOS only. Update Status and Evidence as physical testing progresses.",null,null,null,null,null],[null,null,null,null,null,null],["Metric","Value",null,"Area","Total","Verified"],["Total test cases",108,null,"Pairing & identity",9,0],["Verified by user",8,null,"Discovery & routes",9,0],["Not started",99,null,"Membership",7,0],["In progress",0,null,"Chats & messages",10,4],["Blocked - coding",0,null,"Projects",5,1],["Completion",0.07407407407407407,null,"Knowledge base",8,1],[null,null,null,"Clipboard",5,0],[null,null,null,"Settings & models",5,0],[null,null,null,"Files",26,1],[null,null,null,"Files UI",8,0],[null,null,null,"Activity & notifications",7,0],[null,null,null,"Persistence & conflicts",5,0],[null,null,null,"Desktop capture",3,1],[null,null,null,"Navigation",1,0],[null,null,null,null,null,null],["Current known scope notes",null,null,null,null,null],["Verified so far","iOS → macOS: chat/project sync, messages, thinking, tool results, generated tool artifacts, knowledge document sync and generic file sync. macOS Replay capture toggle ↔ menu-bar state is also verified.",null,null,null,null],["In progress","Shared ambient-directory source is consumed by both hosts. Desktop Pro is d1b6f9d/7c126fe; Mobile Pro is 1665fc43/98be179c and current iOS is physically installed. Manual verification remains.",null,null,null,null],["Coding backlog","No known coding blocker remains in this matrix. Cmd+[ / Cmd+] history is implemented and is ready for manual verification.",null,null,null,null],["Deferred","The five-device hard cap requires a physical multi-device setup.",null,null,null,null]]} +{"kind":"table","sheet":"Summary","address":"A1:F24","rows":24,"cols":6,"values":[["Off Grid AI iOS ↔ macOS Sync Manual Gate",null,null,null,null,null],["Release scope: iOS and macOS only. Update Status and Evidence as physical testing progresses.",null,null,null,null,null],[null,null,null,null,null,null],["Metric","Value",null,"Area","Total","Verified"],["Total test cases",108,null,"Pairing & identity",9,0],["Verified by user",8,null,"Discovery & routes",9,0],["Not started",99,null,"Membership",7,0],["In progress",0,null,"Chats & messages",10,4],["Blocked - coding",0,null,"Projects",5,1],["Completion",0.07407407407407407,null,"Knowledge base",8,1],[null,null,null,"Clipboard",5,0],[null,null,null,"Settings & models",5,0],[null,null,null,"Files",26,1],[null,null,null,"Files UI",8,0],[null,null,null,"Activity & notifications",7,0],[null,null,null,"Persistence & conflicts",5,0],[null,null,null,"Desktop capture",3,1],[null,null,null,"Navigation",1,0],[null,null,null,null,null,null],["Current known scope notes",null,null,null,null,null],["Verified so far","iOS → macOS: chat/project sync, messages, thinking, tool results, generated tool artifacts, knowledge document sync and generic file sync. macOS Replay capture toggle ↔ menu-bar state is also verified.",null,null,null,null],["In progress","Shared ambient-directory source is consumed by both hosts. Desktop Pro is d1b6f9d/7c126fe; Mobile Pro is 1665fc43/98be179c and current iOS is physically installed. Manual verification remains.",null,null,null,null],["Coding backlog","No known coding blocker remains in this matrix. Cmd+[ / Cmd+] history is implemented and is ready for manual verification.",null,null,null,null],["Deferred","The five-device hard cap requires a physical multi-device setup.",null,null,null,null]]} {"kind":"region","sheet":"Summary","address":"D4:F18","rows":15,"cols":3,"nonEmpty":45,"text":45,"maxTextLength":24,"preview":[["Area","Total","Verified"],["Pairing & identity","9","0"],["Discovery & routes","9","0"],["Membership","7","0"],["Chats & messages","10","4"]],"previewAddress":"D4:F8","previewRows":5,"previewCols":3} {"kind":"region","sheet":"Summary","address":"A4:B18","rows":15,"cols":2,"nonEmpty":14,"blank":16,"text":14,"maxTextLength":19,"preview":[["Metric","Value"],["Total test cases","108"],["Verified by user","8"],["Not started","99"],["In progress","0"]],"previewAddress":"A4:B8","previewRows":5,"previewCols":2} {"kind":"region","sheet":"Summary","address":"A20:B24","rows":5,"cols":2,"nonEmpty":9,"blank":1,"text":9,"maxTextLength":201,"preview":[["Current known scope notes",null],["Verified so far","iOS → macOS: chat/project sync, messages, thinking, tool results, generated t..."],["In progress","Shared ambient-directory source is consumed by both hosts. Desktop Pro is d1b..."],["Coding backlog","No known coding blocker remains in this matrix. Cmd+[ / Cmd+] history is impl..."],["Deferred","The five-device hard cap requires a physical multi-device setup."]],"previewAddress":"A20:B24","previewRows":5,"previewCols":2} diff --git a/package-lock.json b/package-lock.json index 4c246611..608c5ffa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,12 +15,17 @@ "@electron-toolkit/preload": "^3.0.2", "@electron-toolkit/utils": "^4.0.0", "@lancedb/lancedb": "^0.30.0", + "@langchain/core": "^1.2.9", + "@langchain/langgraph": "^1.4.12", "@modelcontextprotocol/sdk": "^1.29.0", "@offgrid/clipboard": "file:./packages/clipboard", "@offgrid/design": "file:./packages/design", "@offgrid/models": "file:../shared/packages/models", "@offgrid/rag": "file:./packages/rag", + "@offgrid/speech": "file:../shared/packages/speech", "@offgrid/sync": "file:../shared/packages/sync", + "@offgrid/ui": "file:../shared/packages/ui", + "@offgrid/use": "file:../shared/packages/use", "@phosphor-icons/react": "^2.1.10", "@scure/bip39": "^2.2.0", "@tabler/icons-react": "^3.36.1", @@ -40,13 +45,13 @@ "js-sha512": "^0.9.0", "jszip": "^3.10.1", "kdbxweb": "^2.1.1", - "kokoro-js": "^1.2.1", "mammoth": "^1.8.0", "mdast-util-to-string": "^4.0.0", "motion": "^12.27.1", "node-machine-id": "^1.1.12", "ollama": "^0.6.3", "pdf-parse": "^1.1.1", + "qrcode.react": "^4.2.0", "radix-ui": "^1.6.0", "react-markdown": "^10.1.0", "react-resizable-panels": "^2.1.9", @@ -64,6 +69,7 @@ "@electron-toolkit/eslint-config-ts": "^3.1.0", "@electron-toolkit/tsconfig": "^2.0.0", "@electron/asar": "3.4.1", + "@offgrid/executorch-speech": "file:../executorch-speech", "@playwright/test": "^1.61.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", @@ -96,6 +102,22 @@ "typescript-eslint": "^8.63.0", "vite": "^7.2.6", "vitest": "^4.0.17" + }, + "optionalDependencies": { + "@nut-tree-fork/nut-js": "^4.2.6" + } + }, + "../executorch-speech": { + "name": "@offgrid/executorch-speech", + "version": "0.1.0", + "dev": true, + "license": "MIT", + "devDependencies": { + "@types/node": "^22.19.1", + "react-native": "0.81.5", + "tsx": "^4.20.6", + "typescript": "~5.9.2", + "vitest": "^4.1.10" } }, "../shared/packages/design": { @@ -115,6 +137,15 @@ "extraneous": true, "license": "AGPL-3.0-only" }, + "../shared/packages/speech": { + "name": "@offgrid/speech", + "version": "0.0.1", + "license": "AGPL-3.0-only", + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5.4.0" + } + }, "../shared/packages/sync": { "name": "@offgrid/sync", "version": "0.0.1", @@ -130,6 +161,25 @@ "c8": "^12.0.0" } }, + "../shared/packages/ui": { + "name": "@offgrid/ui", + "version": "0.0.1", + "license": "AGPL-3.0-only" + }, + "../shared/packages/use": { + "name": "@offgrid/use", + "version": "0.0.1", + "license": "AGPL-3.0-only", + "dependencies": { + "@noble/hashes": "1.8.0", + "xstate": "^5.0.0", + "zod": "^4.0.0" + }, + "devDependencies": { + "better-sqlite3": "^12.6.2", + "c8": "^12.0.0" + } + }, "node_modules/@asamuzakjp/css-color": { "version": "5.1.11", "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", @@ -512,6 +562,12 @@ "specificity": "bin/cli.js" } }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "license": "MIT" + }, "node_modules/@csstools/color-helpers": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz", @@ -1905,139 +1961,6 @@ "node": ">=18" } }, - "node_modules/@huggingface/transformers": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.8.1.tgz", - "integrity": "sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==", - "license": "Apache-2.0", - "dependencies": { - "@huggingface/jinja": "^0.5.3", - "onnxruntime-node": "1.21.0", - "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", - "sharp": "^0.34.1" - } - }, - "node_modules/@huggingface/transformers/node_modules/@huggingface/jinja": { - "version": "0.5.9", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.9.tgz", - "integrity": "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@huggingface/transformers/node_modules/chownr": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } - }, - "node_modules/@huggingface/transformers/node_modules/flatbuffers": { - "version": "25.9.23", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", - "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", - "license": "Apache-2.0" - }, - "node_modules/@huggingface/transformers/node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-common": { - "version": "1.21.0", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", - "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==", - "license": "MIT" - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-node": { - "version": "1.21.0", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", - "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==", - "hasInstallScript": true, - "license": "MIT", - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "global-agent": "^3.0.0", - "onnxruntime-common": "1.21.0", - "tar": "^7.0.1" - } - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-web": { - "version": "1.22.0-dev.20250409-89f8206ba4", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", - "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==", - "license": "MIT", - "dependencies": { - "flatbuffers": "^25.1.24", - "guid-typescript": "^1.0.9", - "long": "^5.2.3", - "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", - "platform": "^1.3.6", - "protobufjs": "^7.2.4" - } - }, - "node_modules/@huggingface/transformers/node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.22.0-dev.20250409-89f8206ba4", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", - "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==", - "license": "MIT" - }, - "node_modules/@huggingface/transformers/node_modules/protobufjs": { - "version": "7.6.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", - "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.5", - "@protobufjs/eventemitter": "^1.1.1", - "@protobufjs/fetch": "^1.1.1", - "@protobufjs/float": "^1.0.2", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.1", - "@types/node": ">=13.7.0", - "long": "^5.3.2" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@huggingface/transformers/node_modules/tar": { - "version": "7.5.16", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.16.tgz", - "integrity": "sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/fs-minipass": "^4.0.0", - "chownr": "^3.0.0", - "minipass": "^7.1.2", - "minizlib": "^3.1.0", - "yallist": "^5.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@huggingface/transformers/node_modules/yallist": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=18" - } - }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -2745,6 +2668,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dev": true, "license": "ISC", "dependencies": { "minipass": "^7.0.4" @@ -2753,6 +2677,456 @@ "node": ">=18.0.0" } }, + "node_modules/@jimp/bmp": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.22.12.tgz", + "integrity": "sha512-aeI64HD0npropd+AR76MCcvvRaa+Qck6loCOS03CkkxGHN5/r336qTM5HPUdHKMDOGzqknuVPA8+kK1t03z12g==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "bmp-js": "^0.1.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/core": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.22.12.tgz", + "integrity": "sha512-l0RR0dOPyzMKfjUW1uebzueFEDtCOj9fN6pyTYWWOM/VS4BciXQ1VVrJs8pO3kycGYZxncRKhCoygbNr8eEZQA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "any-base": "^1.1.0", + "buffer": "^5.2.0", + "exif-parser": "^0.1.12", + "file-type": "^16.5.4", + "isomorphic-fetch": "^3.0.0", + "pixelmatch": "^4.0.2", + "tinycolor2": "^1.6.0" + } + }, + "node_modules/@jimp/custom": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.22.12.tgz", + "integrity": "sha512-xcmww1O/JFP2MrlGUMd3Q78S3Qu6W3mYTXYuIqFq33EorgYHV/HqymHfXy9GjiCJ7OI+7lWx6nYFOzU7M4rd1Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/core": "^0.22.12" + } + }, + "node_modules/@jimp/gif": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.22.12.tgz", + "integrity": "sha512-y6BFTJgch9mbor2H234VSjd9iwAhaNf/t3US5qpYIs0TSbAvM02Fbc28IaDETj9+4YB4676sz4RcN/zwhfu1pg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "gifwrap": "^0.10.1", + "omggif": "^1.0.9" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/jpeg": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.22.12.tgz", + "integrity": "sha512-Rq26XC/uQWaQKyb/5lksCTCxXhtY01NJeBN+dQv5yNYedN0i7iYu+fXEoRsfaJ8xZzjoANH8sns7rVP4GE7d/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "jpeg-js": "^0.4.4" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-blit": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.22.12.tgz", + "integrity": "sha512-xslz2ZoFZOPLY8EZ4dC29m168BtDx95D6K80TzgUi8gqT7LY6CsajWO0FAxDwHz6h0eomHMfyGX0stspBrTKnQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-blur": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.22.12.tgz", + "integrity": "sha512-S0vJADTuh1Q9F+cXAwFPlrKWzDj2F9t/9JAbUvaaDuivpyWuImEKXVz5PUZw2NbpuSHjwssbTpOZ8F13iJX4uw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-circle": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.22.12.tgz", + "integrity": "sha512-SWVXx1yiuj5jZtMijqUfvVOJBwOifFn0918ou4ftoHgegc5aHWW5dZbYPjvC9fLpvz7oSlptNl2Sxr1zwofjTg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-color": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.22.12.tgz", + "integrity": "sha512-xImhTE5BpS8xa+mAN6j4sMRWaUgUDLoaGHhJhpC+r7SKKErYDR0WQV4yCE4gP+N0gozD0F3Ka1LUSaMXrn7ZIA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "tinycolor2": "^1.6.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-contain": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.22.12.tgz", + "integrity": "sha512-Eo3DmfixJw3N79lWk8q/0SDYbqmKt1xSTJ69yy8XLYQj9svoBbyRpSnHR+n9hOw5pKXytHwUW6nU4u1wegHNoQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5", + "@jimp/plugin-scale": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-cover": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.22.12.tgz", + "integrity": "sha512-z0w/1xH/v/knZkpTNx+E8a7fnasQ2wHG5ze6y5oL2dhH1UufNua8gLQXlv8/W56+4nJ1brhSd233HBJCo01BXA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-crop": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5", + "@jimp/plugin-scale": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-crop": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.22.12.tgz", + "integrity": "sha512-FNuUN0OVzRCozx8XSgP9MyLGMxNHHJMFt+LJuFjn1mu3k0VQxrzqbN06yIl46TVejhyAhcq5gLzqmSCHvlcBVw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-displace": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.22.12.tgz", + "integrity": "sha512-qpRM8JRicxfK6aPPqKZA6+GzBwUIitiHaZw0QrJ64Ygd3+AsTc7BXr+37k2x7QcyCvmKXY4haUrSIsBug4S3CA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-dither": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.22.12.tgz", + "integrity": "sha512-jYgGdSdSKl1UUEanX8A85v4+QUm+PE8vHFwlamaKk89s+PXQe7eVE3eNeSZX4inCq63EHL7cX580dMqkoC3ZLw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-fisheye": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.22.12.tgz", + "integrity": "sha512-LGuUTsFg+fOp6KBKrmLkX4LfyCy8IIsROwoUvsUPKzutSqMJnsm3JGDW2eOmWIS/jJpPaeaishjlxvczjgII+Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-flip": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.22.12.tgz", + "integrity": "sha512-m251Rop7GN8W0Yo/rF9LWk6kNclngyjIJs/VXHToGQ6EGveOSTSQaX2Isi9f9lCDLxt+inBIb7nlaLLxnvHX8Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-rotate": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-gaussian": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.22.12.tgz", + "integrity": "sha512-sBfbzoOmJ6FczfG2PquiK84NtVGeScw97JsCC3rpQv1PHVWyW+uqWFF53+n3c8Y0P2HWlUjflEla2h/vWShvhg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-invert": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.22.12.tgz", + "integrity": "sha512-N+6rwxdB+7OCR6PYijaA/iizXXodpxOGvT/smd/lxeXsZ/empHmFFFJ/FaXcYh19Tm04dGDaXcNF/dN5nm6+xQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-mask": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.22.12.tgz", + "integrity": "sha512-4AWZg+DomtpUA099jRV8IEZUfn1wLv6+nem4NRJC7L/82vxzLCgXKTxvNvBcNmJjT9yS1LAAmiJGdWKXG63/NA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-normalize": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.22.12.tgz", + "integrity": "sha512-0So0rexQivnWgnhacX4cfkM2223YdExnJTTy6d06WbkfZk5alHUx8MM3yEzwoCN0ErO7oyqEWRnEkGC+As1FtA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-print": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.22.12.tgz", + "integrity": "sha512-c7TnhHlxm87DJeSnwr/XOLjJU/whoiKYY7r21SbuJ5nuH+7a78EW1teOaj5gEr2wYEd7QtkFqGlmyGXY/YclyQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "load-bmfont": "^1.4.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-resize": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.22.12.tgz", + "integrity": "sha512-3NyTPlPbTnGKDIbaBgQ3HbE6wXbAlFfxHVERmrbqAi8R3r6fQPxpCauA8UVDnieg5eo04D0T8nnnNIX//i/sXg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-rotate": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.22.12.tgz", + "integrity": "sha512-9YNEt7BPAFfTls2FGfKBVgwwLUuKqy+E8bDGGEsOqHtbuhbshVGxN2WMZaD4gh5IDWvR+emmmPPWGgaYNYt1gA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5", + "@jimp/plugin-crop": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-scale": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.22.12.tgz", + "integrity": "sha512-dghs92qM6MhHj0HrV2qAwKPMklQtjNpoYgAB94ysYpsXslhRTiPisueSIELRwZGEr0J0VUxpUY7HgJwlSIgGZw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-shadow": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.22.12.tgz", + "integrity": "sha512-FX8mTJuCt7/3zXVoeD/qHlm4YH2bVqBuWQHXSuBK054e7wFRnRnbSLPUqAwSeYP3lWqpuQzJtgiiBxV3+WWwTg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blur": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-threshold": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.22.12.tgz", + "integrity": "sha512-4x5GrQr1a/9L0paBC/MZZJjjgjxLYrqSmWd+e+QfAEPvmRxdRoQ5uKEuNgXnm9/weHQBTnQBQsOY2iFja+XGAw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-color": ">=0.8.0", + "@jimp/plugin-resize": ">=0.8.0" + } + }, + "node_modules/@jimp/plugins": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.22.12.tgz", + "integrity": "sha512-yBJ8vQrDkBbTgQZLty9k4+KtUQdRjsIDJSPjuI21YdVeqZxYywifHl4/XWILoTZsjTUASQcGoH0TuC0N7xm3ww==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/plugin-blit": "^0.22.12", + "@jimp/plugin-blur": "^0.22.12", + "@jimp/plugin-circle": "^0.22.12", + "@jimp/plugin-color": "^0.22.12", + "@jimp/plugin-contain": "^0.22.12", + "@jimp/plugin-cover": "^0.22.12", + "@jimp/plugin-crop": "^0.22.12", + "@jimp/plugin-displace": "^0.22.12", + "@jimp/plugin-dither": "^0.22.12", + "@jimp/plugin-fisheye": "^0.22.12", + "@jimp/plugin-flip": "^0.22.12", + "@jimp/plugin-gaussian": "^0.22.12", + "@jimp/plugin-invert": "^0.22.12", + "@jimp/plugin-mask": "^0.22.12", + "@jimp/plugin-normalize": "^0.22.12", + "@jimp/plugin-print": "^0.22.12", + "@jimp/plugin-resize": "^0.22.12", + "@jimp/plugin-rotate": "^0.22.12", + "@jimp/plugin-scale": "^0.22.12", + "@jimp/plugin-shadow": "^0.22.12", + "@jimp/plugin-threshold": "^0.22.12", + "timm": "^1.6.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/png": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.22.12.tgz", + "integrity": "sha512-Mrp6dr3UTn+aLK8ty/dSKELz+Otdz1v4aAXzV5q53UDD2rbB5joKVJ/ChY310B+eRzNxIovbUF1KVrUsYdE8Hg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.12", + "pngjs": "^6.0.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/tiff": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.22.12.tgz", + "integrity": "sha512-E1LtMh4RyJsoCAfAkBRVSYyZDTtLq9p9LUiiYP0vPtXyxX4BiYBUYihTLSBlCQg5nF2e4OpQg7SPrLdJ66u7jg==", + "license": "MIT", + "optional": true, + "dependencies": { + "utif2": "^4.0.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/types": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.22.12.tgz", + "integrity": "sha512-wwKYzRdElE1MBXFREvCto5s699izFHNVvALUv79GXNbsOVqlwlOxlWJ8DuyOGIXoLP4JW/m30YyuTtfUJgMRMA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/bmp": "^0.22.12", + "@jimp/gif": "^0.22.12", + "@jimp/jpeg": "^0.22.12", + "@jimp/png": "^0.22.12", + "@jimp/tiff": "^0.22.12", + "timm": "^1.6.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/utils": { + "version": "0.22.12", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.22.12.tgz", + "integrity": "sha512-yJ5cWUknGnilBq97ZXOyOS0HhsHOyAyjHwYfHxGbSyMTohgQI6sVyE8KPgDwH8HHW/nMKXk8TrSwAE71zt716Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "regenerator-runtime": "^0.13.3" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -2923,38 +3297,160 @@ "node": ">= 18" } }, - "node_modules/@lancedb/lancedb-win32-arm64-msvc": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.30.0.tgz", - "integrity": "sha512-N2DQg2XBWZirn5jS6kRJUxF679t3sKcIxBwP9zY4Idq5OVLAj0yfLueWIKhYxv8en7pBFYWdgw5j9dTS7XajyQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], + "node_modules/@lancedb/lancedb-win32-arm64-msvc": { + "version": "0.30.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.30.0.tgz", + "integrity": "sha512-N2DQg2XBWZirn5jS6kRJUxF679t3sKcIxBwP9zY4Idq5OVLAj0yfLueWIKhYxv8en7pBFYWdgw5j9dTS7XajyQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@lancedb/lancedb-win32-x64-msvc": { + "version": "0.30.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.30.0.tgz", + "integrity": "sha512-CDgN/ZmYqSlVX2nBJAF2PYEwqBBxotCVORjagmvrd0k5D7RBLlAQUEAR4gDMum2BpYsUkzdTYQpquLjRCVbwbQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 18" + } + }, + "node_modules/@langchain/core": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.9.tgz", + "integrity": "sha512-conzSEj9Zu1AyXJLXsSbgrtxtxinmI1yGqQ5CIJZSoV5rvv+yvQE/vgBnoySpBQ/bl3YPgj2FL/gbDjWykLSfg==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/langgraph": { + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.12.tgz", + "integrity": "sha512-63iH/igH5Fh5fHqmWp09YYWaDKKB9v4RCmYNJBrnQ224rFRbjebgyYW6o5RCczN5FZxIhQj+xT51rrNmG0zi5A==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^1.1.5", + "@langchain/langgraph-sdk": "~1.9.30", + "@langchain/protocol": "^0.0.18", + "@standard-schema/spec": "1.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "zod": "^3.25.32 || ^4.2.0" + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.5.tgz", + "integrity": "sha512-BwDwl5VeTOh6CVuiIPgsUgfK51vTJDMSbFcSCUfjJWsl8/DPdK/mbv+ejxJstkSk/BlSPMP4JfXWcN6jD2ea2Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "1.9.31", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.31.tgz", + "integrity": "sha512-y1sSdq39IPb6mOX43+JiSezVbUdA8EBEJ1gvn91GP0jrLG0EcSApeRDCjRouyDpPXZ51bQXEQhA8CiHM0mzcAw==", + "license": "MIT", + "dependencies": { + "@langchain/protocol": "^0.0.18", + "@types/json-schema": "^7.0.15", + "p-queue": "^9.0.1", + "p-retry": "^7.1.1" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "react": "^18 || ^19", + "react-dom": "^18 || ^19", + "svelte": "^4.0.0 || ^5.0.0", + "vue": "^3.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "svelte": { + "optional": true + }, + "vue": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, "engines": { - "node": ">= 18" + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@lancedb/lancedb-win32-x64-msvc": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.30.0.tgz", - "integrity": "sha512-CDgN/ZmYqSlVX2nBJAF2PYEwqBBxotCVORjagmvrd0k5D7RBLlAQUEAR4gDMum2BpYsUkzdTYQpquLjRCVbwbQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], + "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", "engines": { - "node": ">= 18" + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@langchain/protocol": { + "version": "0.0.18", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.18.tgz", + "integrity": "sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==", + "license": "MIT" + }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -3283,6 +3779,168 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@nut-tree-fork/default-clipboard-provider": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/default-clipboard-provider/-/default-clipboard-provider-4.2.6.tgz", + "integrity": "sha512-Hzqj57rheIMGtsS4zK4//kOhaX5FxMluOiz+4TVaHXx+idZS/bPhZwd8e6o1w1GT0PVJOUIP+4CdUe//k5VRig==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "clipboardy": "2.3.0" + } + }, + "node_modules/@nut-tree-fork/libnut": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/libnut/-/libnut-4.2.6.tgz", + "integrity": "sha512-2FCiTBokMGrMl4eL/trEIO+mtpkXpdPHoVKdTBmW8UBIbhCbrCKmnXb2skWGfVs+U3q7o5EYDjVTNUYaUWbaxQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@nut-tree-fork/libnut-darwin": "2.7.5", + "@nut-tree-fork/libnut-linux": "2.7.5", + "@nut-tree-fork/libnut-win32": "2.7.5" + }, + "engines": { + "node": ">=10.15.3" + } + }, + "node_modules/@nut-tree-fork/libnut-darwin": { + "version": "2.7.5", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/libnut-darwin/-/libnut-darwin-2.7.5.tgz", + "integrity": "sha512-LbqtPtMPTJUcg4XoPP2jsU1wc8flBcGyKTerKsIfK9cD7nBHROnO0QksbrsbSWEpLym8T8fRtuU7XEY83l6Z2Q==", + "cpu": [ + "x64", + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "bindings": "1.5.0" + }, + "engines": { + "node": ">=10.15.3" + }, + "optionalDependencies": { + "@nut-tree-fork/node-mac-permissions": "2.2.1" + } + }, + "node_modules/@nut-tree-fork/libnut-linux": { + "version": "2.7.5", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/libnut-linux/-/libnut-linux-2.7.5.tgz", + "integrity": "sha512-uxaXEcRKnFObAljsoR6tLOBUU1dJ2sctloG6gFgCBGN7+k6Jdv6jZfOuNjd/fpdq2C5WPMm0rtn9EE7h5J3Jcg==", + "cpu": [ + "x64", + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "bindings": "1.5.0" + }, + "engines": { + "node": ">=10.15.3" + }, + "optionalDependencies": { + "@nut-tree-fork/node-mac-permissions": "2.2.1" + } + }, + "node_modules/@nut-tree-fork/libnut-win32": { + "version": "2.7.5", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/libnut-win32/-/libnut-win32-2.7.5.tgz", + "integrity": "sha512-yqC87zvmFcDPwFrRU40DYhN0xmEVM3aSkOuyF0IX+y1x+HWSu/i0PNklATpPBhGid3QVb/TOHuVoaraMrUFCNw==", + "cpu": [ + "x64", + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "bindings": "1.5.0" + }, + "engines": { + "node": ">=10.15.3" + }, + "optionalDependencies": { + "@nut-tree-fork/node-mac-permissions": "2.2.1" + } + }, + "node_modules/@nut-tree-fork/node-mac-permissions": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/node-mac-permissions/-/node-mac-permissions-2.2.1.tgz", + "integrity": "sha512-iSfOTDiBZ7VDa17PoQje5rUaZSvSAaq+XEyXCmhPuQwV5XuNU02Grv6oFhsdpz89w7+UvB/8KX/cX5IYQ5o2Bw==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "1.5.0", + "node-addon-api": "5.0.0" + } + }, + "node_modules/@nut-tree-fork/nut-js": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/nut-js/-/nut-js-4.2.6.tgz", + "integrity": "sha512-aI/WCX7gE1HFGPH3EZP/UWqpNMM1NMoM/EkXqp7pKMgXFCi8e5+o5p+jd/QOYpmALv9bQg7+s69nI7FONbMqDg==", + "cpu": [ + "x64", + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux", + "darwin", + "win32" + ], + "dependencies": { + "@nut-tree-fork/default-clipboard-provider": "4.2.6", + "@nut-tree-fork/libnut": "4.2.6", + "@nut-tree-fork/provider-interfaces": "4.2.6", + "@nut-tree-fork/shared": "4.2.6", + "jimp": "0.22.10", + "node-abort-controller": "3.1.1" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@nut-tree-fork/provider-interfaces": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/provider-interfaces/-/provider-interfaces-4.2.6.tgz", + "integrity": "sha512-brtRegDkLSV0sa5DUAigjWf6hCoamBNPb/hKK9AQlW+j3BxQ/8djaEdEB2cihqUh1ZjEtgPyXRqpCWSdKCX68A==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@nut-tree-fork/shared": "4.2.6" + } + }, + "node_modules/@nut-tree-fork/shared": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@nut-tree-fork/shared/-/shared-4.2.6.tgz", + "integrity": "sha512-xZaa0YtJt/DDDq/i1vZkabjq8HOWzfhXieMai61cMbYD11J6VhAfhV23ZtQEM02WG7nc2LKjl4UwRnQCteikwA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "jimp": "0.22.10", + "node-abort-controller": "3.1.1" + } + }, "node_modules/@offgrid/clipboard": { "resolved": "packages/clipboard", "link": true @@ -3291,6 +3949,10 @@ "resolved": "packages/design", "link": true }, + "node_modules/@offgrid/executorch-speech": { + "resolved": "../executorch-speech", + "link": true + }, "node_modules/@offgrid/models": { "resolved": "../shared/packages/models", "link": true @@ -3299,10 +3961,22 @@ "resolved": "packages/rag", "link": true }, + "node_modules/@offgrid/speech": { + "resolved": "../shared/packages/speech", + "link": true + }, "node_modules/@offgrid/sync": { "resolved": "../shared/packages/sync", "link": true }, + "node_modules/@offgrid/ui": { + "resolved": "../shared/packages/ui", + "link": true + }, + "node_modules/@offgrid/use": { + "resolved": "../shared/packages/use", + "link": true + }, "node_modules/@oxc-parser/binding-android-arm-eabi": { "version": "0.137.0", "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.137.0.tgz", @@ -6027,7 +6701,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "dev": true, "license": "MIT" }, "node_modules/@swc/helpers": { @@ -6397,6 +7070,13 @@ "@testing-library/dom": ">=7.21.4" } }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT", + "optional": true + }, "node_modules/@tybys/wasm-util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", @@ -6572,7 +7252,6 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, "license": "MIT" }, "node_modules/@types/keyv": { @@ -7118,6 +7797,19 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "optional": true, + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -7317,6 +8009,13 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", + "license": "MIT", + "optional": true + }, "node_modules/apache-arrow": { "version": "18.1.0", "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-18.1.0.tgz", @@ -7591,6 +8290,27 @@ "license": "ISC", "optional": true }, + "node_modules/arch": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "optional": true + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -8035,6 +8755,13 @@ "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", "license": "MIT" }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT", + "optional": true + }, "node_modules/body-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", @@ -8103,7 +8830,8 @@ "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/brace-expansion": { "version": "1.1.12", @@ -8183,6 +8911,16 @@ "node": "*" } }, + "node_modules/buffer-equal": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", + "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -8425,6 +9163,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/centra": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/centra/-/centra-2.7.0.tgz", + "integrity": "sha512-PbFMgMSrmgx6uxCdm57RUos9Tc3fclMvhLSATYN39XsDV29B89zZ3KA89jmY0vwSGazyU+uerqwa6t+KaodPcg==", + "license": "MIT", + "optional": true, + "dependencies": { + "follow-redirects": "^1.15.6" + } + }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -8561,6 +9309,21 @@ "node": ">=6" } }, + "node_modules/clipboardy": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/clipboardy/-/clipboardy-2.3.0.tgz", + "integrity": "sha512-mKhiIL2DrQIsuXMgBgnfEHOZOryC7kY7YO//TN6c63wlEm3NG5tz+YgY5rVi29KCmq/QQjKYvM7a19+MDOTHOQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "arch": "^2.1.1", + "execa": "^1.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -9079,6 +9842,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "devOptional": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", @@ -9096,6 +9860,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "devOptional": true, "license": "MIT", "dependencies": { "define-data-property": "^1.0.1", @@ -9228,7 +9993,8 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/detect-node-es": { "version": "1.1.0", @@ -9363,6 +10129,12 @@ "license": "MIT", "peer": true }, + "node_modules/dom-walk": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz", + "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==", + "optional": true + }, "node_modules/dotenv": { "version": "16.6.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", @@ -10045,7 +10817,8 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/esbuild": { "version": "0.25.12", @@ -10109,6 +10882,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "devOptional": true, "license": "MIT", "engines": { "node": ">=10" @@ -10478,13 +11252,39 @@ "node": ">=0.10.0" } }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "license": "MIT", + "optional": true, "engines": { - "node": ">= 0.6" + "node": ">=0.8.x" } }, "node_modules/eventsource": { @@ -10508,6 +11308,124 @@ "node": ">=18.0.0" } }, + "node_modules/execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "license": "MIT", + "optional": true, + "dependencies": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/execa/node_modules/cross-spawn": { + "version": "6.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.6.tgz", + "integrity": "sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==", + "license": "MIT", + "optional": true, + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/execa/node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "license": "MIT", + "optional": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/execa/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC", + "optional": true + }, + "node_modules/execa/node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/execa/node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "license": "MIT", + "optional": true, + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/execa/node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/execa/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/exif-parser": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", + "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==", + "optional": true + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -10738,6 +11656,24 @@ "node": ">=16.0.0" } }, + "node_modules/file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "license": "MIT", + "optional": true, + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, "node_modules/file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -10854,6 +11790,27 @@ "dev": true, "license": "ISC" }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -11591,6 +12548,17 @@ "license": "ISC", "optional": true }, + "node_modules/gifwrap": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.10.1.tgz", + "integrity": "sha512-2760b1vpJHNmLzZ/ubTtNnEx5WApN/PYWJvXvgS+tL1egTTthayFYIQQNi136FLEDcN/IyEY2EcGpIITD6eYUw==", + "license": "MIT", + "optional": true, + "dependencies": { + "image-q": "^4.0.0", + "omggif": "^1.0.10" + } + }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -11645,11 +12613,23 @@ "node": "*" } }, + "node_modules/global": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", + "integrity": "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==", + "license": "MIT", + "optional": true, + "dependencies": { + "min-document": "^2.19.0", + "process": "^0.11.10" + } + }, "node_modules/global-agent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", "license": "BSD-3-Clause", + "optional": true, "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", @@ -11667,6 +12647,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "license": "ISC", + "optional": true, "bin": { "semver": "bin/semver.js" }, @@ -11717,6 +12698,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "devOptional": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -11804,6 +12786,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "devOptional": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0" @@ -12120,6 +13103,23 @@ "node": ">= 4" } }, + "node_modules/image-q": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/image-q/-/image-q-4.0.0.tgz", + "integrity": "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "16.9.1" + } + }, + "node_modules/image-q/node_modules/@types/node": { + "version": "16.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", + "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", + "license": "MIT", + "optional": true + }, "node_modules/immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", @@ -12405,6 +13405,22 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "license": "MIT", + "optional": true, + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -12441,6 +13457,13 @@ "node": ">=8" } }, + "node_modules/is-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz", + "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==", + "license": "MIT", + "optional": true + }, "node_modules/is-generator-function": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", @@ -12534,6 +13557,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-number-object": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.1.1.tgz", @@ -12637,6 +13672,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-string": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", @@ -12734,6 +13779,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "license": "MIT", + "optional": true, + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -12764,6 +13822,17 @@ "node": ">=16" } }, + "node_modules/isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "license": "MIT", + "optional": true, + "dependencies": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", @@ -12884,6 +13953,19 @@ "node": ">=10" } }, + "node_modules/jimp": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.22.10.tgz", + "integrity": "sha512-lCaHIJAgTOsplyJzC1w/laxSxrbSsEBw4byKwXgUdMmh+ayPsnidTblenQm+IvhIs44Gcuvlb6pd2LQ0wcKaKg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@jimp/custom": "^0.22.10", + "@jimp/plugins": "^0.22.10", + "@jimp/types": "^0.22.10", + "regenerator-runtime": "^0.13.3" + } + }, "node_modules/jiti": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", @@ -12902,12 +13984,28 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "license": "BSD-3-Clause", + "optional": true + }, "node_modules/js-sha512": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/js-sha512/-/js-sha512-0.9.0.tgz", "integrity": "sha512-mirki9WS/SUahm+1TbAPkqvbCiCfOAAsyXeHxK1UkullnJVVqoJG2pL9ObvT05CN+tM7fxhfYm0NbXn+1hWoZg==", "license": "MIT" }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -13077,7 +14175,8 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "license": "ISC" + "license": "ISC", + "optional": true }, "node_modules/json5": { "version": "2.2.3", @@ -13276,14 +14375,37 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/kokoro-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/kokoro-js/-/kokoro-js-1.2.1.tgz", - "integrity": "sha512-oq0HZJWis3t8lERkMJh84WLU86dpYD0EuBPtqYnLlQzyFP1OkyBRDcweAqCfhNOpltyN9j/azp1H6uuC47gShw==", - "license": "Apache-2.0", + "node_modules/langsmith": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.9.0.tgz", + "integrity": "sha512-tlg/aG7qezAKY6G3fgADSX7PkRj+JKoF3z7QNkCMsAOvwvuzhiwP9Amn1Z+zAIxuKoWuXQdIjtFN0LVmUC1oUQ==", + "license": "MIT", "dependencies": { - "@huggingface/transformers": "^3.5.1", - "phonemizer": "^1.2.1" + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } } }, "node_modules/lazy-val": { @@ -13564,6 +14686,36 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/load-bmfont": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.2.tgz", + "integrity": "sha512-qElWkmjW9Oq1F9EI5Gt7aD9zcdHb9spJCW1L/dmPf7KzCCEJxq8nhHz5eCgI9aMf7vrG/wyaCqdsI+Iy9ZTlog==", + "license": "MIT", + "optional": true, + "dependencies": { + "buffer-equal": "0.0.1", + "mime": "^1.3.4", + "parse-bmfont-ascii": "^1.0.3", + "parse-bmfont-binary": "^1.0.5", + "parse-bmfont-xml": "^1.1.4", + "phin": "^3.7.1", + "xhr": "^2.0.1", + "xtend": "^4.0.0" + } + }, + "node_modules/load-bmfont/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "optional": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -13783,6 +14935,7 @@ "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", "license": "MIT", + "optional": true, "dependencies": { "escape-string-regexp": "^4.0.0" }, @@ -14731,6 +15884,16 @@ "node": ">=4" } }, + "node_modules/min-document": { + "version": "2.19.2", + "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.2.tgz", + "integrity": "sha512-8S5I8db/uZN8r9HSLFVWPdJCvYOejMcEC82VIzNUc6Zkklf/d1gg2psfE79/vyhWOj4+J8MtwmoOz3TmvaGu5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "dom-walk": "^0.1.0" + } + }, "node_modules/minimatch": { "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", @@ -14783,6 +15946,7 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "devOptional": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" @@ -14904,6 +16068,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "dev": true, "license": "MIT", "dependencies": { "minipass": "^7.1.2" @@ -14991,6 +16156,15 @@ "multicast-dns": "cli.js" } }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -15031,6 +16205,13 @@ "node": ">= 0.6" } }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "license": "MIT", + "optional": true + }, "node_modules/node-abi": { "version": "4.33.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-4.33.0.tgz", @@ -15057,6 +16238,20 @@ "node": ">=10" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT", + "optional": true + }, + "node_modules/node-addon-api": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.0.0.tgz", + "integrity": "sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA==", + "license": "MIT", + "optional": true + }, "node_modules/node-api-version": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-api-version/-/node-api-version-0.2.1.tgz", @@ -15266,6 +16461,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", + "license": "MIT", + "optional": true, + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -15291,6 +16509,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -15391,6 +16610,13 @@ "whatwg-fetch": "^3.6.20" } }, + "node_modules/omggif": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "license": "MIT", + "optional": true + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -15576,6 +16802,15 @@ "node": ">=8" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -15608,6 +16843,49 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -15634,6 +16912,31 @@ "node": ">=6" } }, + "node_modules/parse-bmfont-ascii": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", + "integrity": "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA==", + "license": "MIT", + "optional": true + }, + "node_modules/parse-bmfont-binary": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz", + "integrity": "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA==", + "license": "MIT", + "optional": true + }, + "node_modules/parse-bmfont-xml": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.6.tgz", + "integrity": "sha512-0cEliVMZEhrFDwMh4SxIyVJpqYoOWDJ9P895tFuS+XuNzI5UBmBk5U5O4KuJdTnZpSBI4LFA2+ZiJaiwfSwlMA==", + "license": "MIT", + "optional": true, + "dependencies": { + "xml-parse-from-string": "^1.0.0", + "xml2js": "^0.5.0" + } + }, "node_modules/parse-entities": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", @@ -15659,6 +16962,13 @@ "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", "license": "MIT" }, + "node_modules/parse-headers": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.6.tgz", + "integrity": "sha512-Tz11t3uKztEW5FEVZnj1ox8GKblWn+PvHY9TmJV5Mll2uHEwRdR/5Li1OlXoECjLYkApdhWy44ocONwXLiKO5A==", + "license": "MIT", + "optional": true + }, "node_modules/parse5": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", @@ -15788,17 +17098,39 @@ "url": "https://github.com/sponsors/jet2jet" } }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, - "node_modules/phonemizer": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/phonemizer/-/phonemizer-1.2.1.tgz", - "integrity": "sha512-v0KJ4mi2T4Q7eJQ0W15Xd4G9k4kICSXE8bpDeJ8jisL4RyJhNWsweKTOi88QXFc4r4LZlz5jVL5lCHhkpdT71A==", - "license": "Apache-2.0" + "node_modules/phin": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/phin/-/phin-3.7.1.tgz", + "integrity": "sha512-GEazpTWwTZaEQ9RhL7Nyz0WwqilbqgLahDM3D0hxWwmVDI52nXEybHqiN6/elwpkJBhcuj+WbBu+QfT0uhPGfQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "optional": true, + "dependencies": { + "centra": "^2.7.0" + }, + "engines": { + "node": ">= 8" + } }, "node_modules/picocolors": { "version": "1.1.1", @@ -15818,6 +17150,29 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pixelmatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", + "integrity": "sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA==", + "license": "ISC", + "optional": true, + "dependencies": { + "pngjs": "^3.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pixelmatch/node_modules/pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/pkce-challenge": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", @@ -15926,6 +17281,16 @@ "node": ">=10.4.0" } }, + "node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12.13.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -16138,6 +17503,16 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -16294,6 +17669,15 @@ "node": ">=16.0.0" } }, + "node_modules/qrcode.react": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/qrcode.react/-/qrcode.react-4.2.0.tgz", + "integrity": "sha512-QpgqWi8rD9DsS9EP3z7BT+5lY5SFhsqGjpgW5DY/i3mK4M9DTBNz3ErMi8BWYEfI3L0d8GIbGmcdFAS1uIRGjA==", + "license": "ISC", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/qs": { "version": "6.15.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", @@ -16633,6 +18017,65 @@ "node": ">= 6" } }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.4.tgz", + "integrity": "sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw==", + "license": "MIT", + "optional": true, + "dependencies": { + "readable-stream": "^4.7.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/readable-web-to-node-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/readable-web-to-node-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "optional": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/rechoir": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", @@ -16710,6 +18153,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT", + "optional": true + }, "node_modules/regexp-ast-analysis": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/regexp-ast-analysis/-/regexp-ast-analysis-0.7.1.tgz", @@ -16959,6 +18409,7 @@ "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", "license": "BSD-3-Clause", + "optional": true, "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", @@ -17188,7 +18639,8 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/send": { "version": "1.2.1", @@ -17246,6 +18698,7 @@ "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", "license": "MIT", + "optional": true, "dependencies": { "type-fest": "^0.13.1" }, @@ -17687,7 +19140,8 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true }, "node_modules/stackback": { "version": "0.0.2", @@ -17925,6 +19379,16 @@ "node": ">=4" } }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -17938,6 +19402,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/style-to-js": { "version": "1.1.21", "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", @@ -18273,6 +19755,13 @@ "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", "license": "MIT" }, + "node_modules/timm": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz", + "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", + "license": "MIT", + "optional": true + }, "node_modules/tiny-async-pool": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.3.0.tgz", @@ -18306,6 +19795,13 @@ "dev": true, "license": "MIT" }, + "node_modules/tinycolor2": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "license": "MIT", + "optional": true + }, "node_modules/tinyexec": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", @@ -18391,6 +19887,24 @@ "node": ">=0.6" } }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/tough-cookie": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz", @@ -18533,6 +20047,7 @@ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", "license": "(MIT OR CC0-1.0)", + "optional": true, "engines": { "node": ">=10" }, @@ -19027,6 +20542,16 @@ "dev": true, "license": "(WTFPL OR MIT)" }, + "node_modules/utif2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/utif2/-/utif2-4.1.0.tgz", + "integrity": "sha512-+oknB9FHrJ7oW7A2WZYajOcv4FcDR4CfoGB0dPNfxbi4GO05RRnFmt5oa23+9w32EanrYcSJWspUiJkLMs+37w==", + "license": "MIT", + "optional": true, + "dependencies": { + "pako": "^1.0.11" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -19969,6 +21494,19 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/xhr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.6.0.tgz", + "integrity": "sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA==", + "license": "MIT", + "optional": true, + "dependencies": { + "global": "~4.4.0", + "is-function": "^1.0.1", + "parse-headers": "^2.0.0", + "xtend": "^4.0.0" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", @@ -19979,6 +21517,37 @@ "node": ">=18" } }, + "node_modules/xml-parse-from-string": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz", + "integrity": "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==", + "license": "MIT", + "optional": true + }, + "node_modules/xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "license": "MIT", + "optional": true, + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xml2js/node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=4.0" + } + }, "node_modules/xmlbuilder": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", @@ -19996,6 +21565,16 @@ "dev": true, "license": "MIT" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 330d285a..3894467a 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,10 @@ "gateway": "node scripts/stage-native.mjs && OFFGRID_SERVER_ONLY=1 electron-vite dev", "build": "node scripts/stage-native.mjs && npm run typecheck && electron-vite build", "postinstall": "electron-builder install-app-deps", - "build:unpack": "npm run build && electron-builder --dir", + "prepare:speech-defaults": "node scripts/prepare-default-speech.mjs", + "build:unpack": "npm run build && npm run prepare:speech-defaults && electron-builder --dir", "build:win": "npm run build && electron-builder --win", - "build:mac": "node scripts/stage-native.mjs && electron-vite build && electron-builder --mac", + "build:mac": "node scripts/stage-native.mjs && electron-vite build && npm run prepare:speech-defaults && electron-builder --mac", "build:linux": "electron-vite build && electron-builder --linux", "test:e2e": "electron-vite build && OFFGRID_E2E_HEADLESS=1 playwright test", "test:sync:physical": "node scripts/physical-sync/iosMacKnowledgeSync.mjs", @@ -55,12 +56,17 @@ "@electron-toolkit/preload": "^3.0.2", "@electron-toolkit/utils": "^4.0.0", "@lancedb/lancedb": "^0.30.0", + "@langchain/core": "^1.2.9", + "@langchain/langgraph": "^1.4.12", "@modelcontextprotocol/sdk": "^1.29.0", "@offgrid/clipboard": "file:./packages/clipboard", "@offgrid/design": "file:./packages/design", "@offgrid/models": "file:../shared/packages/models", "@offgrid/rag": "file:./packages/rag", + "@offgrid/speech": "file:../shared/packages/speech", "@offgrid/sync": "file:../shared/packages/sync", + "@offgrid/ui": "file:../shared/packages/ui", + "@offgrid/use": "file:../shared/packages/use", "@phosphor-icons/react": "^2.1.10", "@scure/bip39": "^2.2.0", "@tabler/icons-react": "^3.36.1", @@ -80,13 +86,13 @@ "js-sha512": "^0.9.0", "jszip": "^3.10.1", "kdbxweb": "^2.1.1", - "kokoro-js": "^1.2.1", "mammoth": "^1.8.0", "mdast-util-to-string": "^4.0.0", "motion": "^12.27.1", "node-machine-id": "^1.1.12", "ollama": "^0.6.3", "pdf-parse": "^1.1.1", + "qrcode.react": "^4.2.0", "radix-ui": "^1.6.0", "react-markdown": "^10.1.0", "react-resizable-panels": "^2.1.9", @@ -99,11 +105,15 @@ "tweetnacl-util": "^0.15.1", "unified": "^11.0.5" }, + "optionalDependencies": { + "@nut-tree-fork/nut-js": "^4.2.6" + }, "devDependencies": { "@electron-toolkit/eslint-config-prettier": "^3.0.0", "@electron-toolkit/eslint-config-ts": "^3.1.0", "@electron-toolkit/tsconfig": "^2.0.0", "@electron/asar": "3.4.1", + "@offgrid/executorch-speech": "file:../executorch-speech", "@playwright/test": "^1.61.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", diff --git a/packages/clipboard/package.json b/packages/clipboard/package.json index bb98cf1d..864a6eed 100644 --- a/packages/clipboard/package.json +++ b/packages/clipboard/package.json @@ -2,7 +2,7 @@ "name": "@offgrid/clipboard", "version": "0.0.1", "private": true, - "description": "Off Grid shared clipboard: cross-platform clipboard capture + history engine, embeddable in desktop and mobile. Absorbed from copyclip (MIT).", + "description": "Off Grid AI shared clipboard: cross-platform clipboard capture + history engine, embeddable in desktop and mobile. Absorbed from copyclip (MIT).", "license": "AGPL-3.0-only", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/packages/clipboard/src/index.ts b/packages/clipboard/src/index.ts index 374e820c..29f44f90 100644 --- a/packages/clipboard/src/index.ts +++ b/packages/clipboard/src/index.ts @@ -1,6 +1,6 @@ // @offgrid/clipboard - cross-platform clipboard capture + history engine. // Absorbed from copyclip (https://github.com/alichherawalla/copyclip, MIT) and -// restructured to be embeddable in Off Grid Desktop and Off Grid Mobile. +// restructured to be embeddable in Off Grid AI Desktop and Off Grid AI Mobile. // // The engine is platform-agnostic; platform specifics live behind ClipboardBridge // (see ./adapters/electron for the desktop bridge) and persistence behind diff --git a/packages/design/package.json b/packages/design/package.json index 70ea2d88..767a97ac 100644 --- a/packages/design/package.json +++ b/packages/design/package.json @@ -2,7 +2,7 @@ "name": "@offgrid/design", "version": "0.0.1", "private": true, - "description": "Off Grid design tokens (colors, typography, spacing) shared across desktop, mobile, and sync UIs", + "description": "Off Grid AI design tokens (colors, typography, spacing) shared across desktop, mobile, and sync UIs", "license": "AGPL-3.0-only", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/packages/design/src/index.ts b/packages/design/src/index.ts index 0d592b21..5618b6c8 100644 --- a/packages/design/src/index.ts +++ b/packages/design/src/index.ts @@ -1,4 +1,4 @@ -// Off Grid design tokens, shared across desktop (my-memories), mobile, and sync. +// Off Grid AI design tokens, shared across desktop (my-memories), mobile, and sync. // Mirrors the canonical React Native tokens in: // mobile/src/theme/palettes.ts (COLORS_LIGHT / COLORS_DARK) // mobile/src/constants/index.ts (TYPOGRAPHY / SPACING / FONTS) @@ -69,7 +69,7 @@ export const COLORS_DARK = { export type ThemeColors = typeof COLORS_LIGHT export type ColorToken = keyof ThemeColors -// Monospace font stack. Menlo is the canonical Off Grid face; the rest are +// Monospace font stack. Menlo is the canonical Off Grid AI face; the rest are // graceful fallbacks for non-macOS web environments. export const FONT_MONO = "'Menlo', ui-monospace, SFMono-Regular, 'SF Mono', Consolas, 'Liberation Mono', monospace" diff --git a/packages/design/src/tokens.css b/packages/design/src/tokens.css index 4a1a8005..f6962a9b 100644 --- a/packages/design/src/tokens.css +++ b/packages/design/src/tokens.css @@ -1,5 +1,5 @@ /* - * Off Grid design tokens as CSS custom properties. + * Off Grid AI design tokens as CSS custom properties. * Source of truth: src/index.ts (mirrors the mobile RN tokens). * * Dark is the default theme (the design guide treats pure-black dark mode as diff --git a/packages/design/tailwind-preset.js b/packages/design/tailwind-preset.js index 02168914..e4c19a3f 100644 --- a/packages/design/tailwind-preset.js +++ b/packages/design/tailwind-preset.js @@ -1,5 +1,5 @@ /** - * Off Grid Tailwind preset (Tailwind v3). + * Off Grid AI Tailwind preset (Tailwind v3). * * Maps Tailwind theme tokens onto the --og-rgb-* CSS variables defined in * tokens.css, using the rgb(... / ) pattern so opacity modifiers diff --git a/packages/rag/package.json b/packages/rag/package.json index 32932245..37f59241 100644 --- a/packages/rag/package.json +++ b/packages/rag/package.json @@ -2,7 +2,7 @@ "name": "@offgrid/rag", "version": "0.0.1", "private": true, - "description": "Off Grid projects + RAG engine: pure-TS chunking, vector retrieval, and prompt assembly over injectable embedding / vector-store / content-extraction bridges. Content extraction routes text/PDF/DOCX directly, audio through a transcription model, and video through frame sampling + a vision model. Shared by desktop and mobile.", + "description": "Off Grid AI projects + RAG engine: pure-TS chunking, vector retrieval, and prompt assembly over injectable embedding / vector-store / content-extraction bridges. Content extraction routes text/PDF/DOCX directly, audio through a transcription model, and video through frame sampling + a vision model. Shared by desktop and mobile.", "license": "AGPL-3.0-only", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/packages/rag/src/chunking.ts b/packages/rag/src/chunking.ts index fae0084a..06c0a213 100644 --- a/packages/rag/src/chunking.ts +++ b/packages/rag/src/chunking.ts @@ -1,4 +1,4 @@ -// Paragraph-aware text chunking, ported from Off Grid Mobile (rag/chunking.ts). +// Paragraph-aware text chunking, ported from Off Grid AI Mobile (rag/chunking.ts). // Splits on blank lines to respect paragraph boundaries; long paragraphs fall // back to a fixed-size sliding window with overlap so context isn't lost across // chunk edges. Pure: no platform dependencies. diff --git a/packages/rag/src/retrieval.ts b/packages/rag/src/retrieval.ts index f22677da..5f4db313 100644 --- a/packages/rag/src/retrieval.ts +++ b/packages/rag/src/retrieval.ts @@ -1,6 +1,6 @@ // Retrieval: rank candidate chunks against a query embedding, optionally trim to // a context-window budget, and format the survivors for prompt injection. Ported -// from Off Grid Mobile (rag/retrieval.ts). Pure: scoring only — fetching is the +// from Off Grid AI Mobile (rag/retrieval.ts). Pure: scoring only — fetching is the // VectorStore's job. import { cosineSimilarity } from './vectorMath' diff --git a/packages/rag/src/service.ts b/packages/rag/src/service.ts index e2fc30ec..21558ab3 100644 --- a/packages/rag/src/service.ts +++ b/packages/rag/src/service.ts @@ -1,6 +1,6 @@ // RagService: ties the bridges together. indexDocument extracts -> chunks -> // embeds -> stores; searchProject embeds the query and ranks stored chunks. -// Mirrors Off Grid Mobile's RagService surface so the apps wire it the same way. +// Mirrors Off Grid AI Mobile's RagService surface so the apps wire it the same way. import { chunkText, type ChunkOptions } from './chunking' import { extractContent, type ExtractOptions } from './extract' diff --git a/packages/rag/src/tools.ts b/packages/rag/src/tools.ts index ccf25e26..937ef28d 100644 --- a/packages/rag/src/tools.ts +++ b/packages/rag/src/tools.ts @@ -1,4 +1,4 @@ -// The search_knowledge_base tool, ported from Off Grid Mobile. Exposed to the +// The search_knowledge_base tool, ported from Off Grid AI Mobile. Exposed to the // model during project chats so it can pull from the KB on demand (in addition // to the always-on retrieval that injects context up front). The OpenAI-style // schema works with our local llama-server tool calling and remote providers. diff --git a/packages/rag/src/types.ts b/packages/rag/src/types.ts index a9044096..7a9f51a9 100644 --- a/packages/rag/src/types.ts +++ b/packages/rag/src/types.ts @@ -1,4 +1,4 @@ -// Core data model for Off Grid projects + RAG, mirrored from Off Grid Mobile so +// Core data model for Off Grid AI projects + RAG, mirrored from Off Grid AI Mobile so // desktop and mobile share one shape. The DB-level representation lives in each // platform's VectorStore implementation; these are the engine-facing types. diff --git a/packages/rag/src/vectorMath.ts b/packages/rag/src/vectorMath.ts index ee99e73f..e71e642c 100644 --- a/packages/rag/src/vectorMath.ts +++ b/packages/rag/src/vectorMath.ts @@ -1,4 +1,4 @@ -// Vector math for retrieval, ported from Off Grid Mobile (rag/vectorMath.ts). +// Vector math for retrieval, ported from Off Grid AI Mobile (rag/vectorMath.ts). // Plain-JS cosine similarity over number[] embeddings — no SIMD, no deps. Fine // for the brute-force search the RAG store does over a project's chunks. diff --git a/pro b/pro index 89870379..327cde48 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit 89870379dc57ee588998996c870e256f18323ccc +Subproject commit 327cde48d5711e224afed379467dfa5a44536bf3 diff --git a/resources/bin/actions-helper b/resources/bin/actions-helper new file mode 100755 index 00000000..b2c3ffb9 --- /dev/null +++ b/resources/bin/actions-helper @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd3ba73ec3d6cc94d29264940f11534d0f1a3b1b011153e10d66556514a7b9e7 +size 107456 diff --git a/resources/bin/text-extractor b/resources/bin/text-extractor index 6df945d6..fd70bd83 100755 --- a/resources/bin/text-extractor +++ b/resources/bin/text-extractor @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5e3d3e346fd9ea94b2d1ac2b4980fd676df1e28150ae04dac70ef30d76bb75f2 -size 136936 +oid sha256:c3fbdd344bf1cab678db55193243af6dd8507687d7c0a09f0439ba02460a7fa0 +size 163712 diff --git a/resources/tts-worker.mjs b/resources/tts-worker.mjs deleted file mode 100644 index e6b10bda..00000000 --- a/resources/tts-worker.mjs +++ /dev/null @@ -1,227 +0,0 @@ -// Isolated TTS worker — runs Kokoro-82M via kokoro-js in its OWN process so its -// onnxruntime-node (bundled by @huggingface/transformers) never collides with the -// onnxruntime-node that @xenova/transformers loads in the main process (loading -// two native ORT builds in one process throws "Session already disposed"). -// -// Running it as a short-lived subprocess also means the ~330MB model is only -// resident while speaking and is reclaimed the moment we exit — true swap-in/out. -// -// Launched via Electron's binary with ELECTRON_RUN_AS_NODE=1 so the native ABI -// matches the app. Usage: -// tts-worker.mjs voices -> prints JSON array of voice ids to stdout -// tts-worker.mjs speak -> reads text from stdin, writes WAV to - -import fs from 'node:fs' -import path from 'node:path' - -const MODEL_ID = 'onnx-community/Kokoro-82M-v1.0-ONNX' -const DEFAULT_VOICE = 'af_heart' - -const worker = { - log(event, fields = {}) { - const details = Object.entries(fields) - .map(([key, value]) => `${key}=${JSON.stringify(value)}`) - .join(' ') - process.stderr.write( - `${new Date().toISOString()} INFO [tts-worker] ${event}${details ? ` ${details}` : ''}\n` - ) - }, - - /** - * @param {string} target - * @param {(string | null | undefined)[]} sources - * @returns {boolean} - */ - materializeRuntimeFile(target, sources) { - if (fs.existsSync(target)) return false - const source = sources.find((candidate) => candidate && fs.existsSync(candidate)) - if (!source) return false - fs.mkdirSync(path.dirname(target), { recursive: true }) - try { - fs.linkSync(source, target) - } catch { - fs.copyFileSync(source, target) - } - return true - }, - - /** - * @param {{ cacheDir?: string }} transformersEnv - * @returns {void} - */ - configureWritableCache(transformersEnv) { - const writableCache = process.env.OFFGRID_TTS_CACHE_DIR - if (!writableCache) return - const bundledCache = transformersEnv.cacheDir - const relativeFiles = [ - 'config.json', - 'tokenizer.json', - 'tokenizer_config.json', - 'onnx/model_quantized.onnx' - ] - let materialized = 0 - for (const relative of relativeFiles) { - const target = path.join(writableCache, MODEL_ID, relative) - const bundled = bundledCache ? path.join(bundledCache, MODEL_ID, relative) : null - const downloaded = - relative === 'onnx/model_quantized.onnx' ? process.env.OFFGRID_TTS_MODEL_FILE : null - if (worker.materializeRuntimeFile(target, [downloaded, bundled])) materialized++ - } - transformersEnv.cacheDir = writableCache - worker.log('cache.configured', { writable: true, materialized }) - }, - - // kokoro-js' RawAudio.toWav() emits 32-bit IEEE-float WAV (format 3), which - // Chromium's