From f40a84dc3ca32caae2db03cfb20cbcd6ebb89892 Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:15:43 +0200 Subject: [PATCH 01/12] docs(dictation): design spec and implementation plan (#13) Co-Authored-By: Claude Fable 5.1 --- .../superpowers/plans/2026-09-19-dictation.md | 172 +++++++++++++++++ .../specs/2026-09-19-dictation-design.md | 175 ++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-19-dictation.md create mode 100644 docs/superpowers/specs/2026-09-19-dictation-design.md diff --git a/docs/superpowers/plans/2026-09-19-dictation.md b/docs/superpowers/plans/2026-09-19-dictation.md new file mode 100644 index 0000000..5e57147 --- /dev/null +++ b/docs/superpowers/plans/2026-09-19-dictation.md @@ -0,0 +1,172 @@ +# Dictation Implementation Plan + +**Goal:** Hold Right Alt, speak, release: the words are pasted at the cursor of the shell in front, +transcribed locally by a bundled whisper.cpp, in both Prism Terminal and Prism. + +**Architecture:** Everything lives in `core/`. The renderer captures the mic and owns the hotkey, +the pill and the settings page; main owns the `whisper-server` process, the model/GPU-pack store and +the media pause helper. Hosts inject Electron (`ipcMain`, paths, GPU info, the media permission). + +**Tech stack:** whisper.cpp official Windows binaries (release `b5130`), Web Audio (AudioWorklet), +Node `http` + `child_process`, Windows PowerShell 5.1 for the media helper. NO new npm dependency. + +**Spec:** `docs/superpowers/specs/2026-09-19-dictation-design.md` + +## Global constraints + +- Core lint wall: relative imports only, no `electron`, no `window.prism`, no host `src/`. +- Official binaries only, every download pinned by SHA-256; an unverified file is never run or loaded. +- Off by default: with dictation off there is no listener armed, no process, no download. +- The app NEVER sends Enter or a newline. Text arrives as one bracketed paste. +- Audio is never written to disk and never leaves `127.0.0.1`. +- Shared folder `%LOCALAPPDATA%\PrismDictation\{models,engines}`; setting values stay per app. +- No em-dashes. Every new `[data-pref]` row is in `options.ts` and covered by both apps' parity e2e. +- UI is looked at (screenshots) and measured, not only asserted to exist (#20). +- Branch `feat/13-dictation`, one PR, version 0.2.0. Never merge without the owner's word. + +## File map + +Core, new: +- `core/shared/dictationCatalog.ts`: engine + model + GPU pack entries: id, label, url, bytes, sha256, needsGpu. +- `core/shared/dictationTypes.ts`: `DictationState`, `ModelStatus`, `DownloadProgress`, IPC payloads. +- `core/shared/channels.ts`: add the `dictation:*` channels. +- `core/tools/fetch-whisper.mjs`: build-time fetch of the CPU engine into a given out dir. +- `core/main/dictationStore.ts`: list / download / verify / delete for models and the GPU pack. +- `core/main/dictationEngine.ts`: choose engine, spawn `whisper-server`, health, idle stand-down, fallback. +- `core/main/mediaPause.ts`: the PowerShell helper: `pausePlaying(): Promise`, `resume(token)`. +- `core/main/dictationIpc.ts`: `registerDictationIpc(deps)`. +- `core/preload/api.ts`: extend `createTermApi` with the dictation calls. +- `core/renderer/lib/dictationKey.ts`: the pure hotkey state machine. +- `core/renderer/lib/dictationClean.ts`: pure cleanup. +- `core/renderer/lib/wav.ts`: PCM16 WAV encode, tail window. +- `core/renderer/lib/micCapture.ts` + `micWorklet.js`: 16 kHz mono capture with a level callback. +- `core/renderer/lib/dictationSounds.ts`: two synthesized cues (Web Audio, no asset files). +- `core/renderer/lib/dictationPrefs.ts`: the per-app stores (the `termLook` store shape). +- `core/renderer/lib/useDictation.ts`: the orchestrating hook. +- `core/renderer/components/DictationPill.tsx` +- `core/renderer/settings/Dictation.tsx`: `DictationSettings`. +- Tests beside each pure/main file. + +Core, changed: `host.ts` (the `dictation` seam), `settings/options.ts` (+ `page` field), `package.json` +(exports `./tools/*`, files `tools`), `README.md`. + +Prism Terminal: `package.json` (`fetch:bin`, run before build/package/e2e), `electron-builder.yml` +(`extraResources: vendor/whisper -> bin/whisper`), `.gitignore` (`vendor/`), `src/main/index.ts` +(register IPC, media permission for own window, e2e fake-mic switches), `src/renderer/src/termHost.ts`, +`App.tsx` (mount hook + pill), `TabStrip.tsx` (mic mark), `Settings.tsx` (Dictation tab), `index.css` +(pill keyframes if any), `tools/e2e/run.mjs` (`dictation` scenario, options parity for the page), +`CLAUDE.md`, `README.md`. + +## Tasks + +### Task 0: two spikes that can change the design (throwaway, scratchpad) +- [ ] Fake mic: launch the built app with `--use-fake-device-for-media-stream + --use-file-for-fake-audio-capture=jfk.wav`, call `getUserMedia`, confirm non-silent samples arrive. + If it does not work in Electron, the e2e injects PCM through a test-only seam in `micCapture` instead. +- [ ] Media helper: in Windows PowerShell 5.1, load `GlobalSystemMediaTransportControlsSessionManager`, + list sessions with playback status, pause and resume one. If WinRT cannot be reached, the option + ships as "not available on this PC" and the spec is corrected. +- [ ] Record both results in issue #13. + +### Task 1: catalog, types, channels +- [ ] Test first: every catalog entry has https url, bytes > 0, 64-hex sha256; ids unique; exactly one + CPU-recommended and one GPU-recommended model; GPU models flagged `needsGpu`. +- [ ] Fill SHA-256 and sizes from the sources themselves (GitHub asset digests; Hugging Face LFS + pointers for `ggerganov/whisper.cpp` `ggml-{tiny,base,small,large-v3-turbo,large-v3}.bin`), and + re-verify each by downloading once. Tiny is catalogued as `e2eOnly`. +- [ ] Commit. + +### Task 2: build-time engine fetch +- [ ] `core/tools/fetch-whisper.mjs `: pinned tag + SHA, stamp file so a second run is a no-op, + extracts only `whisper-server.exe` and the DLLs it loads, plus the licence. Monthly-tag lesson + from Prism's ffmpeg does not apply (whisper.cpp keeps its build tags) but the 404 message still + says how to re-pin. +- [ ] Prism Terminal: `fetch:bin` script, wired before `build`-dependent scripts; `extraResources`; + `.gitignore`. CI: confirm `ci.yml` does not need the binary (typecheck, lint, unit only). +- [ ] Verify: packaged app contains `resources/bin/whisper/whisper-server.exe`. Commit. + +### Task 3: the store (main) +Interface: `createDictationStore({ root, fetchImpl })` -> `{ status(): Promise, +download(id, onProgress): Promise, cancel(id), remove(id), gpuPack: { status, download, remove } }`. +- [ ] Tests first against a local HTTP fixture: progress events, resume not attempted (restart), SHA + mismatch deletes the file and returns `checksum`, cancel leaves nothing, disk error surfaces, + `status` reflects files on disk, a `.part` file is never reported as installed. +- [ ] Implement with streams; the GPU pack unzips into `engines/cuda-` via PowerShell + `Expand-Archive` (no new dependency), verified BEFORE unzip. Commit. + +### Task 4: the engine (main) +Interface: `createDictationEngine({ cpuDir, store, gpuVendors, spawnImpl })` -> `{ transcribe(wav: +Buffer, opts: { model, language }): Promise, stop(), state() }`. +- [ ] Tests first with a fake server binary (a node script): picks GPU only when pack installed AND an + NVIDIA vendor id is present; falls back to CPU when the GPU server exits at start, once per + session; restarts when the model or language changes; serialises passes (latest partial wins, a + final is never dropped); stands down after the idle window; `stop()` kills the child. +- [ ] Implement: free port, `--host 127.0.0.1`, readiness poll, multipart POST to `/inference`, + `response_format=json`. Commit. + +### Task 5: media pause (main), per the spike +- [ ] `pausePlaying()` returns a token naming the sessions it paused; `resume(token)` plays only + those. Helper failure resolves to a no-op token. Unit test the protocol with a fake helper. + +### Task 6: IPC + preload + host seam +- [ ] `registerDictationIpc({ ipcMain, send, cpuEngineDir, gpuVendors, sharedRoot })`; channels for + status, download/cancel/remove, progress events, transcribe(partial|final), mediaPause/resume, stop. +- [ ] `TermApi` grows the matching calls; `TermHostConfig.dictation = { canDictate(): boolean }`. +- [ ] `host.test.ts` models both hosts. Commit. + +### Task 7: pure renderer logic +- [ ] `dictationKey.ts`: reducer over `{type:'down'|'up'|'blur'|'tick', code, ctrl, alt, ...}` -> + `idle | arming | listening | stopping`, emitting `start | stop | cancel`. Tests: AltGr+2 on a + Norwegian layout (ControlLeft then AltRight then Digit2) never starts; 200 ms solo hold starts; + release stops; toggle tap/tap; Escape cancels; blur cancels; a rebound chord (Ctrl+Shift+D) works. +- [ ] `dictationClean.ts` with the cases in the spec. `wav.ts`: header bytes, 28 s tail. Commit. + +### Task 8: capture, sounds, prefs, the hook +- [ ] `micCapture.start({ deviceId, onLevel })` -> `{ snapshot(): Float32Array, stop() }`; worklet + downsamples to 16 kHz. A test-only `__setCaptureSource` seam only if Task 0 demands it. +- [ ] `useDictation(activeSessionId)` -> `{ phase, level, liveText, message, sessionId }`: arms the + window listeners only while `dictation-enabled`; partial loop; final -> clean -> the core's paste + path as bracketed text with newlines flattened; sounds; media pause; 5 min cap. +- [ ] xterm's key handler yields the hotkey (the `ownsKey` route), and swallows the Alt keyup. + +### Task 9: the pill and the tab mark +- [ ] Design pass with the design skills (impeccable + the app's theme tokens; it must read on every + terminal theme, light and dark, and over acrylic). Mount/unmount, no opacity-in-place. +- [ ] `DictationPill` over the panel's bottom centre; level meter driven by `onLevel`; live text + ellipsised from the LEFT so the newest words stay visible. TabStrip mic mark via `sessionId`. +- [ ] Measure in the e2e: the pill does not move the terminal's rows. + +### Task 10: the settings page +- [ ] `DictationSettings` with the nine rows; model manager rows with progress; the mic row's live + meter runs only while the page is open. `options.ts` gains `page: 'terminal' | 'dictation'`; + `options.test.ts` extended; Prism Terminal's Settings gets the Dictation tab. +- [ ] Screenshot both themes; measure row layout in the e2e. + +### Task 11: Prism Terminal wiring + e2e +- [ ] main: register IPC, grant `media` to the app's own window only, add the fake-mic switches under + `--e2e`. App: mount hook and pill. termHost: `canDictate`. +- [ ] e2e `dictation` scenario exactly as the spec's Testing section lists; Tiny model cached under + `.e2e-cache/`. Run it RED first where possible (e.g. before the paste is wired). +- [ ] Full gate: typecheck, lint, unit, all e2e. + +### Task 12: docs, version, install +- [ ] CLAUDE.md: the fourth written exception; "a product for other users" rule; the shared folder; + how to re-pin. core/README: the new seam fields and the fetch script. README: the feature, the + hotkey, privacy (local, nothing leaves the machine), the GPU pack. +- [ ] 0.2.0, package, install (poll the exe timestamp), hands-on list for the owner: how it sounds, + AltGr on the real keyboard, pause-media with Spotify/YouTube, GPU pack on the 5090. +- [ ] PR, CI, "merge?". + +### Task 13 (after merge, and only after asking): Prism +- [ ] Tag `core-v0.2.0`; Prism branch: pin, `fetch:bin` calls the core's script into `vendor/whisper`, + `extraResources`, main wiring + permission, termHost `canDictate` (terminal panel showing), the + Dictation rail page, TabStrip mark, e2e `dictation` + parity, `npm run e2e:terminal`, full e2e, + 0.54.0, install, PR. + +## Risks + +- Fake mic may not work under Electron: fallback seam in Task 0. +- WinRT from PowerShell may be unreachable on some PCs: the option degrades to unavailable. +- The 643 MB GPU pack is a big download: progress, cancel, and it is never required. +- A slow laptop CPU makes live text lag: passes are serialised and the pill shows the level meter + regardless, so the mic is visibly alive even when text trails. diff --git a/docs/superpowers/specs/2026-09-19-dictation-design.md b/docs/superpowers/specs/2026-09-19-dictation-design.md new file mode 100644 index 0000000..f2bb0b5 --- /dev/null +++ b/docs/superpowers/specs/2026-09-19-dictation-design.md @@ -0,0 +1,175 @@ +# Dictation: design + +Issue #13. Owner decisions are in that issue (four rounds, 2026-09-19); this spec is what they add up +to. Built ONCE, in `core/`, so it lands in Prism Terminal and in Prism. + +## What it is + +Hold Right Alt, speak, let go: the words appear at the cursor of the shell in front, as if pasted. +Nothing is sent. It runs entirely on the user's machine, in any tab and any shell, and it is off until +the user turns it on. + +It is a product feature for strangers' PCs, not for the owner's: everything it needs is bundled with +the installer or fetched by the app itself, and it works on a fresh Windows install with no GPU. + +## Decided (owner) + +| | | +|---|---| +| Engine | Local whisper.cpp. Official pinned binaries only; nothing we compile. | +| GPU | CPU engine bundled. An "Enable GPU" button downloads the official NVIDIA pack. AMD/Intel stay on CPU in v1. | +| Models | A short list in a model manager, one marked Recommended, one Active. The app downloads them. | +| Model files | One folder shared by both apps: `%LOCALAPPDATA%\PrismDictation`. Setting VALUES stay per app. | +| Language | Auto-detect by default, with a picker to pin one. Multilingual models only. | +| Trigger | Right Alt. Hold-to-talk by default; press-to-start / press-to-stop as the other mode. Rebindable. | +| Output | Bracketed paste into the shell in front. NEVER presses Enter. | +| Scope | Any tab, any shell. | +| Feedback | A pill over the terminal ("Listening..." with a live level meter and the live text, then "Transcribing..."), a mic mark on the tab, a sound on start and stop. | +| Cleanup | Deterministic only. No second model. | +| Media | Sub-option: pause what is playing while dictating, and resume exactly that. | +| Default | Off. Nothing listens, nothing downloads, no server runs until it is switched on. | +| Settings | Prism Terminal: its own Dictation tab. Prism: its own Dictation page under Behaviour. Same shared page. | + +## Measured (2026-09-19, owner's PC) + +- CPU engine zip `whisper-bin-x64.zip` of release `b5130`: 8 MB. Contains `whisper-server.exe`. +- CPU, resident server, Base model, a GROWING clip re-sent: about 0.9 s per pass whatever the length + (2 s to 11 s). So live text can refresh about once a second with no GPU. Large on CPU: 20 s. CPU + means Base or Small. +- NVIDIA pack `whisper-cublas-12.4.0-bin-x64.zip`: 643 MB, sha256 `af520ddd...c272a7`. It RUNS ON AN + RTX 5090 (a card newer than CUDA 12.4): first run 9.1 s (one-time kernel compile), then Large-v3 + through a resident server is 0.36 to 0.50 s per pass on an 11 s clip. So one pack covers old and + new NVIDIA cards, and the 260 MB CUDA 11.8 pack is not needed. +- Partials are provisional (a phrase heard wrong at 6 s was corrected by 8 s). So live text is shown + in the pill and only the FINAL text is pasted: text typed into a prompt cannot be taken back. + +## Architecture + +``` +renderer (core) main (core, host injects electron bits) + useDictation ── hotkey, state ─┐ dictationEngine spawn/stop whisper-server, idle stand-down + micCapture ── 16 kHz PCM ────┼─IPC─▶ dictationIpc transcribe(wav) -> text + DictationPill ── level, text │ dictationStore models + GPU pack: list, download, verify, delete + dictationSounds │ mediaPause ask Windows what is playing; pause; resume those + dictationClean (pure) │ + settings/Dictation.tsx ─────────┘ +``` + +- **Audio never leaves the machine and is never written to disk.** The renderer captures the mic + (`getUserMedia`), downsamples to 16 kHz mono in an AudioWorklet, and hands main a WAV buffer over + IPC. Main posts it to `whisper-server` on `127.0.0.1` at a random free port and returns the text. +- **Live text:** while recording, whenever no pass is in flight, the clip so far (its last 28 s, + Whisper's window) is sent; the answer goes to the pill. On stop, one final pass over the whole clip; + that text is cleaned and pasted. +- **The server is resident only while dictation is ON**, started on the first press, and stands down + after 5 minutes idle (the model reloads in about a second on the next press). It is killed with the + app. Off means no process at all. +- **The engine** is bundled per app in `resources/bin/whisper` by a fetch script that lives in the + core (`core/tools/fetch-whisper.mjs`: pinned tag, pinned SHA-256, like Prism's ffmpeg), so both + apps fetch the same bytes. The NVIDIA pack and the models are downloaded at RUNTIME into the shared + folder, each against a SHA-256 pinned in `core/shared/dictationCatalog.ts`. A download that does not + match is deleted and reported; nothing unverified is ever run or loaded. +- **Engine choice** is automatic: the GPU pack when it is installed and an NVIDIA adapter is present, + else the bundled CPU engine. If the GPU engine fails to start, it falls back to CPU for the session + and says so once in Settings. There is no "use GPU" switch to get wrong. +- **Host seam** (new declared fields, `TermHostConfig.dictation`): `engineDir()` (where the bundled + CPU engine is), `gpuVendors()` (from Electron's `app.getGPUInfo`, no process spawned), and whether + the terminal is in front (`canDictate()`: always true in Prism Terminal with a tab open; in Prism + only while the terminal panel is showing). Main side: `registerDictationIpc({ ipcMain, send, ... })` + beside `registerTermIpc`. The host grants the `media` permission to its own window only. + +## The hotkey + +Right Alt is AltGr on Norwegian and most European layouts, where it types `@ { } [ ] \ | ~`; Windows +reports it as LeftCtrl+RightAlt. So the key is read as a SOLO HOLD: + +- Hold mode: recording starts once Right Alt has been down 200 ms with no other key. Any other key + during the hold cancels silently (that was AltGr typing). Release stops and transcribes. +- Toggle mode: a solo TAP (down and up with no other key) starts; the next solo tap stops. +- Its keyup is swallowed so Alt does not open the window's system menu. xterm never sees the chord. +- Escape while listening cancels: nothing is pasted. +- Rebindable to any single key or chord in Settings; the solo-hold rule applies to bare modifiers. +- Window-level only, never a global hotkey: the text goes into this window's shell, so it listens + only while this window has the focus. A recording is capped at 5 minutes. + +## Models (v1 list) + +| Model | Download | For | +|---|---|---| +| Base | 142 MB | Recommended on CPU. Fast live text on any PC. | +| Small | 466 MB | CPU, more accurate, slower (a pass is about 3x Base). | +| Large v3 Turbo | 1.5 GB | Recommended once the GPU pack is installed. | +| Large v3 | 2.9 GB | GPU. The most accurate. | + +Turbo and Large are listed without the GPU pack but marked "needs the GPU pack", not hidden. The +"Enable GPU" row appears only when an NVIDIA adapter is found. Turning dictation on with no model +downloaded offers the Recommended one; nothing downloads without a click. NVIDIA Parakeet is OUT of +v1: the owner has had trouble with it and it has not been measured here. + +## Cleanup (`dictationClean`, pure, tested) + +Whisper's own punctuation and capitalisation are kept. Removed: engine artifacts (`[BLANK_AUDIO]`, +`(music)`, `[ Silence ]` and kin), immediately doubled words, the fillers "um", "uh", "erm" as whole +words, leading and trailing whitespace, runs of spaces; newlines between segments become single +spaces. A final text that is empty after cleaning pastes nothing and the pill says "Heard nothing". + +## Pause media + +Not the play/pause media key: it toggles, so it would START music that was not playing. A small +resident Windows PowerShell helper (the `dwmHelper` pattern) asks +`GlobalSystemMediaTransportControlsSessionManager` which sessions are Playing, pauses those, and +resumes exactly those when dictation ends. If the helper cannot start, dictation works and the +option does nothing. + +## Settings (`core/renderer/settings/Dictation.tsx`, ids in `options.ts` under page `dictation`) + +`dictation-enabled` (switch, off), `dictation-mode` (Hold / Toggle), `dictation-hotkey` (rebind), +`dictation-mic` (input device, default "System default", with a live level meter so a dead mic is +visible BEFORE recording), `dictation-language` (Auto + list), `dictation-pause-media` (switch, off), +`dictation-sounds` (switch, on), `dictation-model` (the model manager: size, Download with progress, +Delete, Recommended badge, Active marker), `dictation-gpu` (Enable GPU: size, progress, Remove). +Each app's parity e2e covers the page as it does the terminal's. + +## The written exception + +CLAUDE.md's rule is that the app never types into the user's shell. Dictation is the fourth written +exception, with the agent resume, Prism's `Set-Location`, and drop-to-type-path: the user spoke the +text on purpose, it arrives as a bracketed paste, and it never carries a newline or an Enter. + +## Errors the user can meet, and what they see + +No microphone / permission refused: the pill says so, nothing records. Model missing or deleted: +the pill says "Choose a model in Settings". Engine will not start: CPU fallback, then a plain message. +Download fails or fails its checksum: the row says so and offers Retry; the partial file is removed. +Disk full: same row. The shared folder is created on first use and is never removed by an uninstall. + +## Testing + +- Unit (core): the hotkey state machine (AltGr cancels, tap vs hold, Escape), `dictationClean`, WAV + encoding and the 28 s window, the catalog (every entry has a size and a 64-hex SHA), the store + (verify, reject, delete, progress) against a local HTTP fixture, engine choice and fallback, the + server lifecycle with a fake binary, options parity. +- E2E (both apps), REAL end to end: under `--e2e` Chromium is given a fake microphone fed from a WAV + (`--use-fake-device-for-media-stream`, `--use-file-for-fake-audio-capture`), the real bundled + `whisper-server` runs the Tiny model (75 MB, fetched once into the e2e cache against a pinned SHA), + and the scenario asserts that "ask not what your country can do for you" arrives on the prompt line, + that no Enter was sent (the line is still being edited), that the pill showed Listening then + Transcribing, that the tab wore the mic mark, that Escape pastes nothing, and that with dictation + OFF the key does nothing and no server process exists. Layout of the Dictation page is measured and + screenshotted, as #20 taught. +- Not testable by machine: how it sounds, and AltGr on a physical Norwegian keyboard. Both go on the + owner's hands-on list. + +## Out of v1 + +AMD/Intel GPU acceleration (needs a Vulkan engine we would have to build), Parakeet, voice commands +("new line", "send"), a global hotkey, custom vocabulary, English-only model variants, auto-stop on +silence. + +## Delivery + +1. Prism Terminal PR (this repo): the core feature, this app's wiring, docs, version 0.2.0. Installed + locally for hands-on testing before "merge?". +2. Tag `core-v0.2.0`. ASK the owner before pulling it into Prism (standing rule). +3. Prism PR: pin bump, host wiring, the Dictation page, `npm run e2e:terminal` plus the dictation + scenario, full e2e, install. From 7fe96c41f378c7c9132f1ba32318865b1ce35fa3 Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:18:03 +0200 Subject: [PATCH 02/12] feat(dictation): the shared contract (#13) Co-Authored-By: Claude Fable 5.1 --- core/shared/dictationTypes.ts | 100 ++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 core/shared/dictationTypes.ts diff --git a/core/shared/dictationTypes.ts b/core/shared/dictationTypes.ts new file mode 100644 index 0000000..576cb56 --- /dev/null +++ b/core/shared/dictationTypes.ts @@ -0,0 +1,100 @@ +// What dictation speaks across its seams (#13). Owned by the core, so the two +// apps and the two processes can never disagree about a model or a download. + +/** One downloadable thing: a model, or the NVIDIA engine pack. */ +export interface CatalogEntry { + id: string + label: string + /** One line for the model manager: who it is for. */ + note: string + url: string + /** The size of the download, exactly: progress and the checksum both lean on it. */ + bytes: number + /** Lowercase hex. A file that does not match is deleted, never loaded or run. */ + sha256: string + kind: 'model' | 'gpu-pack' + /** Too slow to use without the GPU pack (measured: Large on a fast CPU, 20 s for 11 s). */ + needsGpu?: boolean + /** The pick for a machine with (gpu) or without (cpu) the GPU pack. */ + recommended?: 'cpu' | 'gpu' + /** Catalogued for the e2e only; never shown in the model manager. */ + e2eOnly?: boolean +} + +export type InstallState = 'absent' | 'downloading' | 'installed' + +/** What the model manager draws for one entry. */ +export interface ItemStatus { + id: string + state: InstallState + /** Bytes on disk so far while downloading; `bytes` of the entry when installed. */ + received: number +} + +export interface DownloadProgress { + id: string + received: number + total: number +} + +/** Why a download did not end in an installed file. */ +export type DownloadFailure = 'network' | 'checksum' | 'disk' | 'cancelled' | 'unpack' +export type DownloadResult = { ok: true } | { ok: false; reason: DownloadFailure; detail?: string } + +/** Which engine answered, so Settings can say when the GPU one was passed over. */ +export type EngineKind = 'cpu' | 'gpu' + +export interface TranscribeRequest { + /** A complete 16 kHz mono PCM16 WAV file. Never written to disk. */ + wav: Uint8Array + modelId: string + /** 'auto', or an ISO 639-1 code Whisper knows. */ + language: string + /** A partial may be dropped for a newer one; a final never is. */ + final: boolean +} + +export type TranscribeResult = + | { ok: true; text: string; engine: EngineKind } + | { ok: false; reason: 'no-model' | 'no-engine' | 'engine-failed' | 'superseded'; detail?: string } + +export interface EngineInfo { + /** An NVIDIA adapter is present, so the GPU pack is worth offering. */ + nvidia: boolean + gpuPack: InstallState + /** The GPU engine was tried this session and would not start; CPU is answering. */ + gpuFellBack: boolean +} + +/** Opaque: which media sessions THIS pause paused, so resume plays only those. */ +export type MediaPauseToken = string + +// ---- main-side contracts (the store, the engine, the media helper) ---------- + +/** Models and the GPU pack on disk, in the folder BOTH apps share. */ +export interface DictationStore { + status(): Promise + /** Resolves when the file is verified and in place, or with why it is not. */ + download(id: string, onProgress: (p: DownloadProgress) => void): Promise + cancel(id: string): void + remove(id: string): Promise + /** Absolute path of an INSTALLED, verified model file, else null. */ + modelPath(id: string): string | null + /** The folder holding whisper-server.exe of the installed GPU pack, else null. */ + gpuEngineDir(): string | null +} + +/** The resident whisper-server: started on first use, gone when idle or stopped. */ +export interface DictationEngine { + transcribe(req: TranscribeRequest): Promise + /** Kill the server now (dictation switched off, app quitting). */ + stop(): void + info(): { running: boolean; engine: EngineKind | null; gpuFellBack: boolean } +} + +/** Pause what is PLAYING, and later resume exactly that. Never throws. */ +export interface MediaPause { + pausePlaying(): Promise + resume(token: MediaPauseToken): Promise + dispose(): void +} From f4d9c2593941d7bb55d8b4b262724a6245716ee6 Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:34:17 +0200 Subject: [PATCH 03/12] feat(dictation): local speech to text, built once in the core (#13) Hold Right Alt, speak, release: the words are pasted at the cursor, never sent. whisper.cpp's official CPU engine is bundled (with Microsoft's C++ runtime and the MIT notice beside it), models and the NVIDIA pack are downloaded on request into a folder both apps share, each against a pinned SHA-256. Off by default. Co-Authored-By: Claude Fable 5.1 --- .gitignore | 2 + core/main/__fixtures__/fakeMediaHelper.mjs | 116 ++++ core/main/__fixtures__/fakeWhisperServer.mjs | 123 ++++ core/main/dictationEngine.test.ts | 415 ++++++++++++++ core/main/dictationEngine.ts | 505 +++++++++++++++++ core/main/dictationIpc.ts | 113 ++++ core/main/dictationStore.test.ts | 533 ++++++++++++++++++ core/main/dictationStore.ts | 447 +++++++++++++++ core/main/ipc.ts | 2 +- core/main/mediaPause.test.ts | 223 ++++++++ core/main/mediaPause.ts | 278 +++++++++ core/package.json | 8 +- core/preload/dictationApi.ts | 55 ++ core/renderer/components/DictationPill.tsx | 94 +++ core/renderer/components/DictationTabMark.tsx | 35 ++ core/renderer/components/TerminalPanel.tsx | 20 +- core/renderer/host.ts | 20 + core/renderer/lib/dictation.ts | 338 +++++++++++ core/renderer/lib/dictationClean.test.ts | 192 +++++++ core/renderer/lib/dictationClean.ts | 174 ++++++ core/renderer/lib/dictationKey.test.ts | 464 +++++++++++++++ core/renderer/lib/dictationKey.ts | 392 +++++++++++++ core/renderer/lib/dictationPrefs.ts | 109 ++++ core/renderer/lib/dictationSounds.ts | 55 ++ core/renderer/lib/micCapture.ts | 139 +++++ core/renderer/lib/termBus.ts | 16 + core/renderer/lib/useDictation.ts | 20 + core/renderer/lib/wav.test.ts | 195 +++++++ core/renderer/lib/wav.ts | 117 ++++ core/renderer/settings/Dictation.tsx | 473 ++++++++++++++++ core/renderer/settings/dictationOptions.ts | 40 ++ core/renderer/settings/options.test.ts | 16 +- core/shared/channels.ts | 15 + core/shared/dictationCatalog.test.ts | 175 ++++++ core/shared/dictationCatalog.ts | 228 ++++++++ core/tools/fetch-whisper.mjs | 208 +++++++ electron-builder.yml | 5 + package.json | 5 +- src/main/index.ts | 57 +- src/preload/index.ts | 3 + src/renderer/src/App.tsx | 7 + src/renderer/src/components/Settings.tsx | 19 +- src/renderer/src/components/TabStrip.tsx | 2 + src/renderer/src/termHost.ts | 4 + tools/e2e/run.mjs | 197 ++++++- 45 files changed, 6635 insertions(+), 19 deletions(-) create mode 100644 core/main/__fixtures__/fakeMediaHelper.mjs create mode 100644 core/main/__fixtures__/fakeWhisperServer.mjs create mode 100644 core/main/dictationEngine.test.ts create mode 100644 core/main/dictationEngine.ts create mode 100644 core/main/dictationIpc.ts create mode 100644 core/main/dictationStore.test.ts create mode 100644 core/main/dictationStore.ts create mode 100644 core/main/mediaPause.test.ts create mode 100644 core/main/mediaPause.ts create mode 100644 core/preload/dictationApi.ts create mode 100644 core/renderer/components/DictationPill.tsx create mode 100644 core/renderer/components/DictationTabMark.tsx create mode 100644 core/renderer/lib/dictation.ts create mode 100644 core/renderer/lib/dictationClean.test.ts create mode 100644 core/renderer/lib/dictationClean.ts create mode 100644 core/renderer/lib/dictationKey.test.ts create mode 100644 core/renderer/lib/dictationKey.ts create mode 100644 core/renderer/lib/dictationPrefs.ts create mode 100644 core/renderer/lib/dictationSounds.ts create mode 100644 core/renderer/lib/micCapture.ts create mode 100644 core/renderer/lib/useDictation.ts create mode 100644 core/renderer/lib/wav.test.ts create mode 100644 core/renderer/lib/wav.ts create mode 100644 core/renderer/settings/Dictation.tsx create mode 100644 core/renderer/settings/dictationOptions.ts create mode 100644 core/shared/dictationCatalog.test.ts create mode 100644 core/shared/dictationCatalog.ts create mode 100644 core/tools/fetch-whisper.mjs diff --git a/.gitignore b/.gitignore index da01645..f58b8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dist *.log .e2e-profile .e2e-shots/ +vendor/ +.e2e-cache/ diff --git a/core/main/__fixtures__/fakeMediaHelper.mjs b/core/main/__fixtures__/fakeMediaHelper.mjs new file mode 100644 index 0000000..8fdeac7 --- /dev/null +++ b/core/main/__fixtures__/fakeMediaHelper.mjs @@ -0,0 +1,116 @@ +// A stand-in for the resident PowerShell media helper (#13), so mediaPause's +// tests never touch the owner's real media sessions and never start a real +// powershell.exe. It speaks the same line protocol on stdin and stdout: +// +// pause -> +// resume -> ok +// list -> =,... +// +// and keeps a tiny world of sessions in memory, so a second pause finds nothing +// left to pause and a resume really does put a session back to Playing. +// +// Arguments (all optional): +// --sessions a=Playing,b=Paused the world it starts with +// --log every request line is appended here +// --mode normal|silent|garbage|reverse|crash-once +// --marker crash-once: exit on the first request unless +// this file exists, creating it on the way out +// --delay answer each request this late +// +// Everything is imported rather than taken off a global, because eslint gives a +// bare .mjs under core/ no Node globals. +import { appendFileSync, existsSync, writeFileSync } from 'node:fs' +import process from 'node:process' +import { createInterface } from 'node:readline' +import { setTimeout as later } from 'node:timers' + +function arg(name, fallback) { + const at = process.argv.indexOf(`--${name}`) + return at >= 0 && at + 1 < process.argv.length ? process.argv[at + 1] : fallback +} + +const mode = arg('mode', 'normal') +const logFile = arg('log', '') +const marker = arg('marker', '') +const delay = Number(arg('delay', '0')) + +/** id -> status, in the order given. */ +const sessions = new Map( + arg('sessions', '') + .split(',') + .filter(Boolean) + .map((pair) => { + const cut = pair.lastIndexOf('=') + return [pair.slice(0, cut), pair.slice(cut + 1)] + }) +) + +function answerTo(line) { + const [verb, n, rest] = splitThree(line) + if (verb === 'pause') { + const ids = [] + for (const [id, status] of sessions) { + if (status === 'Playing') { + sessions.set(id, 'Paused') + ids.push(id) + } + } + return `${n} ${ids.join(',')}` + } + if (verb === 'resume') { + for (const id of rest.split(',')) if (sessions.get(id) === 'Paused') sessions.set(id, 'Playing') + return `${n} ok` + } + if (verb === 'list') return `${n} ${[...sessions].map(([id, status]) => `${id}=${status}`).join(',')}` + return null +} + +/** "verb n rest of the line", where the rest may itself hold spaces. */ +function splitThree(line) { + const first = line.indexOf(' ') + if (first < 0) return [line, '', ''] + const second = line.indexOf(' ', first + 1) + if (second < 0) return [line.slice(0, first), line.slice(first + 1), ''] + return [line.slice(0, first), line.slice(first + 1, second), line.slice(second + 1)] +} + +function say(text) { + if (text === null) return + if (delay > 0) later(() => process.stdout.write(`${text}\n`), delay) + else process.stdout.write(`${text}\n`) +} + +/** reverse mode holds the first answer until the second exists, then swaps them. */ +let held = null + +const lines = createInterface({ input: process.stdin }) +lines.on('line', (line) => { + if (logFile) appendFileSync(logFile, `${line}\n`) + if (mode === 'crash-once' && marker && !existsSync(marker)) { + writeFileSync(marker, 'crashed once') + process.exit(1) + } + if (mode === 'silent') return + const answer = answerTo(line) + if (mode === 'garbage') { + // What a real PowerShell can put on stdout around an answer: a stray + // warning, a blank line, and an answer to a request nobody made. + say('WARNING: something nobody asked for') + say('') + say('999999 not-yours') + } + if (mode === 'reverse') { + if (held === null) { + held = answer + return + } + say(answer) + say(held) + held = null + return + } + say(answer) +}) +// Exactly what the real helper does when the app goes away: stdin closes, the +// read loop ends, the process exits. It is why no orphan can outlive its parent. +lines.on('close', () => process.exit(0)) diff --git a/core/main/__fixtures__/fakeWhisperServer.mjs b/core/main/__fixtures__/fakeWhisperServer.mjs new file mode 100644 index 0000000..9fee547 --- /dev/null +++ b/core/main/__fixtures__/fakeWhisperServer.mjs @@ -0,0 +1,123 @@ +// A stand-in for whisper-server.exe, for dictationEngine.test.ts. It speaks the +// two routes the engine uses (GET / and POST /inference, measured against the +// real b5130 server on 2026-09-19) and answers with a description of what it +// RECEIVED, so a test can read the argv, the multipart parts and the order of +// the passes back out of `text` instead of trusting the engine's word for them. +// +// Started the way the real one is: -m --host 127.0.0.1 --port -l . +// Switches, as argv or as the environment variable beside each: +// --fake-exit FAKE_WHISPER_EXIT=1 say why on stderr and exit 3 at once +// (a GPU engine that cannot start) +// --fake-never-ready FAKE_WHISPER_NEVER_READY=1 stay alive and never listen +// --fake-delay FAKE_WHISPER_DELAY= hold every /inference answer this long +// --fake-die-on-inference FAKE_WHISPER_DIE=1 read the upload, say why on stderr, exit 4 +// +// The globals are imported by name because the repo's eslint config gives Node +// globals to tools/ only, and this file may not change that config. +import { Buffer } from 'node:buffer' +import http from 'node:http' +import process from 'node:process' +import { setInterval, setTimeout } from 'node:timers' + +const argv = process.argv.slice(2) +const flag = (name) => argv.includes(name) +const value = (name) => { + const i = argv.indexOf(name) + return i >= 0 && i + 1 < argv.length ? argv[i + 1] : null +} + +const model = value('-m') +const language = value('-l') +const host = value('--host') ?? '127.0.0.1' +const port = Number(value('--port') ?? 0) +const delay = Number(value('--fake-delay') ?? process.env.FAKE_WHISPER_DELAY ?? 0) +const exitNow = flag('--fake-exit') || process.env.FAKE_WHISPER_EXIT === '1' +const neverReady = flag('--fake-never-ready') || process.env.FAKE_WHISPER_NEVER_READY === '1' +const dieOnInference = flag('--fake-die-on-inference') || process.env.FAKE_WHISPER_DIE === '1' + +/** The parts of a multipart/form-data body: name, filename, type and the bytes. */ +function parseMultipart(body, contentType) { + const m = /boundary=(?:"([^"]+)"|([^;]+))/i.exec(contentType ?? '') + if (!m) return [] + const mark = Buffer.from(`--${m[1] ?? m[2]}`) + const parts = [] + let at = body.indexOf(mark) + while (at >= 0) { + const next = body.indexOf(mark, at + mark.length) + if (next < 0) break + // Between two marks: CRLF, the headers, a blank line, the bytes, CRLF. + const chunk = body.subarray(at + mark.length + 2, next - 2) + const split = chunk.indexOf('\r\n\r\n') + if (split >= 0) { + const head = chunk.subarray(0, split).toString('utf8') + parts.push({ + name: /name="([^"]*)"/i.exec(head)?.[1] ?? null, + filename: /filename="([^"]*)"/i.exec(head)?.[1] ?? null, + type: /content-type:\s*(.+)/i.exec(head)?.[1]?.trim() ?? null, + data: chunk.subarray(split + 4) + }) + } + at = next + } + return parts +} + +if (exitNow) { + process.stderr.write('fake: CUDA error: no kernel image is available for execution on the device\n') + process.exit(3) +} else if (neverReady) { + // Alive, and deaf: what a server stuck loading its model looks like from outside. + setInterval(() => {}, 1000) +} else { + let count = 0 + let active = 0 + let maxActive = 0 + const server = http.createServer((req, res) => { + if (req.method === 'GET') { + res.writeHead(200, { 'content-type': 'text/html' }) + res.end('fake whisper server') + return + } + if (req.method !== 'POST' || req.url !== '/inference') { + res.writeHead(404) + res.end() + return + } + const chunks = [] + req.on('data', (c) => chunks.push(c)) + req.on('end', () => { + if (dieOnInference) { + // The whole message goes out before the exit, so the engine's stderr tail has it. + process.stderr.write('fake: ggml_cuda: out of memory\n', () => process.exit(4)) + return + } + active += 1 + maxActive = Math.max(maxActive, active) + count += 1 + const n = count + const parts = parseMultipart(Buffer.concat(chunks), req.headers['content-type']) + const file = parts.find((p) => p.name === 'file') + const format = parts.find((p) => p.name === 'response_format') + setTimeout(() => { + const seen = { + n, + maxActive, + bytes: file ? file.data.length : -1, + filename: file?.filename ?? null, + type: file?.type ?? null, + format: format ? format.data.toString('utf8') : null, + model, + language, + host, + port, + pid: process.pid, + cwd: process.cwd() + } + active -= 1 + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ text: JSON.stringify(seen) })) + }, delay) + }) + }) + server.listen(port, host) +} diff --git a/core/main/dictationEngine.test.ts b/core/main/dictationEngine.test.ts new file mode 100644 index 0000000..7c4d01d --- /dev/null +++ b/core/main/dictationEngine.test.ts @@ -0,0 +1,415 @@ +import { spawn, type ChildProcess } from 'child_process' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { dirname, join } from 'path' +import { fileURLToPath } from 'url' +import { afterEach, describe, expect, it } from 'vitest' +import type { DictationEngine, TranscribeRequest, TranscribeResult } from '../shared/dictationTypes' +import { createDictationEngine, type DictationEngineDeps } from './dictationEngine' + +/** + * The resident server's lifecycle, against a REAL child process and a REAL + * local HTTP server: the fixture is a Node script that speaks whisper-server's + * two routes and answers with what it received. Nothing is mocked, because + * every rule here is about processes and sockets (who is alive, who was + * killed, what went over the wire), and a mock of `spawn` would only prove + * the engine agrees with the mock. + */ + +const FIXTURE = join(dirname(fileURLToPath(import.meta.url)), '__fixtures__', 'fakeWhisperServer.mjs') +const MODELS: Record = { + base: 'C:\\Prism Dictation\\models\\ggml-base.bin', + small: 'C:\\Prism Dictation\\models\\ggml-small.bin' +} +const WAV = new Uint8Array(1234).fill(7) + +/** What the fixture says it saw, read back out of the `text` it answered with. */ +interface Seen { + n: number + maxActive: number + bytes: number + filename: string | null + type: string | null + format: string | null + model: string | null + language: string | null + host: string + port: number + pid: number + cwd: string +} + +interface World { + engine: DictationEngine + cpuDir: string + gpuDir: string + /** Everything the engine launched, in order: which engine folder, and its argv. */ + launched: Array<{ kind: 'cpu' | 'gpu'; exePath: string; args: string[] }> + children: ChildProcess[] + set: (next: Partial) => void +} + +interface Knobs { + cpu: boolean + gpu: boolean + nvidia: boolean | 'throws' + cpuFlags: string[] + gpuFlags: string[] +} + +const engines: DictationEngine[] = [] +const allChildren: ChildProcess[] = [] +const folders: string[] = [] + +function world(knobs: Partial = {}, deps: Partial = {}): World { + const k: Knobs = { cpu: true, gpu: false, nvidia: false, cpuFlags: [], gpuFlags: [], ...knobs } + // A space in the folder on purpose: the real one is under a user's profile. + const cpuDir = mkdtempSync(join(tmpdir(), 'prism dictation cpu ')) + const gpuDir = mkdtempSync(join(tmpdir(), 'prism dictation gpu ')) + folders.push(cpuDir, gpuDir) + const launched: World['launched'] = [] + const children: ChildProcess[] = [] + const engine = createDictationEngine({ + cpuDir: () => (k.cpu ? cpuDir : null), + store: { modelPath: (id) => MODELS[id] ?? null, gpuEngineDir: () => (k.gpu ? gpuDir : null) }, + hasNvidia: async () => { + if (k.nvidia === 'throws') throw new Error('no gpu info') + return k.nvidia + }, + spawnImpl: ((file: string, args: string[], options: object) => { + const child = spawn(file, args, options) + children.push(child) + allChildren.push(child) + return child + }) as typeof spawn, + // The "exe" is the fixture script, run by this very Node. The engine's own + // argv is kept in front, so the fixture is started the way the real one is. + command: (exePath, args) => { + const kind = exePath.startsWith(gpuDir) ? 'gpu' : 'cpu' + launched.push({ kind, exePath, args }) + return { file: process.execPath, args: [FIXTURE, ...args, ...(kind === 'gpu' ? k.gpuFlags : k.cpuFlags)] } + }, + ...deps + }) + engines.push(engine) + return { engine, cpuDir, gpuDir, launched, children, set: (next) => Object.assign(k, next) } +} + +const req = (over: Partial = {}): TranscribeRequest => ({ + wav: WAV, + modelId: 'base', + language: 'auto', + final: true, + ...over +}) + +function seen(r: TranscribeResult): Seen { + if (!r.ok) throw new Error(`expected an answer, got ${r.reason}: ${r.detail ?? ''}`) + return JSON.parse(r.text) as Seen +} + +const gone = (c: ChildProcess): boolean => c.exitCode !== null || c.signalCode !== null +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)) + +async function until(what: () => boolean, ms = 4000): Promise { + const end = Date.now() + ms + while (!what()) { + if (Date.now() > end) throw new Error('timed out waiting') + await sleep(10) + } +} + +afterEach(async () => { + for (const e of engines.splice(0)) e.stop() + // No test may leave a process behind: every child ever launched must be gone. + await until(() => allChildren.every(gone)) + allChildren.splice(0) + for (const f of folders.splice(0)) rmSync(f, { recursive: true, force: true }) +}) + +describe('what can answer', () => { + it('says no-model for a model that is not installed, and starts nothing', async () => { + const w = world() + expect(await w.engine.transcribe(req({ modelId: 'large' }))).toEqual({ ok: false, reason: 'no-model' }) + expect(w.launched).toHaveLength(0) + }) + + it('says no-engine when there is neither a bundled engine nor a usable GPU pack', async () => { + const none = world({ cpu: false }) + expect(await none.engine.transcribe(req())).toEqual({ ok: false, reason: 'no-engine' }) + // A GPU pack without an NVIDIA adapter is not an engine either. + const packOnly = world({ cpu: false, gpu: true, nvidia: false }) + expect(await packOnly.engine.transcribe(req())).toEqual({ ok: false, reason: 'no-engine' }) + expect(packOnly.launched).toHaveLength(0) + }) + + it('takes the GPU pack when it is installed AND an NVIDIA adapter is present', async () => { + const w = world({ gpu: true, nvidia: true }) + const r = await w.engine.transcribe(req()) + expect(r.ok && r.engine).toBe('gpu') + expect(seen(r).cwd).toBe(w.gpuDir) + expect(w.engine.info()).toEqual({ running: true, engine: 'gpu', gpuFellBack: false }) + }) + + it('stays on CPU with the pack installed and no NVIDIA adapter, or an adapter nobody could ask about', async () => { + const w = world({ gpu: true, nvidia: false }) + const r = await w.engine.transcribe(req()) + expect(r.ok && r.engine).toBe('cpu') + const t = world({ gpu: true, nvidia: 'throws' }) + const r2 = await t.engine.transcribe(req()) + expect(r2.ok && r2.engine).toBe('cpu') + expect(t.engine.info().gpuFellBack).toBe(false) + }) +}) + +describe('the server', () => { + it('is not started until the first transcribe, and is started the way the real one is', async () => { + const w = world() + expect(w.engine.info()).toEqual({ running: false, engine: null, gpuFellBack: false }) + expect(w.launched).toHaveLength(0) + + const s = seen(await w.engine.transcribe(req({ language: 'nb' }))) + expect(w.launched).toHaveLength(1) + const { exePath, args } = w.launched[0] + expect(exePath).toBe(join(w.cpuDir, 'whisper-server.exe')) + expect(args).toEqual(['-m', MODELS.base, '--host', '127.0.0.1', '--port', String(s.port), '-l', 'nb']) + expect(s.port).toBeGreaterThan(0) + // Its DLLs are beside it, so it runs IN its own folder. + expect(s.cwd).toBe(w.cpuDir) + expect(w.engine.info()).toEqual({ running: true, engine: 'cpu', gpuFellBack: false }) + }) + + it('is sent the wav as multipart form-data and asked for json', async () => { + const w = world() + const r = await w.engine.transcribe(req()) + const s = seen(r) + expect(s).toMatchObject({ + bytes: WAV.length, + filename: 'audio.wav', + type: 'audio/wav', + format: 'json', + model: MODELS.base, + language: 'auto' + }) + expect(r.ok && r.engine).toBe('cpu') + }) + + it('is reused from one pass to the next', async () => { + const w = world() + const a = seen(await w.engine.transcribe(req({ final: false }))) + const b = seen(await w.engine.transcribe(req())) + expect(b.pid).toBe(a.pid) + expect(b.n).toBe(2) + expect(w.launched).toHaveLength(1) + }) + + it('is replaced when the model, the language or the engine kind changes', async () => { + const w = world({ nvidia: true }) + const first = seen(await w.engine.transcribe(req())) + const model = seen(await w.engine.transcribe(req({ modelId: 'small' }))) + expect(model.pid).not.toBe(first.pid) + expect(model.model).toBe(MODELS.small) + const lang = seen(await w.engine.transcribe(req({ modelId: 'small', language: 'en' }))) + expect(lang.pid).not.toBe(model.pid) + expect(lang.language).toBe('en') + // The GPU pack finishes downloading mid-session: the next pass moves to it. + w.set({ gpu: true }) + const kind = await w.engine.transcribe(req({ modelId: 'small', language: 'en' })) + expect(kind.ok && kind.engine).toBe('gpu') + expect(seen(kind).pid).not.toBe(lang.pid) + expect(w.launched.map((l) => l.kind)).toEqual(['cpu', 'cpu', 'cpu', 'gpu']) + // One server at a time: the three it replaced are gone. + await until(() => w.children.slice(0, 3).every(gone)) + expect(gone(w.children[3])).toBe(false) + }) +}) + +describe('a GPU engine that will not start', () => { + it('falls back to CPU for the life of the engine when it exits at start, and answers from there', async () => { + const w = world({ gpu: true, nvidia: true, gpuFlags: ['--fake-exit'] }) + const r = await w.engine.transcribe(req()) + expect(r.ok && r.engine).toBe('cpu') + expect(w.engine.info()).toEqual({ running: true, engine: 'cpu', gpuFellBack: true }) + // Never again this session: the second pass does not even try it. + const again = await w.engine.transcribe(req()) + expect(again.ok && again.engine).toBe('cpu') + expect(w.launched.map((l) => l.kind)).toEqual(['gpu', 'cpu']) + }) + + it('falls back the same way when it never becomes ready, and the stuck one is killed', async () => { + const w = world({ gpu: true, nvidia: true, gpuFlags: ['--fake-never-ready'] }, { startTimeoutMs: 400 }) + const r = await w.engine.transcribe(req()) + expect(r.ok && r.engine).toBe('cpu') + expect(w.engine.info().gpuFellBack).toBe(true) + await until(() => gone(w.children[0])) + }) + + it('is a failure, not a loop, when there is no CPU engine to fall back to', async () => { + const w = world({ cpu: false, gpu: true, nvidia: true, gpuFlags: ['--fake-exit'] }) + const r = await w.engine.transcribe(req()) + expect(r).toMatchObject({ ok: false, reason: 'engine-failed' }) + expect(!r.ok && r.detail).toContain('CUDA error') + expect(w.launched).toHaveLength(1) + // The GPU engine is out for the session, so now nothing is left at all. + expect(await w.engine.transcribe(req())).toEqual({ ok: false, reason: 'no-engine' }) + expect(w.launched).toHaveLength(1) + }) +}) + +describe('a CPU engine that will not start', () => { + it('fails that request with what the server said, once, and tries afresh on the next', async () => { + const w = world({ cpuFlags: ['--fake-exit'] }) + const r = await w.engine.transcribe(req()) + expect(r).toMatchObject({ ok: false, reason: 'engine-failed' }) + expect(!r.ok && r.detail).toContain('fake: CUDA error') + expect(w.launched).toHaveLength(1) + expect(w.engine.info().running).toBe(false) + + w.set({ cpuFlags: [] }) + expect((await w.engine.transcribe(req())).ok).toBe(true) + expect(w.launched).toHaveLength(2) + }) + + it('fails when the exe cannot be launched at all', async () => { + const w = world({}, { command: () => ({ file: join(tmpdir(), 'no such whisper-server.exe'), args: [] }) }) + const r = await w.engine.transcribe(req()) + expect(r).toMatchObject({ ok: false, reason: 'engine-failed' }) + }) +}) + +describe('passes are serialised to one in flight', () => { + it('drops a waiting partial for a newer one, and never sends two at once', async () => { + const w = world({ cpuFlags: ['--fake-delay', '120'] }) + const a = w.engine.transcribe(req({ final: false })) + const b = w.engine.transcribe(req({ final: false })) + const c = w.engine.transcribe(req({ final: false })) + // b never reaches the server: it answers as soon as c takes its place. + expect(await b).toEqual({ ok: false, reason: 'superseded' }) + expect(seen(await a).n).toBe(1) + const last = seen(await c) + expect(last.n).toBe(2) + expect(last.maxActive).toBe(1) + }) + + it('never drops a final, and lets it jump ahead of a waiting partial', async () => { + const w = world({ cpuFlags: ['--fake-delay', '120'] }) + const inFlight = w.engine.transcribe(req({ final: false })) + const waiting = w.engine.transcribe(req({ final: false })) + const final1 = w.engine.transcribe(req({ final: true })) + // The next recording has begun before the last one's final was answered. + const nextPartial = w.engine.transcribe(req({ final: false })) + const final2 = w.engine.transcribe(req({ final: true })) + const newest = w.engine.transcribe(req({ final: false })) + + // A partial waiting when a final arrives is for a clip the final covers whole. + expect(await waiting).toEqual({ ok: false, reason: 'superseded' }) + expect(await nextPartial).toEqual({ ok: false, reason: 'superseded' }) + expect(seen(await inFlight).n).toBe(1) + expect(seen(await final1).n).toBe(2) + expect(seen(await final2).n).toBe(3) + const end = seen(await newest) + expect(end.n).toBe(4) + expect(end.maxActive).toBe(1) + }) +}) + +describe('standing down', () => { + it('kills the server after idleMs with no request, and starts a fresh one on the next', async () => { + const w = world({}, { idleMs: 150 }) + const first = seen(await w.engine.transcribe(req())) + expect(w.engine.info().running).toBe(true) + await until(() => gone(w.children[0])) + expect(w.engine.info()).toEqual({ running: false, engine: 'cpu', gpuFellBack: false }) + + const second = seen(await w.engine.transcribe(req())) + expect(second.pid).not.toBe(first.pid) + expect(w.launched).toHaveLength(2) + }) + + it('counts the idle time from the LAST request, not the first', async () => { + const w = world({}, { idleMs: 500 }) + await w.engine.transcribe(req()) + await sleep(300) + await w.engine.transcribe(req()) + await sleep(300) + // 600 ms since the first pass, 300 since the second: still up. + expect(w.engine.info().running).toBe(true) + expect(w.launched).toHaveLength(1) + await until(() => gone(w.children[0])) + }) + + it('does not stand down under a pass that takes longer than idleMs', async () => { + const w = world({ cpuFlags: ['--fake-delay', '250'] }, { idleMs: 60 }) + expect((await w.engine.transcribe(req())).ok).toBe(true) + }) +}) + +describe('stop()', () => { + it('kills the server at once and fails what was in flight and what was waiting', async () => { + const w = world({ cpuFlags: ['--fake-delay', '2000'] }) + const inFlight = w.engine.transcribe(req({ final: true })) + const waitingFinal = w.engine.transcribe(req({ final: true })) + const waitingPartial = w.engine.transcribe(req({ final: false })) + // Long enough for the server to be up and holding the first pass. + await until(() => w.children.length === 1) + await sleep(250) + w.engine.stop() + expect(w.engine.info().running).toBe(false) + for (const p of [inFlight, waitingFinal, waitingPartial]) { + expect(await p).toMatchObject({ ok: false, reason: 'engine-failed' }) + } + await until(() => gone(w.children[0])) + }) + + it('leaves the engine usable: switched back on, the next pass starts a server', async () => { + const w = world() + const first = seen(await w.engine.transcribe(req())) + w.engine.stop() + const second = seen(await w.engine.transcribe(req())) + expect(second.pid).not.toBe(first.pid) + }) + + it('during a GPU start is a stop, not a failed GPU: no fallback, no CPU server', async () => { + const w = world({ gpu: true, nvidia: true, gpuFlags: ['--fake-never-ready'] }, { startTimeoutMs: 5000 }) + const p = w.engine.transcribe(req()) + await until(() => w.children.length === 1) + w.engine.stop() + expect(await p).toMatchObject({ ok: false, reason: 'engine-failed' }) + expect(w.engine.info()).toEqual({ running: false, engine: null, gpuFellBack: false }) + expect(w.launched.map((l) => l.kind)).toEqual(['gpu']) + }) +}) + +describe('a server that dies mid-request', () => { + it('fails that request with the tail of its stderr, and the next one gets a fresh server', async () => { + const w = world({ cpuFlags: ['--fake-die-on-inference'] }) + const r = await w.engine.transcribe(req()) + expect(r).toMatchObject({ ok: false, reason: 'engine-failed' }) + expect(!r.ok && r.detail).toContain('out of memory') + expect(w.engine.info().running).toBe(false) + + w.set({ cpuFlags: [] }) + const next = await w.engine.transcribe(req()) + expect(next.ok).toBe(true) + expect(w.launched).toHaveLength(2) + }) + + it('counts against a GPU engine too: the next pass goes to the CPU one', async () => { + const w = world({ gpu: true, nvidia: true, gpuFlags: ['--fake-die-on-inference'] }) + expect(await w.engine.transcribe(req())).toMatchObject({ ok: false, reason: 'engine-failed' }) + expect(w.engine.info().gpuFellBack).toBe(true) + const next = await w.engine.transcribe(req()) + expect(next.ok && next.engine).toBe('cpu') + expect(w.launched.map((l) => l.kind)).toEqual(['gpu', 'cpu']) + }) + + it('does not take the requests waiting behind it down with it', async () => { + const w = world({ cpuFlags: ['--fake-die-on-inference'] }) + const dies = w.engine.transcribe(req()) + const waits = w.engine.transcribe(req()) + await until(() => w.children.length === 1) + w.set({ cpuFlags: [] }) + expect((await dies).ok).toBe(false) + expect((await waits).ok).toBe(true) + }) +}) diff --git a/core/main/dictationEngine.ts b/core/main/dictationEngine.ts new file mode 100644 index 0000000..0cb4c19 --- /dev/null +++ b/core/main/dictationEngine.ts @@ -0,0 +1,505 @@ +import { spawn, type ChildProcess } from 'child_process' +import { randomBytes } from 'crypto' +import { request } from 'http' +import { createServer } from 'net' +import { join } from 'path' +import type { + DictationEngine, + DictationStore, + EngineKind, + TranscribeRequest, + TranscribeResult +} from '../shared/dictationTypes' + +/* ------------------------------------------------------------------ * + * The resident whisper-server (#13), main side. + * + * WHY A SERVER AND NOT A RUN PER CLIP. Loading the model is the slow part + * and the pass is the fast one. MEASURED 2026-09-19 on the owner's PC: a + * resident server answers a Base pass on CPU in about 0.9 s whatever the + * clip's length (2 s to 11 s), and Large-v3 on the NVIDIA pack in 0.36 to + * 0.50 s, where a fresh process pays the model load every time and, on the + * GPU, a 9.1 s kernel compile on its first run. Live text once a second is + * only possible because the process stays up between passes. + * + * WHAT THE REAL SERVER SPEAKS (release b5130, measured the same day): it is + * started as `whisper-server.exe -m --host 127.0.0.1 --port + * -l `, it is ready when GET / answers, and a pass is a + * POST /inference of multipart form-data with the wav as `file` and + * `response_format=json`; the answer is `{ "text": "..." }`. + * + * The audio is never written to disk: it arrives as bytes over IPC and + * leaves as bytes over a loopback socket to a port nobody else was told. + * ------------------------------------------------------------------ */ + +/** The first CUDA run compiles its kernels for about 9 s (MEASURED), so a + * start is given a minute before it is called a failure. */ +export const DICTATION_START_TIMEOUT_MS = 60000 +/** Five minutes idle and the server stands down (owner decision); the model + * reloads in about a second on the next press. */ +export const DICTATION_IDLE_MS = 300000 +/** A pass that has not answered by now never will. Large on a CPU was + * measured at 20 s for an 11 s clip, and a recording is capped at five + * minutes, so two minutes is far past anything a live server takes. */ +export const DICTATION_REQUEST_TIMEOUT_MS = 120000 + +/** How often a starting server is asked whether it is up yet. */ +const READY_POLL_MS = 100 +/** How much of the server's stderr is kept: enough for its last few lines, + * which is where it says why it died. */ +const STDERR_KEEP = 2000 + +export interface DictationEngineDeps { + /** The folder of the bundled CPU engine, or null when the host has none. */ + cpuDir: () => string | null + store: Pick + /** An NVIDIA adapter is present. Asked until it answers, then remembered. */ + hasNvidia: () => Promise + idleMs?: number + startTimeoutMs?: number + requestTimeoutMs?: number + spawnImpl?: typeof spawn + exeName?: string + /** What to actually launch for an engine exe and its argv. The default is + * the exe itself; the tests run a Node script through process.execPath. */ + command?: (exePath: string, args: string[]) => { file: string; args: string[] } + now?: () => number +} + +/** One launched server, from spawn to the moment its process is gone. */ +interface Server { + child: ChildProcess + kind: EngineKind + modelPath: string + language: string + port: number + /** The tail of its stderr: what it said last is why it died. */ + stderr: string + exitCode: number | null + exited: boolean + /** Called once, when it has exited and its stderr has been read out. */ + onExit: Set<() => void> +} + +interface Job { + req: TranscribeRequest + resolve: (r: TranscribeResult) => void +} + +type Launch = { ok: true; server: Server } | { ok: false; detail: string } +type Answer = { ok: true; text: string } | { ok: false; detail: string; died: boolean } + +const STOPPED: TranscribeResult = { ok: false, reason: 'engine-failed', detail: 'dictation was stopped' } + +/** A port nobody is using: listen on 0, read what Windows handed out, let go. */ +function freePort(): Promise { + return new Promise((resolve, reject) => { + const probe = createServer() + probe.once('error', reject) + probe.listen(0, '127.0.0.1', () => { + const address = probe.address() + const port = typeof address === 'object' && address ? address.port : 0 + probe.close(() => (port ? resolve(port) : reject(new Error('no free port')))) + }) + }) +} + +/** Is anybody answering GET / on that port. Any answer short of a 5xx counts: + * the question is whether the server is up, not what its front page says. */ +function answers(port: number): Promise { + return new Promise((resolve) => { + // agent: false, here and for the pass: a kept-alive socket to a server + // that is about to be killed is a reset waiting for the next request. + const req = request({ host: '127.0.0.1', port, path: '/', method: 'GET', agent: false, timeout: 1000 }, (res) => { + res.resume() + resolve((res.statusCode ?? 500) < 500) + }) + req.once('timeout', () => req.destroy(new Error('timeout'))) + req.once('error', () => resolve(false)) + req.end() + }) +} + +/** The upload, built by hand: two parts and a boundary no wav will contain by + * accident (24 random hex digits), so nothing needs FormData or a Blob copy. */ +function multipart(wav: Uint8Array): { body: Buffer; contentType: string } { + const boundary = `----PrismDictation${randomBytes(12).toString('hex')}` + const body = Buffer.concat([ + Buffer.from( + `--${boundary}\r\n` + + 'Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n' + + 'Content-Type: audio/wav\r\n\r\n' + ), + Buffer.from(wav.buffer, wav.byteOffset, wav.byteLength), + Buffer.from( + `\r\n--${boundary}\r\n` + + 'Content-Disposition: form-data; name="response_format"\r\n\r\n' + + `json\r\n--${boundary}--\r\n` + ) + ]) + return { body, contentType: `multipart/form-data; boundary=${boundary}` } +} + +function whyItDied(s: Server): string { + const said = s.stderr.trim() + if (said) return said + return s.exitCode === null ? 'the server exited' : `the server exited with code ${s.exitCode}` +} + +/** + * The engine. Nothing runs until the first `transcribe`; after that ONE + * whisper-server is kept, replaced when the model, the language or the engine + * kind it was started for is no longer what is being asked for, and killed + * after `idleMs` with no request or by `stop()`. + * + * ENGINE CHOICE has no switch to get wrong (owner decision): the GPU pack when + * it is installed AND an NVIDIA adapter is present, else the bundled CPU + * engine. A GPU server that exits or never becomes ready at start is passed + * over FOR THE LIFE OF THIS OBJECT (`gpuFellBack`), the CPU one is started in + * its place and answers the same request: one attempt each, never a loop. + * + * PASSES ARE SERIALISED to one in flight, because the server is one model on + * one device and two passes at once each take twice as long. A FINAL is what + * gets pasted, so it is never dropped and goes ahead of any partial. A PARTIAL + * is only the pill's live text, so at most one waits: a newer one takes its + * place and the older answers 'superseded' without ever being sent. A partial + * waiting when a final ARRIVES is dropped the same way: it was asked before + * the final, so it is for a clip the final covers whole, and sending it would + * only hold up the next recording's first live text. + */ +export function createDictationEngine(deps: DictationEngineDeps): DictationEngine { + const idleMs = deps.idleMs ?? DICTATION_IDLE_MS + const startTimeoutMs = deps.startTimeoutMs ?? DICTATION_START_TIMEOUT_MS + const requestTimeoutMs = deps.requestTimeoutMs ?? DICTATION_REQUEST_TIMEOUT_MS + const spawnImpl = deps.spawnImpl ?? spawn + const exeName = deps.exeName ?? 'whisper-server.exe' + const command = deps.command ?? ((exePath: string, args: string[]) => ({ file: exePath, args })) + const now = deps.now ?? (() => Date.now()) + + let current: Server | null = null + let lastKind: EngineKind | null = null + let gpuFellBack = false + let nvidia: boolean | null = null + /** Bumped by stop(). Everything that awaits compares it before and after: + * a server that went away because it was TOLD to is not a failed engine, + * and must neither start a CPU fallback nor mark the GPU as broken. */ + let epoch = 0 + + let running: Job | null = null + const finals: Job[] = [] + let partial: Job | null = null + let idleTimer: ReturnType | null = null + let lastUsed = 0 + + function kill(s: Server): void { + if (current === s) current = null + // TerminateProcess on Windows: no signal to ignore and no children to + // orphan, since whisper-server spawns nothing of its own. + if (!s.exited) s.child.kill() + } + + function markExited(s: Server): void { + if (s.exited) return + s.exited = true + if (current === s) current = null + for (const f of [...s.onExit]) f() + s.onExit.clear() + } + + /** Wait `ms`, or less if the server goes first. */ + function pause(s: Server, ms: number): Promise { + return new Promise((resolve) => { + const done = (): void => { + clearTimeout(timer) + s.onExit.delete(done) + resolve() + } + const timer = setTimeout(done, ms) + s.onExit.add(done) + if (s.exited) done() + }) + } + + async function hasNvidia(): Promise { + if (nvidia !== null) return nvidia + try { + // Adapters do not come and go in a session, so one answer is kept. A + // question that THREW is not an answer: it reads as no, and is asked again. + nvidia = await deps.hasNvidia() + return nvidia + } catch { + return false + } + } + + async function pick(): Promise<{ kind: EngineKind; dir: string } | null> { + const gpuDir = gpuFellBack ? null : deps.store.gpuEngineDir() + if (gpuDir && (await hasNvidia())) return { kind: 'gpu', dir: gpuDir } + const cpuDir = deps.cpuDir() + return cpuDir ? { kind: 'cpu', dir: cpuDir } : null + } + + async function launch( + kind: EngineKind, + dir: string, + modelPath: string, + language: string, + stopped: () => boolean + ): Promise { + let port: number + try { + port = await freePort() + } catch (e) { + return { ok: false, detail: `no free port: ${(e as Error).message}` } + } + if (stopped()) return { ok: false, detail: 'stopped' } + + const cmd = command(join(dir, exeName), ['-m', modelPath, '--host', '127.0.0.1', '--port', String(port), '-l', language]) + let child: ChildProcess + try { + // An argv array, never a shell string: both folders sit under a user's + // profile and may hold spaces. cwd is the engine's own folder because its + // DLLs (ggml, and cuBLAS for the GPU pack) are beside the exe. stdout is + // dropped rather than piped: a pipe nobody drains fills, and the server + // then blocks on its own logging. + // THE C++ RUNTIME IS BESIDE THE CPU ENGINE (fetch-whisper.mjs copies it + // app-local, since a fresh Windows has no VCOMP140.dll). The GPU pack is + // the official zip and carries none, so the CPU engine's folder goes on + // the front of the child's PATH: Windows looks beside the exe first, then + // along PATH, and the GPU server finds the same four DLLs there. + const runtimeDir = deps.cpuDir() + const env = runtimeDir && runtimeDir !== dir ? { ...process.env, PATH: `${runtimeDir};${process.env.PATH ?? ''}` } : process.env + child = spawnImpl(cmd.file, cmd.args, { cwd: dir, env, windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'] }) + } catch (e) { + return { ok: false, detail: (e as Error).message } + } + const s: Server = { child, kind, modelPath, language, port, stderr: '', exitCode: null, exited: false, onExit: new Set() } + // Current from the moment it exists, so a stop() during the start kills it. + current = s + child.stderr?.setEncoding('utf8') + child.stderr?.on('data', (chunk: string) => { + s.stderr = (s.stderr + chunk).slice(-STDERR_KEEP) + }) + // An exe that could not be launched at all (missing, blocked) says so here + // and may never say anything else. + child.once('error', (e) => { + s.stderr = (s.stderr + e.message).slice(-STDERR_KEEP) + markExited(s) + }) + // Decided on `close`, not `exit`: the reason is read from stderr, and + // `exit` can arrive before stderr has drained (Prism's phone transcoder + // learned this: a fast refusal read as an empty reason). The timer is for + // a pipe something else still holds open, which would leave `close` unsent. + child.once('exit', (code) => { + s.exitCode = code + setTimeout(() => markExited(s), 250).unref() + }) + child.once('close', () => markExited(s)) + + const deadline = now() + startTimeoutMs + for (;;) { + if (s.exited) return { ok: false, detail: whyItDied(s) } + if (await answers(port)) { + // Something answered; make sure it was not the last thing it did. + if (s.exited) return { ok: false, detail: whyItDied(s) } + return { ok: true, server: s } + } + if (stopped()) return { ok: false, detail: 'stopped' } + if (now() >= deadline) { + kill(s) + const said = s.stderr.trim() + return { ok: false, detail: `the server was not ready after ${startTimeoutMs} ms${said ? `: ${said}` : ''}` } + } + await pause(s, READY_POLL_MS) + } + } + + function post(s: Server, wav: Uint8Array): Promise { + return new Promise((resolve) => { + const { body, contentType } = multipart(wav) + let settled = false + let grace: ReturnType | null = null + const settle = (a: Answer): void => { + if (settled) return + settled = true + clearTimeout(timeout) + if (grace) clearTimeout(grace) + s.onExit.delete(died) + req.destroy() + resolve(a) + } + const died = (): void => settle({ ok: false, detail: whyItDied(s), died: true }) + + const req = request( + { + host: '127.0.0.1', + port: s.port, + path: '/inference', + method: 'POST', + agent: false, + headers: { 'Content-Type': contentType, 'Content-Length': body.length } + }, + (res) => { + const chunks: Buffer[] = [] + res.on('data', (c: Buffer) => chunks.push(c)) + res.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf8') + if (res.statusCode !== 200) { + settle({ ok: false, detail: `HTTP ${res.statusCode}: ${raw.slice(0, 300)}`, died: false }) + return + } + try { + const text = (JSON.parse(raw) as { text?: unknown }).text + if (typeof text === 'string') settle({ ok: true, text }) + else settle({ ok: false, detail: `no text in the answer: ${raw.slice(0, 300)}`, died: false }) + } catch { + settle({ ok: false, detail: `not JSON: ${raw.slice(0, 300)}`, died: false }) + } + }) + } + ) + // A dying server is seen TWICE: the socket resets at once, and the + // process is reported gone a moment later with its stderr read out. The + // second is the one worth telling the user, so a socket error waits a + // second for it. A socket error from a server that stays up means it + // cannot be trusted with the next pass either, so it is replaced. + req.once('error', (e) => { + if (settled || grace) return + grace = setTimeout(() => { + settle({ ok: false, detail: e.message, died: false }) + kill(s) + }, 1000) + }) + const timeout = setTimeout(() => { + settle({ ok: false, detail: `no answer after ${requestTimeoutMs} ms`, died: false }) + kill(s) + }, requestTimeoutMs) + s.onExit.add(died) + if (s.exited) died() + else req.end(body) + }) + } + + async function run(req: TranscribeRequest): Promise { + const mine = epoch + const stopped = (): boolean => epoch !== mine + + const modelPath = deps.store.modelPath(req.modelId) + if (!modelPath) return { ok: false, reason: 'no-model' } + const language = req.language || 'auto' + + const want = await pick() + if (stopped()) return STOPPED + if (!want) return { ok: false, reason: 'no-engine' } + + let s = current + if (s && (s.exited || s.kind !== want.kind || s.modelPath !== modelPath || s.language !== language)) { + // The server holds ONE model and ONE language for its whole life: both + // are argv. The port is new each time, so the old one need not be waited for. + kill(s) + s = null + } + if (!s) { + let started = await launch(want.kind, want.dir, modelPath, language, stopped) + if (stopped()) return STOPPED + if (!started.ok && want.kind === 'gpu') { + // Once, and for good: a pack that will not start on this machine now + // will not start on the next press either, and each try can cost the + // whole start timeout. + gpuFellBack = true + const cpuDir = deps.cpuDir() + if (!cpuDir) return { ok: false, reason: 'engine-failed', detail: started.detail } + started = await launch('cpu', cpuDir, modelPath, language, stopped) + if (stopped()) return STOPPED + } + if (!started.ok) return { ok: false, reason: 'engine-failed', detail: started.detail } + s = started.server + } + lastKind = s.kind + + const answer = await post(s, req.wav) + if (stopped()) return STOPPED + if (answer.ok) return { ok: true, text: answer.text, engine: s.kind } + // A GPU server that started and then died UNDER A PASS (a model too big for + // the card's memory does exactly this) would die under the next one too. + // This request is lost either way; the next goes to the CPU engine. + if (answer.died && s.kind === 'gpu') gpuFellBack = true + return { ok: false, reason: 'engine-failed', detail: answer.detail } + } + + function armIdle(): void { + if (idleTimer) clearTimeout(idleTimer) + // unref: a timer five minutes out must not be what keeps a quitting app alive. + idleTimer = setTimeout(() => { + idleTimer = null + if (running || !current) return + const left = lastUsed + idleMs - now() + if (left > 0) armIdle() + else kill(current) + }, Math.max(1, lastUsed + idleMs - now())) + idleTimer.unref() + } + + function pump(): void { + if (running) return + let job = finals.shift() ?? null + if (!job) { + job = partial + partial = null + } + if (!job) { + lastUsed = now() + if (current) armIdle() + return + } + if (idleTimer) { + clearTimeout(idleTimer) + idleTimer = null + } + running = job + const mine = job + run(mine.req) + .catch((e: unknown): TranscribeResult => ({ ok: false, reason: 'engine-failed', detail: String(e) })) + .then((result) => { + if (running === mine) running = null + mine.resolve(result) + pump() + }) + } + + return { + transcribe(req) { + return new Promise((resolve) => { + const job: Job = { req, resolve } + if (partial) { + // Whatever arrives, the partial already waiting is now out of date. + partial.resolve({ ok: false, reason: 'superseded' }) + partial = null + } + if (req.final) finals.push(job) + else partial = job + pump() + }) + }, + + stop() { + epoch += 1 + if (idleTimer) { + clearTimeout(idleTimer) + idleTimer = null + } + for (const job of finals.splice(0)) job.resolve(STOPPED) + if (partial) partial.resolve(STOPPED) + partial = null + // The pass in flight answers for itself: its server going away wakes it, + // and it sees the epoch has moved. + if (current) kill(current) + }, + + info() { + // `engine` is the kind of the server that is up, or of the last one that + // was, so Settings can still say which engine answered after a stand-down. + return { running: current !== null, engine: current ? current.kind : lastKind, gpuFellBack } + } + } +} diff --git a/core/main/dictationIpc.ts b/core/main/dictationIpc.ts new file mode 100644 index 0000000..3681cf2 --- /dev/null +++ b/core/main/dictationIpc.ts @@ -0,0 +1,113 @@ +import { DCH } from '../shared/channels' +import { CATALOG, catalogEntry } from '../shared/dictationCatalog' +import type { + DictationEngine, + DictationStore, + EngineInfo, + MediaPause, + TranscribeRequest, + TranscribeResult +} from '../shared/dictationTypes' +import { createDictationEngine } from './dictationEngine' +import { createDictationStore } from './dictationStore' +import { createMediaPause } from './mediaPause' +import type { IpcMainLike } from './ipc' + +/** + * The main half of DICTATION's bridge (#13), for both hosts. + * + * Nothing here starts until it is asked to: registering costs a few closures. + * The server process exists only between the first transcribe and the idle + * stand-down (or `stop`), the media helper only after the first pause, and a + * download only after a click. "Off means off" is kept by the renderer never + * calling, and by `stop` when the switch goes off. + * + * The host hands in what only it knows: where ITS installer put the CPU + * engine, the folder both apps share, and whether there is an NVIDIA adapter + * (Electron's own GPU info: no process spawned to find out). + */ +export interface DictationIpcDeps { + ipcMain: IpcMainLike + /** Send to the window, if there is one. */ + send(channel: string, ...args: unknown[]): void + /** The folder holding the bundled whisper-server.exe, or null if it is missing. */ + cpuEngineDir(): string | null + /** `%LOCALAPPDATA%\PrismDictation`: models and the GPU pack, shared by both apps. */ + sharedRoot: string + hasNvidia(): Promise + /** Test seams. */ + store?: DictationStore + engine?: DictationEngine + media?: MediaPause +} + +/** The largest WAV worth accepting: 5 minutes of 16 kHz PCM16 mono, and a margin. */ +const MAX_WAV_BYTES = 5 * 60 * 16000 * 2 + 1024 * 1024 + +export function registerDictationIpc(deps: DictationIpcDeps): () => void { + const { ipcMain } = deps + const store = deps.store ?? createDictationStore({ root: deps.sharedRoot, catalog: CATALOG }) + const engine = + deps.engine ?? + createDictationEngine({ cpuDir: deps.cpuEngineDir, store, hasNvidia: deps.hasNvidia }) + const media = deps.media ?? createMediaPause() + + ipcMain.handle(DCH.info, async (): Promise => { + const [nvidia, status] = await Promise.all([deps.hasNvidia().catch(() => false), store.status()]) + return { + nvidia, + gpuPack: status.find((s) => catalogEntry(s.id)?.kind === 'gpu-pack')?.state ?? 'absent', + gpuFellBack: engine.info().gpuFellBack + } + }) + + ipcMain.handle(DCH.status, () => store.status()) + + ipcMain.handle(DCH.download, async (_e: unknown, id: unknown) => { + // Only what the catalog names, by id: the renderer never supplies a URL. + if (typeof id !== 'string' || !catalogEntry(id)) return { ok: false, reason: 'network', detail: 'unknown item' } + const result = await store.download(id, (p) => deps.send(DCH.progress, p)) + // A new GPU pack or model is a different server: the next pass starts it. + if (result.ok) engine.stop() + return result + }) + + ipcMain.on(DCH.cancel, (_e: unknown, id: unknown) => { + if (typeof id === 'string') store.cancel(id) + }) + + ipcMain.handle(DCH.remove, async (_e: unknown, id: unknown) => { + if (typeof id !== 'string' || !catalogEntry(id)) return + // The running server may hold the file open (and IS the pack being removed). + engine.stop() + await store.remove(id) + }) + + ipcMain.handle(DCH.transcribe, async (_e: unknown, req: unknown): Promise => { + const r = req as Partial | null + if ( + !r || + !(r.wav instanceof Uint8Array) || + r.wav.byteLength < 44 || + r.wav.byteLength > MAX_WAV_BYTES || + typeof r.modelId !== 'string' || + typeof r.language !== 'string' || + !/^[a-z]{2,4}$/.test(r.language) + ) + return { ok: false, reason: 'engine-failed', detail: 'bad request' } + return engine.transcribe({ wav: r.wav, modelId: r.modelId, language: r.language, final: r.final === true }) + }) + + ipcMain.on(DCH.stop, () => engine.stop()) + + ipcMain.handle(DCH.mediaPause, () => media.pausePlaying()) + ipcMain.on(DCH.mediaResume, (_e: unknown, token: unknown) => { + if (typeof token === 'string') void media.resume(token) + }) + + /** For the host's quit path: no child may outlive the app. */ + return () => { + engine.stop() + media.dispose() + } +} diff --git a/core/main/dictationStore.test.ts b/core/main/dictationStore.test.ts new file mode 100644 index 0000000..93c2b7d --- /dev/null +++ b/core/main/dictationStore.test.ts @@ -0,0 +1,533 @@ +import { createHash } from 'crypto' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'fs' +import { createServer, type Server } from 'http' +import type { AddressInfo } from 'net' +import { tmpdir } from 'os' +import { dirname, join } from 'path' +import { crc32 } from 'zlib' +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import type { CatalogEntry, DownloadProgress } from '../shared/dictationTypes' +import { createDictationStore, PACK_MARKER, STALE_PART_MS } from './dictationStore' + +/** + * The store's rules, checked against a local http server and a temp folder: + * nothing here reaches the network or the real %LOCALAPPDATA%. The server can + * serve a good file, a corrupt one, a short one, a 404, one that drops the + * line half way and one that stalls for ever (for cancel and the shared + * promise). Unzipping is a fake everywhere but the last test, which runs the + * real Expand-Archive once, on a zip built by hand, in a folder with a space. + */ + +const GOOD = Buffer.alloc(512 * 1024) +for (let i = 0; i < GOOD.length; i++) GOOD[i] = (i * 31 + (i >> 8)) & 0xff +const CORRUPT = Buffer.from(GOOD) +CORRUPT[1000] ^= 0xff +const SHORT = GOOD.subarray(0, GOOD.length / 2) +const PACK = Buffer.from('not really a zip: the fake unzip never opens it') + +const sha = (b: Uint8Array): string => createHash('sha256').update(b).digest('hex') + +/** One stored (uncompressed) member is all a zip needs to be real. */ +function storedZip(name: string, data: Buffer): Buffer { + const n = Buffer.from(name) + const crc = crc32(data) + const local = Buffer.alloc(30) + local.writeUInt32LE(0x04034b50, 0) + local.writeUInt16LE(20, 4) + local.writeUInt32LE(crc, 14) + local.writeUInt32LE(data.length, 18) + local.writeUInt32LE(data.length, 22) + local.writeUInt16LE(n.length, 26) + const central = Buffer.alloc(46) + central.writeUInt32LE(0x02014b50, 0) + central.writeUInt16LE(20, 4) + central.writeUInt16LE(20, 6) + central.writeUInt32LE(crc, 16) + central.writeUInt32LE(data.length, 20) + central.writeUInt32LE(data.length, 24) + central.writeUInt16LE(n.length, 28) + const end = Buffer.alloc(22) + end.writeUInt32LE(0x06054b50, 0) + end.writeUInt16LE(1, 8) + end.writeUInt16LE(1, 10) + end.writeUInt32LE(central.length + n.length, 12) + end.writeUInt32LE(local.length + n.length + data.length, 16) + return Buffer.concat([local, n, data, central, n, end]) +} +const REAL_ZIP = storedZip('Release/whisper-server.exe', Buffer.from('MZ pretend')) + +let server: Server +let base = '' +let hits: Record = {} + +beforeAll(async () => { + server = createServer((req, res) => { + const url = req.url ?? '' + hits[url] = (hits[url] ?? 0) + 1 + const whole = (b: Buffer): void => { + res.writeHead(200, { 'content-length': b.length }) + // In pieces, so progress has more than one chunk to count. + for (let at = 0; at < b.length; at += 64 * 1024) res.write(b.subarray(at, at + 64 * 1024)) + res.end() + } + if (url === '/good') return whole(GOOD) + if (url === '/corrupt') return whole(CORRUPT) + if (url === '/short') return whole(SHORT) + if (url === '/pack') return whole(PACK) + if (url === '/realzip') return whole(REAL_ZIP) + if (url === '/slow') { + // The head and a first mouthful, then nothing, for as long as it is asked. + res.writeHead(200, { 'content-length': GOOD.length }) + res.write(GOOD.subarray(0, 4096)) + return + } + if (url === '/drop') { + res.writeHead(200, { 'content-length': GOOD.length }) + res.write(GOOD.subarray(0, 4096), () => res.destroy()) + return + } + res.writeHead(404).end('no') + }) + await new Promise((done) => server.listen(0, '127.0.0.1', done)) + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` +}) + +afterAll(async () => { + server.closeAllConnections() + await new Promise((done) => server.close(done)) +}) + +const roots: string[] = [] +afterEach(() => { + hits = {} + for (const r of roots.splice(0)) rmSync(r, { recursive: true, force: true, maxRetries: 5 }) +}) + +/** A root that does NOT exist yet, inside a temp folder that does. */ +function freshRoot(): string { + const holder = mkdtempSync(join(tmpdir(), 'prism dictation ')) + roots.push(holder) + return join(holder, 'PrismDictation') +} + +function entry(id: string, path: string, body: Uint8Array, over: Partial = {}): CatalogEntry { + return { id, label: id, note: '', url: base + path, bytes: body.length, sha256: sha(body), kind: 'model', ...over } +} + +const PACK_ENTRY = (path = '/pack', body: Uint8Array = PACK): CatalogEntry => entry('gpu', path, body, { kind: 'gpu-pack' }) +const packDir = (root: string, e: CatalogEntry): string => join(root, 'engines', `${e.id}-${e.sha256.slice(0, 12)}`) + +/** What the official zip looks like once open: the server is under Release/. */ +const fakeUnzip = async (_zip: string, outDir: string): Promise => { + mkdirSync(join(outDir, 'Release'), { recursive: true }) + writeFileSync(join(outDir, 'Release', 'whisper-server.exe'), 'MZ') + writeFileSync(join(outDir, 'Release', 'ggml-cuda.dll'), 'MZ') +} + +async function until(cond: () => boolean): Promise { + const t0 = Date.now() + while (!cond()) { + if (Date.now() - t0 > 4000) throw new Error('waited too long') + await new Promise((r) => setTimeout(r, 5)) + } +} + +const quiet = (): void => {} + +describe('the dictation store: a model', () => { + it('reports every catalog entry as absent in a root that does not exist yet', async () => { + const root = freshRoot() + const store = createDictationStore({ root, catalog: [entry('base', '/good', GOOD), PACK_ENTRY()] }) + expect(await store.status()).toEqual([ + { id: 'base', state: 'absent', received: 0 }, + { id: 'gpu', state: 'absent', received: 0 } + ]) + expect(store.modelPath('base')).toBeNull() + expect(store.gpuEngineDir()).toBeNull() + }) + + it('downloads, verifies and renames into models/.bin, creating the folders', async () => { + const root = freshRoot() + const store = createDictationStore({ root, catalog: [entry('base', '/good', GOOD)] }) + expect(await store.download('base', quiet)).toEqual({ ok: true }) + const file = join(root, 'models', 'base.bin') + expect(store.modelPath('base')).toBe(file) + expect(sha(readFileSync(file))).toBe(sha(GOOD)) + expect(existsSync(file + '.part')).toBe(false) + expect(await store.status()).toEqual([{ id: 'base', state: 'installed', received: GOOD.length }]) + }) + + it('does not fetch again what is already installed', async () => { + const store = createDictationStore({ root: freshRoot(), catalog: [entry('base', '/good', GOOD)] }) + await store.download('base', quiet) + expect(await store.download('base', quiet)).toEqual({ ok: true }) + expect(hits['/good']).toBe(1) + }) + + it('reports progress at most every 100 ms, plus a final one that says it all arrived', async () => { + const seen: DownloadProgress[] = [] + // A clock that never moves: only the first chunk and the final report get out. + const frozen = createDictationStore({ root: freshRoot(), catalog: [entry('base', '/good', GOOD)], now: () => 5000 }) + await frozen.download('base', (p) => seen.push(p)) + expect(seen.length).toBeGreaterThanOrEqual(1) + expect(seen.length).toBeLessThanOrEqual(2) + expect(seen.at(-1)).toEqual({ id: 'base', received: GOOD.length, total: GOOD.length }) + + // A clock that jumps a second per look: every chunk is reported, in order. + const many: DownloadProgress[] = [] + let t = 0 + const moving = createDictationStore({ root: freshRoot(), catalog: [entry('base', '/good', GOOD)], now: () => (t += 1000) }) + await moving.download('base', (p) => many.push(p)) + expect(many.length).toBeGreaterThan(2) + expect(many.map((p) => p.received)).toEqual([...many.map((p) => p.received)].sort((a, b) => a - b)) + expect(many.at(-1)?.received).toBe(GOOD.length) + expect(new Set(many.map((p) => p.received)).size).toBe(many.length) + }) + + it('survives a progress listener that throws', async () => { + const store = createDictationStore({ root: freshRoot(), catalog: [entry('base', '/good', GOOD)] }) + const r = await store.download('base', () => { + throw new Error('a renderer went away') + }) + expect(r).toEqual({ ok: true }) + }) + + it('deletes a file whose sha256 does not match, and says checksum', async () => { + const root = freshRoot() + // The right length, the wrong bytes. + const store = createDictationStore({ root, catalog: [{ ...entry('base', '/corrupt', GOOD) }] }) + const r = await store.download('base', quiet) + expect(r).toMatchObject({ ok: false, reason: 'checksum' }) + expect(readdirSync(join(root, 'models'))).toEqual([]) + expect(store.modelPath('base')).toBeNull() + expect((await store.status())[0].state).toBe('absent') + }) + + it('compares the byte count as well as the hash', async () => { + const root = freshRoot() + // The hash of what is served is right; the size the catalog promises is not. + const tooFew = entry('base', '/short', SHORT, { bytes: GOOD.length }) + const a = createDictationStore({ root, catalog: [tooFew] }) + expect(await a.download('base', quiet)).toMatchObject({ ok: false, reason: 'checksum' }) + expect(readdirSync(join(root, 'models'))).toEqual([]) + + // And a body LONGER than promised is cut off rather than written to the end. + const tooMany = entry('base', '/good', GOOD, { bytes: 100 }) + const b = createDictationStore({ root, catalog: [tooMany] }) + expect(await b.download('base', quiet)).toMatchObject({ ok: false, reason: 'checksum' }) + expect(readdirSync(join(root, 'models'))).toEqual([]) + }) + + it('answers network for a 404, a refused connection and a line that drops', async () => { + const root = freshRoot() + const dead = createServer() + await new Promise((done) => dead.listen(0, '127.0.0.1', done)) + const port = (dead.address() as AddressInfo).port + await new Promise((done) => dead.close(done)) + const store = createDictationStore({ + root, + catalog: [ + entry('gone', '/nothing-here', GOOD), + { ...entry('refused', '/good', GOOD), url: `http://127.0.0.1:${port}/good` }, + entry('dropped', '/drop', GOOD) + ] + }) + expect(await store.download('gone', quiet)).toMatchObject({ ok: false, reason: 'network', detail: 'HTTP 404' }) + expect(await store.download('refused', quiet)).toMatchObject({ ok: false, reason: 'network' }) + expect(await store.download('dropped', quiet)).toMatchObject({ ok: false, reason: 'network' }) + // One ask each: a failure is reported, never retried behind the user's back. + expect(hits['/nothing-here']).toBe(1) + expect(hits['/drop']).toBe(1) + expect(readdirSync(join(root, 'models'))).toEqual([]) + }) + + it('fetches through the fetch it was given, with a signal and nothing else', async () => { + const asked: Array<{ url: string; init: RequestInit | undefined }> = [] + const store = createDictationStore({ + root: freshRoot(), + catalog: [{ ...entry('base', '/good', GOOD), url: 'https://models.invalid/base.bin' }], + fetchImpl: async (url, init) => { + asked.push({ url: String(url), init }) + return new Response(GOOD) + } + }) + expect(await store.download('base', quiet)).toEqual({ ok: true }) + expect(asked.map((a) => a.url)).toEqual(['https://models.invalid/base.bin']) + expect(asked[0].init?.signal).toBeInstanceOf(AbortSignal) + // Redirects are left to fetch: no 'manual', no second ask. + expect(asked[0].init?.redirect).toBeUndefined() + }) + + it('answers network for an id the catalog does not hold', async () => { + const store = createDictationStore({ root: freshRoot(), catalog: [] }) + expect(await store.download('nope', quiet)).toMatchObject({ ok: false, reason: 'network' }) + expect(store.modelPath('nope')).toBeNull() + await store.remove('nope') + store.cancel('nope') + }) + + it('answers disk, with the message, when it cannot write', async () => { + const root = freshRoot() + // A FILE where the models folder has to go. + mkdirSync(root, { recursive: true }) + writeFileSync(join(root, 'models'), 'in the way') + const store = createDictationStore({ root, catalog: [entry('base', '/good', GOOD)] }) + const r = await store.download('base', quiet) + expect(r).toMatchObject({ ok: false, reason: 'disk' }) + expect(r.ok === false && r.detail).toBeTruthy() + }) + + it('shares one promise between two asks, shows the bytes in status, and cancels cleanly', async () => { + const root = freshRoot() + const store = createDictationStore({ root, catalog: [entry('base', '/slow', GOOD)] }) + const first: DownloadProgress[] = [] + const second: DownloadProgress[] = [] + const p1 = store.download('base', (p) => first.push(p)) + const p2 = store.download('base', (p) => second.push(p)) + expect(p2).toBe(p1) + await until(() => first.length > 0) + expect(hits['/slow']).toBe(1) + expect(existsSync(join(root, 'models', 'base.bin.part'))).toBe(true) + const [s] = await store.status() + expect(s.state).toBe('downloading') + expect(s.received).toBeGreaterThan(0) + expect(s.received).toBeLessThan(GOOD.length) + // The part file of a download in flight is not a leftover. + expect(existsSync(join(root, 'models', 'base.bin.part'))).toBe(true) + + store.cancel('base') + expect(await p1).toEqual({ ok: false, reason: 'cancelled' }) + expect(readdirSync(join(root, 'models'))).toEqual([]) + expect(await store.status()).toEqual([{ id: 'base', state: 'absent', received: 0 }]) + }) + + it('starts afresh when asked again straight after a cancel', async () => { + const store = createDictationStore({ root: freshRoot(), catalog: [entry('base', '/slow', GOOD)] }) + const seen: DownloadProgress[] = [] + const p1 = store.download('base', (p) => seen.push(p)) + await until(() => seen.length > 0) + store.cancel('base') + // The cancelled flight is still tidying up: it is not the one handed back. + const p2 = store.download('base', quiet) + expect(p2).not.toBe(p1) + expect(await p1).toEqual({ ok: false, reason: 'cancelled' }) + await until(() => hits['/slow'] === 2) + store.cancel('base') + expect(await p2).toEqual({ ok: false, reason: 'cancelled' }) + }) + + it('calls a wrong-size file absent, and will not hand out its path', async () => { + const root = freshRoot() + mkdirSync(join(root, 'models'), { recursive: true }) + writeFileSync(join(root, 'models', 'base.bin'), 'far too small') + const store = createDictationStore({ root, catalog: [entry('base', '/good', GOOD)] }) + expect(await store.status()).toEqual([{ id: 'base', state: 'absent', received: 0 }]) + expect(store.modelPath('base')).toBeNull() + // And a download puts the real one over it. + expect(await store.download('base', quiet)).toEqual({ ok: true }) + expect(store.modelPath('base')).toBe(join(root, 'models', 'base.bin')) + }) + + it('removes a leftover part file nobody is writing, but not one still warm', async () => { + const root = freshRoot() + const part = join(root, 'models', 'base.bin.part') + mkdirSync(dirname(part), { recursive: true }) + writeFileSync(part, 'half a model') + const now = Date.now() + const store = createDictationStore({ root, catalog: [entry('base', '/good', GOOD)], now: () => now }) + // Written this second: the OTHER app may be the one downloading it. + expect(await store.status()).toEqual([{ id: 'base', state: 'absent', received: 0 }]) + expect(existsSync(part)).toBe(true) + const old = new Date(now - STALE_PART_MS - 1000) + utimesSync(part, old, old) + expect(await store.status()).toEqual([{ id: 'base', state: 'absent', received: 0 }]) + expect(existsSync(part)).toBe(false) + }) + + it('removes an installed model, and a download in flight', async () => { + const root = freshRoot() + const store = createDictationStore({ root, catalog: [entry('base', '/good', GOOD), entry('slow', '/slow', GOOD)] }) + await store.download('base', quiet) + await store.remove('base') + expect(store.modelPath('base')).toBeNull() + expect(existsSync(join(root, 'models', 'base.bin'))).toBe(false) + + const seen: DownloadProgress[] = [] + const p = store.download('slow', (x) => seen.push(x)) + await until(() => seen.length > 0) + await store.remove('slow') + expect(await p).toEqual({ ok: false, reason: 'cancelled' }) + expect(readdirSync(join(root, 'models'))).toEqual([]) + }) + + it('refuses a catalog id that could walk out of the folder', () => { + expect(() => createDictationStore({ root: freshRoot(), catalog: [entry('..\\evil', '/good', GOOD)] })).toThrow() + }) +}) + +describe('the dictation store: the GPU pack', () => { + it('downloads the zip, verifies it, THEN unzips, deletes the zip and writes the marker last', async () => { + const root = freshRoot() + const e = PACK_ENTRY() + const calls: Array<{ zip: string; outDir: string; zipSha: string; marker: boolean }> = [] + const store = createDictationStore({ + root, + catalog: [e], + unzip: async (zip, outDir) => { + calls.push({ zip, outDir, zipSha: sha(readFileSync(zip)), marker: existsSync(join(outDir, PACK_MARKER)) }) + await fakeUnzip(zip, outDir) + } + }) + expect(await store.download('gpu', quiet)).toEqual({ ok: true }) + expect(calls).toEqual([{ zip: join(root, 'engines', 'gpu.zip'), outDir: packDir(root, e), zipSha: e.sha256, marker: false }]) + expect(readdirSync(join(root, 'engines'))).toEqual([`gpu-${e.sha256.slice(0, 12)}`]) + expect(existsSync(join(packDir(root, e), PACK_MARKER))).toBe(true) + expect(store.gpuEngineDir()).toBe(join(packDir(root, e), 'Release')) + expect(await store.status()).toEqual([{ id: 'gpu', state: 'installed', received: PACK.length }]) + // A pack is not a model. + expect(store.modelPath('gpu')).toBeNull() + }) + + it('never unzips a zip that failed its checksum', async () => { + const root = freshRoot() + let unzipped = 0 + const store = createDictationStore({ + root, + catalog: [entry('gpu', '/corrupt', GOOD, { kind: 'gpu-pack' })], + unzip: async () => void (unzipped += 1) + }) + expect(await store.download('gpu', quiet)).toMatchObject({ ok: false, reason: 'checksum' }) + expect(unzipped).toBe(0) + expect(readdirSync(join(root, 'engines'))).toEqual([]) + }) + + it('answers unpack and cleans up when the unzip fails, or holds no server', async () => { + const root = freshRoot() + const e = PACK_ENTRY() + const failing = createDictationStore({ + root, + catalog: [e], + unzip: async (_zip, outDir) => { + mkdirSync(join(outDir, 'Release'), { recursive: true }) + writeFileSync(join(outDir, 'Release', 'half.dll'), 'MZ') + throw new Error('the archive is damaged') + } + }) + expect(await failing.download('gpu', quiet)).toEqual({ ok: false, reason: 'unpack', detail: 'the archive is damaged' }) + expect(readdirSync(join(root, 'engines'))).toEqual([]) + expect(failing.gpuEngineDir()).toBeNull() + + const hollow = createDictationStore({ + root, + catalog: [e], + unzip: async (_zip, outDir) => { + mkdirSync(outDir, { recursive: true }) + } + }) + expect(await hollow.download('gpu', quiet)).toMatchObject({ ok: false, reason: 'unpack' }) + expect(readdirSync(join(root, 'engines'))).toEqual([]) + }) + + it('does not call a half-unzipped folder installed', async () => { + const root = freshRoot() + const e = PACK_ENTRY() + // What a crash in the middle of the unzip leaves: the server is there, the marker is not. + await fakeUnzip('', packDir(root, e)) + const store = createDictationStore({ root, catalog: [e], unzip: fakeUnzip }) + expect(await store.status()).toEqual([{ id: 'gpu', state: 'absent', received: 0 }]) + expect(store.gpuEngineDir()).toBeNull() + // A marker written for some OTHER size of pack does not count either. + writeFileSync(join(packDir(root, e), PACK_MARKER), JSON.stringify({ id: 'gpu', bytes: 1, sha256: e.sha256 })) + expect((await store.status())[0].state).toBe('absent') + expect(store.gpuEngineDir()).toBeNull() + // And the next download starts from a clean folder. + writeFileSync(join(packDir(root, e), 'stale.txt'), 'from the crash') + expect(await store.download('gpu', quiet)).toEqual({ ok: true }) + expect(existsSync(join(packDir(root, e), 'stale.txt'))).toBe(false) + expect(store.gpuEngineDir()).toBe(join(packDir(root, e), 'Release')) + }) + + it('finds the server at the top and two levels down, and no deeper', async () => { + const at = async (rel: string[]): Promise => { + const root = freshRoot() + const e = PACK_ENTRY() + const store = createDictationStore({ + root, + catalog: [e], + unzip: async (_zip, outDir) => { + mkdirSync(join(outDir, ...rel), { recursive: true }) + writeFileSync(join(outDir, ...rel, 'whisper-server.exe'), 'MZ') + } + }) + const r = await store.download('gpu', quiet) + const dir = store.gpuEngineDir() + if (dir) expect(dir).toBe(join(packDir(root, e), ...rel)) + else expect(r).toMatchObject({ ok: false, reason: 'unpack' }) + return dir + } + expect(await at([])).not.toBeNull() + expect(await at(['Release'])).not.toBeNull() + expect(await at(['build', 'Release'])).not.toBeNull() + expect(await at(['a', 'b', 'c'])).toBeNull() + }) + + it('cancels during the download, leaving no zip behind', async () => { + const root = freshRoot() + const store = createDictationStore({ root, catalog: [PACK_ENTRY('/slow', GOOD)], unzip: fakeUnzip }) + const seen: DownloadProgress[] = [] + const p = store.download('gpu', (x) => seen.push(x)) + await until(() => seen.length > 0) + expect(existsSync(join(root, 'engines', 'gpu.zip.part'))).toBe(true) + store.cancel('gpu') + expect(await p).toEqual({ ok: false, reason: 'cancelled' }) + expect(readdirSync(join(root, 'engines'))).toEqual([]) + }) + + it('cancels during the unzip: the signal reaches it, and nothing is left installed', async () => { + const root = freshRoot() + let started = false + const store = createDictationStore({ + root, + catalog: [PACK_ENTRY()], + unzip: (_zip, outDir, signal) => + new Promise((_resolve, reject) => { + mkdirSync(outDir, { recursive: true }) + started = true + signal?.addEventListener('abort', () => reject(new Error('aborted'))) + }) + }) + const p = store.download('gpu', quiet) + await until(() => started) + expect((await store.status())[0]).toEqual({ id: 'gpu', state: 'downloading', received: PACK.length }) + store.cancel('gpu') + expect(await p).toEqual({ ok: false, reason: 'cancelled' }) + expect(readdirSync(join(root, 'engines'))).toEqual([]) + }) + + it('removes the pack, and the folder of a pack the catalog has moved on from', async () => { + const root = freshRoot() + const e = PACK_ENTRY() + const old = join(root, 'engines', 'gpu-0123456789ab') + await fakeUnzip('', old) + const keep = join(root, 'engines', 'gpu-notes') + mkdirSync(keep, { recursive: true }) + const store = createDictationStore({ root, catalog: [e], unzip: fakeUnzip }) + await store.download('gpu', quiet) + // Installing the new pack is what retires the old one. + expect(existsSync(old)).toBe(false) + await store.remove('gpu') + expect(store.gpuEngineDir()).toBeNull() + expect((await store.status())[0].state).toBe('absent') + // Only what the store made is the store's to delete. + expect(readdirSync(join(root, 'engines'))).toEqual(['gpu-notes']) + }) + + it('unzips for real with Expand-Archive, in a path with a space', async () => { + const root = freshRoot() + const e = PACK_ENTRY('/realzip', REAL_ZIP) + const store = createDictationStore({ root, catalog: [e] }) + expect(await store.download('gpu', quiet)).toEqual({ ok: true }) + expect(store.gpuEngineDir()).toBe(join(packDir(root, e), 'Release')) + expect(readdirSync(join(root, 'engines'))).toEqual([`gpu-${e.sha256.slice(0, 12)}`]) + }, 30000) +}) diff --git a/core/main/dictationStore.ts b/core/main/dictationStore.ts new file mode 100644 index 0000000..9d0c617 --- /dev/null +++ b/core/main/dictationStore.ts @@ -0,0 +1,447 @@ +import { execFile } from 'child_process' +import { createHash } from 'crypto' +import { existsSync, readdirSync, readFileSync, statSync } from 'fs' +import { mkdir, open, readdir, rename, rm, stat, writeFile } from 'fs/promises' +import { dirname, join } from 'path' +import type { + CatalogEntry, + DictationStore, + DownloadProgress, + DownloadResult, + ItemStatus +} from '../shared/dictationTypes' + +/** + * The dictation models and the NVIDIA engine pack on disk (#13), in the one + * folder Prism and Prism Terminal share (`%LOCALAPPDATA%\PrismDictation`; the + * host passes it in as `root`, and the tests pass a temp folder). + * + * /models/.bin a model + * /engines/-/ the GPU pack, unzipped + * + * THE ONE PROMISE this file makes: nothing unverified is ever loaded or run. A + * download streams into `.part` while it is hashed and counted, and only + * a file whose byte count AND sha256 both match the catalog is renamed into + * place. So a file under its final name was verified by construction, and the + * cheap question asked afterwards (is it there, is it the catalogued size) is + * enough to call it installed without hashing 2.9 GB on every look. + * + * The pack is a zip, so "in place" takes three more steps: the verified zip is + * unzipped into its folder, the zip is deleted, and a small marker file is + * written LAST. A folder without its marker is a crash half way through an + * unzip and never reads as installed. The folder carries the hash in its name + * so a pack the catalog has moved on from can never be mistaken for the new one. + * + * Nothing here retries: a failure is reported with its reason and the row in + * the model manager offers Retry. Redirects are fetch's own business (the + * model host answers every download with one). + */ + +/** Written last into the pack's folder; its presence IS "the unzip finished". */ +export const PACK_MARKER = 'prism-installed.json' + +/** + * How long a `.part` file must have sat untouched before status() calls it a + * leftover and deletes it. The folder is SHARED by two apps: with no such + * pause, opening the Dictation page in Prism would delete the part file Prism + * Terminal was busy writing, and that download would end in a rename of a file + * that is gone. A part file being written was modified a moment ago; one left + * by a crash was not. A download this store starts itself clears the way at + * once, whatever the age. + */ +export const STALE_PART_MS = 60_000 + +/** "at most about 10 times a second": a 2.9 GB model is about 50 000 chunks. */ +const PROGRESS_EVERY_MS = 100 + +const SERVER_EXE = 'whisper-server.exe' + +/** An id becomes a file name, so it is held to what cannot leave the folder. */ +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/ + +export type Unzip = (zip: string, outDir: string, signal?: AbortSignal) => Promise + +export interface DictationStoreDeps { + /** The shared folder. It need not exist: every folder is made on demand. */ + root: string + /** Handed in, never imported: the integrator owns which catalog this is. */ + catalog: readonly CatalogEntry[] + fetchImpl?: typeof fetch + /** The signal is the cancel button's; an unzip that ignores it is only slower to stop. */ + unzip?: Unzip + now?: () => number +} + +const EXPAND_COMMAND = + "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue'; " + + 'Expand-Archive -LiteralPath $env:PRISM_DICTATION_ZIP -DestinationPath $env:PRISM_DICTATION_OUT -Force' + +/** + * The default unzip: Windows PowerShell's own Expand-Archive, which every + * Windows 10 and 11 has, so the pack needs no unzip library of ours. + * + * The two paths travel in the ENVIRONMENT, not in the command: after + * `-Command` PowerShell joins its remaining arguments into one string and + * parses it again, so a path with a space or a quote in it (a user folder + * named "Ole Hansen") would need quoting that is easy to get wrong. The + * command text is a constant and `-LiteralPath` keeps [ and ] from being read + * as wildcards. The progress bar is switched off because Windows PowerShell + * 5.1 spends most of an Expand-Archive drawing it, even with nobody watching. + * Found by its full path so a stray powershell.exe earlier on PATH is never run. + */ +export const expandArchive: Unzip = (zip, outDir, signal) => + new Promise((resolve, reject) => { + const exe = join(process.env.SystemRoot ?? 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') + execFile( + exe, + ['-NoProfile', '-NonInteractive', '-Command', EXPAND_COMMAND], + { + windowsHide: true, + signal, + maxBuffer: 1024 * 1024, + env: { ...process.env, PRISM_DICTATION_ZIP: zip, PRISM_DICTATION_OUT: outDir } + }, + (err, _stdout, stderr) => { + if (!err) return resolve() + const said = String(stderr).split(/\r?\n/).find((l) => l.trim()) + reject(new Error(said?.trim() || err.message)) + } + ) + }) + +const message = (e: unknown): string => (e instanceof Error ? e.message : String(e)) + +/** A folder can be held for a moment by a virus scanner reading what was just unzipped. */ +const gone = (path: string): Promise => rm(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }) + +interface Flight { + promise: Promise + controller: AbortController + received: number + cancelled: boolean + listeners: Set<(p: DownloadProgress) => void> +} + +export function createDictationStore(deps: DictationStoreDeps): DictationStore { + const { root, catalog } = deps + const fetchImpl = deps.fetchImpl ?? fetch + const unzip = deps.unzip ?? expandArchive + const now = deps.now ?? (() => Date.now()) + for (const e of catalog) { + if (!SAFE_ID.test(e.id)) throw new Error(`dictation catalog: "${e.id}" cannot be a file name`) + } + const flights = new Map() + /** Where the server was last found, so the engine's every ask is not a folder walk. */ + let engineDirMemo: string | null = null + + // The catalog is read on every call rather than indexed once: it is four or + // five entries, and a host that swaps an entry is then simply believed. + const find = (id: string): CatalogEntry | undefined => catalog.find((e) => e.id === id) + const isPack = (e: CatalogEntry): boolean => e.kind === 'gpu-pack' + /** The model itself, or the pack's zip: what `.part` is renamed to. */ + const finalPath = (e: CatalogEntry): string => + isPack(e) ? join(root, 'engines', `${e.id}.zip`) : join(root, 'models', `${e.id}.bin`) + const partPath = (e: CatalogEntry): string => finalPath(e) + '.part' + const packDir = (e: CatalogEntry): string => join(root, 'engines', `${e.id}-${e.sha256.slice(0, 12).toLowerCase()}`) + + /** + * Synchronous on purpose: `modelPath` and `gpuEngineDir` are synchronous in + * the contract, and what is read is one stat, or a marker of under 200 bytes. + * A marker counts only when it was written for THIS catalog entry, so the + * size and hash are in it, not just its existence. + */ + function installedSync(e: CatalogEntry): boolean { + try { + if (!isPack(e)) { + const s = statSync(finalPath(e)) + return s.isFile() && s.size === e.bytes + } + const m = JSON.parse(readFileSync(join(packDir(e), PACK_MARKER), 'utf8')) as Partial + return m.bytes === e.bytes && m.sha256 === e.sha256 + } catch { + return false + } + } + + /** + * The folder that actually holds whisper-server.exe: the official zip nests + * everything under Release/, and a later release may not. Two levels down + * and no further, so a pack that has no server is a quick "no" rather than + * a walk of whatever was in the zip. + */ + function findServer(dir: string, depth = 0): string | null { + if (existsSync(join(dir, SERVER_EXE))) return dir + if (depth >= 2) return null + let names: string[] + try { + names = readdirSync(dir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + } catch { + return null + } + for (const name of names) { + const hit = findServer(join(dir, name), depth + 1) + if (hit) return hit + } + return null + } + + /** Folders of this pack under another hash: a pack the catalog has replaced. 643 MB each. */ + async function sweepPacks(e: CatalogEntry, keep: string | null): Promise { + const engines = join(root, 'engines') + const ours = new RegExp(`^${e.id.replace(/[.]/g, '\\.')}-[0-9a-f]{12}$`) + const names = await readdir(engines).catch(() => [] as string[]) + for (const name of names) { + const path = join(engines, name) + if (ours.test(name) && path !== keep) await gone(path) + } + } + + function report(e: CatalogEntry, flight: Flight): void { + const p: DownloadProgress = { id: e.id, received: flight.received, total: e.bytes } + for (const listen of flight.listeners) { + // A listener is a window that may have closed; the download is not its hostage. + try { + listen(p) + } catch { + /* nothing to tell it */ + } + } + } + + /** One download, start to finish. Never rejects: every way out is a DownloadResult. */ + async function run(e: CatalogEntry, flight: Flight): Promise { + const final = finalPath(e) + const part = partPath(e) + const signal = flight.controller.signal + const cancelled: DownloadResult = { ok: false, reason: 'cancelled' } + try { + if (installedSync(e)) return { ok: true } + // Clear the way: a part file from a crash, and for the pack the zip and + // the half-unzipped folder a crash during the unzip leaves behind. + try { + await mkdir(dirname(final), { recursive: true }) + await gone(part) + if (isPack(e)) { + await gone(final) + await gone(packDir(e)) + } + } catch (err) { + return { ok: false, reason: 'disk', detail: message(err) } + } + + let res: Response + try { + res = await fetchImpl(e.url, { signal }) + } catch (err) { + return flight.cancelled ? cancelled : { ok: false, reason: 'network', detail: message(err) } + } + if (!res.ok || !res.body) { + await res.body?.cancel().catch(() => {}) + return { ok: false, reason: 'network', detail: `HTTP ${res.status}` } + } + + // A FileHandle rather than a write stream: each write is awaited, which + // is the backpressure, and a full disk is a rejected promise on the line + // that caused it rather than an 'error' event somewhere else. + const hash = createHash('sha256') + const reader = res.body.getReader() + let failed: DownloadResult | null = null + let file + try { + file = await open(part, 'w') + } catch (err) { + await reader.cancel().catch(() => {}) + return { ok: false, reason: 'disk', detail: message(err) } + } + let lastReport = -Infinity + let lastReported = -1 + try { + for (;;) { + let chunk: Uint8Array + try { + const step = await reader.read() + if (step.done) break + chunk = step.value + } catch (err) { + failed = flight.cancelled ? cancelled : { ok: false, reason: 'network', detail: message(err) } + break + } + flight.received += chunk.byteLength + // More than the catalog promised can never verify, so it is not + // written to the end first: a wrong url must not be able to fill the disk. + if (flight.received > e.bytes) { + flight.controller.abort() + failed = { ok: false, reason: 'checksum', detail: `more than the expected ${e.bytes} bytes` } + break + } + hash.update(chunk) + try { + await file.write(chunk) + } catch (err) { + flight.controller.abort() + failed = { ok: false, reason: 'disk', detail: message(err) } + break + } + const t = now() + if (t - lastReport >= PROGRESS_EVERY_MS) { + lastReport = t + lastReported = flight.received + report(e, flight) + } + } + } finally { + await file.close().catch(() => {}) + } + // The final word, whatever the clock says, unless the last chunk already said it. + if (!failed && lastReported !== flight.received) report(e, flight) + + if (!failed && flight.cancelled) failed = cancelled + if (!failed) { + const got = hash.digest('hex') + if (flight.received !== e.bytes) { + failed = { ok: false, reason: 'checksum', detail: `${flight.received} bytes, expected ${e.bytes}` } + } else if (got !== e.sha256.toLowerCase()) { + failed = { ok: false, reason: 'checksum', detail: `sha256 ${got}` } + } + } + if (failed) { + await gone(part).catch(() => {}) + return failed + } + + try { + await rename(part, final) + } catch (err) { + await gone(part).catch(() => {}) + return { ok: false, reason: 'disk', detail: message(err) } + } + if (!isPack(e)) return { ok: true } + + // The pack: verified, THEN unzipped. `final` is the zip, and it keeps + // the .zip name because Expand-Archive refuses any other extension. + const dir = packDir(e) + const undo = async (): Promise => { + await gone(final).catch(() => {}) + await gone(dir).catch(() => {}) + } + try { + await unzip(final, dir, signal) + } catch (err) { + await undo() + return flight.cancelled ? cancelled : { ok: false, reason: 'unpack', detail: message(err) } + } + if (flight.cancelled) { + await undo() + return cancelled + } + const server = findServer(dir) + if (!server) { + await undo() + return { ok: false, reason: 'unpack', detail: `no ${SERVER_EXE} in the pack` } + } + try { + await gone(final) + // LAST: from here on, and not a moment before, the folder reads as installed. + await writeFile(join(dir, PACK_MARKER), JSON.stringify({ id: e.id, bytes: e.bytes, sha256: e.sha256 })) + } catch (err) { + await undo() + return { ok: false, reason: 'disk', detail: message(err) } + } + engineDirMemo = server + await sweepPacks(e, dir).catch(() => {}) + return { ok: true } + } catch (err) { + // Nothing above is meant to reach here; if it does, it is still an answer. + await gone(part).catch(() => {}) + return { ok: false, reason: 'disk', detail: message(err) } + } + } + + function download(id: string, onProgress: (p: DownloadProgress) => void): Promise { + const e = find(id) + // The contract has no reason for "no such thing"; nothing can be fetched, so network it is. + if (!e) return Promise.resolve({ ok: false, reason: 'network', detail: `not in the catalog: ${id}` }) + const running = flights.get(id) + if (running) { + // Cancel and then Retry, quicker than the cancelled flight can tidy up: + // the answer owed is a new download, not the old one's 'cancelled'. + if (running.cancelled) return running.promise.then(() => download(id, onProgress)) + // Two windows, one download: both hear the progress, both get the same promise. + running.listeners.add(onProgress) + return running.promise + } + const flight: Flight = { + promise: Promise.resolve({ ok: true }), + controller: new AbortController(), + received: 0, + cancelled: false, + listeners: new Set([onProgress]) + } + flight.promise = run(e, flight).finally(() => { + if (flights.get(id) === flight) flights.delete(id) + }) + flights.set(id, flight) + return flight.promise + } + + function cancel(id: string): void { + const flight = flights.get(id) + if (!flight || flight.cancelled) return + flight.cancelled = true + // run() is what removes the part file, once the handle on it is closed. + flight.controller.abort() + } + + async function status(): Promise { + const out: ItemStatus[] = [] + for (const e of catalog) { + const flight = flights.get(e.id) + if (flight) { + // Through the unzip too: the row stays busy until the pack can be used. + out.push({ id: e.id, state: 'downloading', received: flight.received }) + continue + } + const part = partPath(e) + const left = await stat(part).catch(() => null) + if (left && now() - left.mtimeMs > STALE_PART_MS) await gone(part).catch(() => {}) + const installed = installedSync(e) + out.push({ id: e.id, state: installed ? 'installed' : 'absent', received: installed ? e.bytes : 0 }) + } + return out + } + + async function remove(id: string): Promise { + const e = find(id) + if (!e) return + const flight = flights.get(id) + if (flight) { + cancel(id) + await flight.promise + } + await gone(partPath(e)) + await gone(finalPath(e)) + if (isPack(e)) { + engineDirMemo = null + await gone(packDir(e)) + await sweepPacks(e, null) + } + } + + function modelPath(id: string): string | null { + const e = find(id) + return e && !isPack(e) && installedSync(e) ? finalPath(e) : null + } + + function gpuEngineDir(): string | null { + const e = catalog.find(isPack) + if (!e || !installedSync(e)) return null + const dir = packDir(e) + if (engineDirMemo && engineDirMemo.startsWith(dir) && existsSync(join(engineDirMemo, SERVER_EXE))) return engineDirMemo + engineDirMemo = findServer(dir) + return engineDirMemo + } + + return { status, download, cancel, remove, modelPath, gpuEngineDir } +} diff --git a/core/main/ipc.ts b/core/main/ipc.ts index e71fc30..35e384e 100644 --- a/core/main/ipc.ts +++ b/core/main/ipc.ts @@ -23,7 +23,7 @@ import { cdTerm, killTerm, prewarmShell, resizeTerm, spawnTerm, writeTerm } from * written against Electron's own event classes, which the core does not * import, so the arguments are typed at each handler instead. */ /* eslint-disable @typescript-eslint/no-explicit-any */ -interface IpcMainLike { +export interface IpcMainLike { handle(channel: string, listener: (event: any, ...args: any[]) => unknown): unknown on(channel: string, listener: (event: any, ...args: any[]) => void): unknown } diff --git a/core/main/mediaPause.test.ts b/core/main/mediaPause.test.ts new file mode 100644 index 0000000..67e690f --- /dev/null +++ b/core/main/mediaPause.test.ts @@ -0,0 +1,223 @@ +import { spawn, type ChildProcess } from 'child_process' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { createMediaPause, MEDIA_HELPER_SCRIPT, mediaHelperCommand } from './mediaPause' +import type { MediaPause } from '../shared/dictationTypes' + +/** + * The media pause, checked against a FAKE helper (a Node script speaking the + * same line protocol), never the real PowerShell: a unit test that paused the + * owner's music would be a test nobody runs twice. Every child the module + * starts is recorded through the injected spawn, which is how the last check of + * each test (nothing left running) is made. + */ + +const FIXTURE = join(__dirname, '__fixtures__', 'fakeMediaHelper.mjs') + +let dir = '' +let log = '' +let children: ChildProcess[] = [] +let made: MediaPause[] = [] + +const spawnAndRecord = ((file: string, args: string[], options: object) => { + const child = spawn(file, args, options) + children.push(child) + return child +}) as typeof spawn + +/** A media pause wired to the fake. `flags` are the fixture's own arguments. */ +function media(flags: string[], timeoutMs = 3000): MediaPause { + const m = createMediaPause({ + spawnImpl: spawnAndRecord, + command: { file: process.execPath, args: [FIXTURE, '--log', log, ...flags] }, + timeoutMs + }) + made.push(m) + return m +} + +const sent = (): string[] => (existsSync(log) ? readFileSync(log, 'utf8').split('\n').filter(Boolean) : []) + +function gone(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve() + return new Promise((done) => child.once('exit', () => done())) +} + +async function until(test: () => boolean, ms = 4000): Promise { + const end = Date.now() + ms + while (!test()) { + if (Date.now() > end) throw new Error('waited too long') + await new Promise((r) => setTimeout(r, 20)) + } +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'prism media pause ')) + log = join(dir, 'requests.log') + children = [] + made = [] +}) + +afterEach(async () => { + for (const m of made) m.dispose() + await Promise.all(children.map(gone)) + rmSync(dir, { recursive: true, force: true }) +}) + +describe('the media pause', () => { + it('starts nothing until the first pause, then keeps ONE helper for every call', async () => { + const m = media(['--sessions', 'spotify=Playing']) + expect(children).toHaveLength(0) + expect(await m.pausePlaying()).toBe('spotify') + await m.resume('spotify') + expect(await m.pausePlaying()).toBe('spotify') + expect(children).toHaveLength(1) + expect(sent()).toEqual(['pause 1', 'resume 2 spotify', 'pause 3']) + }) + + it('brings back only what WAS playing, and nothing the second time', async () => { + const m = media(['--sessions', 'spotify=Playing,chrome=Paused,vlc=Playing,edge=Stopped']) + expect(await m.pausePlaying()).toBe('spotify,vlc') + // They are paused now, so there is nothing left for a second pause to claim: + // a token only ever names what THIS pause stopped. + expect(await m.pausePlaying()).toBe('') + }) + + it('resumes exactly the ids of the token, spaces in an id included', async () => { + const m = media(['--sessions', 'Spotify AB.Music!App=Playing,chrome=Playing']) + const token = await m.pausePlaying() + expect(token).toBe('Spotify AB.Music!App,chrome') + await m.resume(token) + expect(sent()[1]).toBe('resume 2 Spotify AB.Music!App,chrome') + }) + + it('does nothing and starts nothing for an empty token', async () => { + const m = media(['--sessions', 'spotify=Playing']) + await m.resume('') + expect(children).toHaveLength(0) + expect(sent()).toEqual([]) + }) + + it('cannot be made to say a second thing by a token with a newline in it', async () => { + const m = media(['--sessions', 'spotify=Playing']) + await m.resume('spotify\npause 9') + expect(sent()).toEqual(['resume 1 spotifypause 9']) + }) + + it('gives an empty token when the helper does not answer in time, and resume still resolves', async () => { + const m = media(['--mode', 'silent'], 150) + const began = Date.now() + expect(await m.pausePlaying()).toBe('') + expect(Date.now() - began).toBeLessThan(2000) + await expect(m.resume('spotify')).resolves.toBeUndefined() + }) + + it('undoes a pause whose answer arrives after its caller was told nothing was paused', async () => { + // The dangerous half of a timeout: the helper was only slow, the music DID + // stop, and the token that would have started it again was never handed out. + const m = media(['--sessions', 'spotify=Playing', '--delay', '400'], 150) + expect(await m.pausePlaying()).toBe('') + await until(() => sent().some((line) => /^resume \d+ spotify$/.test(line))) + }) + + it('stands a helper down after two timeouts in a row, and starts a fresh one', async () => { + const m = media(['--mode', 'silent'], 100) + await m.pausePlaying() + await m.pausePlaying() + await gone(children[0]) + await m.pausePlaying() + expect(children).toHaveLength(2) + }) + + it('ignores lines that are not answers, and answers to questions nobody asked', async () => { + const m = media(['--sessions', 'spotify=Playing', '--mode', 'garbage']) + expect(await m.pausePlaying()).toBe('spotify') + await expect(m.resume('spotify')).resolves.toBeUndefined() + }) + + it('matches an answer to its call by number, not by the order they come back in', async () => { + const m = media(['--sessions', 'spotify=Playing', '--mode', 'reverse']) + const [first, second] = await Promise.all([m.pausePlaying(), m.pausePlaying()]) + expect(first).toBe('spotify') + expect(second).toBe('') + }) + + it('answers empty when the helper dies mid-call, and starts another on the next pause', async () => { + const m = media(['--sessions', 'spotify=Playing', '--mode', 'crash-once', '--marker', join(dir, 'crashed')]) + const began = Date.now() + expect(await m.pausePlaying()).toBe('') + // The crash is what ended the call, not the three-second clock. + expect(Date.now() - began).toBeLessThan(2500) + expect(await m.pausePlaying()).toBe('spotify') + expect(children).toHaveLength(2) + }) + + it('never throws when the helper cannot be started at all', async () => { + const m = createMediaPause({ + command: { file: join(dir, 'no such helper.exe'), args: [] }, + timeoutMs: 500 + }) + made.push(m) + expect(await m.pausePlaying()).toBe('') + await expect(m.resume('spotify')).resolves.toBeUndefined() + }) + + it('never throws when spawn itself throws', async () => { + const m = createMediaPause({ + spawnImpl: (() => { + throw new Error('no') + }) as unknown as typeof spawn, + timeoutMs: 200 + }) + expect(await m.pausePlaying()).toBe('') + await expect(m.resume('spotify')).resolves.toBeUndefined() + m.dispose() + }) + + it('kills the helper on dispose, answers whoever was waiting, and leaves no process behind', async () => { + const m = media(['--mode', 'silent'], 3000) + const waiting = m.pausePlaying() + await until(() => sent().length === 1) + m.dispose() + expect(await waiting).toBe('') + await gone(children[0]) + expect(children[0].exitCode !== null || children[0].signalCode !== null).toBe(true) + expect(() => process.kill(children[0].pid as number, 0)).toThrow() + // Disposing twice, or with nothing running, is nothing. + m.dispose() + }) + + it('can be used again after dispose: the next pause starts a fresh helper', async () => { + const m = media(['--sessions', 'spotify=Playing']) + expect(await m.pausePlaying()).toBe('spotify') + m.dispose() + await gone(children[0]) + expect(await m.pausePlaying()).toBe('spotify') + expect(children).toHaveLength(2) + }) +}) + +describe('the real helper command', () => { + it('is Windows PowerShell with an argv array and no profile', () => { + const { file, args } = mediaHelperCommand() + expect(file).toBe('powershell.exe') + expect(args.slice(0, 5)).toEqual(['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command']) + expect(args[5]).toBe(MEDIA_HELPER_SCRIPT) + }) + + it('carries a script that survives being ONE argument', () => { + // Node quotes an argument for CreateProcess by escaping double quotes, and + // PowerShell's -Command then reads the result a second time: a script with + // no double quote and no newline in it is one that neither can mangle. + expect(MEDIA_HELPER_SCRIPT).not.toMatch(/["\r\n]/) + expect(MEDIA_HELPER_SCRIPT.endsWith('\\')).toBe(false) + }) + + it('never reaches for the play/pause media key, which would START music that was off', () => { + expect(MEDIA_HELPER_SCRIPT).not.toMatch(/keybd_event|SendKeys|TogglePlayPause|MEDIA_PLAY_PAUSE/i) + expect(MEDIA_HELPER_SCRIPT).toContain('TryPauseAsync') + expect(MEDIA_HELPER_SCRIPT).toContain('TryPlayAsync') + }) +}) diff --git a/core/main/mediaPause.ts b/core/main/mediaPause.ts new file mode 100644 index 0000000..9207bdb --- /dev/null +++ b/core/main/mediaPause.ts @@ -0,0 +1,278 @@ +/** + * Pause what is PLAYING while the user dictates, and afterwards resume exactly + * that (#13, the `dictation-pause-media` sub-option). + * + * NOT the play/pause media key. That key TOGGLES, so on a machine where nothing + * was playing it would START music the moment the user began to speak, and the + * second press at the end would stop it again: the feature inverted. Windows + * knows what is actually playing (GlobalSystemMediaTransportControlsSession- + * Manager, the thing behind the volume flyout's media card), so this asks it, + * pauses only the sessions whose status is Playing, and hands back their ids as + * the token. Resume plays those ids and nothing else, so a player the user had + * paused by hand an hour ago stays paused. + * + * The API is WinRT and the core has no native module to reach it with (and may + * not grow one: node-pty is the one native module either app carries). Windows + * PowerShell 5.1 can reach it and is on every Windows 10 and 11, so the work is + * done by ONE resident powershell.exe, the pattern of the app's dwmHelper: it + * is started lazily on the first pause, loads the WinRT projection once, and + * then reads one request per line from stdin. A process per press would pay + * PowerShell's start every time, which is the better part of a second of music + * playing into the microphone. + * + * The line protocol, numbered so that calls in flight together cannot take each + * other's answers: + * + * pause -> + * resume -> ok + * list -> =,... (read-only, for diagnosis) + * + * EVERYTHING FAILS SOFT, by the design's own word: "if the helper cannot start, + * dictation works and the option does nothing". No PowerShell, a helper that + * dies, one that answers nonsense or not at all: the token is empty and resume + * resolves. Nothing here throws and nothing rejects. + */ +import { spawn, type ChildProcess } from 'child_process' +import type { MediaPause, MediaPauseToken } from '../shared/dictationTypes' + +/** + * The helper, as ONE line with no double quote in it. It travels as a single + * argv entry: Node quotes that for CreateProcess by escaping double quotes and + * PowerShell's -Command reads the result a second time, so a script with + * neither a double quote nor a newline is one that neither step can mangle + * (mediaPause.test.ts holds it to that). + * + * The way into WinRT is the one PROVEN on this codebase's target (spiked + * 2026-09-19 under Windows PowerShell 5.1, it listed the sessions correctly): + * load System.Runtime.WindowsRuntime, name the type with its ContentType so the + * projection is loaded, and find the generic AsTask(IAsyncOperation`1) overload + * by REFLECTION, since PowerShell 5.1 has no await and cannot bind a generic + * method on its own. Each await is bounded (2 s): a player that never answers + * TryPauseAsync must not wedge the one loop every later request waits behind. + * + * The projection and the manager are loaded on the first REQUEST and retried on + * the next if that failed, so a machine where it cannot load costs an empty + * answer per press rather than a dead helper. A pause reports an id only when + * TryPauseAsync said true: the token is a promise to start something again, and + * it must not name a session that never stopped. A resume plays only sessions + * that are still Paused, so one the user closed or started again by hand in the + * meantime is left as they left it. + * + * A session is named by its SourceAppUserModelId. Two sessions of one app share + * it, so resuming plays every PAUSED session of that app; and an id holding a + * comma would split into pieces that match nothing (it would simply not resume). + * Neither has been seen: the ids met so far are 'Spotify.exe', 'MSEdge', a + * package family name with '!App', a hex string for Firefox. + */ +export const MEDIA_HELPER_SCRIPT = [ + "$ErrorActionPreference = 'Stop'", + "$ProgressPreference = 'SilentlyContinue'", + // Ids go out and come back as the bytes Node writes, which is UTF-8. No BOM: + // one in front of the first answer would make its request number unreadable. + 'try { $utf8 = New-Object System.Text.UTF8Encoding $false; [Console]::InputEncoding = $utf8; [Console]::OutputEncoding = $utf8 } catch {}', + '$asTask = $null', + '$mgr = $null', + 'function Await($op, $type) { $t = $script:asTask.MakeGenericMethod($type).Invoke($null, @($op)); if ($t.Wait(2000)) { $t.Result } else { $null } }', + 'function Sessions { if ($null -eq $script:mgr) { ' + + 'Add-Type -AssemblyName System.Runtime.WindowsRuntime; ' + + '$null = [Windows.Media.Control.GlobalSystemMediaTransportControlsSessionManager, Windows.Media.Control, ContentType = WindowsRuntime]; ' + + "$script:asTask = [System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object { $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1' } | Select-Object -First 1; " + + '$script:mgr = Await ([Windows.Media.Control.GlobalSystemMediaTransportControlsSessionManager]::RequestAsync()) ([Windows.Media.Control.GlobalSystemMediaTransportControlsSessionManager]) }; ' + + "if ($null -eq $script:mgr) { throw 'no session manager' }; " + + ',@($script:mgr.GetSessions()) }', + 'while ($true) { ' + + '$line = [Console]::In.ReadLine(); if ($null -eq $line) { break }; ' + + "$p = $line.Split(' ', 3); if ($p.Length -lt 2) { continue }; " + + "$verb = $p[0]; $n = $p[1]; $out = $null; $found = New-Object 'System.Collections.Generic.List[string]'; " + + 'try { ' + + "if ($verb -eq 'pause') { $out = ''; foreach ($s in (Sessions)) { try { " + + "if ([string]$s.GetPlaybackInfo().PlaybackStatus -eq 'Playing') { if (Await ($s.TryPauseAsync()) ([bool])) { $id = [string]$s.SourceAppUserModelId; if ($id -and -not $found.Contains($id)) { $found.Add($id) } } } " + + '} catch {} } } ' + + "elseif ($verb -eq 'resume') { $out = 'ok'; if ($p.Length -eq 3) { $want = $p[2].Split(','); foreach ($s in (Sessions)) { try { " + + "if (($want -ccontains [string]$s.SourceAppUserModelId) -and ([string]$s.GetPlaybackInfo().PlaybackStatus -eq 'Paused')) { $null = Await ($s.TryPlayAsync()) ([bool]) } " + + '} catch {} } } } ' + + "elseif ($verb -eq 'list') { $out = ''; foreach ($s in (Sessions)) { try { $found.Add([string]$s.SourceAppUserModelId + '=' + [string]$s.GetPlaybackInfo().PlaybackStatus) } catch {} } } " + + '} catch {}; ' + + // An unknown verb is not answered at all: its caller times out, which is + // the same soft failure as everything else. + 'if ($null -eq $out) { continue }; ' + + "if ($verb -ne 'resume') { $out = $found -join ',' }; " + + "[Console]::Out.WriteLine($n + ' ' + $out); [Console]::Out.Flush() }" +].join('; ') + +/** The real helper: Windows PowerShell 5.1 (never pwsh, which is not on every + * machine and whose WinRT support was removed in 7), as an argv array. */ +export function mediaHelperCommand(): { file: string; args: string[] } { + return { + file: 'powershell.exe', + args: ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', MEDIA_HELPER_SCRIPT] + } +} + +/** What the media pause reaches the outside world through. The defaults are the + * real thing; the tests point `command` at a Node script that speaks the same + * protocol, so no test ever touches the owner's media. */ +export interface MediaPauseDeps { + spawnImpl?: typeof spawn + command?: { file: string; args: string[] } + /** How long one request may take before its caller is answered with nothing. */ + timeoutMs?: number +} + +/** Two unanswered requests in a row and the helper is taken to be wedged, not + * slow. ONE is not enough: the first request of a cold helper pays PowerShell's + * start and the WinRT load, and killing it for that would mean it never warms. */ +const TIMEOUTS_BEFORE_KILL = 2 + +interface Waiting { + done: (payload: string | null) => void + timer: ReturnType +} + +export function createMediaPause(deps: MediaPauseDeps = {}): MediaPause { + const spawnImpl = deps.spawnImpl ?? spawn + const command = deps.command ?? mediaHelperCommand() + const timeoutMs = deps.timeoutMs ?? 3000 + + let child: ChildProcess | null = null + let buffered = '' + let nextNumber = 1 + let timeoutsInARow = 0 + const waiting = new Map() + /** Pauses whose caller has already been told "nothing was paused". */ + const abandonedPauses = new Set() + + /** Answer everybody still waiting on `from` with nothing: it is gone. */ + function lost(from: ChildProcess): void { + if (child !== from) return + child = null + buffered = '' + abandonedPauses.clear() + for (const [n, w] of [...waiting]) { + waiting.delete(n) + clearTimeout(w.timer) + w.done(null) + } + } + + function kill(target: ChildProcess): void { + try { + target.stdin?.end() + target.kill() + } catch { + /* gone already */ + } + } + + function onLine(line: string): void { + // Anything that is not "" or " " is PowerShell + // talking to itself (a warning, a blank line) and is nobody's answer. + const m = /^(\d+)(?: (.*))?$/.exec(line) + if (!m) return + const n = Number(m[1]) + const payload = m[2] ?? '' + const w = waiting.get(n) + if (w) { + waiting.delete(n) + clearTimeout(w.timer) + timeoutsInARow = 0 + w.done(payload) + return + } + // THE DANGEROUS HALF OF A TIMEOUT. The helper was only slow: the music DID + // stop, and the caller was told long ago that nothing had been paused, so + // no token exists that would ever start it again. Undo it here, at once. + if (abandonedPauses.delete(n)) { + timeoutsInARow = 0 + if (payload !== '') void ask('resume', payload) + } + } + + function helper(): ChildProcess | null { + if (child && child.exitCode === null && !child.killed) return child + try { + const started = spawnImpl(command.file, command.args, { + stdio: ['pipe', 'pipe', 'ignore'], + windowsHide: true + }) + child = started + buffered = '' + timeoutsInARow = 0 + // A missing powershell.exe arrives here, as an EVENT, a tick after spawn + // returned something that looked like a process. + started.on('error', () => lost(started)) + started.on('exit', () => lost(started)) + // Writing to a helper that has just died raises EPIPE on the stream, and + // an unheard stream error is an uncaught exception in main. + started.stdin?.on('error', () => {}) + started.stdout?.setEncoding('utf8') + started.stdout?.on('data', (chunk: string) => { + if (child !== started) return + buffered += chunk + let at = buffered.indexOf('\n') + while (at >= 0) { + const line = buffered.slice(0, at) + buffered = buffered.slice(at + 1) + onLine(line.replace(/\r$/, '')) + at = buffered.indexOf('\n') + } + }) + return started + } catch { + child = null + return null + } + } + + /** One request, one answer: the payload after the number, or null for every + * way of not getting one. Never rejects. */ + function ask(verb: 'pause' | 'resume', ids?: string): Promise { + return new Promise((resolve) => { + const target = helper() + if (!target || !target.stdin) return resolve(null) + const n = nextNumber++ + const timer = setTimeout(() => { + if (!waiting.delete(n)) return + if (verb === 'pause') abandonedPauses.add(n) + resolve(null) + timeoutsInARow += 1 + if (timeoutsInARow >= TIMEOUTS_BEFORE_KILL && child === target) kill(target) + }, timeoutMs) + waiting.set(n, { done: resolve, timer }) + try { + target.stdin.write(ids === undefined ? `${verb} ${n}\n` : `${verb} ${n} ${ids}\n`) + } catch { + waiting.delete(n) + clearTimeout(timer) + resolve(null) + } + }) + } + + return { + async pausePlaying(): Promise { + return (await ask('pause')) ?? '' + }, + + async resume(token: MediaPauseToken): Promise { + // The token crosses IPC on its way here, so it is treated as text from + // outside: a line break in it would be a second request of its own. + const ids = String(token ?? '').replace(/[\r\n]/g, '') + // Nothing was paused, so there is nothing to play, and no reason to start + // a PowerShell to find that out. + if (ids.trim() === '') return + await ask('resume', ids) + }, + + /** Kill the helper and answer whoever is waiting. Not final, like the + * engine's stop(): a later pausePlaying() starts a fresh helper. A helper + * that somehow outlived the app would end itself anyway, since its read + * loop stops when stdin closes. */ + dispose(): void { + const target = child + if (!target) return + lost(target) + kill(target) + } + } +} diff --git a/core/package.json b/core/package.json index 88be127..e9a1518 100644 --- a/core/package.json +++ b/core/package.json @@ -1,6 +1,6 @@ { "name": "prism-term-core", - "version": "0.1.0", + "version": "0.2.0", "description": "The terminal shared by Prism Terminal and Prism: pty, shells, agent detection and indicator, themes, links, the panel. TypeScript source, compiled by the host.", "license": "MIT", "private": true, @@ -16,14 +16,18 @@ "./renderer/lib/*": "./renderer/lib/*.ts", "./renderer/components/*": "./renderer/components/*.tsx", "./renderer/settings/options": "./renderer/settings/options.ts", - "./renderer/settings/*": "./renderer/settings/*.tsx" + "./renderer/settings/*": "./renderer/settings/*.tsx", + "./renderer/settings/dictationOptions": "./renderer/settings/dictationOptions.ts", + "./tools/*": "./tools/*" }, "files": [ "main", "preload", "shared", "renderer", + "tools", "!**/*.test.ts", + "!**/__fixtures__/**", "README.md" ], "peerDependencies": { diff --git a/core/preload/dictationApi.ts b/core/preload/dictationApi.ts new file mode 100644 index 0000000..52456e1 --- /dev/null +++ b/core/preload/dictationApi.ts @@ -0,0 +1,55 @@ +import { DCH } from '../shared/channels' +import type { + DownloadProgress, + DownloadResult, + EngineInfo, + ItemStatus, + MediaPauseToken, + TranscribeRequest, + TranscribeResult +} from '../shared/dictationTypes' +import type { IpcRendererLike } from './api' + +/** + * The preload half of DICTATION's bridge (#13), for both hosts. Separate from + * `createTermApi` because dictation is optional: a host that wires it spreads + * this in beside the terminal's, and `registerDictationIpc` is its other half. + */ +export interface DictationApi { + /** Is there an NVIDIA adapter, is the GPU pack in, did the GPU engine fall back. */ + dictationInfo(): Promise + /** Every catalogued model and the GPU pack: absent, downloading or installed. */ + dictationStatus(): Promise + /** Resolves when the file is verified and in place, or with why it is not. */ + dictationDownload(id: string): Promise + dictationCancel(id: string): void + dictationRemove(id: string): Promise + onDictationProgress(cb: (p: DownloadProgress) => void): () => void + /** The WAV goes to the local engine and nowhere else; nothing is written to disk. */ + dictationTranscribe(req: TranscribeRequest): Promise + /** Dictation was switched off: no server process may remain. */ + dictationStop(): void + dictationMediaPause(): Promise + dictationMediaResume(token: MediaPauseToken): void +} + +export function createDictationApi(ipc: IpcRendererLike): DictationApi { + return { + dictationInfo: () => ipc.invoke(DCH.info) as Promise, + dictationStatus: () => ipc.invoke(DCH.status) as Promise, + dictationDownload: (id) => ipc.invoke(DCH.download, id) as Promise, + dictationCancel: (id) => ipc.send(DCH.cancel, id), + dictationRemove: (id) => ipc.invoke(DCH.remove, id) as Promise, + onDictationProgress: (cb) => { + const listener = (_e: unknown, p: DownloadProgress): void => cb(p) + ipc.on(DCH.progress, listener) + return () => { + ipc.removeListener(DCH.progress, listener) + } + }, + dictationTranscribe: (req) => ipc.invoke(DCH.transcribe, req) as Promise, + dictationStop: () => ipc.send(DCH.stop), + dictationMediaPause: () => ipc.invoke(DCH.mediaPause) as Promise, + dictationMediaResume: (token) => ipc.send(DCH.mediaResume, token) + } +} diff --git a/core/renderer/components/DictationPill.tsx b/core/renderer/components/DictationPill.tsx new file mode 100644 index 0000000..642243f --- /dev/null +++ b/core/renderer/components/DictationPill.tsx @@ -0,0 +1,94 @@ +import { useEffect, useRef, type JSX } from 'react' +import { onDictationLevel } from '../lib/dictation' +import { useDictationView } from '../lib/useDictation' + +/** + * What dictation is doing, over the terminal it is dictating into (#13). + * + * ONE capsule, bottom centre, three faces: listening (a live meter and the + * provisional words), transcribing, and a short message when something needs + * saying ("No microphone found"). The owner's complaint about every other + * dictation tool is the reason for the first face: "you see no text before you + * stop, so if your mic's off you won't know until you finish". The METER moves + * the moment sound arrives, before any model has answered, so a dead + * microphone is visible in the first half second. + * + * It MOUNTS and UNMOUNTS (no fade held at opacity 0): an idle terminal carries + * no dictation DOM at all. It is absolutely positioned inside the host's + * terminal box and never takes layout, so the shell's rows do not move when it + * appears (measured in the e2e). The ground is `--p-side-flat`, the OPAQUE + * surface both apps define, because the terminal under it may be acrylic and a + * capsule you can read the prompt through is a smear. + * + * The meter is written straight to the DOM from the level callback: it moves + * at frame rate, and React has no business re-rendering for it. + */ +const BARS = [0.55, 0.85, 1, 0.8, 0.5] as const + +export function DictationPill({ sessionId }: { sessionId: string | null }): JSX.Element | null { + const view = useDictationView() + const bars = useRef>([]) + const listening = view.phase === 'listening' + + useEffect(() => { + if (!listening) return + let shown = 0 + return onDictationLevel((level) => { + // Fast up, slow down: speech is spiky, and a meter that falls as fast as + // it rises flickers instead of reading as loudness. + shown = level > shown ? level : shown * 0.82 + level * 0.18 + bars.current.forEach((el, i) => { + if (el) el.style.transform = `scaleY(${Math.max(0.12, Math.min(1, shown * BARS[i] * 1.25))})` + }) + }) + }, [listening]) + + // Drawn over the session being dictated into, and for a message over + // whichever terminal is in front. + const mine = view.phase === 'idle' ? view.message !== null : view.sessionId === sessionId + if (!mine) return null + + const label = + view.phase === 'listening' ? 'Listening…' : view.phase === 'transcribing' ? 'Transcribing…' : view.message + + return ( +
+
+ {listening ? ( +
+
+ ) +} diff --git a/core/renderer/components/DictationTabMark.tsx b/core/renderer/components/DictationTabMark.tsx new file mode 100644 index 0000000..b53fb88 --- /dev/null +++ b/core/renderer/components/DictationTabMark.tsx @@ -0,0 +1,35 @@ +import type { JSX } from 'react' +import { useDictationView } from '../lib/useDictation' + +/** + * The mic mark on the tab being dictated into (#13). The pill says what + * dictation is doing; this says WHERE the words will land, which matters the + * moment you switch tabs mid-sentence: the paste still goes to the tab you + * started in, and this is how you can tell. + * + * It reads the dictation store itself, so a host's tab strip drops it in + * beside a label and passes nothing but the session: the strip does not + * re-render when someone starts to speak, only this does. + */ +export function DictationTabMark({ sessionId }: { sessionId: string }): JSX.Element | null { + const view = useDictationView() + if (view.phase === 'idle' || view.sessionId !== sessionId) return null + return ( + + + + + ) +} diff --git a/core/renderer/components/TerminalPanel.tsx b/core/renderer/components/TerminalPanel.tsx index 7e4a3f9..d23fc62 100644 --- a/core/renderer/components/TerminalPanel.tsx +++ b/core/renderer/components/TerminalPanel.tsx @@ -5,7 +5,7 @@ import { Unicode11Addon } from '@xterm/addon-unicode11' import { WebLinksAddon } from '@xterm/addon-web-links' import { SearchAddon } from '@xterm/addon-search' import { decidePaste } from '../lib/termPaste' -import { registerPaste, reportCwd, reportTitle } from '../lib/termBus' +import { registerPaste, reportCwd, reportTitle, setTextPaster } from '../lib/termBus' import { parseOsc9 } from '../../shared/termCwd' import { resolveTermTheme, watchTermTheme } from '../lib/termTheme' import { followsHostStyle, paintsGround, termApi, termHost } from '../host' @@ -222,6 +222,24 @@ export function focusTermSession(id: string): void { sessions.get(id)?.term.focus() } +/** + * Paste TEXT THE USER SPOKE into a session (#13), exactly as a Ctrl+V of text + * would arrive: xterm wraps it in the bracketed-paste escape when the shell + * asked for that, so an agent's prompt takes it as one paste. NEVER A NEWLINE: + * `paste` turns one into a carriage return, which is Enter, and dictation must + * never send a command. The caller cleans the text; this strips what is left + * anyway, because the rule is worth two locks. + */ +function pasteSpokenText(id: string, text: string): boolean { + const s = sessions.get(id) + const flat = text.replace(/[\r\n\u2028\u2029]+/g, ' ') + if (!s || !flat) return false + markTouched(id) + s.term.paste(flat) + return true +} +setTextPaster(pasteSpokenText) + /** Kill a session's renderer half: the xterm instance and its element. Main's * pty half is killed separately (term:kill) or already exited. */ export function disposeTermSession(id: string): void { diff --git a/core/renderer/host.ts b/core/renderer/host.ts index 274ac03..6a6c8f4 100644 --- a/core/renderer/host.ts +++ b/core/renderer/host.ts @@ -1,4 +1,5 @@ import type { DetectedAgent, ShellDef } from '../shared/types' +import type { DictationApi } from '../preload/dictationApi' /** * THE HOST SEAM (2026-09-19, #15). @@ -108,6 +109,20 @@ export interface TermHostConfig { * shell's, so claim little: plain Ctrl+W is delete-word in every readline. */ ownsKey(e: KeyboardEvent): boolean + + /** + * DICTATION (#13). Optional: a host that has not wired main's half simply + * leaves it out, and every dictation surface renders nothing. `api` is the + * bridge (`createDictationApi` in the host's preload). `canDictate` answers + * "is there a shell in front to paste into RIGHT NOW": always, given a tab, + * in Prism Terminal; only while the terminal panel is showing in Prism, + * where the same window is mostly a media viewer and Right Alt over a film + * must do nothing. + */ + dictation?: { + api: DictationApi + canDictate(): boolean + } } let host: TermHostConfig | null = null @@ -117,6 +132,11 @@ export function configureTermCore(config: TermHostConfig): void { host = config } +/** Null where the host has not wired dictation: callers render nothing. */ +export function dictationHost(): NonNullable | null { + return host?.dictation ?? null +} + export function termHost(): TermHostConfig { if (!host) throw new Error('prism-term-core: configureTermCore() has not been called by the host') return host diff --git a/core/renderer/lib/dictation.ts b/core/renderer/lib/dictation.ts new file mode 100644 index 0000000..06b8e8a --- /dev/null +++ b/core/renderer/lib/dictation.ts @@ -0,0 +1,338 @@ +import { dictationHost } from '../host' +import { cleanTranscript } from './dictationClean' +import { initialKeyState, reduceKey, type KeyEvt, type KeyState } from './dictationKey' +import { + dictationEnabled, + dictationHotkey, + dictationLanguage, + dictationMic, + dictationMode, + dictationModel, + dictationPauseMedia, + dictationSounds, + onDictationPrefs +} from './dictationPrefs' +import { cancelCue, startCue, stopCue } from './dictationSounds' +import { startCapture, TARGET_RATE, type Capture, type CaptureFailure } from './micCapture' +import { pasteTextInto } from './termBus' +import { encodeWav, tailSamples } from './wav' + +/** + * DICTATION'S CONTROLLER (#13): key -> microphone -> local engine -> paste. + * + * Outside React on purpose. The level meter moves at frame rate and the hook + * that arms this is mounted in each host's App: state held in React there + * would re-render the whole app thirty times a second while someone speaks. + * So this is a small store (`dictationView` + `onDictationView`) for the few + * things that change rarely, and a plain callback (`onDictationLevel`) for the + * one that changes constantly, which the pill writes straight to the DOM. + * + * THE RULES IT KEEPS: + * - Off means off: `armDictation` adds no listener until the setting is on, + * removes them when it goes off, and tells main to kill the server. + * - Live text is PROVISIONAL and only ever shown; the one thing pasted is the + * final pass over the whole clip. Text typed into a prompt cannot be taken + * back, and a partial heard wrong at 6 s is corrected by 8 s (measured). + * - The paste goes to the session that was in front when the recording + * STARTED: switching tabs mid-sentence must not drop your words into + * another agent's prompt. + * - Never Enter, never a newline (see the panel's `pasteSpokenText` and `cleanTranscript`). + */ +export type DictPhase = 'idle' | 'listening' | 'transcribing' + +export interface DictationView { + phase: DictPhase + /** The provisional text while listening. Shown, never pasted. */ + liveText: string + /** A short line for the pill when something needs saying; null otherwise. */ + message: string | null + /** The session being dictated into, for the mic mark on its tab. */ + sessionId: string | null +} + +const IDLE: DictationView = { phase: 'idle', liveText: '', message: null, sessionId: null } +let view: DictationView = IDLE +const viewListeners = new Set<() => void>() +const levelListeners = new Set<(level: number) => void>() + +export const dictationView = (): DictationView => view +export function onDictationView(l: () => void): () => void { + viewListeners.add(l) + return () => { + viewListeners.delete(l) + } +} +/** 0..1, many times a second while listening. Write it to the DOM, not to state. */ +export function onDictationLevel(l: (level: number) => void): () => void { + levelListeners.add(l) + return () => { + levelListeners.delete(l) + } +} +function setView(patch: Partial): void { + view = { ...view, ...patch } + viewListeners.forEach((l) => l()) +} + +/** A recording is capped (spec): nobody dictates for five minutes on purpose, + * and a key held down by a book must not fill memory. */ +const MAX_SECONDS = 300 +/** Whisper's window. A partial only ever needs the newest of it. */ +const PARTIAL_WINDOW_S = 28 +/** Below this there is nothing to transcribe: a brush of the key. */ +const MIN_SECONDS = 0.35 +const PARTIAL_EVERY_MS = 350 +const MESSAGE_MS = 2600 + +const FAILURE_TEXT: Record = { + 'no-device': 'No microphone found', + denied: 'Microphone access was refused', + unsupported: 'The microphone could not be opened', + failed: 'The microphone could not be opened' +} + +interface Recording { + sessionId: string + capture: Capture | null + /** Set when stop or cancel arrived before the mic finished opening. */ + ended: 'stop' | 'cancel' | null + partialTimer: number | null + partialBusy: boolean + lastPartialAt: number + mediaToken: Promise | null +} + +let rec: Recording | null = null +let messageTimer: number | null = null + +function say(message: string): void { + if (messageTimer !== null) clearTimeout(messageTimer) + setView({ ...IDLE, message }) + messageTimer = window.setTimeout(() => { + messageTimer = null + if (view.phase === 'idle') setView({ message: null }) + }, MESSAGE_MS) +} + +function resumeMedia(r: Recording): void { + const api = dictationHost()?.api + if (!api || !r.mediaToken) return + void r.mediaToken.then((t) => { + if (t) api.dictationMediaResume(t) + }) +} + +function endCapture(r: Recording): Float32Array { + if (r.partialTimer !== null) clearInterval(r.partialTimer) + r.partialTimer = null + const samples = r.capture?.snapshot() ?? new Float32Array(0) + r.capture?.stop() + levelListeners.forEach((l) => l(0)) + return samples +} + +async function begin(sessionId: string): Promise { + const host = dictationHost() + if (!host || rec) return + const modelId = dictationModel() + if (!modelId) { + say('Choose a model in Settings') + return + } + const r: Recording = { + sessionId, + capture: null, + ended: null, + partialTimer: null, + partialBusy: false, + lastPartialAt: 0, + mediaToken: dictationPauseMedia() ? host.api.dictationMediaPause().catch(() => '') : null + } + rec = r + if (messageTimer !== null) clearTimeout(messageTimer) + setView({ phase: 'listening', liveText: '', message: null, sessionId }) + if (dictationSounds()) void startCue() + + try { + r.capture = await startCapture({ + deviceId: dictationMic() || undefined, + onLevel: (level) => levelListeners.forEach((l) => l(level)) + }) + } catch (err) { + rec = null + resumeMedia(r) + say(FAILURE_TEXT[(typeof err === 'string' ? err : 'failed') as CaptureFailure] ?? FAILURE_TEXT.failed) + return + } + // The key was released (or Escape pressed) while the mic was still opening. + if (r.ended === 'cancel') return void finishCancel(r) + if (r.ended === 'stop') return void finishStop(r) + + r.partialTimer = window.setInterval(() => { + const cap = r.capture + if (!cap || rec !== r) return + if (cap.seconds() >= MAX_SECONDS) return void stop() + if (r.partialBusy || cap.seconds() < 1 || cap.seconds() - r.lastPartialAt < 0.6) return + r.partialBusy = true + r.lastPartialAt = cap.seconds() + const wav = encodeWav(tailSamples(cap.snapshot(), TARGET_RATE, PARTIAL_WINDOW_S), TARGET_RATE) + void host.api + .dictationTranscribe({ wav, modelId, language: dictationLanguage(), final: false }) + .then((res) => { + // Only while THIS recording is still listening: a partial that lands + // after the final would overwrite the pill with older words. + if (res.ok && rec === r && view.phase === 'listening') setView({ liveText: cleanTranscript(res.text) }) + }) + .catch(() => {}) + .finally(() => { + r.partialBusy = false + }) + }, PARTIAL_EVERY_MS) +} + +function finishCancel(r: Recording): void { + endCapture(r) + if (rec === r) rec = null + resumeMedia(r) + if (dictationSounds()) cancelCue() + setView(IDLE) +} + +async function finishStop(r: Recording): Promise { + const host = dictationHost() + const samples = endCapture(r) + if (dictationSounds()) void stopCue() + if (!host || samples.length < MIN_SECONDS * TARGET_RATE) { + if (rec === r) rec = null + resumeMedia(r) + setView(IDLE) + return + } + setView({ phase: 'transcribing' }) + let res + try { + res = await host.api.dictationTranscribe({ + wav: encodeWav(samples, TARGET_RATE), + modelId: dictationModel(), + language: dictationLanguage(), + final: true + }) + } catch { + res = { ok: false as const, reason: 'engine-failed' as const } + } + if (rec === r) rec = null + resumeMedia(r) + if (!res.ok) { + say( + res.reason === 'no-model' + ? 'Choose a model in Settings' + : res.reason === 'no-engine' + ? 'The speech engine is missing. Reinstall the app.' + : 'The speech engine stopped. Try again.' + ) + return + } + const text = cleanTranscript(res.text) + if (!text) return say('Heard nothing') + // A trailing space, so the next dictation (or the next typed word) does not + // run into this one. Never a newline: that is Enter. + if (!pasteTextInto(r.sessionId, text + ' ')) return say('That tab has closed') + setView(IDLE) +} + +function stop(): void { + const r = rec + if (!r) return + if (!r.capture) r.ended = 'stop' + else void finishStop(r) +} + +function cancel(): void { + const r = rec + if (!r) return + if (!r.capture) r.ended = 'cancel' + else finishCancel(r) +} + +/** + * Arm dictation for this window. `activeSession` answers, at the moment of a + * press, which shell is in front (null: none, so the press does nothing). + * Returns the disposer. Safe to call with dictation off: it then only watches + * the setting. + */ +export function armDictation(activeSession: () => string | null): () => void { + let keyState: KeyState = initialKeyState() + let tick: number | null = null + let armed = false + + const feed = (evt: KeyEvt, dom?: KeyboardEvent): void => { + const out = reduceKey(keyState, evt, { hotkey: dictationHotkey(), mode: dictationMode() }) + keyState = out.state + if (out.swallow && dom) { + // Capture phase, before xterm: the shell never sees the chord, and the + // Alt keyup never reaches the window's system menu. + dom.preventDefault() + dom.stopPropagation() + } + if (tick !== null) { + clearTimeout(tick) + tick = null + } + // The solo-hold rule needs a clock: nothing else happens while a key is + // simply held down. + if (keyState.phase === 'arming') tick = window.setTimeout(() => feed({ type: 'tick', at: performance.now() }), 210) + if (out.action === 'start') { + const host = dictationHost() + const id = host?.canDictate() ? activeSession() : null + if (id) void begin(id) + } else if (out.action === 'stop') stop() + else if (out.action === 'cancel') cancel() + } + + const onKey = (e: KeyboardEvent): void => + feed( + { + type: e.type === 'keydown' ? 'down' : 'up', + code: e.code, + ctrl: e.ctrlKey, + alt: e.altKey, + shift: e.shiftKey, + meta: e.metaKey, + repeat: e.repeat, + at: performance.now() + }, + e + ) + const onBlur = (): void => feed({ type: 'blur', at: performance.now() }) + + const sync = (): void => { + const want = dictationEnabled() && dictationHost() !== null + if (want === armed) return + armed = want + if (want) { + window.addEventListener('keydown', onKey, true) + window.addEventListener('keyup', onKey, true) + window.addEventListener('blur', onBlur) + } else { + window.removeEventListener('keydown', onKey, true) + window.removeEventListener('keyup', onKey, true) + window.removeEventListener('blur', onBlur) + keyState = initialKeyState() + cancel() + // Off means off: no server process may remain. + dictationHost()?.api.dictationStop() + } + } + sync() + const unsub = onDictationPrefs(sync) + return () => { + unsub() + if (tick !== null) clearTimeout(tick) + if (armed) { + window.removeEventListener('keydown', onKey, true) + window.removeEventListener('keyup', onKey, true) + window.removeEventListener('blur', onBlur) + cancel() + } + } +} diff --git a/core/renderer/lib/dictationClean.test.ts b/core/renderer/lib/dictationClean.test.ts new file mode 100644 index 0000000..6e84981 --- /dev/null +++ b/core/renderer/lib/dictationClean.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from 'vitest' +import { cleanTranscript } from './dictationClean' + +describe('what Whisper wrote is kept', () => { + it('leaves punctuation and capitalisation alone', () => { + expect(cleanTranscript(' Ask not what your country can do for you.')).toBe( + 'Ask not what your country can do for you.' + ) + expect(cleanTranscript("Wait, really? Yes! It's 3.5 GB; see: the log.")).toBe( + "Wait, really? Yes! It's 3.5 GB; see: the log." + ) + }) + it('does not capitalise a sentence Whisper left lowercase', () => { + expect(cleanTranscript('git status')).toBe('git status') + }) + it('keeps an ellipsis', () => { + expect(cleanTranscript('Well... maybe.')).toBe('Well... maybe.') + }) +}) + +describe('engine artifacts', () => { + it.each([ + '[BLANK_AUDIO]', + '[ Silence ]', + '[silence]', + '(music)', + '[Music]', + '[MUSIC PLAYING]', + '(applause)', + '[inaudible]', + '[NO_SPEECH]', + '(upbeat music)', + '*sighs*', + '*clears throat*', + '♪', + '♪♪ ♫', + '[ ]', + '(...)', + '[BLANK_AUDIO] [BLANK_AUDIO]', + '[BLANK_AUDIO].' + ])('%s alone is nothing', (raw) => { + expect(cleanTranscript(raw)).toBe('') + }) + + it('takes them out of the middle of speech', () => { + expect(cleanTranscript('Run the tests [BLANK_AUDIO] and then push.')).toBe('Run the tests and then push.') + expect(cleanTranscript('Hello (music) world.')).toBe('Hello world.') + expect(cleanTranscript('Okay *sighs* fine.')).toBe('Okay fine.') + expect(cleanTranscript('♪ Happy birthday to you ♪')).toBe('Happy birthday to you') + }) + + it('does not strip parentheses a speaker dictated around real words', () => { + expect(cleanTranscript('Add a note (see the README) at the end.')).toBe('Add a note (see the README) at the end.') + expect(cleanTranscript('Index [0] and step (1).')).toBe('Index [0] and step (1).') + expect(cleanTranscript('The (music) folder is separate from (my music library).')).toBe( + 'The folder is separate from (my music library).' + ) + }) + + it('does not take a multiplication for a stage direction', () => { + expect(cleanTranscript('2 * 3 * 4')).toBe('2 * 3 * 4') + expect(cleanTranscript('Match *.ts and *.tsx files')).toBe('Match *.ts and *.tsx files') + }) + + it('unwraps a nest of them', () => { + expect(cleanTranscript('[ [music] ] hi')).toBe('hi') + }) +}) + +describe('fillers', () => { + it('removes um, uh, erm, uhm and hmm as whole words, with the comma that follows', () => { + expect(cleanTranscript('I think, um, that works.')).toBe('I think, that works.') + expect(cleanTranscript('So uh we ship it erm today uhm yes hmm.')).toBe('So we ship it today yes.') + expect(cleanTranscript('It is UM, fine.')).toBe('It is fine.') + expect(cleanTranscript('It is ummm, fine, uhh.')).toBe('It is fine.') + }) + it('does not touch words that contain them', () => { + const s = 'The umbrella will hum; Uhura hummed, the drummer erred.' + expect(cleanTranscript(s)).toBe(s) + }) + it('leaves a hyphenated word whole', () => { + expect(cleanTranscript('He said uh-huh twice.')).toBe('He said uh-huh twice.') + }) + it('capitalises a sentence its filler used to lead', () => { + expect(cleanTranscript('Um, so this is the plan.')).toBe('So this is the plan.') + expect(cleanTranscript('uh list the files')).toBe('List the files') + expect(cleanTranscript('Um. Uh, okay then.')).toBe('Okay then.') + expect(cleanTranscript('Done. Um, next one.')).toBe('Done. Next one.') + expect(cleanTranscript('[BLANK_AUDIO] Um, hello.')).toBe('Hello.') + }) + it('leaves no orphaned punctuation behind', () => { + expect(cleanTranscript('Right. Um. Okay.')).toBe('Right. Okay.') + expect(cleanTranscript('That works, uh.')).toBe('That works.') + expect(cleanTranscript('Um.')).toBe('') + expect(cleanTranscript('Um, uh, erm...')).toBe('') + }) +}) + +describe('doubled words', () => { + it('collapses an immediate double, keeping the first', () => { + expect(cleanTranscript('Open the the file.')).toBe('Open the file.') + expect(cleanTranscript('The the file.')).toBe('The file.') + expect(cleanTranscript('I I I think so.')).toBe('I think so.') + expect(cleanTranscript('go to to the the root')).toBe('go to the root') + }) + it('collapses one a filler was hiding', () => { + expect(cleanTranscript('Open the, um, the file.')).toBe('Open the, the file.') + expect(cleanTranscript('Open the um the file.')).toBe('Open the file.') + }) + it('never touches digits', () => { + expect(cleanTranscript('Version 2 2 and 10 10.')).toBe('Version 2 2 and 10 10.') + }) + it('keeps the pairs people say on purpose', () => { + const s = 'I know that that is what he had had. No no, bye bye.' + expect(cleanTranscript(s)).toBe(s) + }) + it('does not see a double inside longer words', () => { + expect(cleanTranscript('the theme, an answer, is isolated')).toBe('the theme, an answer, is isolated') + }) + it('works past ASCII', () => { + expect(cleanTranscript('Jeg så så filen og og åpnet den.')).toBe('Jeg så filen og åpnet den.') + }) + it('does not reach across a full stop or a comma', () => { + expect(cleanTranscript('Yes. Yes, it is.')).toBe('Yes. Yes, it is.') + }) +}) + +describe('whitespace', () => { + it('turns segment breaks and tabs into single spaces, and trims', () => { + expect(cleanTranscript(' First line.\n Second line.\r\n\tThird. ')).toBe('First line. Second line. Third.') + }) + it('removes a space before punctuation', () => { + expect(cleanTranscript('Hello , world ! Is it ? Yes ; here : now .')).toBe('Hello, world! Is it? Yes; here: now.') + }) + it('does not glue a dotfile or a relative path to the word before it', () => { + expect(cleanTranscript('open .gitignore and cd ..')).toBe('open .gitignore and cd ..') + expect(cleanTranscript('.gitignore is listed')).toBe('.gitignore is listed') + expect(cleanTranscript('wait ... what')).toBe('wait... what') + }) + it('returns nothing for nothing', () => { + expect(cleanTranscript('')).toBe('') + expect(cleanTranscript(' \n\t ')).toBe('') + expect(cleanTranscript('...')).toBe('') + }) +}) + +describe('the app never sends Enter', () => { + // Not a fixed list alone: a seeded generator glues the nastiest pieces together + // in orders nobody wrote down, so the rule is tested as a property. + const PIECES = [ + '\n', '\r', '\r\n', '\n\n\n', '\t', '\v', '\f', '\u2028', '\u2029', '\u0085', '\u001b[201~', '\u0000', '\u0007', + '\u00a0', ' ', ' ', 'um', 'Um,', 'uh', '[BLANK_AUDIO]', '(music)', '[', ']', '(', ')', '*', '*sighs*', '♪', + 'the', 'the', 'The', 'word', '.', ',', '!', '?', ';', ':', '...', 'ls -la', 'rm -rf /', '&&', '"', "'", 'å', '日本語', + '\ud83c\udfb5', '\ud83d', 'x'.repeat(40) + ] + function* nasty(count: number): Generator { + let seed = 0x2f6e2b1 + const next = (): number => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0 + return seed + } + for (let i = 0; i < count; i += 1) { + const n = 1 + (next() % 12) + let s = '' + for (let j = 0; j < n; j += 1) s += PIECES[next() % PIECES.length] + (next() % 3 === 0 ? '' : ' ') + yield s + } + } + + it('never returns a line break, a control character or an untrimmed edge', () => { + const inputs = [...PIECES, PIECES.join(''), PIECES.join('\n'), ...nasty(3000)] + for (const raw of inputs) { + const out = cleanTranscript(raw) + const where = JSON.stringify(raw) + expect(/[\n\r\u2028\u2029\u0085\v\f]/.test(out), where).toBe(false) + expect([...out].some((c) => c.charCodeAt(0) < 0x20 || c.charCodeAt(0) === 0x7f), where).toBe(false) + expect(out, where).toBe(out.trim()) + expect(/ {2}/.test(out), where).toBe(false) + } + }) + + it('is stable: cleaning a cleaned text changes nothing', () => { + for (const raw of nasty(1500)) { + const once = cleanTranscript(raw) + expect(cleanTranscript(once), JSON.stringify(raw)).toBe(once) + } + }) + + it('cannot be made to close the bracketed paste it travels in', () => { + expect(cleanTranscript('echo hi\u001b[201~\nrm -rf x')).not.toContain('\u001b') + }) +}) diff --git a/core/renderer/lib/dictationClean.ts b/core/renderer/lib/dictationClean.ts new file mode 100644 index 0000000..7d61685 --- /dev/null +++ b/core/renderer/lib/dictationClean.ts @@ -0,0 +1,174 @@ +/** + * What is done to Whisper's text before it is pasted (#13). Deterministic only + * (owner's decision: no second model), and deliberately timid: text typed into + * a prompt cannot be taken back, so every rule here removes something that is + * certainly not speech, and anything that MIGHT be what the user said is kept. + * Whisper's own punctuation and capitalisation are never rewritten. + * + * ONE RULE IS NOT ABOUT TIDINESS: the result never carries a line break or any + * other control character. Dictation is a written exception to "the app never + * types into the user's shell" on the condition that it never presses Enter, + * and the text travels as a bracketed paste, which an ESC in the text could + * close early. So control characters go first, and the last thing done is a + * whitespace collapse that leaves nothing but single spaces. + */ + +/** Inside [ ], ( ) or * *, lowercased, with _ and - read as spaces. Whisper + * writes these for sound that is not speech; nobody dictates them in brackets. */ +const ARTIFACTS = new Set([ + 'blank audio', + 'silence', + 'no speech', + 'no audio', + 'inaudible', + 'unintelligible', + 'indistinct', + 'crosstalk', + 'music', + 'music playing', + 'applause', + 'clapping', + 'cheering', + 'laughter', + 'laughs', + 'laughing', + 'sighs', + 'sigh', + 'coughs', + 'cough', + 'coughing', + 'sniffs', + 'gasps', + 'groans', + 'breathing', + 'clears throat', + 'noise', + 'background noise', + 'static', + 'beep', + 'click', + 'clicking', + 'typing', + 'keyboard clicking', + 'wind', + 'footsteps', + 'pause' +]) +/** "(upbeat music)", "[dramatic music playing]": one describing word, then music. */ +const MUSIC_CUE = /^(?:[a-z]+ )?music(?: playing| plays| continues| fades)?$/ + +const SQUARE = /\[([^[\]]*)\]/g +const ROUND = /\(([^()]*)\)/g +/** + * A stage direction: one to three plain words hard against their asterisks, + * "*sighs*", "*clears throat*". More liberal than the bracket rule because + * speech never produces a pair of asterisks, while "(music)" could be dictated; + * and still shaped so that "2 * 3 * 4" and "*.ts and *.tsx" are left alone + * (their content has a space at the edge, a digit or a dot). + */ +const STAGE_CUE = /\*[a-z]+(?: [a-z]+){0,2}\*/gi +/** The notes Whisper brackets singing with. Sharps and flats are not in it. */ +const NOTES = /[\u2669-\u266c\u{1f3b5}\u{1f3b6}\u{1f3bc}]/gu + +/** um, umm, uh, uhh, uhm, erm, hm, hmm, in any case and however long drawn. */ +const FILLER = "(?:u+m+|u+h+m*|e+r+m+|h+m+)" +/** What makes the filler a WORD: nothing word-like against it on either side. + * Unicode-aware on purpose (\b is ASCII, and would cut "håum" at the å), and a + * hyphen or an apostrophe counts, so "uh-huh" is one word and stays. */ +const NOT_AFTER = "(?= 0x7f && c <= 0x9f) || c === 0x2028 || c === 0x2029 ? ' ' : ch + } + return out +} + +function isArtifact(content: string): boolean { + if (!HAS_WORD.test(content)) return true // "[ ]", "(...)": noise, not words + const norm = content.toLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim() + return ARTIFACTS.has(norm) || MUSIC_CUE.test(norm) +} + +const dropArtifact = (whole: string, content: string): string => (isArtifact(content) ? ' ' : whole) + +function pass(input: string): string { + let text = input.replace(NOTES, ' ').replace(STAGE_CUE, ' ').replace(SQUARE, dropArtifact).replace(ROUND, dropArtifact) + + // One at a time from the front: each removal can put the NEXT filler at the + // head of the sentence ("Um, uh, okay"), which a single global pass walks past. + for (let prev = ''; prev !== text; ) { + prev = text + text = text.replace(LEADING_FILLER, (_m, lead: string, letter?: string) => lead + (letter ?? '').toUpperCase()) + } + text = text.replace(FILLER_WORD, '') + + // To a standstill, since a global pass pairs "I I I" as (I I) I and stops. + for (let prev = ''; prev !== text; ) { + prev = text + text = text.replace(DOUBLED, (m, word: string) => (DELIBERATE_PAIRS.has(word.toLowerCase()) ? m : word)) + } + + for (let prev = ''; prev !== text; ) { + prev = text + text = text.replace(STRANDED, (_m, a: string, b: string) => (a === ',' ? b : a)) + } + return text.replace(SPACE_BEFORE_MARK, '$1').replace(LEADING_MARKS, '').replace(/\s+/g, ' ').trim() +} + +/** + * Whisper's raw text in, the text to paste out. Returns '' when nothing that was + * said is left (silence, a lone "[BLANK_AUDIO]", a lone "Um."), which the caller + * shows as "Heard nothing" and pastes nothing for. + */ +export function cleanTranscript(raw: string): string { + let text = withoutControls(raw).replace(/\s+/g, ' ').trim() + // A removal can uncover the next one: "[um]" is a word in brackets until the + // filler goes, and an empty bracket after. Run to a standstill, so cleaning a + // cleaned text changes nothing; the cap is only there so a bug cannot hang a paste. + for (let i = 0; i < 8; i += 1) { + const next = pass(text) + if (next === text) break + text = next + } + return HAS_WORD.test(text) ? text : '' +} diff --git a/core/renderer/lib/dictationKey.test.ts b/core/renderer/lib/dictationKey.test.ts new file mode 100644 index 0000000..2e2dd99 --- /dev/null +++ b/core/renderer/lib/dictationKey.test.ts @@ -0,0 +1,464 @@ +import { describe, expect, it } from 'vitest' +import { + DEFAULT_HOTKEY, + formatHotkey, + initialKeyState, + isHotkeyEvent, + keyEvtFromDom, + parseHotkeyFromEvent, + reduceKey, + type DictMode, + type Hotkey, + type KeyAction, + type KeyEvt, + type KeyState +} from './dictationKey' + +type Mods = Partial> & { repeat?: boolean } + +const key = (type: 'down' | 'up', code: string, at: number, m: Mods = {}): KeyEvt => ({ + type, + code, + at, + ctrl: m.ctrl ?? false, + alt: m.alt ?? false, + shift: m.shift ?? false, + meta: m.meta ?? false, + repeat: m.repeat +}) +const down = (code: string, at: number, m?: Mods): KeyEvt => key('down', code, at, m) +const up = (code: string, at: number, m?: Mods): KeyEvt => key('up', code, at, m) +const tick = (at: number): KeyEvt => ({ type: 'tick', at }) +const blur = (at: number): KeyEvt => ({ type: 'blur', at }) + +/** Right Alt as a US layout reports it: the key carries its own alt flag. */ +const raltDown = (at: number, m: Mods = {}): KeyEvt => down('AltRight', at, { alt: true, ...m }) +const raltUp = (at: number): KeyEvt => up('AltRight', at) + +interface Step { + action: KeyAction + swallow: boolean + phase: KeyState['phase'] +} + +/** Feed a whole sequence through the reducer and keep what each event answered. */ +function run( + events: KeyEvt[], + mode: DictMode, + hotkey: Hotkey = DEFAULT_HOTKEY, + holdMs?: number +): { steps: Step[]; actions: KeyAction[]; state: KeyState } { + let state = initialKeyState() + const steps: Step[] = [] + for (const evt of events) { + const r = reduceKey(state, evt, { hotkey, mode, holdMs }) + state = r.state + steps.push({ action: r.action, swallow: r.swallow, phase: r.state.phase }) + } + return { steps, actions: steps.map((s) => s.action).filter((a) => a !== null), state } +} + +const CHORD: Hotkey = { code: 'KeyD', ctrl: true, alt: false, shift: true, meta: false } +const F9: Hotkey = { code: 'F9', ctrl: false, alt: false, shift: false, meta: false } + +describe('the default', () => { + it('is a bare Right Alt', () => { + expect(DEFAULT_HOTKEY).toEqual({ code: 'AltRight', ctrl: false, alt: false, shift: false, meta: false }) + }) + it('starts idle', () => { + expect(initialKeyState().phase).toBe('idle') + }) +}) + +describe('hold mode, bare modifier', () => { + it('a clean 250 ms hold starts, then stops on release', () => { + const r = run([raltDown(0), tick(100), tick(200), raltUp(250)], 'hold') + expect(r.steps.map((s) => s.phase)).toEqual(['arming', 'arming', 'listening', 'idle']) + expect(r.actions).toEqual(['start', 'stop']) + }) + + it('starts on the tick at exactly holdMs, and not one millisecond sooner', () => { + const r = run([raltDown(1000), tick(1199), tick(1200)], 'hold') + expect(r.steps.map((s) => s.action)).toEqual([null, null, 'start']) + }) + + it('honours a holdMs of its own', () => { + const r = run([raltDown(0), tick(200), tick(350)], 'hold', DEFAULT_HOTKEY, 350) + expect(r.steps.map((s) => s.action)).toEqual([null, null, 'start']) + }) + + it('the Norwegian AltGr sequence types an @ and never starts', () => { + // Windows reports AltGr as ControlLeft then AltRight, both flags up on the second. + const t = 5000 + const r = run( + [ + down('ControlLeft', t, { ctrl: true }), + down('AltRight', t + 1, { ctrl: true, alt: true }), + tick(t + 50), + down('Digit2', t + 80, { ctrl: true, alt: true }), + up('Digit2', t + 140, { ctrl: true, alt: true }), + tick(t + 300), + up('ControlLeft', t + 320, { alt: true }), + up('AltRight', t + 321), + tick(t + 600) + ], + 'hold' + ) + expect(r.actions).toEqual([]) + expect(r.state.phase).toBe('idle') + // The @ itself must reach the shell: neither half of Digit2 is swallowed. + expect(r.steps[3].swallow).toBe(false) + expect(r.steps[4].swallow).toBe(false) + // The synthetic Ctrl is not the hotkey either. + expect(r.steps[0].swallow).toBe(false) + }) + + it('AltGr held ALONE on that layout still dictates: its own Ctrl is not another key', () => { + const r = run( + [ + down('ControlLeft', 0, { ctrl: true }), + down('AltRight', 1, { ctrl: true, alt: true }), + // Auto-repeat of the pair while it is held. + down('ControlLeft', 40, { ctrl: true, alt: true, repeat: true }), + down('AltRight', 41, { ctrl: true, alt: true, repeat: true }), + down('ControlLeft', 120, { ctrl: true, alt: true, repeat: true }), + tick(201), + up('ControlLeft', 900, { alt: true }), + up('AltRight', 901) + ], + 'hold' + ) + expect(r.actions).toEqual(['start', 'stop']) + }) + + it('a synthetic Ctrl landing just AFTER Right Alt is forgiven too', () => { + const r = run([raltDown(0), down('ControlLeft', 3, { ctrl: true, alt: true }), tick(200)], 'hold') + expect(r.actions).toEqual(['start']) + }) + + it('a real Left Ctrl pressed well into the arming cancels it', () => { + const r = run([raltDown(0), down('ControlLeft', 120, { ctrl: true, alt: true }), tick(200), tick(400)], 'hold') + expect(r.actions).toEqual([]) + expect(r.state.phase).toBe('idle') + }) + + it('another key during arming cancels silently, and that key is not swallowed', () => { + const r = run([raltDown(0), down('KeyQ', 90, { alt: true }), tick(200), tick(500), raltUp(600)], 'hold') + expect(r.actions).toEqual([]) + expect(r.steps[1]).toEqual({ action: null, swallow: false, phase: 'idle' }) + // The hotkey's own up is still ours, cancelled or not. + expect(r.steps[4].swallow).toBe(true) + }) + + it('does not re-arm from the auto-repeat of a press that was cancelled', () => { + const r = run( + [raltDown(0), down('KeyQ', 50, { alt: true }), raltDown(80, { repeat: true }), raltDown(110), tick(400)], + 'hold' + ) + expect(r.actions).toEqual([]) + expect(r.state.phase).toBe('idle') + expect(r.steps[2].swallow).toBe(true) + }) + + it('release during arming goes back to idle with no action', () => { + const r = run([raltDown(0), raltUp(120), tick(300)], 'hold') + expect(r.actions).toEqual([]) + expect(r.steps[1]).toEqual({ action: null, swallow: true, phase: 'idle' }) + }) + + it('a release nobody ticked before is still only a release', () => { + // The host's timer never fired, so nothing was started and nothing may stop. + const r = run([raltDown(0), raltUp(400)], 'hold') + expect(r.actions).toEqual([]) + }) + + it('auto-repeat of the hotkey is swallowed, changes nothing, and can carry the start', () => { + const r = run([raltDown(0), raltDown(30, { repeat: true }), raltDown(230, { repeat: true }), raltUp(500)], 'hold') + expect(r.steps.map((s) => s.action)).toEqual([null, null, 'start', 'stop']) + expect(r.steps.every((s) => s.swallow)).toBe(true) + }) + + it('another key while LISTENING does not cancel, and is not swallowed', () => { + const r = run([raltDown(0), tick(200), down('KeyJ', 900, { alt: true }), up('KeyJ', 950), raltUp(1500)], 'hold') + expect(r.actions).toEqual(['start', 'stop']) + expect(r.steps[2]).toEqual({ action: null, swallow: false, phase: 'listening' }) + }) + + it('Escape while listening cancels and is swallowed, down and up', () => { + const r = run([raltDown(0), tick(200), down('Escape', 700, { alt: true }), up('Escape', 760), raltUp(900)], 'hold') + expect(r.actions).toEqual(['start', 'cancel']) + expect(r.steps[2]).toEqual({ action: 'cancel', swallow: true, phase: 'idle' }) + expect(r.steps[3].swallow).toBe(true) + // The key is still down after the cancel: its up is ours and stops nothing. + expect(r.steps[4]).toEqual({ action: null, swallow: true, phase: 'idle' }) + }) + + it('Escape when nothing is listening is an ordinary key', () => { + const r = run([down('Escape', 0), up('Escape', 50)], 'hold') + expect(r.steps.every((s) => !s.swallow && s.action === null)).toBe(true) + }) + + it('blur while listening cancels', () => { + const r = run([raltDown(0), tick(200), blur(800)], 'hold') + expect(r.actions).toEqual(['start', 'cancel']) + expect(r.state.phase).toBe('idle') + }) + + it('blur while arming goes to idle without a word', () => { + const r = run([raltDown(0), blur(100), tick(300)], 'hold') + expect(r.actions).toEqual([]) + expect(r.state.phase).toBe('idle') + }) + + it('an up the reducer never saw the down of is not swallowed', () => { + const r = run([raltUp(10)], 'hold') + expect(r.steps[0]).toEqual({ action: null, swallow: false, phase: 'idle' }) + }) + + it('Shift already down means it is not the solo hotkey', () => { + const r = run([down('ShiftLeft', 0, { shift: true }), raltDown(20, { shift: true }), tick(400)], 'hold') + expect(r.actions).toEqual([]) + expect(r.steps[1].swallow).toBe(false) + }) + + it('ticks while idle do nothing', () => { + const r = run([tick(0), tick(1000)], 'hold') + expect(r.steps.every((s) => s.action === null && !s.swallow && s.phase === 'idle')).toBe(true) + }) +}) + +describe('toggle mode, bare modifier', () => { + it('a solo tap starts and the next solo tap stops', () => { + const r = run([raltDown(0), raltUp(90), down('KeyA', 500), up('KeyA', 560), raltDown(3000), raltUp(3120)], 'toggle') + expect(r.steps.map((s) => s.action)).toEqual([null, 'start', null, null, null, 'stop']) + expect(r.steps.map((s) => s.phase)).toEqual(['arming', 'listening', 'listening', 'listening', 'listening', 'idle']) + // Typing while it listens goes to the shell. + expect(r.steps[2].swallow).toBe(false) + expect([0, 1, 4, 5].every((i) => r.steps[i].swallow)).toBe(true) + }) + + it('a slow tap still counts, up to a second', () => { + expect(run([raltDown(0), raltUp(999)], 'toggle').actions).toEqual(['start']) + expect(run([raltDown(0), raltUp(1000)], 'toggle').actions).toEqual([]) + }) + + it('a tap polluted by another key does nothing (AltGr typing)', () => { + const r = run( + [ + down('ControlLeft', 0, { ctrl: true }), + down('AltRight', 1, { ctrl: true, alt: true }), + down('Digit2', 60, { ctrl: true, alt: true }), + up('Digit2', 110, { ctrl: true, alt: true }), + up('ControlLeft', 150, { alt: true }), + up('AltRight', 151) + ], + 'toggle' + ) + expect(r.actions).toEqual([]) + expect(r.state.phase).toBe('idle') + }) + + it('a solo AltGr tap on that layout is a tap', () => { + const r = run( + [ + down('ControlLeft', 0, { ctrl: true }), + down('AltRight', 1, { ctrl: true, alt: true }), + up('ControlLeft', 80, { alt: true }), + up('AltRight', 81) + ], + 'toggle' + ) + expect(r.actions).toEqual(['start']) + }) + + it('a polluted tap while listening does not stop it', () => { + const r = run( + [raltDown(0), raltUp(80), raltDown(1000), down('KeyQ', 1050, { alt: true }), up('KeyQ', 1100), raltUp(1150)], + 'toggle' + ) + expect(r.actions).toEqual(['start']) + expect(r.state.phase).toBe('listening') + }) + + it('never starts from holding the key down', () => { + const r = run([raltDown(0), tick(200), tick(800), tick(1200), raltUp(1500)], 'toggle') + expect(r.actions).toEqual([]) + expect(r.state.phase).toBe('idle') + }) + + it('Escape cancels, and an Escape pressed mid-tap does not let the tap start again', () => { + const r = run([raltDown(0), raltUp(50), raltDown(900), down('Escape', 950, { alt: true }), raltUp(1000)], 'toggle') + expect(r.actions).toEqual(['start', 'cancel']) + expect(r.state.phase).toBe('idle') + }) + + it('blur cancels a listening toggle', () => { + const r = run([raltDown(0), raltUp(50), blur(400)], 'toggle') + expect(r.actions).toEqual(['start', 'cancel']) + }) +}) + +describe('a chord or a plain key', () => { + it('hold: starts on the down at once and stops on the up of the main key', () => { + const r = run( + [ + down('ControlLeft', 0, { ctrl: true }), + down('ShiftLeft', 10, { ctrl: true, shift: true }), + down('KeyD', 20, { ctrl: true, shift: true }), + down('KeyD', 60, { ctrl: true, shift: true, repeat: true }), + // The modifiers may let go first: the main key is what is being held. + up('ShiftLeft', 400, { ctrl: true }), + down('KeyD', 420, { ctrl: true, repeat: true }), + up('KeyD', 900, { ctrl: true }), + up('ControlLeft', 950) + ], + 'hold', + CHORD + ) + expect(r.steps.map((s) => s.action)).toEqual([null, null, 'start', null, null, null, 'stop', null]) + expect(r.steps.map((s) => s.swallow)).toEqual([false, false, true, true, false, true, true, false]) + }) + + it('hold: a plain F9 needs no holdMs and no solo rule', () => { + const r = run([down('F9', 0), down('KeyA', 5), up('F9', 40)], 'hold', F9) + expect(r.actions).toEqual(['start', 'stop']) + }) + + it('toggle: each non-repeat down toggles, repeats and ups do nothing', () => { + const r = run( + [down('F9', 0), down('F9', 40, { repeat: true }), up('F9', 90), down('F9', 2000), up('F9', 2050)], + 'toggle', + F9 + ) + expect(r.steps.map((s) => s.action)).toEqual(['start', null, null, 'stop', null]) + expect(r.steps.every((s) => s.swallow)).toBe(true) + }) + + it('the main key with the wrong modifiers is somebody else\'s key', () => { + const r = run([down('KeyD', 0, { ctrl: true }), down('F9', 10, { shift: true })], 'hold', CHORD) + expect(r.steps.every((s) => s.action === null && !s.swallow)).toBe(true) + }) + + it('Escape and blur cancel here too', () => { + expect(run([down('F9', 0), down('Escape', 300)], 'hold', F9).actions).toEqual(['start', 'cancel']) + expect(run([down('F9', 0), up('F9', 40), blur(300)], 'toggle', F9).actions).toEqual(['start', 'cancel']) + }) + + it('hold: after an Escape the still-held key stops nothing on its way up', () => { + const r = run([down('F9', 0), down('Escape', 300), down('F9', 330, { repeat: true }), up('F9', 600)], 'hold', F9) + expect(r.actions).toEqual(['start', 'cancel']) + expect(r.steps[3]).toEqual({ action: null, swallow: true, phase: 'idle' }) + }) +}) + +describe('isHotkeyEvent', () => { + it('a bare modifier matches itself, its own flag notwithstanding', () => { + expect(isHotkeyEvent(raltDown(0), DEFAULT_HOTKEY)).toBe(true) + expect(isHotkeyEvent(down('AltRight', 0, { ctrl: true, alt: true }), DEFAULT_HOTKEY)).toBe(true) + expect(isHotkeyEvent(down('AltLeft', 0, { alt: true }), DEFAULT_HOTKEY)).toBe(false) + expect(isHotkeyEvent(raltDown(0, { shift: true }), DEFAULT_HOTKEY)).toBe(false) + }) + it('only Right Alt is forgiven a Ctrl: that is AltGr, and no other key has one', () => { + const lalt: Hotkey = { ...DEFAULT_HOTKEY, code: 'AltLeft' } + expect(isHotkeyEvent(down('AltLeft', 0, { ctrl: true, alt: true }), lalt)).toBe(false) + }) + it('an up matches on the code alone, whatever is still held', () => { + expect(isHotkeyEvent(up('AltRight', 0, { ctrl: true }), DEFAULT_HOTKEY)).toBe(true) + expect(isHotkeyEvent(up('KeyD', 0), CHORD)).toBe(true) + }) + it('a chord matches its modifiers exactly', () => { + expect(isHotkeyEvent(down('KeyD', 0, { ctrl: true, shift: true }), CHORD)).toBe(true) + expect(isHotkeyEvent(down('KeyD', 0, { ctrl: true }), CHORD)).toBe(false) + expect(isHotkeyEvent(down('KeyD', 0, { ctrl: true, shift: true, alt: true }), CHORD)).toBe(false) + }) + it('a tick and a blur are nobody\'s key', () => { + expect(isHotkeyEvent(tick(0), DEFAULT_HOTKEY)).toBe(false) + expect(isHotkeyEvent(blur(0), DEFAULT_HOTKEY)).toBe(false) + }) +}) + +describe('formatHotkey', () => { + const h = (code: string, m: Mods = {}): Hotkey => ({ + code, + ctrl: m.ctrl ?? false, + alt: m.alt ?? false, + shift: m.shift ?? false, + meta: m.meta ?? false + }) + it('names a bare modifier by its side', () => { + expect(formatHotkey(DEFAULT_HOTKEY)).toBe('Right Alt') + expect(formatHotkey(h('ControlLeft'))).toBe('Left Ctrl') + expect(formatHotkey(h('ShiftRight'))).toBe('Right Shift') + expect(formatHotkey(h('MetaLeft'))).toBe('Left Win') + }) + it('writes a chord the way Windows does', () => { + expect(formatHotkey(CHORD)).toBe('Ctrl+Shift+D') + expect(formatHotkey(h('Digit2', { ctrl: true, alt: true }))).toBe('Ctrl+Alt+2') + expect(formatHotkey(h('F9', { meta: true }))).toBe('Win+F9') + }) + it('names the keys that are not letters', () => { + expect(formatHotkey(F9)).toBe('F9') + expect(formatHotkey(h('Space', { ctrl: true }))).toBe('Ctrl+Space') + expect(formatHotkey(h('Backquote', { ctrl: true }))).toBe('Ctrl+`') + expect(formatHotkey(h('ArrowUp'))).toBe('Up') + expect(formatHotkey(h('Numpad5'))).toBe('Num 5') + expect(formatHotkey(h('NumpadAdd'))).toBe('Num +') + expect(formatHotkey(h('Pause'))).toBe('Pause') + }) +}) + +describe('parseHotkeyFromEvent', () => { + it('a bare modifier press is that modifier, with its own flag dropped', () => { + expect(parseHotkeyFromEvent(raltDown(0))).toEqual(DEFAULT_HOTKEY) + expect(parseHotkeyFromEvent(down('ControlLeft', 0, { ctrl: true }))).toEqual({ + code: 'ControlLeft', + ctrl: false, + alt: false, + shift: false, + meta: false + }) + }) + it('AltGr is read as Right Alt, not as Ctrl+Right Alt', () => { + expect(parseHotkeyFromEvent(down('AltRight', 0, { ctrl: true, alt: true }))).toEqual(DEFAULT_HOTKEY) + }) + it('a chord keeps its modifiers', () => { + expect(parseHotkeyFromEvent(down('KeyD', 0, { ctrl: true, shift: true }))).toEqual(CHORD) + expect(parseHotkeyFromEvent(down('F9', 0))).toEqual(F9) + }) + it('Escape is the way out of the rebind control, never a binding', () => { + expect(parseHotkeyFromEvent(down('Escape', 0))).toBeNull() + expect(parseHotkeyFromEvent(down('Escape', 0, { ctrl: true }))).toBeNull() + }) + it('a key the browser could not name is no binding', () => { + expect(parseHotkeyFromEvent(down('', 0))).toBeNull() + expect(parseHotkeyFromEvent(down('Unidentified', 0))).toBeNull() + }) + it('takes a DOM keyboard event as it comes', () => { + const dom = { type: 'keydown', code: 'KeyD', ctrlKey: true, altKey: false, shiftKey: true, metaKey: false } + expect(parseHotkeyFromEvent(dom)).toEqual(CHORD) + const altGr = { type: 'keydown', code: 'AltRight', ctrlKey: true, altKey: true, shiftKey: false, metaKey: false } + expect(parseHotkeyFromEvent(altGr)).toEqual(DEFAULT_HOTKEY) + expect(parseHotkeyFromEvent({ ...dom, code: 'Escape' })).toBeNull() + }) + it('round-trips through the reducer: what was bound is what is recognised', () => { + const bound = parseHotkeyFromEvent(down('KeyD', 0, { ctrl: true, shift: true })) + expect(bound).not.toBeNull() + expect(run([down('KeyD', 0, { ctrl: true, shift: true })], 'toggle', bound as Hotkey).actions).toEqual(['start']) + }) +}) + +describe('keyEvtFromDom', () => { + it('copies what the reducer reads off a keyboard event', () => { + const dom = { code: 'AltRight', ctrlKey: true, altKey: true, shiftKey: false, metaKey: false, repeat: true } + expect(keyEvtFromDom('down', dom, 42)).toEqual({ + type: 'down', + code: 'AltRight', + ctrl: true, + alt: true, + shift: false, + meta: false, + repeat: true, + at: 42 + }) + }) +}) diff --git a/core/renderer/lib/dictationKey.ts b/core/renderer/lib/dictationKey.ts new file mode 100644 index 0000000..ed437c9 --- /dev/null +++ b/core/renderer/lib/dictationKey.ts @@ -0,0 +1,392 @@ +/** + * The dictation hotkey (#13): which key events start, stop and cancel a + * recording. A pure reducer, because the hard part of this key cannot be tried + * by machine (AltGr on a physical Norwegian keyboard is on the owner's hands-on + * list) and so every rule has to be provable from a written sequence of events. + * + * WHY IT IS NOT A PLAIN KEYDOWN. The default key is Right Alt, and on Norwegian + * and most European layouts Right Alt is AltGr, the key that types @ { } [ ] \ | ~. + * Windows reports AltGr as ControlLeft DOWN immediately followed by AltRight + * DOWN, so the AltRight event arrives carrying ctrl AND alt, and the user then + * presses Digit2 for an @. A hotkey that fired on the down would start a + * recording every time somebody typed an e-mail address. So a BARE MODIFIER is + * read as a SOLO HOLD (hold mode) or a SOLO TAP (toggle mode): the key alone, + * with nothing else pressed while it decides. A chord or a plain key (Ctrl+Shift+D, + * F9) has no such ambiguity and acts on its down. + * + * The reducer keeps no clock of its own: every event carries `at`, and the host + * sends a 'tick' when the hold time is up (one setTimeout at holdMs is enough). + */ + +export interface Hotkey { + /** KeyboardEvent.code, the PHYSICAL key: the binding must not move with the layout. */ + code: string + ctrl: boolean + alt: boolean + shift: boolean + meta: boolean +} + +export const DEFAULT_HOTKEY: Hotkey = { code: 'AltRight', ctrl: false, alt: false, shift: false, meta: false } + +export type DictMode = 'hold' | 'toggle' + +export type KeyEvt = + | { + type: 'down' | 'up' + code: string + ctrl: boolean + alt: boolean + shift: boolean + meta: boolean + repeat?: boolean + at: number + } + | { type: 'tick'; at: number } + | { type: 'blur'; at: number } + +/** A key and the modifiers held with it, with or without the rest of an event. */ +export type KeyChord = Pick & { type?: 'down' | 'up' } + +export interface KeyState { + /** + * idle: nothing is recording. arming: the bare-modifier hotkey is down and + * still alone, and nothing has been started yet (hold mode is waiting out + * holdMs, toggle mode is waiting for the up that makes it a tap). listening: + * 'start' has been given and neither 'stop' nor 'cancel' has. + */ + phase: 'idle' | 'arming' | 'listening' + /** + * The hotkey is physically down as far as the reducer knows. It outlives the + * phase on purpose: after an Escape or an AltGr cancel the key is still held, + * its auto-repeat must not arm a second press, and its up is still swallowed. + */ + held: boolean + /** When the press that set `held` went down. Meaningless while not held. */ + downAt: number + /** Another key went down during this press, so it is not a solo tap. */ + polluted: boolean + /** This press arrived carrying Ctrl, which is how AltGr arrives: its ControlLeft is its own. */ + altGr: boolean + /** The Escape that cancelled is still down; its up is swallowed with it. */ + escHeld: boolean +} + +export type KeyAction = 'start' | 'stop' | 'cancel' | null + +export interface KeyConfig { + hotkey: Hotkey + mode: DictMode + /** How long a bare modifier must be held alone before it records. Default 200. */ + holdMs?: number +} + +export interface KeyResult { + state: KeyState + action: KeyAction + /** The host must preventDefault AND stop the event reaching xterm. */ + swallow: boolean +} + +/** The spec's figure, not yet tried on a physical AltGr keyboard (owner's + * hands-on list): meant to be longer than the gap between AltGr and the key + * typed with it, and short enough not to feel like a wait. */ +const DEFAULT_HOLD_MS = 200 +/** A tap is a tap up to here; past it the key was held for some other reason. */ +const TAP_MAX_MS = 1000 +/** Windows sends AltGr's ControlLeft and AltRight back to back, out of the one + * physical key press; 50 leaves room for a busy renderer and is still far + * under the time a person needs to press a second key on purpose. */ +const ALTGR_PAIR_MS = 50 + +const MODIFIER_FLAG: Record = { + ControlLeft: 'ctrl', + ControlRight: 'ctrl', + AltLeft: 'alt', + AltRight: 'alt', + ShiftLeft: 'shift', + ShiftRight: 'shift', + MetaLeft: 'meta', + MetaRight: 'meta' +} +const FLAGS = ['ctrl', 'alt', 'shift', 'meta'] as const + +export function initialKeyState(): KeyState { + return { phase: 'idle', held: false, downAt: 0, polluted: false, altGr: false, escHeld: false } +} + +/** A modifier bound on its own, which is what the solo rules apply to. */ +function isBareModifier(h: Hotkey): boolean { + return h.code in MODIFIER_FLAG && !h.ctrl && !h.alt && !h.shift && !h.meta +} + +/** + * Is this event the hotkey? + * + * An UP matches on the code alone: by the time the main key of a chord comes + * up its modifiers may already have, and the up must still be recognised. + * + * A DOWN of a bare modifier ignores the flag the key sets for itself (Right Alt + * arrives with alt already true) and wants every OTHER modifier up, since Shift + * held first is a chord, not the solo key. The one exception is Ctrl on + * AltRight, which is AltGr. It is forgiven without checking that a ControlLeft + * was seen first, deliberately: the strict reading would make the default key + * dead on the owner's own keyboard if that event ever went missing, while the + * loose reading costs only that a real Ctrl+Right Alt, held alone, dictates. + * + * A DOWN of a chord or a plain key matches its modifiers exactly. + */ +export function isHotkeyEvent(evt: KeyEvt | KeyChord, hotkey: Hotkey): boolean { + if (evt.type === 'tick' || evt.type === 'blur') return false + if (evt.code !== hotkey.code) return false + if (evt.type === 'up') return true + if (!isBareModifier(hotkey)) return FLAGS.every((f) => evt[f] === hotkey[f]) + const own = MODIFIER_FLAG[hotkey.code] + return FLAGS.every((f) => f === own || !evt[f] || (f === 'ctrl' && hotkey.code === 'AltRight')) +} + +/** + * One event in, the next state out, plus what the host must DO about it. + * + * What the host owes the reducer: every keydown and keyup of the window (capture + * phase, before xterm), a 'blur' when the window loses focus, and one 'tick' at + * downAt + holdMs while the phase is 'arming'. Hold mode starts ONLY on an event + * at or after holdMs while the key is still down; a release that no tick came + * before is only a release, since a 'start' and a 'stop' cannot both ride on one + * event and a recording that would already be over is not worth starting. + */ +export function reduceKey(state: KeyState, evt: KeyEvt, cfg: KeyConfig): KeyResult { + const holdMs = cfg.holdMs ?? DEFAULT_HOLD_MS + const bare = isBareModifier(cfg.hotkey) + const stay = (swallow = false): KeyResult => ({ state, action: null, swallow }) + /** Hold mode's wait is over: the key has been down, alone, for holdMs. */ + const ripe = bare && cfg.mode === 'hold' && state.phase === 'arming' && evt.at - state.downAt >= holdMs + + if (evt.type === 'blur') { + // The up will never arrive in a window that is not looking, so nothing may + // be left held. A recording that loses its window is cancelled rather than + // pasted: the text would land in a shell the user has walked away from. + return { state: initialKeyState(), action: state.phase === 'listening' ? 'cancel' : null, swallow: false } + } + + if (evt.type === 'tick') { + if (ripe) return { state: { ...state, phase: 'listening' }, action: 'start', swallow: false } + // Toggle mode: held past a tap is not a tap. The key stays `held`, so its up is still ours. + if (state.phase === 'arming' && cfg.mode === 'toggle' && evt.at - state.downAt >= TAP_MAX_MS) { + return { state: { ...state, phase: 'idle', polluted: true }, action: null, swallow: false } + } + return stay() + } + + const own = evt.code === cfg.hotkey.code + + if (evt.type === 'up') { + if (own && state.held) return release(state, evt.at, cfg.mode, bare) + if (evt.code === 'Escape' && state.escHeld) return { state: { ...state, escHeld: false }, action: null, swallow: true } + // Any other up, the synthetic ControlLeft's included, says nothing: only a + // DOWN is somebody pressing a second key. + return stay() + } + + // ---- a key going down ---- + + if (own && state.held) { + // Auto-repeat of the hotkey (or a down with the repeat flag lost): part of + // the same press. It never arms again, but it is an event like any other + // and can carry the start if the host's tick is late. + if (ripe) return { state: { ...state, phase: 'listening' }, action: 'start', swallow: true } + return stay(true) + } + + if (!evt.repeat && isHotkeyEvent(evt, cfg.hotkey)) return press(state, evt, cfg.mode, bare) + + if (evt.code === 'Escape' && state.phase === 'listening') { + // `polluted` so that in toggle mode an Escape pressed MID-TAP does not let + // the tap's own up start a fresh recording a moment after the cancel. + return { state: { ...state, phase: 'idle', polluted: true, escHeld: true }, action: 'cancel', swallow: true } + } + + if (state.held && bare) { + // AltGr's other half is not a second key. On those layouts it precedes + // AltRight, repeats beside it for as long as the pair is held, and is + // forgiven a moment after the down as well in case the order ever flips. + const altGrCtrl = + cfg.hotkey.code === 'AltRight' && + evt.code === 'ControlLeft' && + (state.altGr || evt.at - state.downAt <= ALTGR_PAIR_MS) + if (altGrCtrl) return stay() + if (state.phase === 'arming') { + // Escape means "no" wherever it lands: a late tick must not turn it into a start. + if (evt.code === 'Escape') return { state: { ...state, phase: 'idle', polluted: true }, action: null, swallow: false } + // Held alone for the whole of holdMs, THEN a key: that is dictation that + // began, not AltGr typing. Before it: AltGr typing, cancelled without a + // word, and the key that gave it away goes on to the shell untouched. + if (ripe) return { state: { ...state, phase: 'listening', polluted: true }, action: 'start', swallow: false } + return { state: { ...state, phase: 'idle', polluted: true }, action: null, swallow: false } + } + // While LISTENING a brushed key cancels nothing (owner's rule: the user may + // shift in their seat). It only spoils a toggle tap that is in progress. + if (!state.polluted) return { state: { ...state, polluted: true }, action: null, swallow: false } + } + + return stay() +} + +/** A fresh, matching, non-repeat down of the hotkey. */ +function press(state: KeyState, evt: Extract, mode: DictMode, bare: boolean): KeyResult { + const held: KeyState = { + ...state, + held: true, + downAt: evt.at, + polluted: false, + altGr: bare && evt.code === 'AltRight' && evt.ctrl + } + if (bare) { + // Nothing is decided yet in either mode. From idle the press is 'arming'; + // while listening (a toggle's second tap) the phase holds until the up. + return { state: { ...held, phase: state.phase === 'idle' ? 'arming' : state.phase }, action: null, swallow: true } + } + if (state.phase === 'listening') { + // Toggle: the second press stops. Hold: a press while already listening can + // only follow an up that was never seen; keep listening and let this up stop it. + if (mode === 'toggle') return { state: { ...held, phase: 'idle' }, action: 'stop', swallow: true } + return { state: held, action: null, swallow: true } + } + return { state: { ...held, phase: 'listening' }, action: 'start', swallow: true } +} + +/** The up of a hotkey the reducer saw go down. Always swallowed: an unswallowed + * Alt keyup is what opens the window's system menu. */ +function release(state: KeyState, at: number, mode: DictMode, bare: boolean): KeyResult { + const next: KeyState = { ...state, held: false, polluted: false, altGr: false } + if (mode === 'hold') { + // Arming: let go before holdMs (or before any tick said otherwise). Idle: + // the press was cancelled by Escape or by AltGr typing. Neither stops anything. + if (state.phase === 'listening') return { state: { ...next, phase: 'idle' }, action: 'stop', swallow: true } + return { state: { ...next, phase: 'idle' }, action: null, swallow: true } + } + if (!bare) return { state: next, action: null, swallow: true } + const tap = !state.polluted && at - state.downAt < TAP_MAX_MS + if (state.phase === 'arming') { + return tap + ? { state: { ...next, phase: 'listening' }, action: 'start', swallow: true } + : { state: { ...next, phase: 'idle' }, action: null, swallow: true } + } + if (state.phase === 'listening' && tap) return { state: { ...next, phase: 'idle' }, action: 'stop', swallow: true } + return { state: next, action: null, swallow: true } +} + +// ---- naming and binding ------------------------------------------------------ + +const SIDE_NAMES: Record = { + ControlLeft: 'Left Ctrl', + ControlRight: 'Right Ctrl', + AltLeft: 'Left Alt', + AltRight: 'Right Alt', + ShiftLeft: 'Left Shift', + ShiftRight: 'Right Shift', + MetaLeft: 'Left Win', + MetaRight: 'Right Win' +} + +const KEY_NAMES: Record = { + Backquote: '`', + Minus: '-', + Equal: '=', + BracketLeft: '[', + BracketRight: ']', + Backslash: '\\', + IntlBackslash: '\\', + Semicolon: ';', + Quote: "'", + Comma: ',', + Period: '.', + Slash: '/', + ArrowUp: 'Up', + ArrowDown: 'Down', + ArrowLeft: 'Left', + ArrowRight: 'Right', + PageUp: 'Page Up', + PageDown: 'Page Down', + PrintScreen: 'Print Screen', + ScrollLock: 'Scroll Lock', + CapsLock: 'Caps Lock', + NumLock: 'Num Lock', + ContextMenu: 'Menu', + NumpadAdd: 'Num +', + NumpadSubtract: 'Num -', + NumpadMultiply: 'Num *', + NumpadDivide: 'Num /', + NumpadDecimal: 'Num .', + NumpadEnter: 'Num Enter' +} + +/** The key's name by POSITION (a US legend), since the binding is by position: + * it cannot know what the user's layout prints on that cap, and must not guess. */ +function keyName(code: string): string { + if (code in SIDE_NAMES) return SIDE_NAMES[code] + if (code in KEY_NAMES) return KEY_NAMES[code] + const m = /^(Key|Digit|Numpad)(.+)$/.exec(code) + if (m) return m[1] === 'Numpad' ? `Num ${m[2]}` : m[2] + return code +} + +/** 'Right Alt', 'Ctrl+Shift+D': what Settings shows for a binding. */ +export function formatHotkey(h: Hotkey): string { + const parts: string[] = [] + if (h.ctrl) parts.push('Ctrl') + if (h.alt) parts.push('Alt') + if (h.shift) parts.push('Shift') + if (h.meta) parts.push('Win') + parts.push(keyName(h.code)) + return parts.join('+') +} + +/** + * What this key event would be as a binding, for the rebind control. + * + * A modifier's own event is that modifier BARE, its flags dropped: its own flag + * is the key itself, and the Ctrl that rides on AltRight is AltGr, so the + * Norwegian Right Alt binds as Right Alt rather than as Ctrl+Right Alt. That + * means the control sees 'Left Ctrl' on the way to Ctrl+Shift+D; it should + * commit a bare modifier on that key's UP and a chord on the main key's down. + * Escape is null: it is how the user leaves the control, and as a binding it + * would take away the one key that cancels a recording. + * + * Takes the reducer's own event OR a DOM KeyboardEvent as it comes: the rebind + * control lives in Settings, has no reducer running, and should not have to + * build a KeyEvt only to ask what was pressed. + */ +export function parseHotkeyFromEvent(evt: KeyEvt | KeyChord | DomKeyLike): Hotkey | null { + if ('ctrlKey' in evt) return parseHotkeyFromEvent(keyEvtFromDom('down', evt, 0)) + if (evt.type === 'tick' || evt.type === 'blur') return null + if (!evt.code || evt.code === 'Unidentified' || evt.code === 'Escape') return null + if (evt.code in MODIFIER_FLAG) return { code: evt.code, ctrl: false, alt: false, shift: false, meta: false } + return { code: evt.code, ctrl: evt.ctrl, alt: evt.alt, shift: evt.shift, meta: evt.meta } +} + +/** The fields of a DOM KeyboardEvent the reducer reads, named structurally so + * this file stays free of the DOM. */ +export interface DomKeyLike { + code: string + ctrlKey: boolean + altKey: boolean + shiftKey: boolean + metaKey: boolean + repeat?: boolean +} + +/** A keyboard event as the reducer takes it. `at` is the host's clock + * (performance.now()), the same one its tick is measured on. */ +export function keyEvtFromDom(type: 'down' | 'up', e: DomKeyLike, at: number): KeyEvt { + return { + type, + code: e.code, + ctrl: e.ctrlKey, + alt: e.altKey, + shift: e.shiftKey, + meta: e.metaKey, + repeat: e.repeat ?? false, + at + } +} diff --git a/core/renderer/lib/dictationPrefs.ts b/core/renderer/lib/dictationPrefs.ts new file mode 100644 index 0000000..3a0fdfd --- /dev/null +++ b/core/renderer/lib/dictationPrefs.ts @@ -0,0 +1,109 @@ +import { useSyncExternalStore } from 'react' +import { DEFAULT_HOTKEY, type DictMode, type Hotkey } from './dictationKey' + +/** + * Dictation's settings (#13). VALUES ARE PER APP: each app has its own + * localStorage, so switching it on in Prism Terminal switches nothing on in + * Prism. Only the big files (models, the GPU pack) are shared, by main. + * + * OFF BY DEFAULT, and every reader treats a missing key as off: nothing + * listens, nothing downloads and no server runs until the user says so. + */ +const K = { + enabled: 'prism.dictation.enabled', + mode: 'prism.dictation.mode', + hotkey: 'prism.dictation.hotkey', + mic: 'prism.dictation.mic', + language: 'prism.dictation.language', + pauseMedia: 'prism.dictation.pauseMedia', + sounds: 'prism.dictation.sounds', + model: 'prism.dictation.model' +} as const + +export const DICTATION_KEYS = K + +const listeners = new Set<() => void>() +const notify = (): void => listeners.forEach((l) => l()) +const sub = (l: () => void): (() => void) => { + listeners.add(l) + return () => { + listeners.delete(l) + } +} + +const read = (key: string): string | null => { + try { + return localStorage.getItem(key) + } catch { + return null + } +} +const write = (key: string, value: string): void => { + try { + localStorage.setItem(key, value) + } catch { + /* a full or blocked store loses the setting, not the app */ + } + notify() +} + +export const dictationEnabled = (): boolean => read(K.enabled) === '1' +export const setDictationEnabled = (on: boolean): void => write(K.enabled, on ? '1' : '0') + +/** Hold-to-talk is the default (owner, 2026-09-19). */ +export const dictationMode = (): DictMode => (read(K.mode) === 'toggle' ? 'toggle' : 'hold') +export const setDictationMode = (m: DictMode): void => write(K.mode, m) + +// Parsed once per stored string: useSyncExternalStore needs a STABLE snapshot, +// and a fresh object per read would re-render for ever. +let hotkeyRaw: string | null = null +let hotkeyParsed: Hotkey = DEFAULT_HOTKEY +export function dictationHotkey(): Hotkey { + const raw = read(K.hotkey) + if (raw === hotkeyRaw) return hotkeyParsed + hotkeyRaw = raw + hotkeyParsed = DEFAULT_HOTKEY + if (raw) { + try { + const v = JSON.parse(raw) as Partial + if (typeof v.code === 'string' && v.code) + hotkeyParsed = { code: v.code, ctrl: !!v.ctrl, alt: !!v.alt, shift: !!v.shift, meta: !!v.meta } + } catch { + /* a damaged value reads as the default */ + } + } + return hotkeyParsed +} +export const setDictationHotkey = (h: Hotkey): void => write(K.hotkey, JSON.stringify(h)) + +/** '' is the system default input. */ +export const dictationMic = (): string => read(K.mic) ?? '' +export const setDictationMic = (id: string): void => write(K.mic, id) + +/** 'auto', or a language code Whisper knows. */ +export const dictationLanguage = (): string => { + const v = read(K.language) + return v && /^[a-z]{2,4}$/.test(v) ? v : 'auto' +} +export const setDictationLanguage = (code: string): void => write(K.language, code) + +export const dictationPauseMedia = (): boolean => read(K.pauseMedia) === '1' +export const setDictationPauseMedia = (on: boolean): void => write(K.pauseMedia, on ? '1' : '0') + +/** On unless switched off: the cue is how you know the mic opened without looking. */ +export const dictationSounds = (): boolean => read(K.sounds) !== '0' +export const setDictationSounds = (on: boolean): void => write(K.sounds, on ? '1' : '0') + +/** The ACTIVE model's catalog id, '' when none was ever chosen. */ +export const dictationModel = (): string => read(K.model) ?? '' +export const setDictationModel = (id: string): void => write(K.model, id) + +export const useDictationEnabled = (): boolean => useSyncExternalStore(sub, dictationEnabled) +export const useDictationMode = (): DictMode => useSyncExternalStore(sub, dictationMode) +export const useDictationHotkey = (): Hotkey => useSyncExternalStore(sub, dictationHotkey) +export const useDictationMic = (): string => useSyncExternalStore(sub, dictationMic) +export const useDictationLanguage = (): string => useSyncExternalStore(sub, dictationLanguage) +export const useDictationPauseMedia = (): boolean => useSyncExternalStore(sub, dictationPauseMedia) +export const useDictationSounds = (): boolean => useSyncExternalStore(sub, dictationSounds) +export const useDictationModel = (): string => useSyncExternalStore(sub, dictationModel) +export const onDictationPrefs = sub diff --git a/core/renderer/lib/dictationSounds.ts b/core/renderer/lib/dictationSounds.ts new file mode 100644 index 0000000..a8e70fd --- /dev/null +++ b/core/renderer/lib/dictationSounds.ts @@ -0,0 +1,55 @@ +/** + * The two cues (#13, owner: "plus a microphone sound effect"): a rising pair + * of notes when the mic opens, a falling pair when it closes. SYNTHESISED, so + * there is no asset to bundle, fetch or license, and the same code sounds the + * same in both apps. Short and quiet on purpose: it plays into the room the + * user is about to speak in. It does NOT hold the recording back: the mic opens + * at the same moment, because a cue that delays capture clips the first word of + * anyone who starts talking as they press. The capture asks for echo + * cancellation, which takes this page's own output out of the recording. + */ +let ctx: AudioContext | null = null + +function context(): AudioContext | null { + try { + ctx ??= new AudioContext() + if (ctx.state === 'suspended') void ctx.resume() + return ctx + } catch { + return null + } +} + +function note(ac: AudioContext, freq: number, at: number, len: number): void { + const osc = ac.createOscillator() + const gain = ac.createGain() + osc.type = 'sine' + osc.frequency.value = freq + // A few ms of attack and an exponential tail: a bare sine switched on and + // off clicks, and the click is louder than the note. + gain.gain.setValueAtTime(0.0001, at) + gain.gain.exponentialRampToValueAtTime(0.12, at + 0.012) + gain.gain.exponentialRampToValueAtTime(0.0001, at + len) + osc.connect(gain).connect(ac.destination) + osc.start(at) + osc.stop(at + len + 0.02) +} + +function cue(freqs: [number, number]): Promise { + const ac = context() + if (!ac) return Promise.resolve() + const t = ac.currentTime + 0.005 + note(ac, freqs[0], t, 0.09) + note(ac, freqs[1], t + 0.07, 0.11) + return new Promise((r) => setTimeout(r, 190)) +} + +/** The mic is opening. Resolves when the cue has finished sounding. */ +export const startCue = (): Promise => cue([587.33, 880]) +/** The mic has closed. */ +export const stopCue = (): Promise => cue([880, 587.33]) +/** Cancelled: one low note, so it does not read as "got it". */ +export function cancelCue(): void { + const ac = context() + if (ac) note(ac, 392, ac.currentTime + 0.005, 0.12) +} diff --git a/core/renderer/lib/micCapture.ts b/core/renderer/lib/micCapture.ts new file mode 100644 index 0000000..610924e --- /dev/null +++ b/core/renderer/lib/micCapture.ts @@ -0,0 +1,139 @@ +import { downsample, rms } from './wav' + +/** + * The microphone, as 16 kHz mono samples (#13). Nothing is written anywhere: + * the samples live in this closure until the recording ends. + * + * NO AudioWorklet, deliberately. A worklet is a module loaded by URL, and both + * apps run a `script-src 'self'` policy: a blob: module is refused, and a real + * file would have to be bundled and found by two different hosts' bundlers. + * `MediaStreamTrackProcessor` hands the raw frames to a plain reader on this + * thread instead, with nothing to load. Dictation audio is small (16 kHz mono + * is 32 KB a second) so there is no case for a realtime thread. + */ +export const TARGET_RATE = 16000 + +export interface Capture { + /** Everything heard so far, at 16 kHz. A copy: the caller may keep it. */ + snapshot(): Float32Array + /** Seconds recorded so far. */ + seconds(): number + stop(): void +} + +export type CaptureFailure = 'no-device' | 'denied' | 'unsupported' | 'failed' + +interface AudioDataLike { + sampleRate: number + numberOfFrames: number + numberOfChannels: number + copyTo(dest: Float32Array, opts: { planeIndex: number; format: string }): void + close(): void +} +type ProcessorCtor = new (init: { track: MediaStreamTrack }) => { readable: ReadableStream } + +export function captureFailure(err: unknown): CaptureFailure { + const name = (err as { name?: string } | null)?.name ?? '' + if (name === 'NotFoundError' || name === 'OverconstrainedError') return 'no-device' + if (name === 'NotAllowedError' || name === 'SecurityError') return 'denied' + return 'failed' +} + +/** + * Open the mic. `onLevel` gets a 0..1 loudness per frame, which is the whole + * point of the pill's meter: a dead or muted microphone is visible while you + * speak, not after. Rejects with a CaptureFailure. + */ +export async function startCapture(opts: { + deviceId?: string + onLevel?: (level: number) => void +}): Promise { + const Processor = (globalThis as unknown as { MediaStreamTrackProcessor?: ProcessorCtor }) + .MediaStreamTrackProcessor + if (!Processor || !navigator.mediaDevices?.getUserMedia) throw 'unsupported' satisfies CaptureFailure + + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { + ...(opts.deviceId ? { deviceId: { exact: opts.deviceId } } : {}), + channelCount: 1, + // Takes this page's own output (the start cue) out of the recording. + echoCancellation: true, + // Whisper was trained on raw audio; a denoiser's artefacts cost more + // words than the noise does. Gain control stays: quiet mics are common. + noiseSuppression: false, + autoGainControl: true + } + }) + } catch (err) { + throw captureFailure(err) + } + + const track = stream.getAudioTracks()[0] + if (!track) { + stream.getTracks().forEach((t) => t.stop()) + throw 'no-device' satisfies CaptureFailure + } + + const chunks: Float32Array[] = [] + let total = 0 + let live = true + const reader = new Processor({ track }).readable.getReader() + + void (async () => { + try { + while (live) { + const { value, done } = await reader.read() + if (done || !value) break + const frame = new Float32Array(value.numberOfFrames) + // Plane 0 only: a stereo device's left channel is the voice as well as + // its mix is, and asking for a mix is one more thing to get wrong. + value.copyTo(frame, { planeIndex: 0, format: 'f32-planar' }) + const rate = value.sampleRate + value.close() + if (!live) break + const out = downsample(frame, rate, TARGET_RATE) + chunks.push(out) + total += out.length + // Speech RMS sits around 0.02-0.2; scaled so ordinary talking fills + // most of the meter and silence reads as empty. + opts.onLevel?.(Math.min(1, rms(out) * 6)) + } + } catch { + /* the track ended under us: what was heard so far still stands */ + } + })() + + return { + snapshot() { + const all = new Float32Array(total) + let at = 0 + for (const c of chunks) { + all.set(c, at) + at += c.length + } + return all + }, + seconds: () => total / TARGET_RATE, + stop() { + if (!live) return + live = false + void reader.cancel().catch(() => {}) + stream.getTracks().forEach((t) => t.stop()) + opts.onLevel?.(0) + } + } +} + +/** The input devices, for the picker. Labels are empty until permission was given once. */ +export async function listMics(): Promise> { + try { + const all = await navigator.mediaDevices.enumerateDevices() + return all + .filter((d) => d.kind === 'audioinput' && d.deviceId !== 'default' && d.deviceId !== 'communications') + .map((d, i) => ({ id: d.deviceId, label: d.label || `Microphone ${i + 1}` })) + } catch { + return [] + } +} diff --git a/core/renderer/lib/termBus.ts b/core/renderer/lib/termBus.ts index f79e8ed..2ccb451 100644 --- a/core/renderer/lib/termBus.ts +++ b/core/renderer/lib/termBus.ts @@ -32,6 +32,22 @@ export function pasteInto(sessionId: string): boolean { return true } +/** + * TEXT THE USER SPOKE (#13). The panel owns the xterm instances, and it sits + * behind a lazy boundary in Prism so that xterm stays out of the launch bundle; + * dictation's controller is armed from the app shell, so it must not import the + * panel. The panel hands its paste in when its module loads, and until then + * there is no session to paste into anyway. + */ +let textPaster: ((sessionId: string, text: string) => boolean) | null = null +export function setTextPaster(fn: (sessionId: string, text: string) => boolean): void { + textPaster = fn +} +/** True when the session exists and took the text. */ +export function pasteTextInto(sessionId: string, text: string): boolean { + return textPaster ? textPaster(sessionId, text) : false +} + // Where each shell says it is (#99). TerminalPanel hears the prompt's report // through xterm's OSC parser and posts it here; App listens, because the // tree and the tab root are its to move. diff --git a/core/renderer/lib/useDictation.ts b/core/renderer/lib/useDictation.ts new file mode 100644 index 0000000..4187f81 --- /dev/null +++ b/core/renderer/lib/useDictation.ts @@ -0,0 +1,20 @@ +import { useEffect, useRef, useSyncExternalStore } from 'react' +import { armDictation, dictationView, onDictationView, type DictationView } from './dictation' + +/** + * Arm dictation for this window (#13). Mounted ONCE, by the host's App, with + * the session in front. It returns nothing on purpose: what dictation is doing + * is read where it is drawn (`useDictationView`, in the pill and the tab + * strip), so speaking never re-renders the app shell. + */ +export function useDictationArm(activeSessionId: string | null): void { + const active = useRef(activeSessionId) + useEffect(() => { + active.current = activeSessionId + }, [activeSessionId]) + useEffect(() => armDictation(() => active.current), []) +} + +export function useDictationView(): DictationView { + return useSyncExternalStore(onDictationView, dictationView) +} diff --git a/core/renderer/lib/wav.test.ts b/core/renderer/lib/wav.test.ts new file mode 100644 index 0000000..c6a70ee --- /dev/null +++ b/core/renderer/lib/wav.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it } from 'vitest' +import { downsample, encodeWav, rms, tailSamples } from './wav' + +const ascii = (bytes: Uint8Array, at: number, len: number): string => + String.fromCharCode(...bytes.subarray(at, at + len)) + +/** A sine with a little phase, so no sample sits exactly on zero and a crossing + * is never counted twice or missed. */ +function sine(hz: number, rate: number, seconds: number, amp = 0.8): Float32Array { + const out = new Float32Array(Math.round(rate * seconds)) + for (let i = 0; i < out.length; i += 1) out[i] = amp * Math.sin((2 * Math.PI * hz * i) / rate + 0.3) + return out +} + +function zeroCrossings(s: Float32Array): number { + let n = 0 + for (let i = 1; i < s.length; i += 1) if (s[i - 1] < 0 !== s[i] < 0) n += 1 + return n +} + +describe('encodeWav', () => { + const samples = new Float32Array([0, 0.5, -0.5, 1, -1]) + const wav = encodeWav(samples, 16000) + const view = new DataView(wav.buffer, wav.byteOffset, wav.byteLength) + + it('is 44 bytes of header and two bytes a sample', () => { + expect(wav).toBeInstanceOf(Uint8Array) + expect(wav.length).toBe(44 + samples.length * 2) + expect(encodeWav(new Float32Array(16000), 16000).length).toBe(44 + 32000) + }) + + it('writes the RIFF header field by field', () => { + expect(ascii(wav, 0, 4)).toBe('RIFF') + expect(view.getUint32(4, true)).toBe(36 + 10) // the file, less these first eight bytes + expect(ascii(wav, 8, 4)).toBe('WAVE') + expect(ascii(wav, 12, 4)).toBe('fmt ') + expect(view.getUint32(16, true)).toBe(16) // size of the fmt chunk + expect(view.getUint16(20, true)).toBe(1) // PCM + expect(view.getUint16(22, true)).toBe(1) // mono + expect(view.getUint32(24, true)).toBe(16000) // sample rate + expect(view.getUint32(28, true)).toBe(32000) // bytes a second + expect(view.getUint16(32, true)).toBe(2) // block align + expect(view.getUint16(34, true)).toBe(16) // bits a sample + expect(ascii(wav, 36, 4)).toBe('data') + expect(view.getUint32(40, true)).toBe(10) + }) + + it('carries the rate it was given into both rate fields', () => { + const v = new DataView(encodeWav(new Float32Array(4), 48000).buffer) + expect(v.getUint32(24, true)).toBe(48000) + expect(v.getUint32(28, true)).toBe(96000) + }) + + it('writes little-endian signed 16-bit samples, full scale at the rails', () => { + const at = (i: number): number => view.getInt16(44 + i * 2, true) + expect(at(0)).toBe(0) + expect(at(1)).toBe(16384) + expect(at(2)).toBe(-16384) + expect(at(3)).toBe(32767) + expect(at(4)).toBe(-32768) + }) + + it('clamps what is outside [-1, 1] instead of wrapping it round', () => { + const v = new DataView(encodeWav(new Float32Array([1.7, -3, Infinity, -Infinity, NaN]), 16000).buffer) + const got = [0, 1, 2, 3, 4].map((i) => v.getInt16(44 + i * 2, true)) + expect(got).toEqual([32767, -32768, 32767, -32768, 0]) + }) + + it('an empty clip is a valid, empty file', () => { + const empty = encodeWav(new Float32Array(0), 16000) + const v = new DataView(empty.buffer) + expect(empty.length).toBe(44) + expect(v.getUint32(4, true)).toBe(36) + expect(v.getUint32(40, true)).toBe(0) + }) + + it('reads a subarray from where it starts, not from the start of its buffer', () => { + const whole = new Float32Array([0.25, 0.25, -1, 1]) + const v = new DataView(encodeWav(whole.subarray(2), 16000).buffer) + expect(v.getUint32(40, true)).toBe(4) + expect(v.getInt16(44, true)).toBe(-32768) + expect(v.getInt16(46, true)).toBe(32767) + }) +}) + +describe('tailSamples', () => { + const ramp = Float32Array.from({ length: 100 }, (_, i) => i) + + it('is the last N seconds', () => { + const tail = tailSamples(ramp, 10, 3) + expect(tail.length).toBe(30) + expect(tail[0]).toBe(70) + expect(tail[29]).toBe(99) + }) + + it("is Whisper's 28 s window of a long clip", () => { + const clip = new Float32Array(16000 * 40) + expect(tailSamples(clip, 16000, 28).length).toBe(16000 * 28) + }) + + it('is the whole clip when the clip is shorter than the window', () => { + const tail = tailSamples(ramp, 10, 28) + expect(Array.from(tail)).toEqual(Array.from(ramp)) + expect(tailSamples(ramp, 10, 10).length).toBe(100) + }) + + it('is a copy: the recording goes on growing underneath it', () => { + const source = Float32Array.from([1, 2, 3, 4]) + const tail = tailSamples(source, 1, 2) + const all = tailSamples(source, 1, 99) + source.fill(9) + expect(Array.from(tail)).toEqual([3, 4]) + expect(Array.from(all)).toEqual([1, 2, 3, 4]) + expect(tail.buffer).not.toBe(source.buffer) + }) + + it('is empty for no time, negative time and an empty clip', () => { + expect(tailSamples(ramp, 10, 0).length).toBe(0) + expect(tailSamples(ramp, 10, -1).length).toBe(0) + expect(tailSamples(ramp, 10, NaN).length).toBe(0) + expect(tailSamples(new Float32Array(0), 16000, 28).length).toBe(0) + }) + + it('rounds a fraction of a sample down', () => { + expect(tailSamples(ramp, 10, 0.25).length).toBe(2) + }) +}) + +describe('downsample', () => { + it('hands the input back when the rates match', () => { + const input = sine(440, 16000, 0.1) + expect(downsample(input, 16000, 16000)).toBe(input) + }) + + it.each([ + [48000, 16000], + [44100, 16000], + [96000, 16000], + [32000, 16000] + ])('%i to %i: a second in is a second out', (from, to) => { + expect(downsample(new Float32Array(from), from, to).length).toBe(to) + expect(downsample(new Float32Array(from / 2), from, to).length).toBe(to / 2) + }) + + it.each([48000, 44100])('a 1 kHz sine at %i stays a 1 kHz sine', (from) => { + const out = downsample(sine(1000, from, 1), from, 16000) + // Two crossings a cycle; the ends of the clip may cost one. + expect(Math.abs(zeroCrossings(out) - 2000)).toBeLessThanOrEqual(2) + // And it is still a sine of about that size, not a thinned-out or clipped one. + expect(rms(out)).toBeGreaterThan((0.8 / Math.SQRT2) * 0.9) + expect(rms(out)).toBeLessThan((0.8 / Math.SQRT2) * 1.01) + }) + + it('averages, so steady level in is the same level out', () => { + const out = downsample(new Float32Array(44100).fill(0.5), 44100, 16000) + expect(Math.min(...out)).toBeCloseTo(0.5, 6) + expect(Math.max(...out)).toBeCloseTo(0.5, 6) + }) + + it('averages each window rather than picking a sample out of it', () => { + expect(Array.from(downsample(Float32Array.from([1, 0, 0, 1, 0, 0]), 48000, 16000))).toEqual([ + expect.closeTo(1 / 3, 6), + expect.closeTo(1 / 3, 6) + ]) + }) + + it('takes a slow microphone up to 16 kHz as well', () => { + const out = downsample(sine(300, 8000, 1), 8000, 16000) + expect(out.length).toBe(16000) + expect(Math.abs(zeroCrossings(out) - 600)).toBeLessThanOrEqual(2) + }) + + it('copes with nothing, and refuses a rate that is not one', () => { + expect(downsample(new Float32Array(0), 48000, 16000).length).toBe(0) + expect(() => downsample(new Float32Array(4), 0, 16000)).toThrow(RangeError) + expect(() => downsample(new Float32Array(4), 48000, -1)).toThrow(RangeError) + }) +}) + +describe('rms', () => { + it('is 0 for silence and for nothing at all', () => { + expect(rms(new Float32Array(1000))).toBe(0) + expect(rms(new Float32Array(0))).toBe(0) + }) + it('is the level of a steady signal, whichever way it points', () => { + expect(rms(new Float32Array(100).fill(0.5))).toBeCloseTo(0.5, 6) + expect(rms(new Float32Array(100).fill(-0.5))).toBeCloseTo(0.5, 6) + }) + it('is amplitude over root two for a sine', () => { + expect(rms(sine(1000, 16000, 1, 1))).toBeCloseTo(Math.SQRT1_2, 3) + }) + it('is 1 for a full-scale square wave', () => { + expect(rms(Float32Array.from({ length: 64 }, (_, i) => (i % 2 ? 1 : -1)))).toBeCloseTo(1, 6) + }) +}) diff --git a/core/renderer/lib/wav.ts b/core/renderer/lib/wav.ts new file mode 100644 index 0000000..fee01c3 --- /dev/null +++ b/core/renderer/lib/wav.ts @@ -0,0 +1,117 @@ +/** + * The arithmetic between the microphone and whisper-server (#13): samples to a + * WAV file, the window of a long clip, the rate Whisper wants, a level for the + * meter. Pure on purpose. The capture itself (getUserMedia, the worklet) cannot + * run under a unit test, so everything that CAN be wrong about the bytes lives + * here, where it can be checked field by field. + * + * The WAV is built in memory and handed to main over IPC. Nothing here, or + * anywhere downstream, writes audio to disk (the spec's rule). + */ + +const HEADER_BYTES = 44 +/** PCM16: what whisper-server reads without converting, and half the IPC + * payload of float32 (a 28 s window is 0.9 MB instead of 1.8). */ +const BYTES_PER_SAMPLE = 2 + +/** + * A complete mono PCM16 little-endian WAV file: the canonical 44-byte RIFF + * header, then the samples. Anything outside [-1, 1] is CLAMPED: a hot + * microphone that wraps round instead turns a loud syllable into a full-scale + * click, which Whisper hears as a word. NaN is written as silence. + */ +export function encodeWav(samples: Float32Array, sampleRate: number): Uint8Array { + const dataBytes = samples.length * BYTES_PER_SAMPLE + const out = new Uint8Array(HEADER_BYTES + dataBytes) + const view = new DataView(out.buffer) + const tag = (at: number, text: string): void => { + for (let i = 0; i < text.length; i += 1) out[at + i] = text.charCodeAt(i) + } + + tag(0, 'RIFF') + view.setUint32(4, HEADER_BYTES - 8 + dataBytes, true) // the file, less 'RIFF' and this field + tag(8, 'WAVE') + tag(12, 'fmt ') + view.setUint32(16, 16, true) // the fmt chunk's own size + view.setUint16(20, 1, true) // format 1: integer PCM + view.setUint16(22, 1, true) // channels + view.setUint32(24, sampleRate, true) + view.setUint32(28, sampleRate * BYTES_PER_SAMPLE, true) // bytes a second + view.setUint16(32, BYTES_PER_SAMPLE, true) // block align: one frame of one channel + view.setUint16(34, 16, true) // bits a sample + tag(36, 'data') + view.setUint32(40, dataBytes, true) + + for (let i = 0; i < samples.length; i += 1) { + // Scaled by 32768 so -1 is the bottom rail exactly, then clamped, since + // +1 would otherwise be 32768, one past the top of an int16. + const scaled = Math.round(samples[i] * 0x8000) + const pcm = scaled > 0x7fff ? 0x7fff : scaled < -0x8000 ? -0x8000 : scaled || 0 + view.setInt16(HEADER_BYTES + i * BYTES_PER_SAMPLE, pcm, true) + } + return out +} + +/** + * The last `seconds` of a clip, for the live text: Whisper looks at 30 s and the + * spec sends the last 28. Always a COPY, even when it is the whole clip: the + * recording goes on growing while this pass is in flight, and a view would be + * the capture's own buffer changing underneath the encoder. + */ +export function tailSamples(samples: Float32Array, sampleRate: number, seconds: number): Float32Array { + const want = Math.floor(seconds * sampleRate) + if (!(want > 0)) return new Float32Array(0) // zero, negative and NaN alike + return samples.slice(Math.max(0, samples.length - want)) +} + +/** + * From the rate the audio hardware runs at (48000 or 44100, whatever the + * AudioContext was given) to the 16000 Whisper wants. + * + * Each output sample is the AVERAGE of the input samples that fall in its + * slot. That is a box filter, not a proper low-pass, and it is enough here on + * purpose: what aliases down is what a voice has above 8 kHz, which is little, + * and the alternative is a windowed-sinc kernel run on every block of a live + * recording. Picking every third sample instead would fold all of it into the + * speech band unweakened. 44100 is not a whole multiple, so the slots are 2 and + * 3 samples wide by turns; the edges are computed from the index each time and + * never accumulated, so a five-minute clip does not drift. + * + * Returns the INPUT ITSELF when the rates already match (no copy: the caller is + * on the audio path). A slower source, which a headset in hands-free mode can + * be, is brought up by linear interpolation rather than refused. + */ +export function downsample(input: Float32Array, fromRate: number, toRate: number): Float32Array { + if (!(fromRate > 0) || !(toRate > 0)) throw new RangeError(`downsample: not a sample rate: ${fromRate} to ${toRate}`) + if (fromRate === toRate) return input + const ratio = fromRate / toRate + const out = new Float32Array(Math.floor(input.length / ratio)) + + if (ratio < 1) { + for (let i = 0; i < out.length; i += 1) { + const pos = i * ratio + const a = Math.floor(pos) + const b = Math.min(a + 1, input.length - 1) + out[i] = input[a] + (input[b] - input[a]) * (pos - a) + } + return out + } + + for (let i = 0; i < out.length; i += 1) { + const start = Math.floor(i * ratio) + const end = Math.min(input.length, Math.max(start + 1, Math.floor((i + 1) * ratio))) + let sum = 0 + for (let j = start; j < end; j += 1) sum += input[j] + out[i] = sum / (end - start) + } + return out +} + +/** Root mean square, 0 to 1 for a signal inside the rails: the level meter's + * number, and how a dead microphone shows up as one before anything is recorded. */ +export function rms(samples: Float32Array): number { + if (samples.length === 0) return 0 + let sum = 0 + for (let i = 0; i < samples.length; i += 1) sum += samples[i] * samples[i] + return Math.sqrt(sum / samples.length) +} diff --git a/core/renderer/settings/Dictation.tsx b/core/renderer/settings/Dictation.tsx new file mode 100644 index 0000000..6dfebbb --- /dev/null +++ b/core/renderer/settings/Dictation.tsx @@ -0,0 +1,473 @@ +import { useCallback, useEffect, useRef, useState, type JSX } from 'react' +import { dictationHost } from '../host' +import { catalogEntry, LANGUAGES, recommendedModel, visibleModels } from '../../shared/dictationCatalog' +import type { CatalogEntry, DownloadFailure, EngineInfo, ItemStatus } from '../../shared/dictationTypes' +import { DEFAULT_HOTKEY, formatHotkey, parseHotkeyFromEvent, type Hotkey } from '../lib/dictationKey' +import { + setDictationEnabled, + setDictationHotkey, + setDictationLanguage, + setDictationMic, + setDictationMode, + setDictationModel, + setDictationPauseMedia, + setDictationSounds, + useDictationEnabled, + useDictationHotkey, + useDictationLanguage, + useDictationMic, + useDictationMode, + useDictationModel, + useDictationPauseMedia, + useDictationSounds +} from '../lib/dictationPrefs' +import { listMics, startCapture, type Capture } from '../lib/micCapture' +import { Pref, Segmented, Select, Switch } from './fields' + +/** + * DICTATION'S SETTINGS PAGE, for both hosts (#13). Prism Terminal shows it as + * its own tab, Prism as its own page under Behaviour (owner, 2026-09-19); the + * page itself is this, once. Values are per app; the model files it manages + * are shared by both apps, which is why a model downloaded in one shows as + * installed in the other. + * + * NOTHING HERE STARTS ANYTHING BY ITSELF. Opening the page does not open the + * microphone (the meter is behind a Test button: a mic that lights up because + * you looked at a settings page is a privacy bug), and nothing downloads + * without a click. + */ +const ROWS = 'border-t border-[color:var(--p-line)]' +const GPU_ID = 'gpu-pack' + +const FAILURES: Record = { + network: 'The download failed. Check the connection and try again.', + checksum: 'The file did not match its checksum and was deleted. Try again.', + disk: 'It could not be written to disk.', + cancelled: '', + unpack: 'It could not be unpacked.' +} + +/** A copy of a record without one key. */ +function without(rec: Record, key: string): Record { + const next = { ...rec } + delete next[key] + return next +} + +function size(bytes: number): string { + return bytes >= 1e9 ? `${(bytes / 1e9).toFixed(1)} GB` : `${Math.round(bytes / 1e6)} MB` +} + +const button = + 'rounded-md border border-[color:var(--p-divider)] bg-[var(--p-control)] px-3 py-1 text-[11.5px] font-semibold text-[var(--p-text)] transition hover:bg-[var(--p-hover)] disabled:opacity-50' +const primary = + 'rounded-md bg-[var(--p-accent)] px-3 py-1 text-[11.5px] font-semibold text-[var(--p-on-accent)] transition hover:brightness-110 disabled:opacity-50' + +/** Click, then press the key or chord. A bare modifier (Right Alt, the default) + * is taken on its RELEASE, so that Ctrl+Shift+D is not captured as "Ctrl". */ +function HotkeyField({ value, disabled }: { value: Hotkey; disabled: boolean }): JSX.Element { + const [listening, setListening] = useState(false) + useEffect(() => { + if (!listening) return + let bare: Hotkey | null = null + const isModifier = (code: string): boolean => /^(Alt|Control|Shift|Meta)(Left|Right)$/.test(code) + const down = (e: KeyboardEvent): void => { + e.preventDefault() + e.stopPropagation() + if (e.repeat) return + if (e.code === 'Escape') return setListening(false) + const hk = parseHotkeyFromEvent(e) + if (!hk) return + if (isModifier(e.code)) bare = hk + else { + setDictationHotkey(hk) + setListening(false) + } + } + const up = (e: KeyboardEvent): void => { + e.preventDefault() + e.stopPropagation() + if (bare && bare.code === e.code) { + setDictationHotkey(bare) + setListening(false) + } + } + const stop = (): void => setListening(false) + window.addEventListener('keydown', down, true) + window.addEventListener('keyup', up, true) + window.addEventListener('blur', stop) + return () => { + window.removeEventListener('keydown', down, true) + window.removeEventListener('keyup', up, true) + window.removeEventListener('blur', stop) + } + }, [listening]) + const isDefault = JSON.stringify(value) === JSON.stringify(DEFAULT_HOTKEY) + return ( +
+ {!isDefault && !listening && ( + + )} + +
+ ) +} + +/** The mic picker, and a meter behind a button: a dead or muted microphone is + * found here, in ten seconds, instead of after a lost dictation. */ +function MicField({ value, disabled }: { value: string; disabled: boolean }): JSX.Element { + const [mics, setMics] = useState>([]) + const [testing, setTesting] = useState(false) + const [problem, setProblem] = useState(null) + const bar = useRef(null) + useEffect(() => { + void listMics().then(setMics) + }, [testing]) + useEffect(() => { + if (!testing) return + let cap: Capture | null = null + let dead = false + let shown = 0 + void startCapture({ + deviceId: value || undefined, + onLevel: (l) => { + shown = l > shown ? l : shown * 0.85 + l * 0.15 + if (bar.current) bar.current.style.transform = `scaleX(${Math.min(1, shown)})` + } + }) + .then((c) => { + if (dead) c.stop() + else cap = c + }) + .catch((err) => { + setProblem(err === 'denied' ? 'Microphone access was refused.' : 'No working microphone was found.') + setTesting(false) + }) + const timer = window.setTimeout(() => setTesting(false), 10000) + return () => { + dead = true + clearTimeout(timer) + cap?.stop() + } + }, [testing, value]) + return ( +
+ {testing ? ( + + + + ) : ( + problem && {problem} + )} + + ({ id: l.code, name: l.name }))} + /> + + + + + + + +
+ +
+

Models

+

+ Bigger hears better and needs more from the PC. Downloaded once, used by Prism and Prism Terminal alike. +

+
+ {visibleModels().map((m) => ( + download(m.id)} + onCancel={() => api.dictationCancel(m.id)} + onRemove={() => remove(m.id)} + onUse={() => setDictationModel(m.id)} + /> + ))} +
+
+ + {/* Offered only where it can work: the official GPU engine is NVIDIA's. + Everyone else runs the bundled CPU engine, which needs no setup. */} + {gpu && info?.nvidia && ( +
+

GPU acceleration

+

+ {info.gpuFellBack + ? 'The GPU engine would not start on this PC, so the CPU is answering. Removing and enabling it again may help.' + : 'An NVIDIA card was found. The GPU engine makes the large models answer in under a second.'} +

+
+ download(GPU_ID)} + onCancel={() => api.dictationCancel(GPU_ID)} + onRemove={() => remove(GPU_ID)} + /> +
+
+ )} + + ) +} diff --git a/core/renderer/settings/dictationOptions.ts b/core/renderer/settings/dictationOptions.ts new file mode 100644 index 0000000..3355d1e --- /dev/null +++ b/core/renderer/settings/dictationOptions.ts @@ -0,0 +1,40 @@ +import { DICTATION_KEYS } from '../lib/dictationPrefs' + +/** + * EVERY DICTATION OPTION, BY ID (#13): the same deal as `options.ts` makes for + * the terminal's. The Dictation page renders these rows as `data-pref=""` + * and each app's e2e asserts that its page shows this list, so an option can + * never exist in one app and not the other. + * + * A list of its own rather than more rows in `TERMINAL_OPTIONS`: each app's + * terminal parity check reads that file as TEXT, and dictation lives on a page + * of its own in both apps. + */ +export interface DictationOption { + id: string + label: string + type: 'switch' | 'choice' | 'key' | 'manager' + /** The localStorage key behind it; null where the row manages FILES, which + * both apps share, rather than a per-app value. */ + key: string | null + /** Absent = every PC. */ + onlyWhere?: 'an NVIDIA adapter is present' +} + +export const DICTATION_OPTIONS: readonly DictationOption[] = [ + { id: 'dictation-enabled', label: 'Dictation', type: 'switch', key: DICTATION_KEYS.enabled }, + { id: 'dictation-mode', label: 'Trigger', type: 'choice', key: DICTATION_KEYS.mode }, + { id: 'dictation-hotkey', label: 'Key', type: 'key', key: DICTATION_KEYS.hotkey }, + { id: 'dictation-mic', label: 'Microphone', type: 'choice', key: DICTATION_KEYS.mic }, + { id: 'dictation-language', label: 'Language', type: 'choice', key: DICTATION_KEYS.language }, + { id: 'dictation-pause-media', label: 'Pause media while dictating', type: 'switch', key: DICTATION_KEYS.pauseMedia }, + { id: 'dictation-sounds', label: 'Sounds', type: 'switch', key: DICTATION_KEYS.sounds }, + { id: 'dictation-model', label: 'Models', type: 'manager', key: DICTATION_KEYS.model }, + // The official GPU engine is NVIDIA's: the row is offered only where it can work. + { id: 'dictation-gpu', label: 'GPU acceleration', type: 'manager', key: null, onlyWhere: 'an NVIDIA adapter is present' } +] + +/** The option ids a PC should be showing. */ +export function dictationOptionIds(pc: { nvidia: boolean }): string[] { + return DICTATION_OPTIONS.filter((o) => !o.onlyWhere || pc.nvidia).map((o) => o.id) +} diff --git a/core/renderer/settings/options.test.ts b/core/renderer/settings/options.test.ts index f0771e4..b286008 100644 --- a/core/renderer/settings/options.test.ts +++ b/core/renderer/settings/options.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { readdirSync, readFileSync } from 'fs' import { join } from 'path' +import { DICTATION_OPTIONS, dictationOptionIds } from './dictationOptions' import { TERMINAL_OPTIONS, terminalOptionIds } from './options' // The list and the sections must say the same thing: the list is what each @@ -20,7 +21,7 @@ const rendered = new Set( describe('the terminal options list', () => { it('names every row the shared sections render, and nothing they do not', () => { - expect([...rendered].sort()).toEqual(TERMINAL_OPTIONS.map((o) => o.id).sort()) + expect([...rendered].sort()).toEqual([...TERMINAL_OPTIONS, ...DICTATION_OPTIONS].map((o) => o.id).sort()) }) it('has one storage key per option, all under prism.term', () => { @@ -34,4 +35,17 @@ describe('the terminal options list', () => { const prism = terminalOptionIds({ windowAcrylic: false }) expect(all.filter((id) => !prism.includes(id))).toEqual(['term-opacity']) }) + + it('gives dictation one key per option, under prism.dictation, and only the file managers none', () => { + const keys = DICTATION_OPTIONS.map((o) => o.key).filter((k): k is string => k !== null) + expect(new Set(keys).size).toBe(keys.length) + for (const k of keys) expect(k.startsWith('prism.dictation.')).toBe(true) + expect(DICTATION_OPTIONS.filter((o) => o.key === null).map((o) => o.id)).toEqual(['dictation-gpu']) + }) + + it('offers the GPU row only where an NVIDIA adapter is present', () => { + const all = dictationOptionIds({ nvidia: true }) + const without = dictationOptionIds({ nvidia: false }) + expect(all.filter((id) => !without.includes(id))).toEqual(['dictation-gpu']) + }) }) diff --git a/core/shared/channels.ts b/core/shared/channels.ts index b86aa7a..9265924 100644 --- a/core/shared/channels.ts +++ b/core/shared/channels.ts @@ -17,3 +17,18 @@ export const CH = { clipboardRead: 'clipboard:read', openExternal: 'shell:open-external' } as const + +/** Dictation's channels (#13). A table of its own: dictation is optional, and + * a host wires it with a separate call on each side of the bridge. */ +export const DCH = { + info: 'dictation:info', + status: 'dictation:status', + download: 'dictation:download', + cancel: 'dictation:cancel', + remove: 'dictation:remove', + progress: 'dictation:progress', + transcribe: 'dictation:transcribe', + stop: 'dictation:stop', + mediaPause: 'dictation:media-pause', + mediaResume: 'dictation:media-resume' +} as const diff --git a/core/shared/dictationCatalog.test.ts b/core/shared/dictationCatalog.test.ts new file mode 100644 index 0000000..e0a7101 --- /dev/null +++ b/core/shared/dictationCatalog.test.ts @@ -0,0 +1,175 @@ +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { + CATALOG, + ENGINE, + LANGUAGES, + catalogEntry, + recommendedModel, + visibleModels +} from './dictationCatalog' + +const HEX64 = /^[0-9a-f]{64}$/ + +describe('CATALOG', () => { + it('gives every entry an https url, an exact size and a lowercase 64-hex checksum', () => { + for (const e of CATALOG) { + expect(e.url, e.id).toMatch(/^https:\/\//) + expect(Number.isInteger(e.bytes) && e.bytes > 0, e.id).toBe(true) + expect(e.sha256, e.id).toMatch(HEX64) + expect(e.label.length, e.id).toBeGreaterThan(0) + // One short line: the model manager has a row, not a paragraph. + expect(e.note.length, e.id).toBeGreaterThan(0) + expect(e.note.length, e.id).toBeLessThanOrEqual(80) + expect(e.note, e.id).not.toMatch(/\n/) + } + }) + + it('has unique ids, and no two entries share a url or a checksum', () => { + for (const key of ['id', 'url', 'sha256'] as const) { + const seen = CATALOG.map((e) => e[key]) + expect(new Set(seen).size, key).toBe(seen.length) + } + }) + + it('holds the five models and the one GPU pack the design names', () => { + expect(CATALOG.map((e) => e.id).sort()).toEqual( + ['base', 'gpu-pack', 'large-v3', 'large-v3-turbo', 'small', 'tiny'].sort() + ) + expect(CATALOG.filter((e) => e.kind === 'gpu-pack').map((e) => e.id)).toEqual(['gpu-pack']) + const label = (id: string): string | undefined => catalogEntry(id)?.label + expect(label('tiny')).toBe('Tiny') + expect(label('base')).toBe('Base') + expect(label('small')).toBe('Small') + expect(label('large-v3-turbo')).toBe('Large v3 Turbo') + expect(label('large-v3')).toBe('Large v3') + }) + + it('pins every model to a COMMIT of the Hugging Face repo, never to main', () => { + const models = CATALOG.filter((e) => e.kind === 'model') + expect(models.length).toBe(5) + for (const e of models) { + expect(e.url, e.id).not.toContain('/resolve/main/') + expect(e.url, e.id).toMatch( + /^https:\/\/huggingface\.co\/ggerganov\/whisper\.cpp\/resolve\/[0-9a-f]{40}\/ggml-[a-z0-9.-]+\.bin$/ + ) + expect(e.url.endsWith(`/ggml-${e.id}.bin`), e.id).toBe(true) + } + }) + + it('takes the GPU pack from the same pinned release as the engine', () => { + const pack = catalogEntry('gpu-pack') + expect(pack?.url).toBe( + `https://github.com/ggml-org/whisper.cpp/releases/download/${ENGINE.tag}/whisper-cublas-12.4.0-bin-x64.zip` + ) + expect(pack?.sha256).toBe('af520ddd034d985b55dfeea3e465ed93653ba2aee1a55e865033edc548c272a7') + // The pack is not a model: it must never be offered as one, or recommended. + expect(pack?.recommended).toBeUndefined() + expect(pack?.needsGpu).toBeUndefined() + }) + + it('flags the models a CPU cannot keep up with, and only those', () => { + const needs = CATALOG.filter((e) => e.needsGpu) + .map((e) => e.id) + .sort() + expect(needs).toEqual(['large-v3', 'large-v3-turbo']) + }) +}) + +describe('visibleModels', () => { + it('is the model manager list: models only, the e2e one left out, smallest first', () => { + const ids = visibleModels().map((e) => e.id) + expect(ids).toEqual(['base', 'small', 'large-v3-turbo', 'large-v3']) + expect(catalogEntry('tiny')?.e2eOnly).toBe(true) + expect(ids).not.toContain('tiny') + expect(ids).not.toContain('gpu-pack') + }) + + it('marks exactly one pick for a CPU and one for the GPU pack', () => { + const cpu = visibleModels().filter((e) => e.recommended === 'cpu') + const gpu = visibleModels().filter((e) => e.recommended === 'gpu') + expect(cpu.map((e) => e.id)).toEqual(['base']) + expect(gpu.map((e) => e.id)).toEqual(['large-v3-turbo']) + // A CPU pick that needs the GPU would be a recommendation nobody can use. + expect(cpu[0].needsGpu).toBeFalsy() + expect(gpu[0].needsGpu).toBe(true) + // Nothing outside the visible list carries a badge. + expect(CATALOG.filter((e) => e.recommended).length).toBe(2) + }) +}) + +describe('catalogEntry and recommendedModel', () => { + it('finds an entry by id and answers undefined for anything else', () => { + expect(catalogEntry('base')?.bytes).toBe(147951465) + expect(catalogEntry('gpu-pack')?.kind).toBe('gpu-pack') + expect(catalogEntry('')).toBeUndefined() + expect(catalogEntry('BASE')).toBeUndefined() + expect(catalogEntry('parakeet')).toBeUndefined() + }) + + it('recommends by what the machine can run', () => { + expect(recommendedModel(false).id).toBe('base') + expect(recommendedModel(true).id).toBe('large-v3-turbo') + }) +}) + +describe('LANGUAGES', () => { + it('starts with auto-detect and lists each Whisper code once', () => { + expect(LANGUAGES[0]).toEqual({ code: 'auto', name: 'Auto-detect' }) + const codes = LANGUAGES.map((l) => l.code) + expect(new Set(codes).size).toBe(codes.length) + for (const l of LANGUAGES.slice(1)) { + expect(l.code, l.name).toMatch(/^[a-z]{2}$/) + expect(l.name.length, l.code).toBeGreaterThan(0) + } + for (const must of 'en no sv da de fr es it pt nl pl fi uk ru tr ar hi ja ko zh'.split(' ')) + expect(codes, must).toContain(must) + expect(LANGUAGES.length).toBeGreaterThanOrEqual(21) + expect(LANGUAGES.length).toBeLessThanOrEqual(35) + }) + + it('names are unique too, so the picker never shows one language twice', () => { + const names = LANGUAGES.map((l) => l.name) + expect(new Set(names).size).toBe(names.length) + }) +}) + +describe('ENGINE', () => { + it('is the pinned official CPU build', () => { + expect(ENGINE.tag).toBe('b5130') + expect(ENGINE.asset).toBe('whisper-bin-x64.zip') + expect(ENGINE.url).toBe( + 'https://github.com/ggml-org/whisper.cpp/releases/download/b5130/whisper-bin-x64.zip' + ) + expect(ENGINE.sha256).toBe('f9ec6c52a2e949b62ab51fa21d0d497958f9e41c3010c157c4e42932d5316f3c') + expect(ENGINE.bytes).toBeGreaterThan(0) + expect(ENGINE.files).toContain('whisper-server.exe') + expect(new Set(ENGINE.files).size).toBe(ENGINE.files.length) + // Flattened names: what the engine folder holds, not where the zip kept them. + for (const f of ENGINE.files) expect(f, f).not.toMatch(/[\\/]/) + }) + + // The fetch script runs before any build exists, so it cannot import this + // catalog and carries its own copy of the pin. Two copies drift silently + // (the app would verify one archive and the build would bundle another), so + // the script is read as TEXT here and held to the same values. + it('says the same as core/tools/fetch-whisper.mjs', () => { + const here = dirname(fileURLToPath(import.meta.url)) + const src = readFileSync(join(here, '..', 'tools', 'fetch-whisper.mjs'), 'utf8') + const one = (name: string): string | undefined => + new RegExp(`^const ${name} = '([^']+)'`, 'm').exec(src)?.[1] + expect(one('TAG')).toBe(ENGINE.tag) + expect(one('ASSET')).toBe(ENGINE.asset) + expect(one('SHA256')).toBe(ENGINE.sha256) + expect(Number(/^const BYTES = (\d+)/m.exec(src)?.[1])).toBe(ENGINE.bytes) + const block = /^const FILES = \[([\s\S]*?)\]/m.exec(src)?.[1] ?? '' + const files = [...block.matchAll(/'([^']+)'/g)].map((m) => m[1]) + expect(files).toEqual([...ENGINE.files]) + // The url is built from the pin in the script, as it is here. + expect(src).toContain( + 'https://github.com/ggml-org/whisper.cpp/releases/download/${TAG}/${ASSET}' + ) + }) +}) diff --git a/core/shared/dictationCatalog.ts b/core/shared/dictationCatalog.ts new file mode 100644 index 0000000..ee3f374 --- /dev/null +++ b/core/shared/dictationCatalog.ts @@ -0,0 +1,228 @@ +/** + * Everything dictation downloads, and the one thing it bundles (#13). Pure + * data plus lookups, shared by main (which verifies against it) and the + * renderer (which draws the model manager from it). + * + * THE RULE THIS FILE EXISTS FOR: nothing is run or loaded unless its bytes + * match a SHA-256 written down here. So every url below must name bytes that + * CANNOT change under its checksum: a GitHub release asset by tag, and a + * Hugging Face file by COMMIT, never `resolve/main`. A url that follows a + * branch is a download that starts failing its checksum the day upstream + * re-uploads, on every user's machine at once, with nothing we can ship fast + * enough to help. + * + * Sizes and checksums were read from the upstream APIs on 2026-09-19 (the + * GitHub release API's `size` and `digest`; Hugging Face's `lfs.size` and + * `lfs.oid`), and two of them were CROSS-CHECKED against files already on the + * owner's disk with sha256sum: ggml-base.bin and ggml-large-v3.bin both matched + * byte for byte. The commit-pinned urls were each asked for with a HEAD request + * and answered 200 with the same size and the same hash in `X-Linked-ETag`. + */ +import type { CatalogEntry } from './dictationTypes' + +/** The whisper.cpp release both the bundled CPU engine and the GPU pack come from. */ +const TAG = 'b5130' +const RELEASE = `https://github.com/ggml-org/whisper.cpp/releases/download/${TAG}` + +/** + * The newest commit of huggingface.co/ggerganov/whisper.cpp on 2026-09-19 + * (the repo has not moved since 2024-10-29). Every model url is resolved + * against it, so a later re-upload to `main` changes nothing here. + */ +const MODELS_COMMIT = '5359861c739e955e79d9a303bcbc70fb988958b1' +const model = (file: string): string => + `https://huggingface.co/ggerganov/whisper.cpp/resolve/${MODELS_COMMIT}/${file}` + +/** + * The CPU engine each app BUNDLES, fetched at build time by + * `core/tools/fetch-whisper.mjs`. That script runs before anything is compiled, + * so it cannot import this file and carries its own copy of the pin; the + * catalog test reads the script as text and fails if the two ever disagree. + * + * `files` is the MINIMAL set whisper-server.exe needs, MEASURED rather than + * guessed: those thirteen were copied alone into an empty folder and the server + * started, loaded Base and transcribed a clip (answering on its port 190 ms + * after spawn). They are the zip's `Release/` folder minus 27 files of other + * programs (the CLI, the benchmarks, the tests, Parakeet, llama.dll, SDL2): + * 10.8 MB kept of 21.8 MB. + * + * ALL NINE `ggml-cpu-*.dll` STAY, and that is not caution. ggml picks its CPU + * backend at run time by what the processor supports (this machine loaded + * `ggml-cpu-cascadelake.dll`; an older one falls to haswell, sandybridge, sse42 + * or plain x64), so which one is "needed" is a fact about the USER's PC, not + * about ours. Dropping the ones this machine did not load would ship an engine + * that works here and nowhere else. + * + * The names are FLATTENED (what the engine folder holds), not where the zip + * kept them. The official zip carries NO licence file, so none is listed. + */ +export const ENGINE = { + tag: TAG, + asset: 'whisper-bin-x64.zip', + url: `${RELEASE}/whisper-bin-x64.zip`, + bytes: 8573270, + sha256: 'f9ec6c52a2e949b62ab51fa21d0d497958f9e41c3010c157c4e42932d5316f3c', + files: [ + 'whisper-server.exe', + 'whisper.dll', + 'ggml.dll', + 'ggml-base.dll', + 'ggml-cpu-alderlake.dll', + 'ggml-cpu-cannonlake.dll', + 'ggml-cpu-cascadelake.dll', + 'ggml-cpu-haswell.dll', + 'ggml-cpu-icelake.dll', + 'ggml-cpu-sandybridge.dll', + 'ggml-cpu-skylakex.dll', + 'ggml-cpu-sse42.dll', + 'ggml-cpu-x64.dll' + ] +} as const satisfies { + tag: string + asset: string + url: string + bytes: number + sha256: string + files: readonly string[] +} + +/** + * Multilingual models only (owner decision: auto-detect is the default, so an + * English-only variant would be a model that silently ignores the language + * picker). In the order the model manager shows them, smallest first, with + * the e2e's model and the GPU pack after the ones a user picks from. + * + * The notes say who a model is FOR and leave "Recommended" and "needs the GPU + * pack" to the badges the manager draws from `recommended` and `needsGpu`: a + * note repeating a badge is the same thing said twice on one row. + */ +export const CATALOG: readonly CatalogEntry[] = [ + { + id: 'base', + label: 'Base', + note: 'Fast live text on any PC.', + url: model('ggml-base.bin'), + bytes: 147951465, + sha256: '60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe', + kind: 'model', + // MEASURED: about 0.9 s a pass on a CPU whatever the clip's length, which + // is what lets the pill's text refresh once a second with no GPU at all. + recommended: 'cpu' + }, + { + id: 'small', + label: 'Small', + note: 'More accurate without a GPU, slower: a pass takes about 3x Base.', + url: model('ggml-small.bin'), + bytes: 487601967, + sha256: '1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b', + kind: 'model' + }, + { + id: 'large-v3-turbo', + label: 'Large v3 Turbo', + note: 'Close to Large v3 at about half the download.', + url: model('ggml-large-v3-turbo.bin'), + bytes: 1624555275, + sha256: '1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69', + kind: 'model', + needsGpu: true, + recommended: 'gpu' + }, + { + id: 'large-v3', + label: 'Large v3', + note: 'The most accurate.', + url: model('ggml-large-v3.bin'), + bytes: 3095033483, + sha256: '64d182b440b98d5203c4f9bd541544d84c605196c4f7b845dfa11fb23594d1e2', + kind: 'model', + // MEASURED: 20 s for an 11 s clip on a fast CPU, 0.36 to 0.50 s through + // the GPU pack. Listed without the pack, but never usable without it. + needsGpu: true + }, + { + id: 'tiny', + label: 'Tiny', + note: 'For the end-to-end test: small enough to fetch once and run anywhere.', + url: model('ggml-tiny.bin'), + bytes: 77691713, + sha256: 'be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21', + kind: 'model', + e2eOnly: true + }, + { + id: 'gpu-pack', + label: 'NVIDIA GPU engine', + note: 'The official CUDA 12.4 build. Runs the Large models in under a second.', + // ONE pack for every NVIDIA card (measured: it runs on an RTX 5090, a card + // newer than CUDA 12.4, after a one-time 9.1 s kernel compile), which is + // why the 273 MB CUDA 11.8 pack of the same release is not catalogued. + url: `${RELEASE}/whisper-cublas-12.4.0-bin-x64.zip`, + bytes: 674539285, + sha256: 'af520ddd034d985b55dfeea3e465ed93653ba2aee1a55e865033edc548c272a7', + kind: 'gpu-pack' + } +] + +/** The entry of that exact id (ids are lowercase), or undefined. */ +export function catalogEntry(id: string): CatalogEntry | undefined { + return CATALOG.find((e) => e.id === id) +} + +/** What the model manager lists: models a user picks from, in catalog order. */ +export function visibleModels(): CatalogEntry[] { + return CATALOG.filter((e) => e.kind === 'model' && !e.e2eOnly) +} + +/** + * The model to offer when dictation is switched on with none downloaded, and + * the one wearing the Recommended badge. `hasGpuPack` means the pack is + * INSTALLED, not merely that an NVIDIA card is present: until it is, the GPU + * pick would be a 1.5 GB download that then transcribes at CPU speed. + */ +export function recommendedModel(hasGpuPack: boolean): CatalogEntry { + const want = hasGpuPack ? 'gpu' : 'cpu' + // The test holds the catalog to exactly one of each, so the fallback is for + // the type checker and for a catalog edited wrongly, never for a user. + return visibleModels().find((e) => e.recommended === want) ?? visibleModels()[0] +} + +/** + * The language picker. 'auto' is Whisper's own detection and the default; the + * rest are ISO 639-1 codes exactly as whisper.cpp's `-l` takes them (so + * Norwegian is `no`, Hebrew is `he`, Indonesian is `id`). Whisper knows 99 + * languages; this is the short list of the ones it does WELL, the Nordic ones + * first after English because that is where both apps' first users are, then + * alphabetical. A language missing here still works through Auto-detect. + */ +export const LANGUAGES: readonly { code: string; name: string }[] = [ + { code: 'auto', name: 'Auto-detect' }, + { code: 'en', name: 'English' }, + { code: 'no', name: 'Norwegian' }, + { code: 'sv', name: 'Swedish' }, + { code: 'da', name: 'Danish' }, + { code: 'fi', name: 'Finnish' }, + { code: 'ar', name: 'Arabic' }, + { code: 'zh', name: 'Chinese' }, + { code: 'cs', name: 'Czech' }, + { code: 'nl', name: 'Dutch' }, + { code: 'fr', name: 'French' }, + { code: 'de', name: 'German' }, + { code: 'el', name: 'Greek' }, + { code: 'he', name: 'Hebrew' }, + { code: 'hi', name: 'Hindi' }, + { code: 'hu', name: 'Hungarian' }, + { code: 'id', name: 'Indonesian' }, + { code: 'it', name: 'Italian' }, + { code: 'ja', name: 'Japanese' }, + { code: 'ko', name: 'Korean' }, + { code: 'pl', name: 'Polish' }, + { code: 'pt', name: 'Portuguese' }, + { code: 'ro', name: 'Romanian' }, + { code: 'ru', name: 'Russian' }, + { code: 'es', name: 'Spanish' }, + { code: 'tr', name: 'Turkish' }, + { code: 'uk', name: 'Ukrainian' }, + { code: 'vi', name: 'Vietnamese' } +] diff --git a/core/tools/fetch-whisper.mjs b/core/tools/fetch-whisper.mjs new file mode 100644 index 0000000..927101f --- /dev/null +++ b/core/tools/fetch-whisper.mjs @@ -0,0 +1,208 @@ +// Fetch the whisper.cpp CPU engine dictation bundles (#13), at build time. +// +// node fetch-whisper.mjs +// +// It lives in the CORE, not in an app, so Prism Terminal and Prism fetch the +// same bytes: each app's build runs it with its own (the folder its +// electron-builder config copies to resources/bin/whisper). +// +// The engine is the OFFICIAL whisper.cpp release zip, pinned by tag and +// verified by SHA-256 (owner decision: official pinned binaries only, nothing +// we compile). The binaries are NOT committed: this is what a fresh clone and +// CI run before packaging, and re-running is free once is populated. +// +// Plain Node, no dependencies, on purpose: it runs before `npm run build`, in +// both apps, and in a package (core-dist) that has no install step at all. +// Which is also why it cannot import core/shared/dictationCatalog.ts. +/* global console, fetch */ +import { Buffer } from 'node:buffer' +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import process from 'node:process' + +// THESE FIVE MUST SAY WHAT `ENGINE` IN core/shared/dictationCatalog.ts SAYS. +// Two copies of a pin drift silently, so dictationCatalog.test.ts reads this +// file as text and fails when they disagree. Keep each on one line, in this +// shape, or that test cannot find it. +const TAG = 'b5130' +const ASSET = 'whisper-bin-x64.zip' +const SHA256 = 'f9ec6c52a2e949b62ab51fa21d0d497958f9e41c3010c157c4e42932d5316f3c' +const BYTES = 8573270 +// The minimal set whisper-server.exe runs on, MEASURED: these thirteen alone +// in an empty folder start, load a model and transcribe. Every ggml-cpu-*.dll +// stays, because ggml picks one at run time by what the USER's processor +// supports (cascadelake on the machine this was measured on; haswell, +// sandybridge, sse42 or plain x64 on older ones). The other 27 files of the +// zip are other programs (the CLI, benchmarks, tests, Parakeet, SDL2). +const FILES = [ + 'whisper-server.exe', + 'whisper.dll', + 'ggml.dll', + 'ggml-base.dll', + 'ggml-cpu-alderlake.dll', + 'ggml-cpu-cannonlake.dll', + 'ggml-cpu-cascadelake.dll', + 'ggml-cpu-haswell.dll', + 'ggml-cpu-icelake.dll', + 'ggml-cpu-sandybridge.dll', + 'ggml-cpu-skylakex.dll', + 'ggml-cpu-sse42.dll', + 'ggml-cpu-x64.dll' +] +// Where the zip keeps them. They land FLATTENED in , beside each other, +// which is what the DLL loader needs and what the engine resolver expects. +/** Microsoft's C++ runtime, copied app-local beside the engine (see below). */ +const RUNTIME = ['msvcp140.dll', 'vcruntime140.dll', 'vcruntime140_1.dll', 'vcomp140.dll'] +const ZIP_DIR = 'Release' +const URL = `https://github.com/ggml-org/whisper.cpp/releases/download/${TAG}/${ASSET}` + +const fail = (msg) => { + console.error('whisper: ' + msg) + process.exit(1) +} + +if (process.platform !== 'win32') + fail('this is the Windows x64 engine; nothing to fetch on ' + process.platform) +if (!process.argv[2]) fail('usage: node fetch-whisper.mjs ') +const OUT = resolve(process.argv[2]) + +const stamp = join(OUT, '.source') +if ( + existsSync(join(OUT, 'whisper-server.exe')) && + existsSync(stamp) && + readFileSync(stamp, 'utf8').trim() === SHA256 +) { + console.log(`whisper: ${OUT} already holds ${ASSET} of ${TAG}`) + process.exit(0) +} + +console.log(`whisper: downloading ${ASSET} of ${TAG} (${(BYTES / 1e6).toFixed(1)} MB)`) +let res +try { + res = await fetch(URL) +} catch (e) { + fail(`download failed (${e?.cause?.code ?? e?.message ?? e})\n ${URL}`) +} +if (!res.ok) { + console.error(`whisper: download failed (${res.status} ${res.statusText})\n ${URL}`) + // GitHub keeps a release's assets for as long as the release exists, so a + // 404 means upstream removed or renamed it, not that this machine is broken. + // The pin is deliberate: the fix is to re-pin, never to follow `latest` and + // lose the checksum. + if (res.status === 404) + console.error( + 'whisper: that release asset is gone upstream. Re-pin TAG, ASSET, SHA256 and BYTES above from\n' + + ' https://github.com/ggml-org/whisper.cpp/releases (the whisper-bin-x64.zip of a release; its\n' + + ' sha256 and size are on the release page and in the GitHub API as `digest` and `size`),\n' + + ' check FILES against the new zip, and make ENGINE in core/shared/dictationCatalog.ts say the\n' + + ' same, with the GPU pack of the SAME tag. dictationCatalog.test.ts holds the two together.' + ) + process.exit(1) +} +const buf = Buffer.from(await res.arrayBuffer()) + +const got = createHash('sha256').update(buf).digest('hex') +if (got !== SHA256) + fail( + `SHA-256 mismatch, refusing to unpack\n expected ${SHA256}\n got ${got}` + + (buf.length !== BYTES + ? `\n (${buf.length} bytes, expected ${BYTES}: a cut-off download?)` + : '') + ) + +// Windows' OWN tar, by full path. It is bsdtar (libarchive, in the box since +// Windows 10 1803) and reads zip. A bare `tar` is whatever PATH finds first, +// and under Git Bash, where npm scripts are often run, that is GNU tar, which +// does not: MEASURED, "This does not look like a tar archive". +const TAR = join(process.env.SystemRoot ?? 'C:\\Windows', 'System32', 'tar.exe') +if (!existsSync(TAR)) + fail(`${TAR} is missing; it is what unpacks the zip (Windows 10 1803 or later)`) + +const tmp = mkdtempSync(join(tmpdir(), 'prism-whisper-')) +let taken = 0 +// A failure in here THROWS rather than exits: process.exit skips `finally`, +// and the temp folder (the zip plus 22 MB unpacked) would be left behind. +let problem = null +try { + const zip = join(tmp, ASSET) + writeFileSync(zip, buf) + const tree = join(tmp, 'x') + mkdirSync(tree) + // argv, never a shell string: both paths may hold spaces. + execFileSync(TAR, ['-xf', zip, '-C', tree], { + stdio: ['ignore', 'ignore', 'inherit'], + windowsHide: true + }) + + // Checked BEFORE anything in is touched, so a zip that has changed + // shape cannot leave half an engine behind a stamp-less folder. + for (const name of FILES) + if (!existsSync(join(tree, ZIP_DIR, name))) + throw new Error(`${ZIP_DIR}/${name} is missing from the archive`) + + mkdirSync(OUT, { recursive: true }) + // is the caller's folder, so it is never removed wholesale. What + // goes is only what a previous pin could have left: an engine DLL that is no + // longer in FILES would otherwise still be found, and loaded, by ggml. + for (const old of readdirSync(OUT)) + if (old === '.source' || /^(whisper|ggml)[\w.-]*\.(exe|dll)$/i.test(old)) + rmSync(join(OUT, old), { force: true }) + for (const name of FILES) { + copyFileSync(join(tree, ZIP_DIR, name), join(OUT, name)) + taken++ + } + + // THE C++ RUNTIME TRAVELS WITH THE ENGINE. Every file above imports + // MSVCP140 / VCRUNTIME140 / VCRUNTIME140_1, and ggml imports VCOMP140 + // (OpenMP); MEASURED from their import tables, and neither official zip + // carries any of them. They are on most PCs because something else installed + // the Visual C++ redistributable, which is exactly the kind of luck a + // product for strangers' machines cannot lean on: without them the server + // dies at launch with no message. Microsoft licenses these four for + // APP-LOCAL deployment (a copy beside the exe), so that is what ships. + // Taken from the BUILD machine, and only if Windows itself vouches for them: + // a valid Authenticode signature from Microsoft. A build machine without + // them fails the build here, loudly, rather than shipping an engine that + // starts only where the developer happens to sit. + const SYS = join(process.env.SystemRoot ?? 'C:\\Windows', 'System32') + for (const name of RUNTIME) { + const from = join(SYS, name) + if (!existsSync(from)) throw new Error(`${from} is missing: install the Visual C++ 2015-2022 x64 redistributable on the build machine`) + const signer = execFileSync( + join(SYS, 'WindowsPowerShell', 'v1.0', 'powershell.exe'), + ['-NoProfile', '-NonInteractive', '-Command', '$s = Get-AuthenticodeSignature -LiteralPath $env:PRISM_DLL; "$($s.Status)|$($s.SignerCertificate.Subject)"'], + { encoding: 'utf8', windowsHide: true, env: { ...process.env, PRISM_DLL: from } } + ).trim() + if (!/^Valid\|.*O=Microsoft Corporation/.test(signer)) throw new Error(`${from} is not validly signed by Microsoft (${signer})`) + copyFileSync(from, join(OUT, name)) + taken++ + } + + // whisper.cpp is MIT, and MIT asks that its notice travel with every copy. + // The official zip has no licence file in it, so it is fetched from the same + // tag the binaries were built from. + const lic = await fetch(`https://raw.githubusercontent.com/ggml-org/whisper.cpp/${TAG}/LICENSE`) + if (!lic.ok) throw new Error(`the whisper.cpp LICENSE could not be fetched (${lic.status})`) + writeFileSync(join(OUT, 'LICENSE-whisper.cpp.txt'), await lic.text()) + taken++ + // Last, so the stamp only ever vouches for a complete folder. + writeFileSync(stamp, SHA256 + '\n') +} catch (e) { + problem = e?.message ?? String(e) +} finally { + rmSync(tmp, { recursive: true, force: true }) +} +if (problem) fail('could not unpack: ' + problem) +console.log(`whisper: unpacked ${taken} files into ${OUT}`) diff --git a/electron-builder.yml b/electron-builder.yml index e2ad2d8..2b12350 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -25,6 +25,11 @@ npmRebuild: false extraResources: - from: build/icon.ico to: icon.ico + # The speech engine for dictation (#13): whisper.cpp's official CPU build, + # fetched by `npm run fetch:bin` against a pinned SHA-256. Beside the app, not + # in the asar: it is an exe and its DLLs, and Windows has to be able to run it. + - from: vendor/whisper + to: bin/whisper win: icon: build/icon.ico executableName: PrismTerminal diff --git a/package.json b/package.json index 98d2b9c..b34004b 100644 --- a/package.json +++ b/package.json @@ -15,10 +15,11 @@ "typecheck": "npm run typecheck:node && npm run typecheck:web", "test": "vitest run --passWithNoTests", "test:watch": "vitest", - "e2e": "npm run build && node tools/e2e/run.mjs", + "fetch:bin": "node core/tools/fetch-whisper.mjs vendor/whisper", + "e2e": "npm run fetch:bin && npm run build && node tools/e2e/run.mjs", "lint": "eslint .", "format": "prettier --write .", - "package": "electron-vite build && electron-builder --win" + "package": "npm run fetch:bin && electron-vite build && electron-builder --win" }, "dependencies": { "@xterm/addon-fit": "^0.11.0", diff --git a/src/main/index.ts b/src/main/index.ts index e4c5271..69612b4 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1,11 +1,13 @@ -import { app, BrowserWindow, clipboard, dialog, ipcMain, Menu, shell } from 'electron' +import { app, BrowserWindow, clipboard, dialog, ipcMain, Menu, session, shell } from 'electron' import pkg from '../../package.json' +import { existsSync } from 'fs' import { stat } from 'fs/promises' import { homedir } from 'os' import { dirname, join } from 'path' import type { Restored, SavedTabs, UpdateInfo } from '@shared/types' import { claudeSessions } from '@core/main/agentResume' import { registerTermIpc } from '@core/main/ipc' +import { registerDictationIpc } from '@core/main/dictationIpc' import { planRestore } from './planRestore' import { foldersFromArgv } from './argv' import { acrylicOk, createMaterial } from './material' @@ -42,6 +44,16 @@ const E2E = process.argv.includes('--e2e') if (!app.commandLine.hasSwitch('user-data-dir')) app.setPath('userData', join(app.getPath('appData'), 'PrismTerminal')) +// THE E2E SPEAKS THROUGH A FAKE MICROPHONE (#13): Chromium plays a WAV into +// getUserMedia, so the dictation scenario runs the REAL capture, the REAL +// engine and the REAL paste with nobody in the room (MEASURED under Electron: +// the fake device delivers the file's audio, peak 0.78). Only under --e2e. +if (E2E && process.env.PT_E2E_MIC) { + app.commandLine.appendSwitch('use-fake-device-for-media-stream') + app.commandLine.appendSwitch('use-fake-ui-for-media-stream') + app.commandLine.appendSwitch('use-file-for-fake-audio-capture', process.env.PT_E2E_MIC) +} + /** * Two things Windows needs told before the first window exists (2026-08-30). * @@ -102,6 +114,8 @@ const isDir = (p: unknown): Promise => * hands its folder over and ends. * ------------------------------------------------------------------ */ let quitting = false // app.quit() is under way +/** Kills dictation's children (the speech server, the media helper). Set by wireIpc. */ +let stopDictation: () => void = () => {} let quitWanted = false // a quit the close question interrupted; confirm resumes it let agentBusy = false // mirrored from the renderer: an agent is WORKING let closeAgreed = false // the renderer's "go ahead" for the close in flight @@ -352,6 +366,27 @@ function wireIpc(): void { // Prism's reroot. This app never moves a shell it did not start there. mayCd: () => false }) + // DICTATION (#13) is the core's too. What is this app's own: where ITS + // installer put the CPU engine, the folder it shares with Prism, and the GPU + // question, answered by Electron's own adapter list (no process spawned). + stopDictation = registerDictationIpc({ + ipcMain, + send, + cpuEngineDir: () => { + const dir = + process.env.PT_WHISPER_DIR ?? + (app.isPackaged ? join(process.resourcesPath, 'bin', 'whisper') : join(app.getAppPath(), 'vendor', 'whisper')) + return existsSync(join(dir, 'whisper-server.exe')) ? dir : null + }, + sharedRoot: + process.env.PT_DICTATION_ROOT ?? + join(process.env.LOCALAPPDATA ?? join(homedir(), 'AppData', 'Local'), 'PrismDictation'), + hasNvidia: async () => { + if (process.env.PT_E2E_NVIDIA) return process.env.PT_E2E_NVIDIA === '1' + const info = (await app.getGPUInfo('basic')) as { gpuDevice?: Array<{ vendorId?: number }> } + return (info.gpuDevice ?? []).some((d) => d.vendorId === 0x10de) + } + }) // A tab's "Show in File Explorer": a folder that exists, and nothing else. ipcMain.on('shell:show-folder', (_e, p: string) => { void isDir(p).then((ok) => { @@ -512,11 +547,31 @@ if (!app.requestSingleInstanceLock()) { // Every shell dies with the app; a pty with no window is an orphan. app.on('will-quit', () => { stopDwmHelper() + stopDictation() killAll() tabs.flush() }) app.whenReady().then(() => { + // WHAT THE PAGE MAY ASK CHROMIUM FOR, as a closed list. Electron's default + // is to grant everything; with dictation (#13) the page now asks for the + // MICROPHONE, so the answer is written down: audio capture for this app's + // own window, the clipboard the terminal already used, and nothing else. + // Never the camera, never another origin. + const own = (wc: Electron.WebContents | null): boolean => !!wc && wc === mainWindow?.webContents + const granted = new Set(['media', 'clipboard-read', 'clipboard-sanitized-write', 'fullscreen']) + session.defaultSession.setPermissionRequestHandler((wc, permission, callback, details) => { + if (!own(wc) || !granted.has(permission)) return callback(false) + if (permission === 'media') { + const types = (details as { mediaTypes?: string[] }).mediaTypes ?? [] + return callback(types.length > 0 && types.every((t) => t === 'audio')) + } + callback(true) + }) + session.defaultSession.setPermissionCheckHandler((wc, permission, _origin, details) => { + if (!own(wc) || !granted.has(permission)) return false + return permission !== 'media' || (details as { mediaType?: string }).mediaType !== 'video' + }) wireIpc() createWindow() // Warm the terminal's fixed costs shortly after launch: the native module diff --git a/src/preload/index.ts b/src/preload/index.ts index a160729..cd46196 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,5 +1,6 @@ import { contextBridge, ipcRenderer, webUtils } from 'electron' import { createTermApi } from '@core/preload/api' +import { createDictationApi } from '@core/preload/dictationApi' import type { Restored, SavedTabs, UpdateInfo } from '@shared/types' // The renderer's whole view of the machine. SANDBOXED: nothing here touches @@ -20,6 +21,8 @@ const api = { // THE TERMINAL'S BRIDGE IS THE CORE'S (core/preload/api): the member names, // their signatures and the channel names exist once, for this app and Prism. ...createTermApi(ipcRenderer), + // ...and so is dictation's (#13). + ...createDictationApi(ipcRenderer), /** The real path of a File from a drop (the sandbox hides `File.path`). */ getDroppedPath: (file: File): string => webUtils.getPathForFile(file), /** A dropped path as the folder a tab would open in: a folder is itself, a diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index d3998e7..c0bb153 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -24,6 +24,8 @@ import { type TabState } from './lib/tabs' import { useAgentIndicator } from '@core/renderer/lib/useAgentIndicator' +import { useDictationArm } from '@core/renderer/lib/useDictation' +import { DictationPill } from '@core/renderer/components/DictationPill' import { humanFor, workingFor } from '@core/renderer/lib/agentClock' import { forgetSession, markResume, markTouched } from '@core/renderer/lib/termActivity' import { onCwd, pasteInto } from '@core/renderer/lib/termBus' @@ -80,6 +82,10 @@ export default function App(): JSX.Element { const active = tabs.find((t) => t.id === activeId) ?? null const activeShell = active && active.kind !== 'settings' ? active : null const indicator = useAgentIndicator(activeShell ? activeShell.id : null) + // Dictation (#13) is the core's. It is armed here because this is where + // "which shell is in front" is known; it renders nothing and re-renders + // nothing, and with the setting off it listens to nothing. + useDictationArm(activeShell ? activeShell.id : null) const findOpen = !!activeShell && findFor === activeShell.id const { agentIds, workingIds, doneIds, agentKinds } = indicator @@ -364,6 +370,7 @@ export default function App(): JSX.Element { shellId={savedShellId()} /> )} + {activeShell && findOpen && ( setFindFor(null)} /> )} diff --git a/src/renderer/src/components/Settings.tsx b/src/renderer/src/components/Settings.tsx index 66b42ca..4b74b62 100644 --- a/src/renderer/src/components/Settings.tsx +++ b/src/renderer/src/components/Settings.tsx @@ -1,6 +1,7 @@ import { useEffect, useState, type JSX, type ReactNode } from 'react' import { setNewTabMode, useNewTabFolder, useNewTabMode } from '../lib/newTabPrefs' import { Pref, ROWS, ROW_BUTTON, Segmented, Switch } from '@core/renderer/settings/fields' +import { DictationSettings } from '@core/renderer/settings/Dictation' import { TerminalAppearanceSettings } from '@core/renderer/settings/TerminalAppearance' import { AgentIndicatorSetting, ShellSetting } from '@core/renderer/settings/TerminalBehaviour' @@ -119,7 +120,7 @@ function GeneralTab(): JSX.Element { /* ---------- page shell ---------- */ -type TabId = 'general' | 'appearance' +type TabId = 'general' | 'appearance' | 'dictation' const Ico = ({ d }: { d: string }): JSX.Element => ( ( ) -// How the app behaves, then what it looks like. +// How the app behaves, what it looks like, then the one optional feature +// that is big enough for a page of its own (#13). const TABS: Array<{ id: TabId; label: string; icon: ReactNode }> = [ { id: 'general', @@ -152,6 +154,11 @@ const TABS: Array<{ id: TabId; label: string; icon: ReactNode }> = [ icon: ( ) + }, + { + id: 'dictation', + label: 'Dictation', + icon: } ] @@ -210,7 +217,13 @@ export default function Settings(): JSX.Element {
- {tab === 'general' ? : } + {tab === 'general' ? ( + + ) : tab === 'appearance' ? ( + + ) : ( + + )}
diff --git a/src/renderer/src/components/TabStrip.tsx b/src/renderer/src/components/TabStrip.tsx index 33f7dc4..b32b5e2 100644 --- a/src/renderer/src/components/TabStrip.tsx +++ b/src/renderer/src/components/TabStrip.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState, type JSX, type MouseEvent, type PointerEvent } from 'react' import { tabLabels, type Tab } from '../lib/tabs' +import { DictationTabMark } from '@core/renderer/components/DictationTabMark' import { useAgentIndicator } from '@core/renderer/lib/termLook' import { useAgentColors } from '@core/renderer/lib/agentColors' import { contrastRatio } from '@core/renderer/lib/termAnsi' @@ -388,6 +389,7 @@ export function TabStrip({ )} )} + {t.kind !== 'settings' && } ) : ( - )} diff --git a/package-lock.json b/package-lock.json index 8c963d2..e472f93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "prism-terminal", - "version": "0.1.1", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "prism-terminal", - "version": "0.1.1", + "version": "0.2.0", "license": "MIT", "dependencies": { "@xterm/addon-fit": "^0.11.0", diff --git a/package.json b/package.json index b34004b..97e8a2e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "prism-terminal", "productName": "Prism Terminal", - "version": "0.1.1", + "version": "0.2.0", "description": "A tabbed Windows terminal for AI CLIs.", "main": "./out/main/index.js", "author": "Max", diff --git a/tools/e2e/run.mjs b/tools/e2e/run.mjs index 363fd15..5a524b3 100644 --- a/tools/e2e/run.mjs +++ b/tools/e2e/run.mjs @@ -147,10 +147,15 @@ async function cached(name, url, sha256) { return file } function tinyModel() { + // The catalog builds its model urls from ONE pinned Hugging Face commit, so + // that is what is read here: the same commit, the same file, the same sha. const src = readFileSync(resolve(process.cwd(), 'core/shared/dictationCatalog.ts'), 'utf8') + const commit = src.match(/const MODELS_COMMIT = '([0-9a-f]{40})'/)?.[1] const block = src.slice(src.indexOf("id: 'tiny'")) - const pick = (re) => block.match(re)?.[1] - return { url: pick(/url:\s*'([^']+)'/) ?? pick(/url:\s*\n?\s*'([^']+)'/), sha256: pick(/sha256:\s*\n?\s*'([0-9a-f]{64})'/) } + const file = block.match(/url:\s*model\('([^']+)'\)/)?.[1] + const sha256 = block.match(/sha256:\s*'([0-9a-f]{64})'/)?.[1] + if (!commit || !file || !sha256) throw new Error('the catalog no longer spells the tiny model the way the e2e reads it') + return { url: `https://huggingface.co/ggerganov/whisper.cpp/resolve/${commit}/${file}`, sha256 } } /** Speech servers started out of THIS checkout's engine folder, and no others: * the owner's own dictation tool runs a whisper-server of its own. */ @@ -759,7 +764,9 @@ const scenarios = { ok(moved, 'the level meter moves while the clip plays: a live mic is visible at once') const live = await until(async () => ((await page.locator('[data-dictation-live]').textContent().catch(() => '')) ?? '').trim(), 15000) ok(!!live, `live text appears while still listening ("${live}")`) - await sleep(9000) // let the whole sentence play + await sleep(4000) + await page.screenshot({ path: resolve(process.cwd(), '.e2e-shots/dictation-listening.png') }).catch(() => {}) + await sleep(5000) // let the whole sentence play const before = await termText(page) await page.keyboard.up('AltRight') ok(await until(async () => (await pill().getAttribute('data-dictation-pill').catch(() => null)) === 'transcribing', 4000), 'releasing says Transcribing') From 82b6df996473e5ae76b0cde78414a698d0dc9453 Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:42:50 +0200 Subject: [PATCH 05/12] build(dictation): keep the e2e cache out of the installer; the e2e can drive the packaged app (#13) Co-Authored-By: Claude Fable 5.1 --- electron-builder.yml | 8 ++++++++ tools/e2e/run.mjs | 24 ++++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/electron-builder.yml b/electron-builder.yml index 2b12350..e29eab6 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -13,6 +13,14 @@ files: - '!build/**' - '!assets/**' - '!dist/**' + # The e2e's cache (a 75 MB test model), its screenshots, the core's source + # (compiled into out/ already) and the speech engine, which ships through + # extraResources below: left in, the first alone added 73 MB to the installer + # (MEASURED, 110 MB -> 184 MB, the day dictation landed). + - '!.e2e-cache/**' + - '!.e2e-shots/**' + - '!vendor/**' + - '!core/**' # node-pty is the app's one native module. Its .node binaries, the conpty # helper dir and winpty-agent.exe must live on the real file system - Windows # cannot spawn an exe or load a DLL out of an asar archive. diff --git a/tools/e2e/run.mjs b/tools/e2e/run.mjs index 5a524b3..5e8e5f0 100644 --- a/tools/e2e/run.mjs +++ b/tools/e2e/run.mjs @@ -2,6 +2,9 @@ // // npm run e2e every scenario // npm run e2e -- theme only the scenarios whose name contains "theme" +// PT_E2E_PACKAGED=1 drive dist/win-unpacked (after `npm run package`) instead of out/: +// what proves the INSTALLER's layout, e.g. that the speech engine is +// where the packaged app looks for it // // OFFSCREEN and UNFOCUSED, as Prism's is. Electron has no headless mode and a // truly hidden window stops answering clicks, so each window is PARKED @@ -21,9 +24,14 @@ import { createRequire } from 'module' const require = createRequire(import.meta.url) const electronPath = require('electron') const MAIN = resolve(process.cwd(), 'out/main/index.js') +const PACKAGED = process.env.PT_E2E_PACKAGED === '1' ? resolve(process.cwd(), 'dist/win-unpacked/PrismTerminal.exe') : null const PROFILE_NAME = 'pt-e2e-profile' const only = process.argv.slice(2).filter((a) => !a.startsWith('-')) +if (PACKAGED && !existsSync(PACKAGED)) { + console.error('dist/win-unpacked/PrismTerminal.exe is missing: run "npm run package" first.') + process.exit(1) +} if (!existsSync(MAIN)) { console.error('out/main/index.js is missing: run "npm run build" first (npm run e2e does).') process.exit(1) @@ -47,7 +55,7 @@ function reapStrays() { [ '-NoProfile', '-Command', - `Get-CimInstance Win32_Process -Filter "Name='electron.exe'" | ` + + `Get-CimInstance Win32_Process | Where-Object { $_.Name -eq 'electron.exe' -or $_.Name -eq 'PrismTerminal.exe' } | ` + `Where-Object { $_.CommandLine -like '*${PROFILE_NAME}*' } | ` + 'ForEach-Object { $_.ProcessId }' ], @@ -77,7 +85,8 @@ function world() { async function launch(w, { args = [], pick, env = {} } = {}) { const app = await electron.launch({ - args: [MAIN, `--user-data-dir=${w.profile}`, '--e2e', ...args], + ...(PACKAGED ? { executablePath: PACKAGED } : {}), + args: [...(PACKAGED ? [] : [MAIN]), `--user-data-dir=${w.profile}`, '--e2e', ...args], env: { ...process.env, ...(pick ? { PT_E2E_PICK: pick } : {}), ...env } }) const page = await app.firstWindow() @@ -163,7 +172,7 @@ function ourSpeechServers() { try { const out = execFileSync( 'powershell.exe', - ['-NoProfile', '-Command', `(Get-CimInstance Win32_Process -Filter "Name='whisper-server.exe'" | Where-Object { $_.ExecutablePath -like '*\\vendor\\whisper\\*' } | Measure-Object).Count`], + ['-NoProfile', '-Command', `(Get-CimInstance Win32_Process -Filter "Name='whisper-server.exe'" | Where-Object { $_.ExecutablePath -like '*\\vendor\\whisper\\*' -or $_.ExecutablePath -like '*\\win-unpacked\\resources\\bin\\whisper\\*' } | Measure-Object).Count`], { encoding: 'utf8', windowsHide: true } ) return Number(out.trim()) || 0 @@ -694,11 +703,14 @@ const scenarios = { const root = join(w.profile, 'dictation') mkdirSync(join(root, 'models'), { recursive: true }) copyFileSync(model, join(root, 'models', 'tiny.bin')) - const engine = resolve(process.cwd(), 'vendor/whisper') - ok(existsSync(join(engine, 'whisper-server.exe')), 'the speech engine was fetched into vendor/whisper') + // Unpackaged, the engine is the fetched folder. PACKAGED, the app is told + // nothing: it has to find the engine where the installer put it. + const engine = PACKAGED ? resolve(process.cwd(), 'dist/win-unpacked/resources/bin/whisper') : resolve(process.cwd(), 'vendor/whisper') + ok(existsSync(join(engine, 'whisper-server.exe')), `the speech engine is in ${PACKAGED ? 'the packaged app' : 'vendor/whisper'}`) + ok(existsSync(join(engine, 'vcomp140.dll')) && existsSync(join(engine, 'LICENSE-whisper.cpp.txt')), 'with the C++ runtime and the MIT notice beside it') const { app, page } = await launch(w, { args: [w.alpha], - env: { PT_E2E_MIC: clip, PT_DICTATION_ROOT: root, PT_WHISPER_DIR: engine, PT_E2E_NVIDIA: '0' } + env: { PT_E2E_MIC: clip, PT_DICTATION_ROOT: root, PT_E2E_NVIDIA: '0', ...(PACKAGED ? {} : { PT_WHISPER_DIR: engine }) } }) await until(async () => (await tabLabels(page)).length === 1) await page.waitForFunction( From 1829ba3c74ed764239e1b510184e4d5d257bf1d9 Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:31 +0200 Subject: [PATCH 06/12] feat(dictation): the model manager reads as a list of things: vendor marks, full names, Uninstall, Enable / Disable (#13) Owner's notes of 2026-09-19. Disabling GPU acceleration is a per-app setting that stops THIS app using the engine and keeps the 675 MB on disk; Uninstall is what frees it. Also fixes a race in the links e2e (it could read the span in the frame before the link decoration was repainted). Co-Authored-By: Claude Fable 5.1 --- core/main/dictationEngine.ts | 7 +- core/main/dictationIpc.ts | 8 +- core/renderer/lib/dictation.ts | 6 +- core/renderer/lib/dictationPrefs.ts | 10 +- core/renderer/settings/Dictation.tsx | 185 ++++++++++++++------- core/renderer/settings/VendorMark.tsx | 34 ++++ core/renderer/settings/dictationOptions.ts | 8 +- core/renderer/settings/options.test.ts | 5 +- core/shared/dictationCatalog.test.ts | 10 +- core/shared/dictationCatalog.ts | 12 +- core/shared/dictationTypes.ts | 3 + tools/e2e/run.mjs | 81 ++++++++- 12 files changed, 284 insertions(+), 85 deletions(-) create mode 100644 core/renderer/settings/VendorMark.tsx diff --git a/core/main/dictationEngine.ts b/core/main/dictationEngine.ts index 0cb4c19..aefc0e9 100644 --- a/core/main/dictationEngine.ts +++ b/core/main/dictationEngine.ts @@ -232,8 +232,8 @@ export function createDictationEngine(deps: DictationEngineDeps): DictationEngin } } - async function pick(): Promise<{ kind: EngineKind; dir: string } | null> { - const gpuDir = gpuFellBack ? null : deps.store.gpuEngineDir() + async function pick(useGpu: boolean): Promise<{ kind: EngineKind; dir: string } | null> { + const gpuDir = gpuFellBack || !useGpu ? null : deps.store.gpuEngineDir() if (gpuDir && (await hasNvidia())) return { kind: 'gpu', dir: gpuDir } const cpuDir = deps.cpuDir() return cpuDir ? { kind: 'cpu', dir: cpuDir } : null @@ -388,7 +388,8 @@ export function createDictationEngine(deps: DictationEngineDeps): DictationEngin if (!modelPath) return { ok: false, reason: 'no-model' } const language = req.language || 'auto' - const want = await pick() + // Disabled in Settings is the user's word, and it wins over a pack on disk. + const want = await pick(req.useGpu !== false) if (stopped()) return STOPPED if (!want) return { ok: false, reason: 'no-engine' } diff --git a/core/main/dictationIpc.ts b/core/main/dictationIpc.ts index 3681cf2..86f7fa5 100644 --- a/core/main/dictationIpc.ts +++ b/core/main/dictationIpc.ts @@ -95,7 +95,13 @@ export function registerDictationIpc(deps: DictationIpcDeps): () => void { !/^[a-z]{2,4}$/.test(r.language) ) return { ok: false, reason: 'engine-failed', detail: 'bad request' } - return engine.transcribe({ wav: r.wav, modelId: r.modelId, language: r.language, final: r.final === true }) + return engine.transcribe({ + wav: r.wav, + modelId: r.modelId, + language: r.language, + final: r.final === true, + useGpu: r.useGpu !== false + }) }) ipcMain.on(DCH.stop, () => engine.stop()) diff --git a/core/renderer/lib/dictation.ts b/core/renderer/lib/dictation.ts index 06b8e8a..e6b6d3c 100644 --- a/core/renderer/lib/dictation.ts +++ b/core/renderer/lib/dictation.ts @@ -3,6 +3,7 @@ import { cleanTranscript } from './dictationClean' import { initialKeyState, reduceKey, type KeyEvt, type KeyState } from './dictationKey' import { dictationEnabled, + dictationGpu, dictationHotkey, dictationLanguage, dictationMic, @@ -177,7 +178,7 @@ async function begin(sessionId: string): Promise { r.lastPartialAt = cap.seconds() const wav = encodeWav(tailSamples(cap.snapshot(), TARGET_RATE, PARTIAL_WINDOW_S), TARGET_RATE) void host.api - .dictationTranscribe({ wav, modelId, language: dictationLanguage(), final: false }) + .dictationTranscribe({ wav, modelId, language: dictationLanguage(), final: false, useGpu: dictationGpu() }) .then((res) => { // Only while THIS recording is still listening: a partial that lands // after the final would overwrite the pill with older words. @@ -215,7 +216,8 @@ async function finishStop(r: Recording): Promise { wav: encodeWav(samples, TARGET_RATE), modelId: dictationModel(), language: dictationLanguage(), - final: true + final: true, + useGpu: dictationGpu() }) } catch { res = { ok: false as const, reason: 'engine-failed' as const } diff --git a/core/renderer/lib/dictationPrefs.ts b/core/renderer/lib/dictationPrefs.ts index 3a0fdfd..ae60d25 100644 --- a/core/renderer/lib/dictationPrefs.ts +++ b/core/renderer/lib/dictationPrefs.ts @@ -17,7 +17,8 @@ const K = { language: 'prism.dictation.language', pauseMedia: 'prism.dictation.pauseMedia', sounds: 'prism.dictation.sounds', - model: 'prism.dictation.model' + model: 'prism.dictation.model', + gpu: 'prism.dictation.gpu' } as const export const DICTATION_KEYS = K @@ -98,6 +99,12 @@ export const setDictationSounds = (on: boolean): void => write(K.sounds, on ? '1 export const dictationModel = (): string => read(K.model) ?? '' export const setDictationModel = (id: string): void => write(K.model, id) +/** GPU acceleration is ON unless switched off: installing the pack is the + * user saying they want it. Disabling keeps the files (675 MB is not something + * to download twice on a whim, and the other app may be using them). */ +export const dictationGpu = (): boolean => read(K.gpu) !== '0' +export const setDictationGpu = (on: boolean): void => write(K.gpu, on ? '1' : '0') + export const useDictationEnabled = (): boolean => useSyncExternalStore(sub, dictationEnabled) export const useDictationMode = (): DictMode => useSyncExternalStore(sub, dictationMode) export const useDictationHotkey = (): Hotkey => useSyncExternalStore(sub, dictationHotkey) @@ -106,4 +113,5 @@ export const useDictationLanguage = (): string => useSyncExternalStore(sub, dict export const useDictationPauseMedia = (): boolean => useSyncExternalStore(sub, dictationPauseMedia) export const useDictationSounds = (): boolean => useSyncExternalStore(sub, dictationSounds) export const useDictationModel = (): string => useSyncExternalStore(sub, dictationModel) +export const useDictationGpu = (): boolean => useSyncExternalStore(sub, dictationGpu) export const onDictationPrefs = sub diff --git a/core/renderer/settings/Dictation.tsx b/core/renderer/settings/Dictation.tsx index 5e1e735..3830eb7 100644 --- a/core/renderer/settings/Dictation.tsx +++ b/core/renderer/settings/Dictation.tsx @@ -1,10 +1,11 @@ -import { useCallback, useEffect, useRef, useState, type JSX } from 'react' +import { useCallback, useEffect, useRef, useState, type JSX, type ReactNode } from 'react' import { dictationHost } from '../host' import { catalogEntry, LANGUAGES, recommendedModel, visibleModels } from '../../shared/dictationCatalog' import type { CatalogEntry, DownloadFailure, EngineInfo, ItemStatus } from '../../shared/dictationTypes' import { DEFAULT_HOTKEY, formatHotkey, parseHotkeyFromEvent, type Hotkey } from '../lib/dictationKey' import { setDictationEnabled, + setDictationGpu, setDictationHotkey, setDictationLanguage, setDictationMic, @@ -13,6 +14,7 @@ import { setDictationPauseMedia, setDictationSounds, useDictationEnabled, + useDictationGpu, useDictationHotkey, useDictationLanguage, useDictationMic, @@ -23,6 +25,7 @@ import { } from '../lib/dictationPrefs' import { listMics, startCapture, type Capture } from '../lib/micCapture' import { Pref, Segmented, Select, Switch } from './fields' +import { VendorMark } from './VendorMark' /** * DICTATION'S SETTINGS PAGE, for both hosts (#13). Prism Terminal shows it as @@ -201,31 +204,60 @@ function MicField({ value, disabled }: { value: string; disabled: boolean }): JS ) } -/** One downloadable thing: a model, or the GPU pack. */ +/** "X Uninstall" (owner, 2026-09-19): it frees the disk, and says so by name. + * Quiet until hovered, because it is the one destructive control in the row. */ +function Uninstall({ onClick, what }: { onClick: () => void; what: string }): JSX.Element { + return ( + + ) +} + +/** + * One downloadable thing: a model, or the GPU engine. The vendor's mark leads + * the row (owner, 2026-09-19), then the FULL name ("Whisper Base", not "Base": + * a bare size word says nothing to someone who has never met Whisper), its + * size and what it is for. What is in use is marked on the row itself, by a + * badge and a faint fill, so the list reads as a state and not as a menu. + */ function ItemRow({ entry, status, progress, failure, - active, + badge, recommended, warn, + getLabel, + getPrimary, onDownload, onCancel, - onRemove, - onUse + installed }: { entry: CatalogEntry status: ItemStatus | undefined progress: number | undefined failure: DownloadFailure | undefined - active: boolean + /** "Active" on the model in use, "Enabled" on a GPU engine that is on. */ + badge: string | null recommended: boolean warn: string | null + /** The button that fetches it: "Download", or "Enable" for the GPU engine. */ + getLabel: string + getPrimary: boolean onDownload: () => void onCancel: () => void - onRemove: () => void - onUse?: () => void + /** The controls once it is on disk. */ + installed: ReactNode }): JSX.Element { const state = status?.state ?? 'absent' const received = progress ?? status?.received ?? 0 @@ -234,21 +266,29 @@ function ItemRow({
-
+ +
- {entry.label} - {size(entry.bytes)} + + {entry.label} + + {size(entry.bytes)} {recommended && ( - + Recommended )} - {active && ( - - Active + {badge && ( + + {badge} )}
@@ -256,14 +296,11 @@ function ItemRow({ {failure && FAILURES[failure] ? FAILURES[failure] : (warn ?? entry.note)}

-
+
{state === 'downloading' ? ( <> - + {pct}% ) : state === 'installed' ? ( - <> - {onUse && !active && ( - - )} - - + installed ) : ( // One filled button per list: the recommended pick (and the GPU - // pack, alone in its own). Four accent buttons in a column is a wall. - )}
@@ -303,6 +331,7 @@ export function DictationSettings(): JSX.Element | null { const pauseMedia = useDictationPauseMedia() const sounds = useDictationSounds() const model = useDictationModel() + const gpuWanted = useDictationGpu() const [status, setStatus] = useState([]) const [info, setInfo] = useState(null) @@ -326,6 +355,8 @@ export function DictationSettings(): JSX.Element | null { const stateOf = (id: string): ItemStatus | undefined => status.find((s) => s.id === id) const installed = (id: string): boolean => stateOf(id)?.state === 'installed' const gpuIn = installed(GPU_ID) + /** In use: on disk AND not switched off in this app. */ + const gpuOn = gpuIn && gpuWanted const download = (id: string): void => { setFailures((prev) => without(prev, id)) @@ -345,7 +376,7 @@ export function DictationSettings(): JSX.Element | null { }) } - const rec = recommendedModel(gpuIn) + const rec = recommendedModel(gpuOn) const noModel = !installed(model) const gpu = catalogEntry(GPU_ID) @@ -424,23 +455,36 @@ export function DictationSettings(): JSX.Element | null {

Bigger hears better and needs more from the PC. Downloaded once, used by Prism and Prism Terminal alike.

-
- {visibleModels().map((m) => ( - download(m.id)} - onCancel={() => api.dictationCancel(m.id)} - onRemove={() => remove(m.id)} - onUse={() => setDictationModel(m.id)} - /> - ))} +
+ {visibleModels().map((m) => { + const active = m.id === model && installed(m.id) + return ( + download(m.id)} + onCancel={() => api.dictationCancel(m.id)} + installed={ + <> + {!active && ( + + )} + remove(m.id)} /> + + } + /> + ) + })}
@@ -454,18 +498,43 @@ export function DictationSettings(): JSX.Element | null { ? 'The GPU engine would not start on this PC, so the CPU is answering. Removing and enabling it again may help.' : 'An NVIDIA card was found. The GPU engine makes the large models answer in under a second.'}

-
+
+ {/* ENABLE and DISABLE, not install and remove (owner, 2026-09-19). + Enabling fetches the engine the first time; disabling only stops + THIS app using it, because 675 MB is not something to download + again on a whim and the other app may be using it. Freeing the + disk is Uninstall, the same control a model has. */} download(GPU_ID)} + warn={gpuIn && !gpuWanted ? 'Disabled: dictation is using the CPU.' : null} + getLabel="Enable" + getPrimary + onDownload={() => { + setDictationGpu(true) + download(GPU_ID) + }} onCancel={() => api.dictationCancel(GPU_ID)} - onRemove={() => remove(GPU_ID)} + installed={ + <> + + remove(GPU_ID)} /> + + } />
diff --git a/core/renderer/settings/VendorMark.tsx b/core/renderer/settings/VendorMark.tsx new file mode 100644 index 0000000..d2b3f9c --- /dev/null +++ b/core/renderer/settings/VendorMark.tsx @@ -0,0 +1,34 @@ +import type { JSX } from 'react' + +/** + * Whose model a row is (#13; owner, 2026-09-19: "have a logo for the model on + * the left side, like an icon on each row: the OpenAI logo, and the NVIDIA logo + * for the GPU acceleration"). Whisper is OpenAI's model and the GPU engine is + * NVIDIA's CUDA build, so the mark says at a glance what a row IS before its + * name is read, and it is where a second model family would show itself. + * + * The outlines are simple-icons' (CC0), drawn in the row's own ink rather than + * in brand colours: a settings page wears the user's theme, and a green and a + * black logo on forty different grounds would be the only thing on the page + * that ignored it. They name a vendor; they are not an endorsement. + */ +const MARKS = { + openai: { title: 'OpenAI', d: 'M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z' }, + nvidia: { title: 'NVIDIA', d: 'M8.948 8.798v-1.43a6.7 6.7 0 0 1 .424-.018c3.922-.124 6.493 3.374 6.493 3.374s-2.774 3.851-5.75 3.851c-.398 0-.787-.062-1.158-.185v-4.346c1.528.185 1.837.857 2.747 2.385l2.04-1.714s-1.492-1.952-4-1.952a6.016 6.016 0 0 0-.796.035m0-4.735v2.138l.424-.027c5.45-.185 9.01 4.47 9.01 4.47s-4.08 4.964-8.33 4.964c-.37 0-.733-.035-1.095-.097v1.325c.3.035.61.062.91.062 3.957 0 6.82-2.023 9.593-4.408.459.371 2.34 1.263 2.73 1.652-2.633 2.208-8.772 3.984-12.253 3.984-.335 0-.653-.018-.971-.053v1.864H24V4.063zm0 10.326v1.131c-3.657-.654-4.673-4.46-4.673-4.46s1.758-1.944 4.673-2.262v1.237H8.94c-1.528-.186-2.73 1.245-2.73 1.245s.68 2.412 2.739 3.11M2.456 10.9s2.164-3.197 6.5-3.533V6.201C4.153 6.59 0 10.653 0 10.653s2.35 6.802 8.948 7.42v-1.237c-4.84-.6-6.492-5.936-6.492-5.936z' } +} as const + +export type Vendor = keyof typeof MARKS + +export function VendorMark({ vendor }: { vendor: Vendor }): JSX.Element { + const m = MARKS[vendor] + return ( + + + + + + ) +} diff --git a/core/renderer/settings/dictationOptions.ts b/core/renderer/settings/dictationOptions.ts index 3355d1e..a435f18 100644 --- a/core/renderer/settings/dictationOptions.ts +++ b/core/renderer/settings/dictationOptions.ts @@ -14,9 +14,9 @@ export interface DictationOption { id: string label: string type: 'switch' | 'choice' | 'key' | 'manager' - /** The localStorage key behind it; null where the row manages FILES, which - * both apps share, rather than a per-app value. */ - key: string | null + /** The localStorage key behind it. The two managers also manage FILES, which + * both apps share; the key is the per-app half (which model, GPU on or off). */ + key: string /** Absent = every PC. */ onlyWhere?: 'an NVIDIA adapter is present' } @@ -31,7 +31,7 @@ export const DICTATION_OPTIONS: readonly DictationOption[] = [ { id: 'dictation-sounds', label: 'Sounds', type: 'switch', key: DICTATION_KEYS.sounds }, { id: 'dictation-model', label: 'Models', type: 'manager', key: DICTATION_KEYS.model }, // The official GPU engine is NVIDIA's: the row is offered only where it can work. - { id: 'dictation-gpu', label: 'GPU acceleration', type: 'manager', key: null, onlyWhere: 'an NVIDIA adapter is present' } + { id: 'dictation-gpu', label: 'GPU acceleration', type: 'manager', key: DICTATION_KEYS.gpu, onlyWhere: 'an NVIDIA adapter is present' } ] /** The option ids a PC should be showing. */ diff --git a/core/renderer/settings/options.test.ts b/core/renderer/settings/options.test.ts index b286008..cd51688 100644 --- a/core/renderer/settings/options.test.ts +++ b/core/renderer/settings/options.test.ts @@ -36,11 +36,10 @@ describe('the terminal options list', () => { expect(all.filter((id) => !prism.includes(id))).toEqual(['term-opacity']) }) - it('gives dictation one key per option, under prism.dictation, and only the file managers none', () => { - const keys = DICTATION_OPTIONS.map((o) => o.key).filter((k): k is string => k !== null) + it('gives dictation one key per option, under prism.dictation', () => { + const keys = DICTATION_OPTIONS.map((o) => o.key) expect(new Set(keys).size).toBe(keys.length) for (const k of keys) expect(k.startsWith('prism.dictation.')).toBe(true) - expect(DICTATION_OPTIONS.filter((o) => o.key === null).map((o) => o.id)).toEqual(['dictation-gpu']) }) it('offers the GPU row only where an NVIDIA adapter is present', () => { diff --git a/core/shared/dictationCatalog.test.ts b/core/shared/dictationCatalog.test.ts index e0a7101..c737c53 100644 --- a/core/shared/dictationCatalog.test.ts +++ b/core/shared/dictationCatalog.test.ts @@ -40,11 +40,11 @@ describe('CATALOG', () => { ) expect(CATALOG.filter((e) => e.kind === 'gpu-pack').map((e) => e.id)).toEqual(['gpu-pack']) const label = (id: string): string | undefined => catalogEntry(id)?.label - expect(label('tiny')).toBe('Tiny') - expect(label('base')).toBe('Base') - expect(label('small')).toBe('Small') - expect(label('large-v3-turbo')).toBe('Large v3 Turbo') - expect(label('large-v3')).toBe('Large v3') + expect(label('tiny')).toBe('Whisper Tiny') + expect(label('base')).toBe('Whisper Base') + expect(label('small')).toBe('Whisper Small') + expect(label('large-v3-turbo')).toBe('Whisper Large v3 Turbo') + expect(label('large-v3')).toBe('Whisper Large v3') }) it('pins every model to a COMMIT of the Hugging Face repo, never to main', () => { diff --git a/core/shared/dictationCatalog.ts b/core/shared/dictationCatalog.ts index ee3f374..24467e9 100644 --- a/core/shared/dictationCatalog.ts +++ b/core/shared/dictationCatalog.ts @@ -99,7 +99,7 @@ export const ENGINE = { export const CATALOG: readonly CatalogEntry[] = [ { id: 'base', - label: 'Base', + label: 'Whisper Base', note: 'Fast live text on any PC.', url: model('ggml-base.bin'), bytes: 147951465, @@ -111,7 +111,7 @@ export const CATALOG: readonly CatalogEntry[] = [ }, { id: 'small', - label: 'Small', + label: 'Whisper Small', note: 'More accurate without a GPU, slower: a pass takes about 3x Base.', url: model('ggml-small.bin'), bytes: 487601967, @@ -120,7 +120,7 @@ export const CATALOG: readonly CatalogEntry[] = [ }, { id: 'large-v3-turbo', - label: 'Large v3 Turbo', + label: 'Whisper Large v3 Turbo', note: 'Close to Large v3 at about half the download.', url: model('ggml-large-v3-turbo.bin'), bytes: 1624555275, @@ -131,7 +131,7 @@ export const CATALOG: readonly CatalogEntry[] = [ }, { id: 'large-v3', - label: 'Large v3', + label: 'Whisper Large v3', note: 'The most accurate.', url: model('ggml-large-v3.bin'), bytes: 3095033483, @@ -143,7 +143,7 @@ export const CATALOG: readonly CatalogEntry[] = [ }, { id: 'tiny', - label: 'Tiny', + label: 'Whisper Tiny', note: 'For the end-to-end test: small enough to fetch once and run anywhere.', url: model('ggml-tiny.bin'), bytes: 77691713, @@ -153,7 +153,7 @@ export const CATALOG: readonly CatalogEntry[] = [ }, { id: 'gpu-pack', - label: 'NVIDIA GPU engine', + label: 'NVIDIA GPU acceleration', note: 'The official CUDA 12.4 build. Runs the Large models in under a second.', // ONE pack for every NVIDIA card (measured: it runs on an RTX 5090, a card // newer than CUDA 12.4, after a one-time 9.1 s kernel compile), which is diff --git a/core/shared/dictationTypes.ts b/core/shared/dictationTypes.ts index 576cb56..71bae23 100644 --- a/core/shared/dictationTypes.ts +++ b/core/shared/dictationTypes.ts @@ -52,6 +52,9 @@ export interface TranscribeRequest { language: string /** A partial may be dropped for a newer one; a final never is. */ final: boolean + /** False when the user has DISABLED GPU acceleration in this app: the pack + * may still be on disk (the other app may use it), it is just not used. */ + useGpu?: boolean } export type TranscribeResult = diff --git a/tools/e2e/run.mjs b/tools/e2e/run.mjs index 5e8e5f0..a0dc551 100644 --- a/tools/e2e/run.mjs +++ b/tools/e2e/run.mjs @@ -16,7 +16,7 @@ import { _electron as electron } from 'playwright-core' import { execFileSync, spawn } from 'child_process' import { createHash } from 'crypto' -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'fs' +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, truncateSync, writeFileSync } from 'fs' import { tmpdir } from 'os' import { join, resolve } from 'path' import { createRequire } from 'module' @@ -558,7 +558,11 @@ const scenarios = { const hi = Math.max(lum(...ink), lum(...ground)) const lo = Math.min(lum(...ink), lum(...ground)) const ratio = (hi + 0.05) / (lo + 0.05) - return ink.join(',') === '78,161,255' || ratio < 4.5 + // Wait for the DECORATED colour. After a theme switch there is a frame or + // two where the row is repainted and the link's decoration is not back yet: + // the span then wears the plain text ink, which reads fine and is not a + // link colour at all (MEASURED: 62,62,62 accepted two runs in three). + return ink.join(',') === '78,161,255' || ratio < 4.5 || !(ink[2] > ink[0]) ? null : { rgb: ink.join(','), ratio, blue: ink[2] > ink[0] } }), @@ -814,6 +818,79 @@ const scenarios = { ok(await until(() => ourSpeechServers() === 0, 8000), 'and none outlives the app') }, + /** + * THE MODEL MANAGER, WITH THINGS IN IT (#13, owner's notes of 2026-09-19): a + * model and the GPU engine are STAGED as installed (a sparse file of the + * catalogued size, a pack folder with its marker), so the rows can be read + * in the states a user actually lives in, without a gigabyte of downloads. + */ + async dictationPage(ok) { + const w = world() + const src = readFileSync(resolve(process.cwd(), 'core/shared/dictationCatalog.ts'), 'utf8') + const entry = (id) => { + const block = src.slice(src.indexOf(`id: '${id}'`)) + return { bytes: Number(block.match(/bytes:\s*(\d+)/)[1]), sha256: block.match(/sha256:\s*\n?\s*'([0-9a-f]{64})'/)[1] } + } + const root = join(w.profile, 'dictation') + mkdirSync(join(root, 'models'), { recursive: true }) + const base = join(root, 'models', 'base.bin') + writeFileSync(base, '') + truncateSync(base, entry('base').bytes) + const gpu = entry('gpu-pack') + const pack = join(root, 'engines', `gpu-pack-${gpu.sha256.slice(0, 12)}`) + mkdirSync(join(pack, 'Release'), { recursive: true }) + writeFileSync(join(pack, 'Release', 'whisper-server.exe'), '') + writeFileSync(join(pack, 'prism-installed.json'), JSON.stringify({ id: 'gpu-pack', bytes: gpu.bytes, sha256: gpu.sha256 })) + + const { app, page } = await launch(w, { args: [w.alpha], env: { PT_E2E_NVIDIA: '1', PT_DICTATION_ROOT: root } }) + await until(async () => (await tabLabels(page)).length === 1) + await page.evaluate(() => { + localStorage.setItem('prism.dictation.enabled', '1') + localStorage.setItem('prism.dictation.model', 'base') + }) + await page.locator('[data-title-settings]').click() + await page.locator('[data-settings-tab="dictation"]').click() + await page.waitForSelector('[data-dictation-item="gpu-pack"][data-state="installed"]', { timeout: 8000 }) + ok((await page.locator('[data-settings-tab="dictation"]').getAttribute('aria-current')) === 'page' && (await page.locator('[data-settings-tab="general"]').getAttribute('aria-current')) === null, 'the rail marks Dictation as the page in front') + await page.mouse.move(900, 300) + await sleep(400) + + const names = await page.evaluate(() => [...document.querySelectorAll('[data-dictation-item] [data-item-name]')].map((e) => e.textContent.trim())) + ok(names.length === 5 && names.slice(0, 4).every((n) => n.startsWith('Whisper ')), `models carry their full names (${JSON.stringify(names)})`) + const marks = await page.evaluate(() => [...document.querySelectorAll('[data-dictation-item] [data-vendor]')].map((e) => e.getAttribute('data-vendor'))) + ok(marks.filter((m) => m === 'openai').length === 4 && marks.filter((m) => m === 'nvidia').length === 1, `every row leads with its vendor's mark (${marks.join(',')})`) + const mark = await page.evaluate(() => { + const row = document.querySelector('[data-dictation-item="base"]').getBoundingClientRect() + const m = document.querySelector('[data-dictation-item="base"] [data-vendor]').getBoundingClientRect() + const name = document.querySelector('[data-dictation-item="base"] [data-item-name]').getBoundingClientRect() + return { left: m.left < name.left, centred: Math.abs(m.top + m.height / 2 - (row.top + row.height / 2)) < 2, size: Math.round(m.width) } + }) + ok(mark.left && mark.centred && mark.size >= 28, `the mark sits left of the name, centred on the row (${JSON.stringify(mark)})`) + + const row = (id) => page.locator(`[data-dictation-item="${id}"]`) + ok(((await row('base').locator('[data-item-badge]').textContent()) ?? '').trim() === 'Active', 'the model in use says Active') + ok((await row('base').locator('[data-uninstall]').count()) === 1, 'an installed model offers Uninstall') + const words = ((await page.locator('[data-dictation-settings]').textContent()) ?? '') + ok(!/\bDelete\b|\bRemove\b/.test(words), 'and nothing on the page says Delete or Remove') + + ok(((await row('gpu-pack').locator('[data-item-badge]').textContent()) ?? '').trim() === 'Enabled', 'the GPU engine says Enabled') + ok(((await row('gpu-pack').locator('[data-gpu-toggle]').textContent()) ?? '').trim() === 'Disable', 'and offers Disable') + ok((await row('large-v3-turbo').locator('text=Recommended').count()) === 1, 'with the GPU on, Turbo is the recommended model') + await page.screenshot({ path: resolve(process.cwd(), '.e2e-shots/dictation-models.png') }).catch(() => {}) + + await row('gpu-pack').locator('[data-gpu-toggle]').click() + ok(await until(async () => ((await row('gpu-pack').locator('[data-gpu-toggle]').textContent()) ?? '').trim() === 'Enable', 4000), 'Disable turns into Enable') + ok((await row('gpu-pack').locator('[data-item-badge]').count()) === 0, 'the Enabled badge goes') + ok(existsSync(join(pack, 'prism-installed.json')), 'and the engine stays on disk: disabling is not uninstalling') + ok((await row('base').locator('text=Recommended').count()) === 1, 'with the GPU off, Base is the recommended model again') + ok((await page.evaluate(() => localStorage.getItem('prism.dictation.gpu'))) === '0', 'the choice is this app\'s own setting') + + await row('gpu-pack').locator('[data-uninstall]').click() + ok(await until(async () => (await row('gpu-pack').getAttribute('data-state')) === 'absent', 8000), 'Uninstall takes the engine off the disk') + ok(!existsSync(pack), 'for real') + await app.close().catch(() => {}) + }, + /** `exit` closes the tab it was typed in. */ async exitClosesTab(ok) { const w = world() From 99e6588807e7d01f1903c3d9d6b1da2fbe4dd818 Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:05:12 +0200 Subject: [PATCH 07/12] fix(dictation): every model's Download button is the same; the badge is what recommends (#13) Co-Authored-By: Claude Fable 5.1 --- core/renderer/settings/Dictation.tsx | 8 +++++--- tools/e2e/run.mjs | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/renderer/settings/Dictation.tsx b/core/renderer/settings/Dictation.tsx index 3830eb7..206ac2a 100644 --- a/core/renderer/settings/Dictation.tsx +++ b/core/renderer/settings/Dictation.tsx @@ -310,8 +310,6 @@ function ItemRow({ ) : state === 'installed' ? ( installed ) : ( - // One filled button per list: the recommended pick (and the GPU - // engine, alone in its own). Four accent buttons in a column is a wall. @@ -469,7 +467,11 @@ export function DictationSettings(): JSX.Element | null { recommended={m.id === rec?.id} warn={m.needsGpu && !gpuOn ? 'Slow without GPU acceleration: seconds per sentence on a CPU.' : null} getLabel="Download" - getPrimary={m.id === rec?.id} + // The same quiet button on every row (owner, 2026-09-19: "the + // recommended shouldn't have a different download button"). The + // badge is what recommends; a second signal on the button made + // one row shout. + getPrimary={false} onDownload={() => download(m.id)} onCancel={() => api.dictationCancel(m.id)} installed={ diff --git a/tools/e2e/run.mjs b/tools/e2e/run.mjs index a0dc551..f7fd349 100644 --- a/tools/e2e/run.mjs +++ b/tools/e2e/run.mjs @@ -876,6 +876,8 @@ const scenarios = { ok(((await row('gpu-pack').locator('[data-item-badge]').textContent()) ?? '').trim() === 'Enabled', 'the GPU engine says Enabled') ok(((await row('gpu-pack').locator('[data-gpu-toggle]').textContent()) ?? '').trim() === 'Disable', 'and offers Disable') ok((await row('large-v3-turbo').locator('text=Recommended').count()) === 1, 'with the GPU on, Turbo is the recommended model') + const fills = await page.evaluate(() => ['small', 'large-v3-turbo', 'large-v3'].map((id) => getComputedStyle(document.querySelector(`[data-dictation-item="${id}"] button`)).backgroundColor)) + ok(new Set(fills).size === 1, `and its Download button is the same as every other (${fills.join(' | ')})`) await page.screenshot({ path: resolve(process.cwd(), '.e2e-shots/dictation-models.png') }).catch(() => {}) await row('gpu-pack').locator('[data-gpu-toggle]').click() From 940796fb972433b8b49bb5482e064d2372fd16da Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:07:41 +0200 Subject: [PATCH 08/12] fix(dictation): GPU acceleration is Enable / Disable and nothing else: enable downloads it, disable uninstalls it (#13) Co-Authored-By: Claude Fable 5.1 --- core/main/dictationEngine.ts | 7 ++-- core/main/dictationIpc.ts | 8 +--- core/renderer/lib/dictation.ts | 6 +-- core/renderer/lib/dictationPrefs.ts | 10 +---- core/renderer/settings/Dictation.tsx | 47 ++++++++-------------- core/renderer/settings/dictationOptions.ts | 8 ++-- core/renderer/settings/options.test.ts | 5 ++- core/shared/dictationTypes.ts | 3 -- tools/e2e/run.mjs | 12 +++--- 9 files changed, 36 insertions(+), 70 deletions(-) diff --git a/core/main/dictationEngine.ts b/core/main/dictationEngine.ts index aefc0e9..0cb4c19 100644 --- a/core/main/dictationEngine.ts +++ b/core/main/dictationEngine.ts @@ -232,8 +232,8 @@ export function createDictationEngine(deps: DictationEngineDeps): DictationEngin } } - async function pick(useGpu: boolean): Promise<{ kind: EngineKind; dir: string } | null> { - const gpuDir = gpuFellBack || !useGpu ? null : deps.store.gpuEngineDir() + async function pick(): Promise<{ kind: EngineKind; dir: string } | null> { + const gpuDir = gpuFellBack ? null : deps.store.gpuEngineDir() if (gpuDir && (await hasNvidia())) return { kind: 'gpu', dir: gpuDir } const cpuDir = deps.cpuDir() return cpuDir ? { kind: 'cpu', dir: cpuDir } : null @@ -388,8 +388,7 @@ export function createDictationEngine(deps: DictationEngineDeps): DictationEngin if (!modelPath) return { ok: false, reason: 'no-model' } const language = req.language || 'auto' - // Disabled in Settings is the user's word, and it wins over a pack on disk. - const want = await pick(req.useGpu !== false) + const want = await pick() if (stopped()) return STOPPED if (!want) return { ok: false, reason: 'no-engine' } diff --git a/core/main/dictationIpc.ts b/core/main/dictationIpc.ts index 86f7fa5..3681cf2 100644 --- a/core/main/dictationIpc.ts +++ b/core/main/dictationIpc.ts @@ -95,13 +95,7 @@ export function registerDictationIpc(deps: DictationIpcDeps): () => void { !/^[a-z]{2,4}$/.test(r.language) ) return { ok: false, reason: 'engine-failed', detail: 'bad request' } - return engine.transcribe({ - wav: r.wav, - modelId: r.modelId, - language: r.language, - final: r.final === true, - useGpu: r.useGpu !== false - }) + return engine.transcribe({ wav: r.wav, modelId: r.modelId, language: r.language, final: r.final === true }) }) ipcMain.on(DCH.stop, () => engine.stop()) diff --git a/core/renderer/lib/dictation.ts b/core/renderer/lib/dictation.ts index e6b6d3c..06b8e8a 100644 --- a/core/renderer/lib/dictation.ts +++ b/core/renderer/lib/dictation.ts @@ -3,7 +3,6 @@ import { cleanTranscript } from './dictationClean' import { initialKeyState, reduceKey, type KeyEvt, type KeyState } from './dictationKey' import { dictationEnabled, - dictationGpu, dictationHotkey, dictationLanguage, dictationMic, @@ -178,7 +177,7 @@ async function begin(sessionId: string): Promise { r.lastPartialAt = cap.seconds() const wav = encodeWav(tailSamples(cap.snapshot(), TARGET_RATE, PARTIAL_WINDOW_S), TARGET_RATE) void host.api - .dictationTranscribe({ wav, modelId, language: dictationLanguage(), final: false, useGpu: dictationGpu() }) + .dictationTranscribe({ wav, modelId, language: dictationLanguage(), final: false }) .then((res) => { // Only while THIS recording is still listening: a partial that lands // after the final would overwrite the pill with older words. @@ -216,8 +215,7 @@ async function finishStop(r: Recording): Promise { wav: encodeWav(samples, TARGET_RATE), modelId: dictationModel(), language: dictationLanguage(), - final: true, - useGpu: dictationGpu() + final: true }) } catch { res = { ok: false as const, reason: 'engine-failed' as const } diff --git a/core/renderer/lib/dictationPrefs.ts b/core/renderer/lib/dictationPrefs.ts index ae60d25..3a0fdfd 100644 --- a/core/renderer/lib/dictationPrefs.ts +++ b/core/renderer/lib/dictationPrefs.ts @@ -17,8 +17,7 @@ const K = { language: 'prism.dictation.language', pauseMedia: 'prism.dictation.pauseMedia', sounds: 'prism.dictation.sounds', - model: 'prism.dictation.model', - gpu: 'prism.dictation.gpu' + model: 'prism.dictation.model' } as const export const DICTATION_KEYS = K @@ -99,12 +98,6 @@ export const setDictationSounds = (on: boolean): void => write(K.sounds, on ? '1 export const dictationModel = (): string => read(K.model) ?? '' export const setDictationModel = (id: string): void => write(K.model, id) -/** GPU acceleration is ON unless switched off: installing the pack is the - * user saying they want it. Disabling keeps the files (675 MB is not something - * to download twice on a whim, and the other app may be using them). */ -export const dictationGpu = (): boolean => read(K.gpu) !== '0' -export const setDictationGpu = (on: boolean): void => write(K.gpu, on ? '1' : '0') - export const useDictationEnabled = (): boolean => useSyncExternalStore(sub, dictationEnabled) export const useDictationMode = (): DictMode => useSyncExternalStore(sub, dictationMode) export const useDictationHotkey = (): Hotkey => useSyncExternalStore(sub, dictationHotkey) @@ -113,5 +106,4 @@ export const useDictationLanguage = (): string => useSyncExternalStore(sub, dict export const useDictationPauseMedia = (): boolean => useSyncExternalStore(sub, dictationPauseMedia) export const useDictationSounds = (): boolean => useSyncExternalStore(sub, dictationSounds) export const useDictationModel = (): string => useSyncExternalStore(sub, dictationModel) -export const useDictationGpu = (): boolean => useSyncExternalStore(sub, dictationGpu) export const onDictationPrefs = sub diff --git a/core/renderer/settings/Dictation.tsx b/core/renderer/settings/Dictation.tsx index 206ac2a..46f2e79 100644 --- a/core/renderer/settings/Dictation.tsx +++ b/core/renderer/settings/Dictation.tsx @@ -5,7 +5,6 @@ import type { CatalogEntry, DownloadFailure, EngineInfo, ItemStatus } from '../. import { DEFAULT_HOTKEY, formatHotkey, parseHotkeyFromEvent, type Hotkey } from '../lib/dictationKey' import { setDictationEnabled, - setDictationGpu, setDictationHotkey, setDictationLanguage, setDictationMic, @@ -14,7 +13,6 @@ import { setDictationPauseMedia, setDictationSounds, useDictationEnabled, - useDictationGpu, useDictationHotkey, useDictationLanguage, useDictationMic, @@ -329,7 +327,6 @@ export function DictationSettings(): JSX.Element | null { const pauseMedia = useDictationPauseMedia() const sounds = useDictationSounds() const model = useDictationModel() - const gpuWanted = useDictationGpu() const [status, setStatus] = useState([]) const [info, setInfo] = useState(null) @@ -352,9 +349,8 @@ export function DictationSettings(): JSX.Element | null { const stateOf = (id: string): ItemStatus | undefined => status.find((s) => s.id === id) const installed = (id: string): boolean => stateOf(id)?.state === 'installed' - const gpuIn = installed(GPU_ID) - /** In use: on disk AND not switched off in this app. */ - const gpuOn = gpuIn && gpuWanted + /** GPU acceleration is ON exactly when its engine is on disk. */ + const gpuOn = installed(GPU_ID) const download = (id: string): void => { setFailures((prev) => without(prev, id)) @@ -501,11 +497,11 @@ export function DictationSettings(): JSX.Element | null { : 'An NVIDIA card was found. The GPU engine makes the large models answer in under a second.'}

- {/* ENABLE and DISABLE, not install and remove (owner, 2026-09-19). - Enabling fetches the engine the first time; disabling only stops - THIS app using it, because 675 MB is not something to download - again on a whim and the other app may be using it. Freeing the - disk is Uninstall, the same control a model has. */} + {/* ENABLE and DISABLE, and nothing else (owner, 2026-09-19: "enable + downloads it, disable uninstalls it"). One control with one + meaning: the engine is either on this PC and in use, or it is + neither. A separate on/off beside an Uninstall was built first + and was two ideas where the owner wanted one. */} { - setDictationGpu(true) - download(GPU_ID) - }} + onDownload={() => download(GPU_ID)} onCancel={() => api.dictationCancel(GPU_ID)} installed={ - <> - - remove(GPU_ID)} /> - + } />
diff --git a/core/renderer/settings/dictationOptions.ts b/core/renderer/settings/dictationOptions.ts index a435f18..50b1068 100644 --- a/core/renderer/settings/dictationOptions.ts +++ b/core/renderer/settings/dictationOptions.ts @@ -14,9 +14,9 @@ export interface DictationOption { id: string label: string type: 'switch' | 'choice' | 'key' | 'manager' - /** The localStorage key behind it. The two managers also manage FILES, which - * both apps share; the key is the per-app half (which model, GPU on or off). */ - key: string + /** The localStorage key behind it; null where the row manages only FILES, + * which both apps share: GPU acceleration is on when its engine is on disk. */ + key: string | null /** Absent = every PC. */ onlyWhere?: 'an NVIDIA adapter is present' } @@ -31,7 +31,7 @@ export const DICTATION_OPTIONS: readonly DictationOption[] = [ { id: 'dictation-sounds', label: 'Sounds', type: 'switch', key: DICTATION_KEYS.sounds }, { id: 'dictation-model', label: 'Models', type: 'manager', key: DICTATION_KEYS.model }, // The official GPU engine is NVIDIA's: the row is offered only where it can work. - { id: 'dictation-gpu', label: 'GPU acceleration', type: 'manager', key: DICTATION_KEYS.gpu, onlyWhere: 'an NVIDIA adapter is present' } + { id: 'dictation-gpu', label: 'GPU acceleration', type: 'manager', key: null, onlyWhere: 'an NVIDIA adapter is present' } ] /** The option ids a PC should be showing. */ diff --git a/core/renderer/settings/options.test.ts b/core/renderer/settings/options.test.ts index cd51688..d5a4f4f 100644 --- a/core/renderer/settings/options.test.ts +++ b/core/renderer/settings/options.test.ts @@ -36,10 +36,11 @@ describe('the terminal options list', () => { expect(all.filter((id) => !prism.includes(id))).toEqual(['term-opacity']) }) - it('gives dictation one key per option, under prism.dictation', () => { - const keys = DICTATION_OPTIONS.map((o) => o.key) + it('gives dictation one key per option, under prism.dictation, and only the GPU row none', () => { + const keys = DICTATION_OPTIONS.map((o) => o.key).filter((k): k is string => k !== null) expect(new Set(keys).size).toBe(keys.length) for (const k of keys) expect(k.startsWith('prism.dictation.')).toBe(true) + expect(DICTATION_OPTIONS.filter((o) => o.key === null).map((o) => o.id)).toEqual(['dictation-gpu']) }) it('offers the GPU row only where an NVIDIA adapter is present', () => { diff --git a/core/shared/dictationTypes.ts b/core/shared/dictationTypes.ts index 71bae23..576cb56 100644 --- a/core/shared/dictationTypes.ts +++ b/core/shared/dictationTypes.ts @@ -52,9 +52,6 @@ export interface TranscribeRequest { language: string /** A partial may be dropped for a newer one; a final never is. */ final: boolean - /** False when the user has DISABLED GPU acceleration in this app: the pack - * may still be on disk (the other app may use it), it is just not used. */ - useGpu?: boolean } export type TranscribeResult = diff --git a/tools/e2e/run.mjs b/tools/e2e/run.mjs index f7fd349..d6c08ce 100644 --- a/tools/e2e/run.mjs +++ b/tools/e2e/run.mjs @@ -880,16 +880,14 @@ const scenarios = { ok(new Set(fills).size === 1, `and its Download button is the same as every other (${fills.join(' | ')})`) await page.screenshot({ path: resolve(process.cwd(), '.e2e-shots/dictation-models.png') }).catch(() => {}) + // DISABLE UNINSTALLS IT, ENABLE DOWNLOADS IT (owner, 2026-09-19): one control. + ok((await row('gpu-pack').locator('[data-uninstall]').count()) === 0, 'the GPU row has no second control beside Disable') await row('gpu-pack').locator('[data-gpu-toggle]').click() - ok(await until(async () => ((await row('gpu-pack').locator('[data-gpu-toggle]').textContent()) ?? '').trim() === 'Enable', 4000), 'Disable turns into Enable') + ok(await until(async () => (await row('gpu-pack').getAttribute('data-state')) === 'absent', 8000), 'Disable takes the engine off the disk') + ok(!existsSync(pack), 'for real') ok((await row('gpu-pack').locator('[data-item-badge]').count()) === 0, 'the Enabled badge goes') - ok(existsSync(join(pack, 'prism-installed.json')), 'and the engine stays on disk: disabling is not uninstalling') + ok(((await row('gpu-pack').locator('button').textContent()) ?? '').trim() === 'Enable', 'and the button offers Enable again') ok((await row('base').locator('text=Recommended').count()) === 1, 'with the GPU off, Base is the recommended model again') - ok((await page.evaluate(() => localStorage.getItem('prism.dictation.gpu'))) === '0', 'the choice is this app\'s own setting') - - await row('gpu-pack').locator('[data-uninstall]').click() - ok(await until(async () => (await row('gpu-pack').getAttribute('data-state')) === 'absent', 8000), 'Uninstall takes the engine off the disk') - ok(!existsSync(pack), 'for real') await app.close().catch(() => {}) }, From 352da6b4106adbcb83fc551590285989d0f2d0eb Mon Sep 17 00:00:00 2001 From: Max <112043822+Maxaubert@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:10:59 +0200 Subject: [PATCH 09/12] fix(dictation): Uninstall is a button, the same one Download is (#13) Co-Authored-By: Claude Fable 5.1 --- core/renderer/settings/Dictation.tsx | 6 ++++-- tools/e2e/run.mjs | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/renderer/settings/Dictation.tsx b/core/renderer/settings/Dictation.tsx index 46f2e79..ba2c56a 100644 --- a/core/renderer/settings/Dictation.tsx +++ b/core/renderer/settings/Dictation.tsx @@ -203,14 +203,16 @@ function MicField({ value, disabled }: { value: string; disabled: boolean }): JS } /** "X Uninstall" (owner, 2026-09-19): it frees the disk, and says so by name. - * Quiet until hovered, because it is the one destructive control in the row. */ + * A BUTTON, the same one Download is: a first cut drew it as bare text to keep + * the destructive control quiet, and it read as a label rather than something + * to press (owner, the same evening). */ function Uninstall({ onClick, what }: { onClick: () => void; what: string }): JSX.Element { return (