vein: speech-to-text core — streaming dictation with hotwords

mcp/Dockerfile

@@ -13,7 +13,7 @@ FROM node:22-bookworm-slim
 WORKDIR /usr/src/app
 
 # Install dependencies
-RUN apt update && apt install -y sed git git-lfs vim tree ripgrep wget curl jq python3 pandoc python3-pip python3-venv zip unzip poppler-utils ffmpeg && \
+RUN apt update && apt install -y sed git git-lfs vim tree ripgrep wget curl jq python3 pandoc python3-pip python3-venv zip unzip poppler-utils ffmpeg bzip2 && \
     git lfs install --system && \
     ln -s /usr/bin/python3 /usr/bin/python
 

mcp/src/index.ts

@@ -24,7 +24,7 @@ import * as l from "./graph/learnings.js";
 import * as uploads from "./graph/uploads.js";
 import * as gitree from "./gitree/routes.js";
 import * as gitreeProposals from "./gitree/proposals.js";
-import { mountLab } from "./lab/mount.js";
+import { mountLab, attachLabAudio } from "./lab/mount.js";
 import { loadModelPricing } from "./aieo/src/index.js";
 import path from "path";
 import { fileURLToPath } from "url";
@@ -447,6 +447,9 @@ const server = app.listen(port, host, () => {
   );
 });
 
+// Dictation WebSocket for the lab UI (upgrades bypass Express).
+attachLabAudio(server);
+
 process.on("SIGTERM", () => void gracefulShutdown("SIGTERM"));
 process.on("SIGINT", () => void gracefulShutdown("SIGINT"));
 

mcp/src/lab/mount.ts

@@ -1,7 +1,16 @@
 import type { Express, Request, Response, NextFunction } from "express";
+import type { IncomingMessage, Server } from "node:http";
+import type { Duplex } from "node:stream";
 import { getRequestListener } from "@hono/node-server";
+import { createAudioUpgradeHandler, type AudioUpgradeHandler } from "vein";
 import { createLabVein } from "./createLabVein.js";
 
+/** The one lab vein, built on first use (HTTP request or dictation upgrade). */
+let labVeinP: ReturnType<typeof createLabVein> | null = null;
+function labVein() {
+  return (labVeinP ??= createLabVein({ serveUi: true }));
+}
+
 /**
  * Bridge a (lazily-built) vein Hono app into Express. The instance is
  * created on the first request to its mount path, so mcp boot is never
@@ -22,6 +31,23 @@ function bridge(factory: () => Promise<{ app: { fetch: any } }>) {
   };
 }
 
+/** Does this request carry the lab credential? (`labAuth` without the
+ *  response side, so the WebSocket upgrade can apply the same rule.) */
+function labAuthorized(req: { header(name: string): string | undefined }): boolean {
+  const apiToken = process.env.API_TOKEN;
+  if (!apiToken) return true;
+  if (req.header("x-api-token") === apiToken) return true;
+  const header = req.header("authorization") ?? "";
+  if (header.startsWith("Basic ")) {
+    const decoded = Buffer.from(header.slice(6), "base64").toString();
+    const sep = decoded.indexOf(":");
+    const user = decoded.slice(0, sep);
+    const pass = decoded.slice(sep + 1);
+    if (sep > 0 && user === "admin" && pass === apiToken) return true;
+  }
+  return false;
+}
+
 /**
  * Mount the single lab vein under `/lab` (API + run-streaming SSE). All
  * experiments share this one instance — they're groups of workflows
@@ -44,17 +70,7 @@ function bridge(factory: () => Promise<{ app: { fetch: any } }>) {
  * rest of mcp for server-to-server callers.
  */
 function labAuth(req: Request, res: Response, next: NextFunction): void {
-  const apiToken = process.env.API_TOKEN;
-  if (!apiToken) return next();
-  if (req.header("x-api-token") === apiToken) return next();
-  const header = req.header("authorization") ?? "";
-  if (header.startsWith("Basic ")) {
-    const decoded = Buffer.from(header.slice(6), "base64").toString();
-    const sep = decoded.indexOf(":");
-    const user = decoded.slice(0, sep);
-    const pass = decoded.slice(sep + 1);
-    if (sep > 0 && user === "admin" && pass === apiToken) return next();
-  }
+  if (labAuthorized(req)) return next();
   res.set("WWW-Authenticate", 'Basic realm="stakgraph-lab"');
   res.status(401).json({ error: "Unauthorized" });
 }
@@ -69,5 +85,46 @@ export function mountLab(app: Express): void {
     if (req.path === "/lab/") return next();
     res.redirect(308, "/lab/");
   });
-  app.use("/lab", labAuth, bridge(() => createLabVein({ serveUi: true })));
+  app.use("/lab", labAuth, bridge(labVein));
+}
+
+const LAB_AUDIO_STREAM = "/lab/audio/stream";
+
+/**
+ * Dictation over `/lab/audio/stream` (vein `src/audio/ws.ts`). A WebSocket
+ * upgrade never enters Express, so the bridge above can't carry it: hook the
+ * Node server's `upgrade` event, apply the lab credential (browsers resend
+ * cached Basic auth on same-origin handshakes, so the UI's one-time prompt
+ * covers it), then hand the socket to vein. Built lazily like the bridge —
+ * the first dictation boots the lab vein if a request hasn't already.
+ * Other upgrade paths get a 404 rather than a socket left hanging.
+ */
+export function attachLabAudio(server: Server): void {
+  let handlerP: Promise<AudioUpgradeHandler | null> | null = null;
+  const handler = () =>
+    (handlerP ??= labVein().then((vein) =>
+      vein.stt
+        ? createAudioUpgradeHandler(vein.stt, { basePath: "/lab", authorize: () => true })
+        : null,
+    ));
+  const reject = (socket: Duplex, status: string) => {
+    socket.write(`HTTP/1.1 ${status}\r\nConnection: close\r\n\r\n`);
+    socket.destroy();
+  };
+  server.on("upgrade", (req: IncomingMessage, socket: Duplex, head: Buffer) => {
+    const url = new URL(req.url ?? "/", "http://localhost");
+    if (url.pathname !== LAB_AUDIO_STREAM) return reject(socket, "404 Not Found");
+    const header = (name: string) => req.headers[name.toLowerCase()] as string | undefined;
+    if (!labAuthorized({ header })) return reject(socket, "401 Unauthorized");
+    handler()
+      .then((h) => {
+        if (!h) return reject(socket, "501 Not Implemented");
+        if (socket.destroyed) return;
+        h.handle(req, socket, head);
+      })
+      .catch((e) => {
+        console.error("[lab] dictation upgrade failed:", e);
+        reject(socket, "500 Internal Server Error");
+      });
+  });
 }

mcp/yarn.lock

@@ -5996,6 +5996,48 @@ shebang-regex@^3.0.0:
   resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz"
   integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
 
+sherpa-onnx-darwin-arm64@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.13.7.tgz#2b46e237bf28e8d15a9de49c6dcc5a3d5f22da09"
+  integrity sha512-5NCE50hAvr3n2pdett0SgfPBJXaFZE0bqHwbHyiq+IKZ8Ids0l4M0VrG+ImGYIafCwie+oC3uAJ+pKj9xg/k+w==
+
+sherpa-onnx-darwin-x64@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-darwin-x64/-/sherpa-onnx-darwin-x64-1.13.7.tgz#8d0ac16d47973a8b25eaf74fcad246005aaa8250"
+  integrity sha512-N3o+T+wn9WaQmsKV5DD8bTHdo+WN2+sXwmZcGJZiDjtOMR2zFz7uVCZnYCmEAMgvChC+oHcF5RvEEKcRCAu6Pw==
+
+sherpa-onnx-linux-arm64@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-linux-arm64/-/sherpa-onnx-linux-arm64-1.13.7.tgz#e1726c9748252d07923d00f81c0c3fd4767583c0"
+  integrity sha512-TFCVpXyTh69buhOtTS8KIfkRXOVKY4Y1qjAktSItrKS4A0chnnrlXO5bKWoNAPeI6fMxTF/uvMYbYgcvjEMfNg==
+
+sherpa-onnx-linux-x64@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-linux-x64/-/sherpa-onnx-linux-x64-1.13.7.tgz#6766b8dc29b5f46cce44f59196b3252f71aa38c5"
+  integrity sha512-npmxn5WwmAmlthgBhmbZ33t3i2j4mJwQt46dMEb3j7d41y1/uJrjrVAfa/DkvV+vn49ZWfcQ2UEWDipaZBVhuw==
+
+sherpa-onnx-node@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-node/-/sherpa-onnx-node-1.13.7.tgz#825bd6e386a9c86ebb6e5c70083b85ef7f14185d"
+  integrity sha512-0XGV7arGngBCnol0m8OLyqlnaUm19Q1KmetVj1DDBdymXa1upmAHZDwNdN47gjsEhqE5hXUEyc1vRQoXrNhNVg==
+  optionalDependencies:
+    sherpa-onnx-darwin-arm64 "^1.13.7"
+    sherpa-onnx-darwin-x64 "^1.13.7"
+    sherpa-onnx-linux-arm64 "^1.13.7"
+    sherpa-onnx-linux-x64 "^1.13.7"
+    sherpa-onnx-win-ia32 "^1.13.7"
+    sherpa-onnx-win-x64 "^1.13.7"
+
+sherpa-onnx-win-ia32@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-win-ia32/-/sherpa-onnx-win-ia32-1.13.7.tgz#d9c9ed4f459e33667be20e88ed00ee51c04f61b1"
+  integrity sha512-sTwtpxPQ76XLn0giAbvknIDEDKD3XXi2mo2AVROEucf1pIK1DjQl+LjLkalTeFoQqbC4J3xGx/g+xgcHQD1dsw==
+
+sherpa-onnx-win-x64@^1.13.7:
+  version "1.13.7"
+  resolved "https://registry.yarnpkg.com/sherpa-onnx-win-x64/-/sherpa-onnx-win-x64-1.13.7.tgz#51a5baf11aefef39f37727f5e50c26082d450610"
+  integrity sha512-wBV1o+/zgsMrOjfCFIgGrH6S28xq6CqRCLSavCOjTZ6cqr80yGc07DUHxqsHFPZvfoJU+2JF5L2l3gyWFWoWdQ==
+
 side-channel-list@^1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz"
@@ -6594,7 +6636,10 @@ vary@^1, vary@^1.1.2:
     system-canvas "^0.2.22"
     system-canvas-react "^0.2.22"
     uuid "^11.1.0"
+    ws "^8.21.3"
     zod "^4.3.6"
+  optionalDependencies:
+    sherpa-onnx-node "^1.13.7"
 
 web-streams-polyfill@4.0.0-beta.3:
   version "4.0.0-beta.3"
@@ -6676,6 +6721,11 @@ ws@^8.18.0:
   resolved "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz"
   integrity sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==
 
+ws@^8.21.3:
+  version "8.21.3"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.3.tgz#660b4faddb6a3e575c86e078126919961f4de4fc"
+  integrity sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==
+
 y18n@^5.0.5:
   version "5.0.8"
   resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"

vein/AGENTS.md

@@ -61,6 +61,13 @@ vein/
 │   │   │                  #                   validate_workflow (static YAML check, no publish — src/validate.ts)
 │   │   ├── stepHelpers.ts # lsSteps / searchSteps / readStepSource (filesystem-style browser)
 │   │   └── schemaHelpers.ts # Zod → FieldDesc[] (for get_step schema rendering)
+│   ├── audio/             # speech-to-text over sherpa-onnx (plans/local-desktop-and-stt.md §4). Streaming dictation is the product surface; workflows learn AROUND it (hotword lists, "dream cycles" §4.8), no STT step in v1
+│   │   ├── stt.ts         # createStt(): model download+verify, recognizer cache, streams (PCM in → partial/final out), two-recognizer mode (fast greedy partials + hotword-capable finals), batch transcribe; sherpa is an optionalDependency, lazy-imported, fakeable via `engine`
+│   │   ├── models.ts      # catalog: id → release URL + sha256 + chunk latency + hotwords?; VEIN_MODEL_DIR/stt/<id>
+│   │   ├── hotwords.ts    # contextual biasing: list format, synthesized bpe.vocab from tokens.txt (REQUIRED with modelingUnit "bpe" — unset = cjkchar = silent no-op), named lists under <dataDir>/audio/hotwords
+│   │   ├── sessions.ts    # <dataDir>/audio/sessions/<id>.jsonl: finals + user corrections (the dream cycle's training data)
+│   │   ├── ws.ts          # GET /audio/stream WebSocket (raw `ws` on the Node server — @hono/node-ws doesn't support node-server 2.x); Bearer or ?key=
+│   │   └── routes.ts      # /audio/models (+ SSE download), /audio/transcribe (WAV body), /audio/hotwords/:name, /audio/sessions/:id (+ corrections)
 │   ├── graph/             # jarvis-compatible Neo4j graph backend over bolt, no jarvis in the loop (plans/jarvis-graph-compat.md). Opt-in via openGraphBackend
 │   │   ├── bolt.ts        # neo4j-driver wrapper; int() for Integer writes (plain JS numbers write as FLOAT)
 │   │   ├── vein-schemas.ts# the 9 Vein node types + 14-row edge registry (label registry in plans/generic-storage.md); author-time checks
@@ -121,6 +128,11 @@ npm run dev                 # starts Hono server on :3000
 docker run -d --name vein-neo4j-test -p 7688:7687 -e NEO4J_AUTH=neo4j/veintest neo4j:5
 VEIN_TEST_NEO4J_URI=bolt://localhost:7688 VEIN_TEST_NEO4J_PASSWORD=veintest npm run test:graph
 
+# Speech-to-text live test — needs the sherpa addon (optionalDependency,
+# installed by `npm install` on supported platforms) and downloads the 57 MB
+# kroko model into VEIN_TEST_STT_MODEL_DIR (temp dir when unset).
+VEIN_TEST_STT=1 npm run test:stt
+
 # Web UI (dev mode with HMR)
 cd vein/web
 npm install
@@ -151,7 +163,9 @@ cd vein && npm run dev        # serves API + UI on :3000
 | `VEIN_GRAPH_NAMESPACE` | `default`   | jarvis namespace every Vein node is written into |
 | `VEIN_GRAPH_EMBEDDINGS` | (on)       | `off` disables the local MiniLM embedder (vectors stay NULL; search is fulltext-only) |
 | `VEIN_GRAPH_SEED_ONTOLOGY` | (off)   | `1` seeds the bundled jarvis ontology (151 schemas + edge schemas + indexes, add-only) on first open, so a standalone Neo4j can host jarvis-typed data (Document, EvalSet, Concept, …) with no jarvis process. No-op on a jarvis-seeded DB. |
-| `VEIN_MODEL_CACHE`  | `~/.cache/vein-models` | Where the embedding model's ONNX files are cached |
+| `VEIN_MODEL_DIR`    | `~/.cache/vein-models` | Local model files: MiniLM's ONNX cache and STT models under `stt/<id>/`. `VEIN_MODEL_CACHE` is the older alias. |
+| `VEIN_STT_MODEL`    | `zipformer-en-kroko` | Finals recognizer for `/audio/stream` + `/audio/transcribe` (hotword-capable) |
+| `VEIN_STT_PARTIAL_MODEL` | `nemo-fast-conformer-en-80ms` | Fast greedy recognizer whose output is shown as live partials; `off` for single-recognizer streams |
 
 ## Auth
 

vein/package.json

@@ -22,7 +22,8 @@
     "build:web": "npm --prefix web run build",
     "dev": "npm run build:web && tsx --env-file=.env src/server.ts",
     "start": "node build/server.js",
-    "test": "tsx --test src/expr.test.ts src/core.test.ts src/runner.test.ts src/run-control.test.ts src/control-flow.test.ts src/store.test.ts src/workspace.test.ts src/storage-conformance.test.ts src/integration.test.ts src/services.test.ts src/cassette.test.ts src/run-step.test.ts src/createVein.test.ts src/authoring.test.ts src/ai-integration.test.ts src/chat-store.test.ts src/chat-endpoints.test.ts src/steps/registry.test.ts src/steps/core/agent.test.ts src/steps/core/pack.test.ts src/auth.test.ts src/secret-store.test.ts src/artifacts.test.ts src/slack.test.ts src/gdrive.test.ts src/html-extract.test.ts src/shell.test.ts src/validate.test.ts",
+    "test": "tsx --test src/expr.test.ts src/core.test.ts src/runner.test.ts src/run-control.test.ts src/control-flow.test.ts src/store.test.ts src/workspace.test.ts src/storage-conformance.test.ts src/integration.test.ts src/services.test.ts src/cassette.test.ts src/run-step.test.ts src/createVein.test.ts src/authoring.test.ts src/ai-integration.test.ts src/chat-store.test.ts src/chat-endpoints.test.ts src/steps/registry.test.ts src/steps/core/agent.test.ts src/steps/core/pack.test.ts src/auth.test.ts src/secret-store.test.ts src/artifacts.test.ts src/slack.test.ts src/gdrive.test.ts src/html-extract.test.ts src/shell.test.ts src/validate.test.ts src/model-dir.test.ts src/audio/hotwords.test.ts src/audio/stt.test.ts src/audio/ws.test.ts",
+    "test:stt": "VEIN_TEST_STT=1 tsx --test src/audio/stt.live.test.ts",
     "test:graph": "tsx --test --test-concurrency=1 \"src/graph/*.test.ts\" \"src/steps/lib/graph/*.test.ts\""
   },
   "dependencies": {
@@ -41,14 +42,19 @@
     "system-canvas": "^0.2.22",
     "system-canvas-react": "^0.2.22",
     "uuid": "^11.1.0",
+    "ws": "^8.21.3",
     "zod": "^4.3.6"
   },
   "devDependencies": {
     "@types/html-to-text": "^9.0.4",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.19.0",
     "@types/uuid": "^10.0.0",
+    "@types/ws": "^8.18.1",
     "tsx": "^4.21.0",
     "typescript": "^5.9.0"
+  },
+  "optionalDependencies": {
+    "sherpa-onnx-node": "^1.13.7"
   }
 }