diff --git a/mcp/package.json b/mcp/package.json index 15bd56e8d..454e845bc 100644 --- a/mcp/package.json +++ b/mcp/package.json @@ -85,6 +85,7 @@ "neo4j-driver": "5.28.3", "node-cache": "5.1.2", "p-queue": "^6.6.2", + "pg": "8.13.1", "redis": "^6.2.1", "simple-git": "3.30.0", "tsx": "4.21.0", @@ -103,6 +104,7 @@ "@types/jsonwebtoken": "9.0.10", "@types/node": "22.19.9", "@types/node-cache": "4.2.5", + "@types/pg": "8.11.10", "@types/xmldom": "0.1.34", "clipboardy": "4.0.0", "esbuild": "0.25.12", diff --git a/mcp/src/tools/server.ts b/mcp/src/tools/server.ts index 01b102f71..1d12e94ae 100644 --- a/mcp/src/tools/server.ts +++ b/mcp/src/tools/server.ts @@ -4,7 +4,8 @@ import { bearerToken } from "./utils.js"; import { Express } from "express"; import * as stakgraph from "./stakgraph/index.js"; import * as stagehand from "./stagehand/tools.js"; -import { getMcpTools } from "./utils.js"; +import * as verify from "./verify/index.js"; +import { getMcpTools, use_stagehand } from "./utils.js"; import { CallToolRequestSchema, ListToolsRequestSchema, @@ -36,7 +37,8 @@ export function graph_mcp_routes(app: Express) { } graphServer.setRequestHandler(ListToolsRequestSchema, async () => { - return { tools: getMcpTools() }; + const tools = getMcpTools(); + return { tools: use_stagehand() ? [...tools, ...verify.VERIFY_TOOLS] : tools }; }); graphServer.setRequestHandler(CallToolRequestSchema, async (request, extra) => { @@ -61,9 +63,31 @@ graphServer.setRequestHandler(CallToolRequestSchema, async (request, extra) => { case stakgraph.GetRulesFilesTool.name: { return await stakgraph.getRulesFiles(); } + case verify.HttpRequestTool.name: { + const sid = extra.sessionId || "default-session-id"; + return await verify.httpRequest(sid, args || {}); + } + case verify.SampleTool.name: { + const sid = extra.sessionId || "default-session-id"; + return await verify.sampleUrl(sid, args || {}); + } + case verify.DbQueryTool.name: { + const sid = extra.sessionId || "default-session-id"; + return await verify.dbQuery(sid, args || {}); + } + case verify.RunCommandTool.name: { + const sid = extra.sessionId || "default-session-id"; + return await verify.runCommand(sid, args || {}); + } + case verify.SubmitVerdictTool.name: { + const sid = extra.sessionId || "default-session-id"; + return await verify.submitVerdict(sid, args || {}); + } default: if (name.startsWith("stagehand_")) { - return await stagehand.call(name, args || {}, extra.sessionId); + const sid = extra.sessionId || "default-session-id"; + const result = await stagehand.call(name, args || {}, extra.sessionId); + return verify.tagEvidence(sid, name, result); } throw new Error(`Unknown tool: ${name}`); } diff --git a/mcp/src/tools/stagehand/core.ts b/mcp/src/tools/stagehand/core.ts index f6998a14e..c83dc50e0 100644 --- a/mcp/src/tools/stagehand/core.ts +++ b/mcp/src/tools/stagehand/core.ts @@ -1,5 +1,6 @@ -import { Stagehand, Page } from "@browserbasehq/stagehand"; -import { getProvider } from "./providers.js"; +import { Stagehand, Page, AISdkClient } from "@browserbasehq/stagehand"; +import { resolveBrowserModel } from "./providers.js"; +import { getModelDetails } from "../../aieo/src/provider.js"; let STATE: { [sessionId: string]: { @@ -56,8 +57,9 @@ export async function getOrCreateStagehand(sessionIdMaybe?: string) { return STATE[sessionId].stagehand; } - let provider = getProvider(); - console.log("initializing stagehand!", provider.model); + const { model: modelName, apiKey } = resolveBrowserModel(); + console.log("initializing stagehand!", modelName); + const { model } = getModelDetails(modelName, apiKey); const sh = new Stagehand({ env: "LOCAL", domSettleTimeout: 60000, @@ -65,10 +67,7 @@ export async function getOrCreateStagehand(sessionIdMaybe?: string) { headless: true, viewport: { width: 1024, height: 768 }, }, - model: { - modelName: provider.model, - apiKey: process.env[provider.api_key_env_var_name], - }, + llmClient: new AISdkClient({ model: model as any }), }); await sh.init(); @@ -80,25 +79,9 @@ export async function getOrCreateStagehand(sessionIdMaybe?: string) { networkEntries: [], }; - // Set up console log listener on the active page + // Set up console + network capture on the active page via CDP const page = getActivePage(sh); - page.on("console", (msg) => { - const location = msg.location(); - addConsoleLog(sessionId, { - timestamp: new Date().toISOString(), - type: msg.type(), - text: msg.text(), - location: { - url: location.url || "", - lineNumber: location.lineNumber || 0, - columnNumber: location.columnNumber || 0, - }, - }); - }); - - // Note: Network monitoring via request/response events is not available - // in the new Stagehand V3 API. The Page class only exposes "console" events. - // Network monitoring would need to be implemented via CDP directly if needed. + attachCapture(page as any, sessionId); // Check if we need to evict old sessions (LRU) if (Object.keys(STATE).length > MAX_SESSIONS) { @@ -112,6 +95,108 @@ export async function getOrCreateStagehand(sessionIdMaybe?: string) { return sh; } +const NET_RESOURCE: Record = { XHR: "xhr", Fetch: "fetch", Document: "document" }; +const PENDING: { [sessionId: string]: Map } = {}; + +// Attach CDP-based console + network capture to a page's main session. +// Populates the session's console logs (incl. uncaught exceptions) and network entries. +function attachCapture(page: any, sessionId: string): void { + try { + page.on("console", (msg: any) => { + const location = msg.location?.() || {}; + addConsoleLog(sessionId, { + timestamp: new Date().toISOString(), + type: msg.type?.() || "log", + text: msg.text?.() || "", + location: { + url: location.url || "", + lineNumber: location.lineNumber || 0, + columnNumber: location.columnNumber || 0, + }, + }); + }); + + const session = page?.mainFrame?.()?.session; + if (!session) return; + PENDING[sessionId] = new Map(); + + const pushLog = (type: string, text: string, url = "") => + addConsoleLog(sessionId, { + timestamp: new Date().toISOString(), + type, + text, + location: { url, lineNumber: 0, columnNumber: 0 }, + }); + + session.send("Runtime.enable").catch(() => {}); + session.on("Runtime.exceptionThrown", (p: any) => { + const d = p?.exceptionDetails; + pushLog("error", d?.exception?.description ?? d?.text ?? "uncaught exception", d?.url ?? ""); + }); + + session.send("Log.enable").catch(() => {}); + session.on("Log.entryAdded", (p: any) => { + const e = p?.entry; + if (e && (e.level === "error" || e.level === "warning")) { + pushLog(e.level, e.text ?? "", e.url ?? ""); + } + }); + + session.send("Network.enable").catch(() => {}); + session.on("Network.requestWillBeSent", (p: any) => { + const type = p?.type ?? ""; + if (p?.redirectResponse && NET_RESOURCE[type]) { + addNetworkEntry(sessionId, { + id: `${p.requestId}-r`, + timestamp: new Date().toISOString(), + type: "response", + method: p?.request?.method ?? "GET", + url: p?.redirectResponse?.url ?? p?.request?.url ?? "", + status: p?.redirectResponse?.status ?? 0, + resourceType: NET_RESOURCE[type], + }); + } + PENDING[sessionId]?.set(p.requestId, { + method: p?.request?.method ?? "GET", + url: p?.request?.url ?? "", + type, + }); + }); + session.on("Network.responseReceived", (p: any) => { + const type = p?.type ?? PENDING[sessionId]?.get(p?.requestId)?.type ?? ""; + const req = PENDING[sessionId]?.get(p?.requestId); + PENDING[sessionId]?.delete(p?.requestId); + if (!NET_RESOURCE[type]) return; + addNetworkEntry(sessionId, { + id: p?.requestId ?? "", + timestamp: new Date().toISOString(), + type: "response", + method: req?.method ?? "GET", + url: p?.response?.url ?? req?.url ?? "", + status: p?.response?.status ?? 0, + resourceType: NET_RESOURCE[type], + }); + }); + session.on("Network.loadingFailed", (p: any) => { + const req = PENDING[sessionId]?.get(p?.requestId); + const type = req?.type ?? p?.type ?? ""; + PENDING[sessionId]?.delete(p?.requestId); + if (!NET_RESOURCE[type]) return; + addNetworkEntry(sessionId, { + id: p?.requestId ?? "", + timestamp: new Date().toISOString(), + type: "response", + method: req?.method ?? "", + url: req?.url ?? "(unknown)", + status: 0, + resourceType: NET_RESOURCE[type], + }); + }); + } catch { + /* a CDP quirk must never fail session creation */ + } +} + export function addConsoleLog(sessionId: string, log: ConsoleLog): void { // Add to global logs (backward compatibility) if (!STATE[sessionId]) { diff --git a/mcp/src/tools/stagehand/providers.ts b/mcp/src/tools/stagehand/providers.ts index acbb70863..81a787d89 100644 --- a/mcp/src/tools/stagehand/providers.ts +++ b/mcp/src/tools/stagehand/providers.ts @@ -10,13 +10,18 @@ export interface ProviderData { export const PROVIDER_MODELS: Record = { anthropic: { name: "anthropic", - model: "claude-3-7-sonnet-latest", + // Frontier model, matched to aieo (MODELS.anthropic.sonnet = "claude-sonnet-5"). + // aieo-format ("provider/model") so it resolves through getModelDetails. + model: "anthropic/claude-sonnet-5", + // Computer-use / agent model stays a native Stagehand string (used only by + // the stagehand_agent CUA path, which does not go through aieo). computer_use_model: "claude-sonnet-4-20250514", api_key_env_var_name: "ANTHROPIC_API_KEY", }, openai: { name: "openai", - model: "gpt-4o", + // Frontier model, matched to aieo (MODELS.openai.gpt = "gpt-5"). + model: "openai/gpt-5", computer_use_model: "computer-use-preview", api_key_env_var_name: "OPENAI_API_KEY", }, @@ -29,3 +34,20 @@ export function getProvider(arg?: "anthropic" | "openai"): ProviderData { } return provider; } + +function apiKeyForModel(model: string): string { + if (model.startsWith("openrouter/")) return process.env.OPENROUTER_API_KEY || ""; + if (model.startsWith("openai/")) return process.env.OPENAI_API_KEY || ""; + return process.env.ANTHROPIC_API_KEY || ""; +} + +/** + * Single source of truth for the browser's LLM. Returns an aieo-format model + * string + its key, resolved once and always driven through getModelDetails — + * no separate "Stagehand built-in vs aieo" branch. The default comes from the + * configured provider; swap it in one place (or wire a registry) here. + */ +export function resolveBrowserModel(): { model: string; apiKey: string } { + const model = getProvider().model; + return { model, apiKey: apiKeyForModel(model) }; +} diff --git a/mcp/src/tools/verify/agent.ts b/mcp/src/tools/verify/agent.ts new file mode 100644 index 000000000..823ff1947 --- /dev/null +++ b/mcp/src/tools/verify/agent.ts @@ -0,0 +1,121 @@ +import { ToolLoopAgent, stepCountIs, tool, jsonSchema, StopCondition } from "ai"; +import { getModelDetails, getProviderOptions } from "../../aieo/src/provider.js"; +import * as stagehand from "../stagehand/tools.js"; +import * as verify from "./index.js"; +import { selectConcept, conceptHint } from "./concepts.js"; + +const SYSTEM = `You are verifying whether a specific TASK actually works in a running application. You did not write this code; you audit it. You never modify anything. + +Use the tools available to exercise the running app and gather evidence, then call submit_verdict to end. + +SCOPE: Judge ONLY what THIS task claims. Do not invent or verify additional claims the task did not make — e.g. do not check database persistence for a task that is only about a network call. Extra observations belong in observations[], never as claims that can fail the task. + +DISCIPLINE (the core): +- Mark a claim "works" ONLY if you CAPTURED proof for it and cite the evidence id (evN) a probe tool returned, in that claim's proof[]. A works claim with no captured evidence id is rejected. +- "It looks right" or "the UI said success" is NOT proof — confirm the underlying behavior (the actual network status, the console, the persisted data). +- If you cannot tell, use unknown. If it is broken, say why. Be honest; an unjustified "works" is the worst outcome. +- submit_verdict is the only way to finish.`; + +export interface VerifyInput { + taskId: string; + taskPrompt: string; + diff: string; + appUrl: string; + notes?: string | null; + model: string; + apiKey: string; + sessionId: string; + maxTurns?: number; +} + +function toText(result: any): string { + const items = (result?.content as Array<{ type: string; text?: string }>) || []; + const parts: string[] = []; + for (const it of items) { + if (it.type === "text" && it.text) parts.push(it.text); + else if (it.type === "image") parts.push("[image captured]"); + } + return parts.join("\n") || "(no output)"; +} + +function buildTools(sessionId: string) { + const defs = [...stagehand.TOOLS, ...verify.VERIFY_TOOLS]; + const out: Record = {}; + for (const t of defs) { + const name = t.name; + out[name] = tool({ + description: t.description as string, + inputSchema: jsonSchema(t.inputSchema as any), + execute: async (args: any) => { + let result: any; + if (name.startsWith("stagehand_")) { + result = verify.tagEvidence(sessionId, name, await stagehand.call(name, args || {}, sessionId)); + } else if (name === verify.HttpRequestTool.name) { + result = await verify.httpRequest(sessionId, args || {}); + } else if (name === verify.SampleTool.name) { + result = await verify.sampleUrl(sessionId, args || {}); + } else if (name === verify.DbQueryTool.name) { + result = await verify.dbQuery(sessionId, args || {}); + } else if (name === verify.RunCommandTool.name) { + result = await verify.runCommand(sessionId, args || {}); + } else if (name === verify.SubmitVerdictTool.name) { + result = await verify.submitVerdict(sessionId, args || {}); + } else { + result = { content: [{ type: "text", text: `unknown tool ${name}` }] }; + } + return toText(result); + }, + }); + } + return out; +} + +const stopAfterVerdict: StopCondition = ({ steps }) => { + for (const s of steps as any[]) + for (const item of s.content || []) + if (item.type === "tool-call" && item.toolName === "submit_verdict") return true; + return false; +}; + +export async function runVerification(input: VerifyInput): Promise { + const started = Date.now(); + const concept = selectConcept(input.taskPrompt, input.diff || ""); + const { model, modelId, provider } = getModelDetails(input.model, input.apiKey, undefined, undefined, undefined, 300000); + + const agent = new ToolLoopAgent({ + model, + instructions: SYSTEM, + tools: buildTools(input.sessionId), + stopWhen: [stopAfterVerdict, stepCountIs(input.maxTurns ?? 50) as StopCondition], + providerOptions: getProviderOptions(provider as any, undefined, modelId) as any, + maxOutputTokens: 64000, + }); + + const diffBlock = input.diff ? `\n\nThe diff of the change:\n${input.diff.slice(0, 6000)}` : ""; + const notesBlock = input.notes ? `\nNotes: ${input.notes}` : ""; + const userPrompt = + `Task to verify: ${input.taskPrompt}\n\n` + + `The application is running at ${input.appUrl}.${notesBlock}${diffBlock}\n\n` + + `${conceptHint(concept)}\n\n` + + `Verify whether the task actually works, capturing evidence, then call submit_verdict.`; + + console.error(`[verify] taskId=${input.taskId} concept=${concept.id} model=${modelId}`); + try { + await agent.generate({ prompt: userPrompt }); + } catch (err: any) { + console.error(`[verify] taskId=${input.taskId} generate error: ${err?.message ?? String(err)}`); + } + + const verdict: any = verify.getVerdict(input.sessionId) || { + overall: "unknown", + claims: [], + observations: ["agent ended without submit_verdict"], + summary: "no verdict", + evidence: [], + }; + verdict.taskId = input.taskId; + verdict.startedAt = new Date(started).toISOString(); + verdict.finishedAt = new Date().toISOString(); + verify.resetVerifySession(input.sessionId); + return verdict; +} diff --git a/mcp/src/tools/verify/concepts.ts b/mcp/src/tools/verify/concepts.ts new file mode 100644 index 000000000..f76e68d07 --- /dev/null +++ b/mcp/src/tools/verify/concepts.ts @@ -0,0 +1,57 @@ +export interface VerificationConcept { + id: string; + name: string; + match: string[]; + procedure: string; +} + +export const VERIFICATION_CONCEPTS: VerificationConcept[] = [ + { + id: "backend-endpoint", + name: "Backend endpoint / response contract", + match: ["endpoint", "route.ts", "/api/", "get ", "post ", "http", "status", "response", "json", "nextresponse", "params", "query"], + procedure: + "This change is about an API endpoint. Prefer verify_http_request over the browser: call the endpoint for the documented params and a couple of edge cases, assert the status code and the response shape, and exercise each documented filter/param (namespace, query, type, limit, flags). Confirm the contract the diff claims — including that optional flags actually change the response and defaults behave. Cite the http evidence id for each claim.", + }, + { + id: "frontend-interaction", + name: "Frontend interaction / rendered behaviour", + match: ["button", "page.tsx", "click", "onclick", "form", "submit", "renders", "component", "usestate", "useeffect", "ui", "input", "modal"], + procedure: + "This change is about a user-facing interaction. Open the page and drive the interaction with the browser. Use stagehand_network_activity to confirm the underlying request actually fired and returned the expected status (a success message in the UI is not proof). Use stagehand_logs to catch runtime/console errors a screenshot hides. Take a screenshot of the outcome. Cite network / console / screenshot evidence ids.", + }, + { + id: "data-persistence", + name: "Data persistence / read-after-write", + match: ["persist", "save", "database", "insert", "update", "create", "store", "prisma", "$executeraw", "db.", "row", "record"], + procedure: + "This change writes data. Perform the write through the app, then INDEPENDENTLY confirm it persisted with verify_db_query (SELECT the row) or an independent re-read via a different path. A 200 or a success toast is not proof of persistence. Cite the db evidence id showing the row is (or is not) there.", + }, +]; + +const DEFAULT_CONCEPT: VerificationConcept = { + id: "general", + name: "General verification", + match: [], + procedure: + "Choose the cheapest sufficient probe for each claim: an API claim → verify_http_request; a UI claim → browser + stagehand_network_activity + stagehand_logs; a persistence claim → verify_db_query. Capture and cite evidence for every claim you mark works.", +}; + +export function selectConcept(taskPrompt: string, diff: string): VerificationConcept { + const hay = `${taskPrompt}\n${diff}`.toLowerCase(); + let best: VerificationConcept | null = null; + let bestScore = 0; + for (const c of VERIFICATION_CONCEPTS) { + let score = 0; + for (const kw of c.match) if (hay.includes(kw)) score++; + if (score > bestScore) { + bestScore = score; + best = c; + } + } + return bestScore > 0 && best ? best : DEFAULT_CONCEPT; +} + +export function conceptHint(concept: VerificationConcept): string { + return `SUGGESTED CHECKS — from the "${concept.name}" verification concept (hints, not a rigid script; adapt to what the diff actually claims):\n${concept.procedure}`; +} diff --git a/mcp/src/tools/verify/index.ts b/mcp/src/tools/verify/index.ts new file mode 100644 index 000000000..4245e2d2c --- /dev/null +++ b/mcp/src/tools/verify/index.ts @@ -0,0 +1,370 @@ +import { z } from "zod"; +import pg from "pg"; +import { exec } from "node:child_process"; +import { promisify } from "node:util"; +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { Tool } from "../types.js"; +import { parseSchema } from "../utils.js"; + +const execAsync = promisify(exec); + +const MUTATING: Array<[RegExp, string]> = [ + [/\bgit\b[^\n|;&]*\b(commit|push|reset|checkout|merge|rebase|add|clean|stash)\b/, "git write command"], + [/\brm\b/, "rm"], + [/\bmv\b/, "mv"], + [/\b(mkdir|rmdir|truncate|chmod|chown|ln)\b/, "filesystem mutation"], + [/\bsed\b[^\n|;&]*-i\b/, "in-place sed"], + [/\b(npm|yarn|pnpm|pip|pip3|cargo|apt|apt-get|brew)\b[^\n|;&]*\b(install|add|remove|uninstall|update|upgrade)\b/, "package install"], + [/\bkill\b|\bpkill\b/, "process kill"], +]; + +function commandRejection(cmd: string): string | undefined { + for (const [re, label] of MUTATING) { + if (re.test(cmd)) return `run_command rejected: ${label} is not allowed — this tool only inspects, it never mutates.`; + } + return undefined; +} + +interface EvidenceRecord { + id: string; + kind: string; + summary: string; + data: string; +} + +interface SessionEvidence { + records: EvidenceRecord[]; + strong: Set; +} + +const EVIDENCE: Record = {}; +const VERDICTS: Record = {}; + +function sess(sessionId: string): SessionEvidence { + if (!EVIDENCE[sessionId]) EVIDENCE[sessionId] = { records: [], strong: new Set() }; + return EVIDENCE[sessionId]; +} + +export function pushEvidence( + sessionId: string, + kind: string, + summary: string, + data: string, + strong = false, +): string { + const s = sess(sessionId); + const id = `ev${s.records.length + 1}`; + s.records.push({ id, kind, summary, data }); + if (strong) s.strong.add(id); + return id; +} + +export function resetVerifySession(sessionId: string): void { + delete EVIDENCE[sessionId]; + delete VERDICTS[sessionId]; +} + +export function getVerdict(sessionId: string): unknown { + return VERDICTS[sessionId]; +} + +const PROBE_KIND: Record = { + stagehand_network_activity: "network", + stagehand_logs: "console", + stagehand_screenshot: "screenshot", + stagehand_extract: "dom", +}; + +function text(t: string): CallToolResult { + return { content: [{ type: "text", text: t }] }; +} + +function payloadOf(result: CallToolResult): { data: string; text: string } { + const items = (result.content as Array<{ type: string; text?: string; data?: string }>) || []; + const texts: string[] = []; + let img = ""; + for (const it of items) { + if (it.type === "text" && it.text) texts.push(it.text); + else if (it.type === "image" && it.data) img = it.data; + } + return { data: img || texts.join("\n"), text: texts.join("\n") }; +} + +// Tag a stagehand probe result with a captured evidence id the agent can cite. +export function tagEvidence(sessionId: string, toolName: string, result: CallToolResult): CallToolResult { + const kind = PROBE_KIND[toolName]; + if (!kind) return result; + const { data } = payloadOf(result); + const id = pushEvidence(sessionId, kind, `${kind} via ${toolName}`, data.slice(0, 4000), true); + return { + ...result, + content: [ + ...((result.content as Array) || []), + { type: "text", text: `[captured probe evidence ${id} (${kind}) — cite ${id} in proof[]]` }, + ], + } as CallToolResult; +} + +// --------------------------------------------------------------------------- +// http_request — timed HTTP probe against the running app / its API +// --------------------------------------------------------------------------- + +const HttpRequestSchema = z.object({ + url: z.string().describe("Absolute URL to request."), + method: z.string().optional().describe("HTTP method (default GET)."), + headers: z.record(z.string(), z.string()).optional(), + body: z.string().optional().describe("Raw request body, if any."), +}); + +export const HttpRequestTool: Tool = { + name: "verify_http_request", + description: + "Make a timed HTTP request against the running app or its API. Returns status, elapsed ms, response headers, and a snippet of the body. Captures an http evidence record and returns its id to cite in proof[]. Prefer this over driving the browser when the check is about an API's status/shape.", + inputSchema: parseSchema(HttpRequestSchema), +}; + +export async function httpRequest(sessionId: string, args: Record): Promise { + const { url, method, headers, body } = HttpRequestSchema.parse(args); + const start = Date.now(); + try { + const resp = await fetch(url, { method: method ?? "GET", headers, body: body ?? undefined }); + const ms = Date.now() - start; + const bodyText = await resp.text(); + const respHeaders: Record = {}; + resp.headers.forEach((v, k) => (respHeaders[k] = v)); + const bodySnippet = bodyText.slice(0, 2000); + const id = pushEvidence( + sessionId, + "http", + `HTTP ${method ?? "GET"} ${url} -> ${resp.status} in ${ms}ms`, + JSON.stringify({ status: resp.status, ms, headers: respHeaders, bodySnippet }), + true, + ); + return text(JSON.stringify({ id, status: resp.status, ms, headers: respHeaders, bodySnippet })); + } catch (err: any) { + const ms = Date.now() - start; + const message = err?.message ?? String(err); + const id = pushEvidence( + sessionId, + "http", + `HTTP ${method ?? "GET"} ${url} -> request failed in ${ms}ms`, + JSON.stringify({ status: 0, ms, bodySnippet: `request failed: ${message}` }), + true, + ); + return text(JSON.stringify({ id, status: 0, ms, bodySnippet: `request failed: ${message}` })); + } +} + +// --------------------------------------------------------------------------- +// sample — timing over n requests +// --------------------------------------------------------------------------- + +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length)); + return sorted[idx]; +} + +const SampleSchema = z.object({ + url: z.string().describe("Absolute URL to sample."), + n: z.number().describe("Number of requests to make."), +}); + +export const SampleTool: Tool = { + name: "verify_sample", + description: + "Call a URL n times and measure timing. Returns count, median ms, p95 ms, and the samples. Captures a timing evidence record and returns its id to cite in proof[]. Use for performance/timing claims.", + inputSchema: parseSchema(SampleSchema), +}; + +export async function sampleUrl(sessionId: string, args: Record): Promise { + const { url, n } = SampleSchema.parse(args); + const count = Math.max(1, Math.min(50, Math.floor(n))); + const samples: number[] = []; + for (let i = 0; i < count; i++) { + const start = Date.now(); + try { + const resp = await fetch(url, { method: "GET" }); + await resp.arrayBuffer(); + } catch { + /* record the elapsed time of the failed attempt */ + } + samples.push(Date.now() - start); + } + const sorted = [...samples].sort((a, b) => a - b); + const medianMs = percentile(sorted, 50); + const p95Ms = percentile(sorted, 95); + const id = pushEvidence( + sessionId, + "timing", + `sampled ${url} n=${count} median=${medianMs}ms p95=${p95Ms}ms`, + JSON.stringify({ count, medianMs, p95Ms, samples }), + true, + ); + return text(JSON.stringify({ id, count, medianMs, p95Ms, samples })); +} + +// --------------------------------------------------------------------------- +// db_query — read-only Postgres probe (read-after-write proof) +// --------------------------------------------------------------------------- + +const DbQuerySchema = z.object({ + query: z.string().describe("A single read-only SELECT statement."), +}); + +const DB_ROW_CAP = 50; + +export const DbQueryTool: Tool = { + name: "verify_db_query", + description: + "Run a READ-ONLY SQL query (Postgres) against the app database to independently confirm state persisted — e.g. after a write through the UI, SELECT the row to prove it exists. Enforced read-only. Captures a db evidence record and returns its id to cite in proof[]. Only available when a database URL is configured.", + inputSchema: parseSchema(DbQuerySchema), +}; + +export async function dbQuery(sessionId: string, args: Record): Promise { + const { query } = DbQuerySchema.parse(args); + const url = process.env.AUDIT_DB_URL || process.env.DATABASE_URL; + if (!url) return text(JSON.stringify({ unavailable: true, message: "no database configured" })); + const client = new pg.Client({ connectionString: url, connectionTimeoutMillis: 5000 }); + try { + await client.connect(); + await client.query("SET default_transaction_read_only = on"); + await client.query("BEGIN"); + await client.query("SET TRANSACTION READ ONLY"); + await client.query("SET LOCAL statement_timeout = '5s'"); + const res = await client.query({ text: query, values: [] }); + await client.query("ROLLBACK"); + const rows = res.rows.slice(0, DB_ROW_CAP); + const id = pushEvidence( + sessionId, + "db", + `db_query rows=${res.rowCount ?? rows.length}`, + JSON.stringify({ rowCount: res.rowCount, rows }), + true, + ); + return text(JSON.stringify({ id, rowCount: res.rowCount, rows })); + } catch (err: any) { + const message = err?.message ?? String(err); + const id = pushEvidence(sessionId, "db", "db_query failed", message, true); + return text(JSON.stringify({ id, error: `db_query failed: ${message}` })); + } finally { + await client.end().catch(() => {}); + } +} + +// --------------------------------------------------------------------------- +// run_command — run a shell command to inspect the running system / a CLI +// --------------------------------------------------------------------------- + +const RunCommandSchema = z.object({ + cmd: z.string().describe("The shell command to run. Mutating commands (git write, rm, installs, etc.) are rejected."), + cwd: z.string().optional().describe("Working directory to run in."), + timeoutMs: z.number().optional().describe("Timeout in ms (default 120000)."), +}); + +export const RunCommandTool: Tool = { + name: "verify_run_command", + description: + "Run a shell command to inspect the system or exercise a CLI (e.g. run a tool to prove a fix works). Mutating commands are rejected. Returns stdout/stderr/exit code. Captures a command evidence record and returns its id to cite in proof[].", + inputSchema: parseSchema(RunCommandSchema), +}; + +export async function runCommand(sessionId: string, args: Record): Promise { + const { cmd, cwd, timeoutMs } = RunCommandSchema.parse(args); + const rejection = commandRejection(cmd); + if (rejection) return text(JSON.stringify({ rejected: true, message: rejection })); + try { + const { stdout, stderr } = await execAsync(cmd, { + cwd: cwd || undefined, + timeout: timeoutMs && timeoutMs > 0 ? timeoutMs : 120000, + maxBuffer: 4 * 1024 * 1024, + }); + const out = `stdout:\n${(stdout || "").slice(0, 4000)}\nstderr:\n${(stderr || "").slice(0, 2000)}`; + const id = pushEvidence(sessionId, "command", `run: ${cmd.slice(0, 120)} -> exit 0`, out, true); + return text(JSON.stringify({ id, exit: 0, stdout: (stdout || "").slice(0, 4000), stderr: (stderr || "").slice(0, 2000) })); + } catch (err: any) { + const out = `exit ${err?.code ?? "?"}\nstdout:\n${(err?.stdout || "").slice(0, 4000)}\nstderr:\n${(err?.stderr || err?.message || "").slice(0, 2000)}`; + const id = pushEvidence(sessionId, "command", `run: ${cmd.slice(0, 120)} -> exit ${err?.code ?? "?"}`, out, true); + return text(JSON.stringify({ id, exit: err?.code ?? 1, stdout: (err?.stdout || "").slice(0, 4000), stderr: (err?.stderr || err?.message || "").slice(0, 2000) })); + } +} + +// --------------------------------------------------------------------------- +// submit_verdict — the shared proof contract (guard + reconcile) +// --------------------------------------------------------------------------- + +const OutcomeSchema = z.enum(["works", "broken", "unknown"]); + +const ClaimSchema = z.object({ + claim: z.string().describe("The specific thing the task claimed to do."), + verdict: OutcomeSchema, + proof: z + .array(z.string()) + .describe( + "Captured evidence ids that back this verdict — the ev ids returned by probe tools (verify_http_request, verify_sample, verify_db_query, stagehand_network_activity, stagehand_logs, stagehand_screenshot, stagehand_extract). A works verdict with no such id is downgraded to unknown.", + ), + reasoning: z.string(), +}); + +const VerdictSchema = z.object({ + overall: OutcomeSchema, + claims: z.array(ClaimSchema), + observations: z.array(z.string()), + summary: z.string(), +}); + +export const SubmitVerdictTool: Tool = { + name: "submit_verdict", + description: + "Submit the final verdict and END. A claim may be marked works ONLY if its proof[] cites at least one captured probe-evidence id; a works claim with no such id is downgraded to unknown and overall follows. This is the terminal tool.", + inputSchema: parseSchema(VerdictSchema), +}; + +export async function submitVerdict(sessionId: string, args: Record): Promise { + const input = VerdictSchema.parse(args); + const strong = sess(sessionId).strong; + const notes: string[] = []; + + const claims = input.claims.map((c) => { + if (c.verdict !== "works") return c; + const backed = c.proof.filter((id) => strong.has(id)); + if (backed.length === 0) { + notes.push(`Guard: claim "${c.claim}" marked works with no captured proof; downgraded to unknown.`); + return { + ...c, + verdict: "unknown" as const, + proof: backed, + reasoning: `${c.reasoning} [guard: no captured proof backed this works claim]`, + }; + } + return { ...c, proof: backed }; + }); + + const hasBroken = claims.some((c) => c.verdict === "broken"); + const allWorks = claims.length > 0 && claims.every((c) => c.verdict === "works"); + let overall = input.overall; + if (overall === "works" && !allWorks) { + overall = hasBroken ? "broken" : "unknown"; + notes.push(`Guard: overall downgraded from works to ${overall} because not every claim is backed as works.`); + } + + const verdict = { + overall, + claims, + observations: notes.length > 0 ? [...input.observations, ...notes] : input.observations, + summary: input.summary, + evidence: sess(sessionId).records, + }; + VERDICTS[sessionId] = verdict; + if (process.env.AUDIT_VERDICT_OUT) { + try { + const fs = await import("node:fs"); + fs.writeFileSync(process.env.AUDIT_VERDICT_OUT, JSON.stringify(verdict, null, 2)); + } catch { + /* best effort */ + } + } + return text(JSON.stringify(verdict)); +} + +export const VERIFY_TOOLS: Tool[] = [HttpRequestTool, SampleTool, DbQueryTool, RunCommandTool, SubmitVerdictTool]; diff --git a/mcp/src/tools/verify/stdio-server.ts b/mcp/src/tools/verify/stdio-server.ts new file mode 100644 index 000000000..07f9feedd --- /dev/null +++ b/mcp/src/tools/verify/stdio-server.ts @@ -0,0 +1,23 @@ +// Stdio MCP transport for the verification tools. A generic agent (e.g. goose, +// launched by staklink in the pod) spawns this as an MCP extension and drives +// the shared verify tools + submit_verdict over stdio. +// +// stdout is the MCP JSON-RPC channel: anything the app writes there (browser +// launch, stagehand, deps) corrupts the protocol for a strict client, so guard +// stdout to pass only JSON-RPC frames and route everything else to stderr. +const realStdoutWrite = process.stdout.write.bind(process.stdout); +process.stdout.write = ((chunk: any, ...rest: any[]): boolean => { + const s = typeof chunk === "string" ? chunk : Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk); + if (s.startsWith("{")) return realStdoutWrite(chunk, ...rest); + return (process.stderr.write as any)(chunk, ...rest); +}) as typeof process.stdout.write; + +console.log = (...a: unknown[]) => console.error(...a); +console.info = (...a: unknown[]) => console.error(...a); +console.warn = (...a: unknown[]) => console.error(...a); + +const { StdioServerTransport } = await import("@modelcontextprotocol/sdk/server/stdio.js"); +const { graphServer } = await import("../server.js"); + +const transport = new StdioServerTransport(); +await graphServer.connect(transport);