feat: Auditor agent — pod-resident verification agent (MVP)

mcp/src/auditor/agent.ts

@@ -0,0 +1,174 @@
+import { ToolLoopAgent, StopCondition, stepCountIs } from "ai";
+import { getModelDetails, getProviderOptions } from "../aieo/src/provider.js";
+import { AuditBrowser } from "./browser.js";
+
+function maxOutputTokensFor(provider?: string): number {
+  const env = Number(process.env.MAX_OUTPUT_TOKENS);
+  if (env > 0) return env;
+  return provider === "anthropic" ? 128_000 : 64_000;
+}
+import { getAuditorTools, AuditorTools, END_OF_AUDIT } from "./tools.js";
+import { AUDITOR_SYSTEM_PROMPT } from "./prompt.js";
+import {
+  AuditJob,
+  EvidenceCollector,
+  EvidenceKind,
+  EvidenceRecord,
+  Verdict,
+} from "./types.js";
+
+const DEFAULT_MAX_TURNS =
+  parseInt(process.env.AUDIT_MAX_TURNS || "", 10) || 40;
+
+const RUN_TIMEOUT_MS =
+  parseInt(process.env.AUDIT_RUN_TIMEOUT_MS || "", 10) || 900_000;
+const MODEL_TIMEOUT_MS =
+  parseInt(process.env.AUDIT_MODEL_TIMEOUT_MS || "", 10) || 300_000;
+
+function createCollector(): EvidenceCollector {
+  const records: EvidenceRecord[] = [];
+  const strongIds = new Set<string>();
+  return {
+    records,
+    strongIds,
+    verdict: undefined,
+    push(kind: EvidenceKind, summary: string, data?: string, strong = false): string {
+      const id = `ev${records.length + 1}`;
+      records.push({ id, kind, summary, data: data ?? "" });
+      if (strong) strongIds.add(id);
+      return id;
+    },
+  };
+}
+
+function hasEndMarker(): StopCondition<AuditorTools> {
+  return ({ steps }) => {
+    for (const step of steps) {
+      for (const item of step.content) {
+        if (item.type === "text" && item.text?.includes(END_OF_AUDIT)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  };
+}
+
+export interface PreparedAuditor {
+  run(): Promise<Verdict>;
+}
+
+export function prepareAuditor(
+  job: AuditJob,
+  maxTurns: number = DEFAULT_MAX_TURNS,
+): PreparedAuditor {
+  const startedAt = new Date().toISOString();
+  const collector = createCollector();
+  const logs: string[] = [];
+
+  const abortController = new AbortController();
+
+  const modelName = job.model.model ?? job.model.provider;
+  const { model, modelId, provider } = getModelDetails(
+    modelName,
+    job.model.apiKey,
+    job.model.host,
+    undefined,
+    abortController.signal,
+    MODEL_TIMEOUT_MS,
+  );
+
+  const browser = new AuditBrowser(model);
+
+  const tools = getAuditorTools({ deck: job.deck, collector, browser });
+
+  const stopWhen: StopCondition<AuditorTools>[] = [
+    hasEndMarker(),
+    stepCountIs(maxTurns) as StopCondition<AuditorTools>,
+  ];
+
+  const agent = new ToolLoopAgent<never, AuditorTools>({
+    model,
+    instructions: AUDITOR_SYSTEM_PROMPT,
+    tools,
+    stopWhen,
+    stopSequences: [END_OF_AUDIT],
+    providerOptions: getProviderOptions(provider as any, undefined, modelId) as any,
+    maxOutputTokens: maxOutputTokensFor(provider),
+    onStepFinish: (sf) => {
+      for (const item of sf.content) {
+        if (item.type === "tool-call") {
+          logs.push(`tool_call ${item.toolName}`);
+          console.log(`[auditor] taskId=${job.taskId} tool_call ${item.toolName}`);
+        } else if (item.type === "text" && item.text) {
+          const line = item.text.slice(0, 200).replace(/\n/g, " ");
+          logs.push(`text ${line}`);
+        }
+      }
+    },
+  });
+
+  async function run(): Promise<Verdict> {
+    const timer = setTimeout(() => abortController.abort(), RUN_TIMEOUT_MS);
+    let runError: string | undefined;
+
+    const userPrompt =
+      `Audit task ${job.taskId}. Load the task and its diff, the feature context, and the map, ` +
+      `then exercise the RUNNING app to reach an evidence-backed verdict. ` +
+      `Capture proof for anything you mark works, and call submit_verdict when done.`;
+
+    console.log(
+      `[auditor] run_start taskId=${job.taskId} model=${modelId} provider=${provider} appUrl=${job.deck.map.appUrl}`,
+    );
+
+    try {
+      await agent.generate({
+        prompt: userPrompt,
+        abortSignal: abortController.signal,
+      });
+    } catch (err: any) {
+      runError = err?.message ?? String(err);
+      console.error(`[auditor] run_error taskId=${job.taskId}: ${runError}`);
+    } finally {
+      clearTimeout(timer);
+      await browser.close().catch(() => {});
+    }
+
+    const finishedAt = new Date().toISOString();
+    const submitted = collector.verdict;
+
+    if (submitted) {
+      return {
+        taskId: job.taskId,
+        overall: submitted.overall,
+        claims: submitted.claims,
+        observations: submitted.observations,
+        summary: submitted.summary,
+        evidence: collector.records,
+        startedAt,
+        finishedAt,
+        ...(runError ? { error: runError } : {}),
+      };
+    }
+
+    return {
+      taskId: job.taskId,
+      overall: "unknown",
+      claims: [],
+      observations: [
+        `Auditor ended without submitting a verdict after ${logs.length} logged step events.`,
+        ...(collector.records.length > 0
+          ? [`${collector.records.length} evidence records were captured.`]
+          : []),
+      ],
+      summary:
+        "The audit ended before submit_verdict was called; no honest verdict could be produced.",
+      evidence: collector.records,
+      startedAt,
+      finishedAt,
+      error: runError ?? "no verdict submitted",
+    };
+  }
+
+  return { run };
+}

mcp/src/auditor/browser.ts

@@ -0,0 +1,83 @@
+import { Stagehand, AISdkClient } from "@browserbasehq/stagehand";
+import type { LanguageModel } from "ai";
+
+export class AuditBrowser {
+  private stagehand?: Stagehand;
+  private readonly model: LanguageModel;
+
+  constructor(model: LanguageModel) {
+    this.model = model;
+  }
+
+  private async ensure(): Promise<Stagehand> {
+    if (this.stagehand) return this.stagehand;
+    const sh = new Stagehand({
+      env: "LOCAL",
+      domSettleTimeout: 60000,
+      localBrowserLaunchOptions: {
+        headless: true,
+        viewport: { width: 1024, height: 768 },
+      },
+      llmClient: new AISdkClient({ model: this.model as any }),
+    });
+    await sh.init();
+    this.stagehand = sh;
+    return sh;
+  }
+
+  private async page(sh: Stagehand) {
+    const page = sh.context.activePage();
+    if (!page) throw new Error("no active page available");
+    return page;
+  }
+
+  async open(url: string): Promise<{ url: string; ok: boolean }> {
+    const sh = await this.ensure();
+    const page = await this.page(sh);
+    await page.goto(url);
+    return { url, ok: true };
+  }
+
+  async act(action: string): Promise<{ action: string; result: unknown }> {
+    const sh = await this.ensure();
+    const result = await sh.act(action);
+    return { action, result };
+  }
+
+  async observe(
+    instruction: string,
+  ): Promise<{ instruction: string; observations: unknown }> {
+    const sh = await this.ensure();
+    const observations = await sh.observe(instruction);
+    return { instruction, observations };
+  }
+
+  async extract(
+    instruction: string,
+  ): Promise<{ instruction: string; extraction: unknown }> {
+    const sh = await this.ensure();
+    const extraction = await sh.extract(instruction);
+    return { instruction, extraction };
+  }
+
+  async currentUrl(): Promise<string> {
+    const sh = await this.ensure();
+    const page = await this.page(sh);
+    return page.url();
+  }
+
+  async screenshot(): Promise<string> {
+    const sh = await this.ensure();
+    const page = await this.page(sh);
+    const buffer = await page.screenshot({ fullPage: false });
+    return buffer.toString("base64");
+  }
+
+  async close(): Promise<void> {
+    if (this.stagehand) {
+      const sh = this.stagehand;
+      this.stagehand = undefined;
+      await sh.close();
+    }
+  }
+}

mcp/src/auditor/prompt.ts

@@ -0,0 +1,19 @@
+export const AUDITOR_SYSTEM_PROMPT = `You are an independent Auditor. Your job is to determine, with evidence, whether the specific TASK was actually solved in the running application. You did not write this code; you audit it. You never modify anything.
+
+SCOPE
+Judge ONLY what THIS task was asked to do (its prompt + its diff). The feature context is background — do not hold this task accountable for parts of the feature other tasks own. You may add feature-level notes as observations, never as failures of this task. Start by reading the task, its diff, the feature context, and the map so you know what "solved" means and where the running app is.
+
+METHOD FREEDOM
+Decide for yourself how to prove it — drive the UI, call APIs, read logs, measure timing. There is no fixed pipeline and no checklist: compose the tools however the task's nature demands. Use the cheapest sufficient method; reserve the browser for genuinely visual checks — an API call, a log line, or a timing sample is faster and stronger when the task is not about pixels.
+
+ACCESS
+You are NOT told how to log in or navigate — every app differs. Discover it by driving the app: snapshot the page to see what is there, try the app's own dev/mock/offline mode, and fill the visible login form with placeholder credentials. The app's base URL is in the map. If access is genuinely blocked, say so as the reason for an unknown verdict — do not invent steps you were told.
+
+EPISTEMICS (the core)
+Only mark a claim works if you CAPTURED proof it works, using a PROBE tool — http_request, sample, read_logs, browser_extract, browser_screenshot, or browser_current_url — and cite the evidence id it returned in proof[]. A note from the capture tool is NOT proof and will not back a works claim: a works claim with no probe-captured proof id is automatically downgraded to unknown, and overall follows. "It compiles" or "looks right" is NOT proof. Prefer cheap http/log/timing probes when a claim can be checked without the UI; reach for a screenshot when the check is genuinely visual. If you cannot reach the app or cannot tell, mark unknown and describe what happened. If it is genuinely broken, mark broken with the reason. Be honest — an unjustified works is the worst possible outcome. Inspect what actually happened; never assume.
+
+PERSISTENCE
+If a tool call fails or a page looks empty, wait briefly and retry, or try browser_observe to find elements and browser_act to interact, before concluding. Do not give up after one failure.
+
+FINISHING
+Work toward a verdict efficiently. As soon as you have captured proof for each claim the task makes, STOP probing and call submit_verdict — do not keep exploring once the evidence is sufficient. Over-exploration burns the turn budget and risks the run ending with NO verdict, which is the worst outcome. A focused audit of a handful of claims rarely needs more than a dozen or so tool calls. Call submit_verdict with per-claim verdicts, each backed by captured evidence ids, plus a holistic overall verdict and a short summary. submit_verdict is the only way to end the audit; if you never call it, the audit fails as unknown.`;

mcp/src/auditor/report.ts

@@ -0,0 +1,45 @@
+import { fetch } from "undici";
+import { Verdict } from "./types.js";
+
+const REPORT_ATTEMPTS =
+  parseInt(process.env.AUDIT_REPORT_ATTEMPTS || "", 10) || 4;
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function reportVerdict(
+  responseUrl: string,
+  callbackApiKey: string,
+  verdict: Verdict,
+): Promise<void> {
+  for (let attempt = 1; attempt <= REPORT_ATTEMPTS; attempt++) {
+    try {
+      const resp = await fetch(responseUrl, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          "x-api-key": callbackApiKey,
+        },
+        body: JSON.stringify(verdict),
+      });
+      if (resp.ok) {
+        console.log(
+          `[auditor] reported taskId=${verdict.taskId} overall=${verdict.overall} status=${resp.status}`,
+        );
+        return;
+      }
+      console.error(
+        `[auditor] report non-ok taskId=${verdict.taskId} status=${resp.status} attempt=${attempt}/${REPORT_ATTEMPTS}`,
+      );
+    } catch (err: any) {
+      console.error(
+        `[auditor] report attempt ${attempt}/${REPORT_ATTEMPTS} failed taskId=${verdict.taskId}: ${err?.message ?? String(err)}`,
+      );
+    }
+    if (attempt < REPORT_ATTEMPTS) await sleep(500 * 2 ** (attempt - 1));
+  }
+  console.error(
+    `[auditor] report gave up taskId=${verdict.taskId} after ${REPORT_ATTEMPTS} attempts`,
+  );
+}

mcp/src/auditor/run.ts

@@ -0,0 +1,66 @@
+import { prepareAuditor } from "./agent.js";
+import { reportVerdict } from "./report.js";
+import { AuditJob, Verdict } from "./types.js";
+
+async function readStdin(): Promise<string> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of process.stdin) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks).toString("utf8");
+}
+
+function fallbackVerdict(
+  taskId: string,
+  error: string,
+  startedAt: string,
+): Verdict {
+  return {
+    taskId,
+    overall: "unknown",
+    claims: [],
+    observations: [],
+    summary: "The auditor could not complete.",
+    evidence: [],
+    startedAt,
+    finishedAt: new Date().toISOString(),
+    error,
+  };
+}
+
+async function main(): Promise<void> {
+  const startedAt = new Date().toISOString();
+  let job: AuditJob | undefined;
+
+  try {
+    const raw = await readStdin();
+    job = JSON.parse(raw) as AuditJob;
+  } catch (err: any) {
+    console.error(
+      `[auditor] failed to read/parse job from stdin: ${err?.message ?? String(err)}`,
+    );
+    return;
+  }
+
+  let verdict: Verdict;
+  try {
+    verdict = await prepareAuditor(job).run();
+  } catch (err: any) {
+    verdict = fallbackVerdict(
+      job.taskId,
+      err?.message ?? String(err),
+      startedAt,
+    );
+  }
+
+  if (job.responseUrl && job.callbackApiKey) {
+    await reportVerdict(job.responseUrl, job.callbackApiKey, verdict);
+  } else {
+    console.error("[auditor] no responseUrl/callbackApiKey — verdict not reported");
+    console.log(JSON.stringify(verdict));
+  }
+}
+
+main().catch((err) => {
+  console.error(`[auditor] fatal: ${err?.message ?? String(err)}`);
+});