From f4ab2be64efe56938850e82773ac0ff753a29c41 Mon Sep 17 00:00:00 2001
From: Chris Huber <chubes@chubes.net>
Date: Mon, 4 May 2026 08:08:33 -0400
Subject: [PATCH] CLI: capture eval-runner failures and timeouts in result
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Extends the eval-runner result JSON with two new fields that any
consumer of `EVAL_RUNNER_RESULT_FILE` can read:

- `error: string | null` — set when an exception is thrown inside
  the message loop (auth blip, MCP crash, network error, SDK throw,
  etc.). Today those exceptions propagate out of `runEval()` to
  `main()`'s top-level catch, which emits a stripped-down
  `{ success: false, error }` JSON without the timings, tools, turns,
  or partial state that were captured before the throw. With this
  change, the exception is caught inside `runEval()`, so consumers
  get the full structured result alongside the error message.
- `timedOut: boolean` — flipped by the timeout callback before it
  calls `query.interrupt()`. Today timeouts are indistinguishable
  from clean model failures: both surface as `success: false` with
  no other differentiator. Consumers can now tell them apart and
  attribute time-budget exhaustion correctly.

Together these address the `finalError` ask in Automattic/studio#3262
section 1 ("First actionable failure") and complete the failure-
visibility story started by #3273 (which added `firstToolError`,
`toolEvents`, `phaseTimingsMs`).

The change is generic — it benefits any consumer of the eval-runner's
structured output (Studio's own scripts, `npm run eval`, future
internal CI, third-party benchmark harnesses).

Originally drafted on the experimental Static Site Importer branch
(#3309); split out for upstream review because the eval-runner
changes are independently valuable and shouldn't ship gated on
that experiment.

Refs: #3262, #3273, #3309

## AI assistance
- **AI assistance:** Yes
- **Tool(s):** Claude Code (Sonnet 4.5)
- **Used for:** Drafted the catch wrapper, the `timedOut` flag,
  and the result-shape extension under Chris's direction. Chris
  reviewed the diff, the issue framing, and the split rationale.
---
 apps/cli/ai/eval-runner.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts
index 28f099ecd5..bf7d33ed66 100644
--- a/apps/cli/ai/eval-runner.ts
+++ b/apps/cli/ai/eval-runner.ts
@@ -178,6 +178,8 @@ async function runEval( input: EvalRunnerInput ) {
 	let turnIndex = 0;
 	let numTurns: number | null = null;
 	let success = false;
+	let error: string | null = null;
+	let timedOut = false;
 
 	phaseStartedAt = Date.now();
 	const query = startAiAgent( {
@@ -189,7 +191,10 @@ async function runEval( input: EvalRunnerInput ) {
 	const queryStartedAt = Date.now();
 	let turnStart = queryStartedAt;
 
-	const timeout = setTimeout( () => void query.interrupt(), input.timeoutMs ?? 300000 );
+	const timeout = setTimeout( () => {
+		timedOut = true;
+		void query.interrupt();
+	}, input.timeoutMs ?? 300000 );
 
 	try {
 		for await ( const message of query ) {
@@ -250,6 +255,8 @@ async function runEval( input: EvalRunnerInput ) {
 				numTurns = message.num_turns ?? null;
 			}
 		}
+	} catch ( caught ) {
+		error = caught instanceof Error ? caught.message : String( caught );
 	} finally {
 		clearTimeout( timeout );
 	}
@@ -257,6 +264,8 @@ async function runEval( input: EvalRunnerInput ) {
 
 	return {
 		success,
+		error,
+		timedOut,
 		numTurns,
 		phaseTimingsMs,
 		turnDurationsMs,