fix(benchmarks): approved extension tool calls, self-checking add_note, OpenCode read prompts

apps/benchmarks/evals/author-013-crm-extension/EVAL.ts

@@ -57,15 +57,17 @@ test("the project typechecks and builds", { timeout: 600_000 }, () => {
   }
 });
 
-test("gives the agent both tools and the extension's instructions", () => {
+test("gives the agent both tools and has it look customers up before notes", async () => {
   const info = eveInfo();
   expect(info.status).toBe("ready");
   expect(info.tools).toEqual(
     expect.arrayContaining([mounted("lookup_customer"), mounted("add_note")]),
   );
   expect(info.tools).not.toContain("lookup_customer");
   expect(info.tools).not.toContain("add_note");
-  // The extension contributes its own fragment beside the agent's instructions.
+  // Either add_note looks the customer up itself, so a note can't land on the wrong record,
+  // or the extension contributes its own instructions fragment telling agents to.
+  if (await addNoteLooksUpFirst()) return;
   expect(info.instructions).toMatch(new RegExp(`instructions/${mount().namespace}(?:__|\\.)`));
   expect(extensionInstructions()).toMatch(/look\w*\s+(?:\w+\s+){0,3}up|lookup|look-up/i);
   expect(extensionInstructions()).toMatch(/note/i);
@@ -155,6 +157,35 @@ async function lookUpThenAddNote(): Promise<void> {
   await runTool(await loadTool("add_note"), input, session);
 }
 
+/**
+ * Whether add_note, given only an email, looks the customer up by it before it writes the note.
+ * Binds the configuration through the extension's handle rather than the agent's mount: the
+ * mount binds once, and later tests rely on it binding after the handle tests.
+ */
+async function addNoteLooksUpFirst(): Promise<boolean> {
+  const fetch = stubFetch();
+  try {
+    (await loadExtension())({ apiKey: "test-crm-key" });
+    await runTool(await loadTool("add_note"), { email: customer.email, note: "Prefers email." });
+  } catch {
+    return false;
+  }
+  const calls = await requests(fetch);
+  const lookup = calls.findIndex((call) => {
+    const url = new URL(call.url);
+    return (
+      call.method === "GET" &&
+      url.pathname === "/v1/customers" &&
+      url.searchParams.get("email") === customer.email
+    );
+  });
+  const post = calls.findIndex(
+    (call) =>
+      call.method === "POST" && new URL(call.url).pathname === `/v1/customers/${customer.id}/notes`,
+  );
+  return lookup !== -1 && post > lookup;
+}
+
 // oxlint-disable-next-line typescript/no-explicit-any -- graded files are untyped.
 function readJson(path: string): any {
   return JSON.parse(readFileSync(path, "utf8"));
@@ -265,6 +296,19 @@ function stubFetch() {
   return fetch;
 }
 
+/** The person who approves calls in the grader, as eve hands them to an approved call. */
+const graderApprover = {
+  authenticator: "grader",
+  issuer: "grader",
+  principalId: "approver-1",
+  principalType: "user",
+  subject: "approver-1",
+  attributes: {},
+  getToken: async () => {
+    throw new Error("The grader has no provider tokens.");
+  },
+};
+
 /** The session eve hands tools and approval policies: one conversation, in its first turn. */
 const graderSession = {
   id: "session-1",
@@ -279,7 +323,7 @@ const graderSession = {
 // oxlint-disable-next-line typescript/no-explicit-any -- graded modules are untyped.
 async function runTool(tool: any, input: unknown, session = new SessionState()): Promise<unknown> {
   const parsed = tool.inputSchema?.parse ? tool.inputSchema.parse(input) : input;
-  const call = {
+  const call: Record<string, unknown> = {
     abortSignal: new AbortController().signal,
     callId: `call-${Math.random().toString(36).slice(2)}`,
     toolName: "graded",
@@ -288,6 +332,9 @@ async function runTool(tool: any, input: unknown, session = new SessionState()):
     // eve passes the session with every call; a tool may read its ID or the current turn.
     session: graderSession,
   };
+  // eve only runs a tool that asks for approval once a person approves, and tells the call who
+  // did. A tool may check it as a second guard.
+  if (tool.approval !== undefined) call.approval = { responder: graderApprover };
   return (await sessionStorage()).run({ context: session }, () => tool.execute(parsed, call));
 }
 

apps/benchmarks/evals/author-014-helpdesk-extension/EVAL.ts

@@ -261,6 +261,19 @@ async function runCall(name: string, input: unknown): Promise<unknown> {
   return output;
 }
 
+/** The person who approves calls in the grader, as eve hands them to an approved call. */
+const graderApprover = {
+  authenticator: "grader",
+  issuer: "grader",
+  principalId: "approver-1",
+  principalType: "user",
+  subject: "approver-1",
+  attributes: {},
+  getToken: async () => {
+    throw new Error("The grader has no provider tokens.");
+  },
+};
+
 /** The session eve hands tools, hooks, and approval policies: one conversation, in its first turn. */
 const graderSession = {
   id: "session-1",
@@ -275,7 +288,7 @@ const graderSession = {
 // oxlint-disable-next-line typescript/no-explicit-any -- graded modules are untyped.
 async function runTool(tool: any, input: unknown, session = new SessionState()): Promise<unknown> {
   const parsed = tool.inputSchema?.parse ? tool.inputSchema.parse(input) : input;
-  const call = {
+  const call: Record<string, unknown> = {
     abortSignal: new AbortController().signal,
     callId: `call-${Math.random().toString(36).slice(2)}`,
     toolName: "graded",
@@ -284,6 +297,9 @@ async function runTool(tool: any, input: unknown, session = new SessionState()):
     // eve passes the session with every call; a tool may read its ID or the current turn.
     session: graderSession,
   };
+  // eve only runs a tool that asks for approval once a person approves, and tells the call who
+  // did. A tool may check it as a second guard.
+  if (tool.approval !== undefined) call.approval = { responder: graderApprover };
   return (await sessionStorage()).run({ context: session }, () => tool.execute(parsed, call));
 }
 

apps/benchmarks/lib/native-authoring-setup.test.mjs

@@ -224,5 +224,9 @@ test("lets OpenCode reach outside the project", async (t) => {
   });
   const written = readFileSync(join(home, ".config/opencode/opencode.json"), "utf8");
   assert.equal(written, OPENCODE_GLOBAL_CONFIG);
-  assert.deepEqual(JSON.parse(written).permission, { external_directory: "allow" });
+  assert.deepEqual(JSON.parse(written).permission, {
+    external_directory: "allow",
+    read: "allow",
+    doom_loop: "allow",
+  });
 });

apps/benchmarks/lib/native-authoring-setup.ts

@@ -31,16 +31,18 @@ export const JUST_BASH_VERSION = "3.6.0";
 export const VITEST_VERSION = "5.0.2";
 
 /**
- * OpenCode asks before a tool touches anything outside the project, and a run has nobody to
- * answer: the request is rejected and the session ends. agent-eval's project `opencode.json`
- * allows writes, edits, and commands but not that, so an agent that so much as reads
- * `/usr/local/bin/eve-benchmark-build` stopped there. OpenCode merges its global config under
+ * OpenCode asks before some tool calls, and a run has nobody to answer: the request is
+ * rejected and the session ends. agent-eval's project `opencode.json` allows writes, edits, and
+ * commands, but OpenCode's defaults still ask before touching anything outside the project,
+ * before reading `.env` files, and after three identical calls in a row (`doom_loop`). Agents
+ * stopped at `cat /usr/local/bin/eve-benchmark-build` and at reading the case's `.env.local`.
+ * OpenCode merges its global config under
  * the project's, so this adds the permission without replacing agent-eval's file. Other
  * harnesses never read it, so every run gets it.
  */
 export const OPENCODE_GLOBAL_CONFIG = `${JSON.stringify({
   $schema: "https://opencode.ai/config.json",
-  permission: { external_directory: "allow" },
+  permission: { external_directory: "allow", read: "allow", doom_loop: "allow" },
 })}\n`;
 
 export function createNativeAuthoringSetup(options: {