fix(benchmarks): extension tool sessions, CRM base URL, and invoice principal types

apps/benchmarks/evals/author-012-invoice-assistant/EVAL.ts

@@ -75,11 +75,17 @@ test("runs and deploys the agent with the Next.js app through withEve", () => {
   expect(source).toMatch(/export\s+default\s+withEve\s*\(/);
 });
 
+/** Principal types eve and its docs use for callers that aren't a signed-in person. */
+const nonPersonTypes = ["anonymous", "app", "local-dev", "runtime", "service"];
+
 test("admits signed-in Tally users as themselves", async () => {
   for (const user of [maya, sam]) {
     const { status, auth } = await authenticate({ cookie: sessionCookie(user) });
     expect(status, user.userId).toBe(202);
-    expect(auth?.principalType, user.userId).toBe("user");
+    // eve leaves `principalType` free-form; "user" is the convention, but `tally_user` still
+    // names a person. Only non-person types fail.
+    expect(auth?.principalType, user.userId).toMatch(/\S/);
+    expect(nonPersonTypes, user.userId).not.toContain(auth?.principalType);
     expect(String(auth?.principalId), user.userId).toContain(user.userId);
   }
 });

apps/benchmarks/evals/author-013-crm-extension/CASE.ts

@@ -12,7 +12,7 @@ export default defineAuthoringCase({
       [
         "We're going to have several agents that work with Acme CRM, and I don't want each one re-implementing the integration. Build it once as a shared package, `@acme/crm` in `packages/crm/` of this repo, that any of our agents can add to pick up the CRM capabilities, and add it to this agent first.",
         "",
-        "- Each agent brings its own Acme API key, and some teams will point at our Acme sandbox instead of production (`https://api.acme-crm.dev`). Call those settings `apiKey` and `baseUrl`. This agent's key is in `CRM_API_KEY`, already set in `.env.local`.",
+        "- Each agent brings its own Acme API key. Agents use our production API, `https://api.acme-crm.dev`, unless they set another base URL, since some teams point at our Acme sandbox instead. Call those settings `apiKey` and `baseUrl`. This agent's key is in `CRM_API_KEY`, already set in `.env.local`.",
         '- Agents need to look up a customer by email (`lookup_customer`) and add a note to a customer\'s record (`add_note`). From Acme\'s API docs: `GET /v1/customers?email=<email>` returns the customer, and `POST /v1/customers/<customerId>/notes` with the JSON body `{ "note": "..." }` adds a note. Both take `Authorization: Bearer <apiKey>`.',
         "- Notes go straight onto the customer's record, so a person has to approve every note before it's written, in any agent that uses the package.",
         "- Agents using it should always look the customer up before writing a note, so notes don't land on the wrong record.",

apps/benchmarks/evals/author-013-crm-extension/EVAL.ts

@@ -117,7 +117,7 @@ test("add_note always asks for approval", async () => {
   expect(typeof policy).toBe("function");
   for (const note of ["short", "x".repeat(500)]) {
     const toolInput = { customerId: customer.id, email: customer.email, note };
-    const request = { callId: "call-1", session: {}, toolInput, toolName: "x" };
+    const request = { callId: "call-1", session: graderSession, toolInput, toolName: "x" };
     const decision = await (
       await sessionStorage()
     ).run({ context: session }, () => policy(request));
@@ -265,6 +265,13 @@ function stubFetch() {
   return fetch;
 }
 
+/** The session eve hands tools and approval policies: one conversation, in its first turn. */
+const graderSession = {
+  id: "session-1",
+  auth: { current: null, initiator: null },
+  turn: { id: "turn-1", sequence: 0 },
+};
+
 /**
  * Runs a tool call the way eve does: with a call context, inside a session whose state the
  * tools can read and write. Pass the same session to calls that belong to one conversation.
@@ -278,6 +285,8 @@ async function runTool(tool: any, input: unknown, session = new SessionState()):
     toolName: "graded",
     messages: [],
     channel: { kind: "eve" },
+    // eve passes the session with every call; a tool may read its ID or the current turn.
+    session: graderSession,
   };
   return (await sessionStorage()).run({ context: session }, () => tool.execute(parsed, call));
 }

apps/benchmarks/evals/author-014-helpdesk-extension/EVAL.ts

@@ -65,7 +65,12 @@ test("asks for approval before closing a ticket", async () => {
   const policy = typeof close.approval === "function" ? close.approval : close.approval?.request;
   expect(typeof policy).toBe("function");
   const toolInput = { ticketId: "T-7", resolution: "Refund issued." };
-  const decision = await policy({ callId: "call-1", session: {}, toolInput, toolName: "x" });
+  const decision = await policy({
+    callId: "call-1",
+    session: graderSession,
+    toolInput,
+    toolName: "x",
+  });
   expect(requiresApproval(decision)).toBe(true);
 
   // Once approved, the call still closes the ticket in the helpdesk.
@@ -238,7 +243,7 @@ async function runCall(name: string, input: unknown): Promise<unknown> {
   );
   const ctx = {
     agent: { name: "support" },
-    session: { id: "session-1", auth: { current: null, initiator: null } },
+    session: graderSession,
     position: { line: facts.length - 1, index: 0 },
     view,
     turn: { id: "turn-1" },
@@ -256,6 +261,13 @@ async function runCall(name: string, input: unknown): Promise<unknown> {
   return output;
 }
 
+/** The session eve hands tools, hooks, and approval policies: one conversation, in its first turn. */
+const graderSession = {
+  id: "session-1",
+  auth: { current: null, initiator: null },
+  turn: { id: "turn-1", sequence: 0 },
+};
+
 /**
  * Runs a tool call the way eve does: with a call context, inside a session whose state the
  * tools can read and write. Pass the same session to calls that belong to one conversation.
@@ -269,6 +281,8 @@ async function runTool(tool: any, input: unknown, session = new SessionState()):
     toolName: "graded",
     messages: [],
     channel: { kind: "eve" },
+    // eve passes the session with every call; a tool may read its ID or the current turn.
+    session: graderSession,
   };
   return (await sessionStorage()).run({ context: session }, () => tool.execute(parsed, call));
 }