diff --git a/apps/dbagent/README.md b/apps/dbagent/README.md index be60dd08..32f0ee68 100644 --- a/apps/dbagent/README.md +++ b/apps/dbagent/README.md @@ -78,3 +78,17 @@ vim .env.eval Update you `.env.local` file to contain: `EVAL=true` Ensure you have docker installed and run: `pnpm run eval` + +Each eval writes replay artifacts under the configured `EVAL_FOLDER`: + +- `human.txt`: readable prompt, answer, and tool-result transcript +- `replay.json`: structured replay manifest with model metadata, prompts, tool calls, tool results, and failure diagnostics +- `response.json`: raw Vercel AI SDK response for deep debugging +- `evalResult.json`: pass/fail result for the case + +The test-run folder also includes `evalResults.csv`. In addition to pass/fail and UI links, the CSV includes diagnostic columns for: + +- `classifications`: high-level failure categories such as `missing-expected-tool`, `unexpected-tool-call`, `tool-error`, `no-tool-result`, `malformed-request`, or `empty-final-answer` +- `expected_tools`, `observed_tools`, `missing_expected_tools`, and `unexpected_tools` + +This makes model/provider regressions easier to triage without opening every raw trace. For example, when a tool-choice eval fails, first filter `evalResults.csv` by `missing-expected-tool` or `unexpected-tool-call`, then open the linked eval UI and inspect `replay.json` to see the exact prompt, model, tool-call sequence, arguments, and result/error previews. diff --git a/apps/dbagent/src/app/api/evals/route.ts b/apps/dbagent/src/app/api/evals/route.ts index 17caf799..658af302 100644 --- a/apps/dbagent/src/app/api/evals/route.ts +++ b/apps/dbagent/src/app/api/evals/route.ts @@ -3,6 +3,7 @@ import { NextRequest } from 'next/server'; import path from 'path'; import { z } from 'zod'; import { evalResponseSchema } from '~/evals/api-schemas'; +import { EVAL_REPLAY_FILE_NAME, EVAL_RESULT_FILE_NAME } from '~/evals/lib/consts'; import { env } from '~/lib/env/server'; export async function GET(request: NextRequest) { @@ -43,21 +44,12 @@ export async function GET(request: NextRequest) { }) ); - filesWithContents.sort((a, b) => { - if (a.fileName === 'human.txt') { - return -1; - } - if (b.fileName === 'human.txt') { - return 1; - } - if (a.fileName === 'evalResult.json') { - return 1; - } - if (b.fileName === 'evalResult.json') { - return -1; - } - return 0; - }); + const fileOrder = ['human.txt', EVAL_REPLAY_FILE_NAME, 'response.json', EVAL_RESULT_FILE_NAME]; + const getFileOrder = (fileName: string) => { + const index = fileOrder.indexOf(fileName); + return index === -1 ? fileOrder.length : index; + }; + filesWithContents.sort((a, b) => getFileOrder(a.fileName) - getFileOrder(b.fileName)); const response = evalResponseSchema.parse({ files: filesWithContents }); diff --git a/apps/dbagent/src/evals/chat/tool-choice.test.ts b/apps/dbagent/src/evals/chat/tool-choice.test.ts index 995388c4..03dd2b7d 100644 --- a/apps/dbagent/src/evals/chat/tool-choice.test.ts +++ b/apps/dbagent/src/evals/chat/tool-choice.test.ts @@ -168,7 +168,14 @@ describe.concurrent('tool_choice', () => { const result = await evalChat({ messages: [{ role: 'user', content: prompt }], dbConnection: dbConfig.connectionString, - expect + expect, + traceMetadata: { + scenario: 'tool-choice', + toolPolicy: { + expectedToolCalls: toolCalls, + allowOtherTools + } + } }); const allToolCalls = result.steps.flatMap((step) => step.toolCalls); diff --git a/apps/dbagent/src/evals/eval-reporter.ts b/apps/dbagent/src/evals/eval-reporter.ts index e3b39143..46d235b4 100644 --- a/apps/dbagent/src/evals/eval-reporter.ts +++ b/apps/dbagent/src/evals/eval-reporter.ts @@ -5,8 +5,13 @@ import { TestCase } from 'vitest/node'; import { Reporter } from 'vitest/reporters'; import { delay } from '~/utils/delay'; import { env } from '../lib/env/eval'; -import { EVAL_RESULT_FILE_NAME, EVAL_RESULTS_CSV_FILE_NAME, EVAL_RESULTS_FILE_NAME } from './lib/consts'; -import { EvalResult, evalResultSchema, evalSummarySchema } from './lib/schemas'; +import { + EVAL_REPLAY_FILE_NAME, + EVAL_RESULT_FILE_NAME, + EVAL_RESULTS_CSV_FILE_NAME, + EVAL_RESULTS_FILE_NAME +} from './lib/consts'; +import { evalReplayManifestSchema, EvalResult, evalResultSchema, evalSummarySchema } from './lib/schemas'; import { ensureTestRunTraceFolderExists, ensureTraceFolderExists, testNameToEvalId } from './lib/test-id'; const getEnv = () => { @@ -37,9 +42,18 @@ export default class EvalReporter implements Reporter { fs.writeFileSync(path.join(evalTraceFolder, EVAL_RESULTS_FILE_NAME), JSON.stringify(testResults, null, 2)); const csvTestResults = testResults.map((testResult) => { + const replayFile = testResult.logFiles.find((logFile) => path.basename(logFile) === EVAL_REPLAY_FILE_NAME); + const replay = replayFile + ? evalReplayManifestSchema.parse(JSON.parse(fs.readFileSync(replayFile, 'utf-8'))) + : undefined; const result: any = { id: testResult.id, result: testResult.result, + classifications: replay?.diagnostics.classifications.join('|') ?? '', + expected_tools: replay?.diagnostics.expectedToolCalls.join('|') ?? '', + observed_tools: replay?.diagnostics.observedToolCalls.join('|') ?? '', + missing_expected_tools: replay?.diagnostics.missingExpectedToolCalls.join('|') ?? '', + unexpected_tools: replay?.diagnostics.unexpectedToolCalls.join('|') ?? '', ui: `http://localhost:4001/evals?folder=${evalTraceFolder}&evalId=${testResult.id}` }; testResult.logFiles.forEach((logFile, index) => { diff --git a/apps/dbagent/src/evals/lib/chat-runner.ts b/apps/dbagent/src/evals/lib/chat-runner.ts index 3fab4297..773ae642 100644 --- a/apps/dbagent/src/evals/lib/chat-runner.ts +++ b/apps/dbagent/src/evals/lib/chat-runner.ts @@ -6,16 +6,19 @@ import { getTools } from '~/lib/ai/tools'; import { Connection, Project } from '~/lib/db/schema'; import { env } from '~/lib/env/eval'; import { getTargetDbPool } from '~/lib/targetdb/db'; +import { EvalTraceMetadata } from './schemas'; import { traceVercelAiResponse } from './trace'; export const evalChat = async ({ messages, dbConnection, - expect + expect, + traceMetadata }: { messages: CoreMessage[] | Omit[]; dbConnection: string; expect: ExpectStatic; + traceMetadata?: EvalTraceMetadata; }) => { const project: Project = { id: 'projectId', @@ -41,7 +44,7 @@ export const evalChat = async ({ tools, messages }); - traceVercelAiResponse(response, expect); + traceVercelAiResponse(response, expect, traceMetadata); return response; } finally { await targetDb.end(); diff --git a/apps/dbagent/src/evals/lib/consts.ts b/apps/dbagent/src/evals/lib/consts.ts index 91c3a7ef..cdb04e4b 100644 --- a/apps/dbagent/src/evals/lib/consts.ts +++ b/apps/dbagent/src/evals/lib/consts.ts @@ -1,3 +1,4 @@ export const EVAL_RESULT_FILE_NAME = 'evalResult.json'; export const EVAL_RESULTS_FILE_NAME = 'evalResults.json'; export const EVAL_RESULTS_CSV_FILE_NAME = 'evalResults.csv'; +export const EVAL_REPLAY_FILE_NAME = 'replay.json'; diff --git a/apps/dbagent/src/evals/lib/schemas.ts b/apps/dbagent/src/evals/lib/schemas.ts index 08e59d4a..72ba431f 100644 --- a/apps/dbagent/src/evals/lib/schemas.ts +++ b/apps/dbagent/src/evals/lib/schemas.ts @@ -17,3 +17,85 @@ export const evalSummarySchema = z }) .strict(); export type EvalSummary = z.infer; + +export const evalToolPolicySchema = z + .object({ + expectedToolCalls: z.array(z.string()).default([]), + allowOtherTools: z.boolean().default(true) + }) + .strict(); +export type EvalToolPolicy = z.infer; + +export const evalTraceMetadataSchema = z + .object({ + scenario: z.string().optional(), + toolPolicy: evalToolPolicySchema.optional() + }) + .strict(); +export type EvalTraceMetadata = z.infer; + +export const evalToolCallSchema = z + .object({ + step: z.number().int().positive(), + toolCallId: z.string().optional(), + toolName: z.string(), + args: z.unknown().optional(), + hasResult: z.boolean(), + resultPreview: z.string().optional(), + error: z.string().optional() + }) + .strict(); +export type EvalToolCall = z.infer; + +export const evalFailureClassificationSchema = z.enum([ + 'malformed-request', + 'missing-system-prompt', + 'missing-user-prompt', + 'missing-expected-tool', + 'unexpected-tool-call', + 'tool-error', + 'no-tool-result', + 'empty-final-answer' +]); +export type EvalFailureClassification = z.infer; + +export const evalReplayManifestSchema = z + .object({ + schemaVersion: z.literal(1), + id: z.string(), + metadata: evalTraceMetadataSchema, + provider: z + .object({ + model: z.string().optional(), + requestBodyCaptured: z.boolean() + }) + .strict(), + prompts: z + .object({ + system: z.string().optional(), + user: z.string().optional() + }) + .strict(), + steps: z.array( + z + .object({ + index: z.number().int().positive(), + text: z.string(), + toolCalls: z.array(evalToolCallSchema) + }) + .strict() + ), + finalAnswer: z.string(), + diagnostics: z + .object({ + classifications: z.array(evalFailureClassificationSchema), + expectedToolCalls: z.array(z.string()), + observedToolCalls: z.array(z.string()), + missingExpectedToolCalls: z.array(z.string()), + unexpectedToolCalls: z.array(z.string()), + toolErrors: z.array(evalToolCallSchema) + }) + .strict() + }) + .strict(); +export type EvalReplayManifest = z.infer; diff --git a/apps/dbagent/src/evals/lib/trace.test.ts b/apps/dbagent/src/evals/lib/trace.test.ts new file mode 100644 index 00000000..931843e3 --- /dev/null +++ b/apps/dbagent/src/evals/lib/trace.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from 'vitest'; +import { buildReplayManifest } from './trace'; + +const response = { + request: { + body: JSON.stringify({ + model: 'test-provider/test-model', + system: [{ text: 'You are a PostgreSQL agent.' }], + messages: [{ content: [{ text: 'Describe the dogs table' }] }] + }) + }, + text: 'The dogs table has an id and name column.', + steps: [ + { + text: '', + toolCalls: [ + { + toolCallId: 'call-1', + toolName: 'describeTable', + args: { table: 'dogs' } + } + ], + toolResults: [ + { + toolCallId: 'call-1', + toolName: 'describeTable', + args: { table: 'dogs' }, + result: { columns: ['id', 'name'] } + } + ] + } + ] +} as any; + +describe('buildReplayManifest', () => { + it('captures prompts, model metadata and tool calls for replayable eval diagnostics', () => { + const manifest = buildReplayManifest({ + id: 'describe_table', + response, + metadata: { + scenario: 'tool-choice', + toolPolicy: { + expectedToolCalls: ['describeTable'], + allowOtherTools: false + } + } + }); + + expect(manifest.provider.model).toBe('test-provider/test-model'); + expect(manifest.prompts.user).toBe('Describe the dogs table'); + expect(manifest.steps[0]?.toolCalls[0]).toMatchObject({ + step: 1, + toolName: 'describeTable', + hasResult: true + }); + expect(manifest.diagnostics.classifications).toEqual([]); + }); + + it('classifies missing expected tools and disallowed extra tools', () => { + const manifest = buildReplayManifest({ + id: 'wrong_tool', + response, + metadata: { + toolPolicy: { + expectedToolCalls: ['getTablesAndInstanceInfo'], + allowOtherTools: false + } + } + }); + + expect(manifest.diagnostics.classifications).toEqual(['missing-expected-tool', 'unexpected-tool-call']); + expect(manifest.diagnostics.missingExpectedToolCalls).toEqual(['getTablesAndInstanceInfo']); + expect(manifest.diagnostics.unexpectedToolCalls).toEqual(['describeTable']); + }); + + it('classifies malformed provider request bodies and tool failures', () => { + const manifest = buildReplayManifest({ + id: 'tool_error', + response: { + ...response, + request: { body: '{not-json' }, + text: '', + steps: [ + { + text: '', + toolCalls: [{ toolCallId: 'call-1', toolName: 'safeExplainQuery', args: { query: 'select * from dogs' } }], + toolResults: [ + { + toolCallId: 'call-1', + toolName: 'safeExplainQuery', + result: { error: 'permission denied' } + } + ] + } + ] + } + }); + + expect(manifest.diagnostics.classifications).toEqual([ + 'malformed-request', + 'missing-system-prompt', + 'missing-user-prompt', + 'tool-error', + 'empty-final-answer' + ]); + expect(manifest.diagnostics.toolErrors[0]).toMatchObject({ + toolName: 'safeExplainQuery', + error: 'permission denied' + }); + }); + + it('classifies tool calls that never produce a result', () => { + const manifest = buildReplayManifest({ + id: 'missing_tool_result', + response: { + ...response, + steps: [ + { + text: '', + toolCalls: [{ toolCallId: 'call-1', toolName: 'getSlowQueries', args: {} }], + toolResults: [] + } + ] + } + }); + + expect(manifest.diagnostics.classifications).toEqual(['no-tool-result']); + }); +}); diff --git a/apps/dbagent/src/evals/lib/trace.ts b/apps/dbagent/src/evals/lib/trace.ts index e32fd3bd..d8f6e6ff 100644 --- a/apps/dbagent/src/evals/lib/trace.ts +++ b/apps/dbagent/src/evals/lib/trace.ts @@ -2,10 +2,28 @@ import { generateText } from 'ai'; import fs from 'fs'; import path from 'path'; import { ExpectStatic } from 'vitest'; +import { EVAL_REPLAY_FILE_NAME } from './consts'; +import { + EvalFailureClassification, + EvalReplayManifest, + EvalToolCall, + EvalTraceMetadata, + evalReplayManifestSchema +} from './schemas'; import { ensureTraceFolderExistsExpect } from './test-id'; type GenerateTextResponse = Awaited>; +const MAX_PREVIEW_LENGTH = 1000; + +const preview = (value: unknown) => { + const serialized = typeof value === 'string' ? value : JSON.stringify(value); + if (!serialized) { + return undefined; + } + return serialized.length > MAX_PREVIEW_LENGTH ? `${serialized.slice(0, MAX_PREVIEW_LENGTH)}...` : serialized; +}; + const toolResultToHuman = (toolResult: any) => { return ` ${toolResult.toolName} with args: ${JSON.stringify(toolResult.args)} @@ -39,7 +57,109 @@ const getUserPromptFromResponse = (response: GenerateTextResponse) => { return body.messages[0].content[0].text; }; -export const traceVercelAiResponse = (response: GenerateTextResponse, expect: ExpectStatic) => { +const parseRequestBodySafe = (response: Pick) => { + try { + if (!response.request.body) { + return { parsed: undefined, error: 'No request body found in response' }; + } + return { parsed: JSON.parse(response.request.body), error: undefined }; + } catch (error) { + return { parsed: undefined, error: error instanceof Error ? error.message : String(error) }; + } +}; + +const getPromptText = (body: any, field: 'system' | 'messages') => { + if (field === 'system') { + return body?.system?.[0]?.text; + } + return body?.messages?.[0]?.content?.[0]?.text; +}; + +const normalizeToolCalls = (response: Pick): EvalToolCall[] => { + return response.steps.flatMap((step, stepIndex) => { + const resultsById = new Map((step.toolResults ?? []).map((result: any) => [result.toolCallId, result])); + return (step.toolCalls ?? []).map((toolCall: any) => { + const toolResult = resultsById.get(toolCall.toolCallId); + const error = toolResult?.error ?? toolResult?.result?.error; + return { + step: stepIndex + 1, + toolCallId: toolCall.toolCallId, + toolName: toolCall.toolName, + args: toolCall.args, + hasResult: Boolean(toolResult), + resultPreview: preview(toolResult?.result), + error: error ? preview(error) : undefined + }; + }); + }); +}; + +export const buildReplayManifest = ({ + id, + response, + metadata = {} +}: { + id: string; + response: Pick; + metadata?: EvalTraceMetadata; +}): EvalReplayManifest => { + const { parsed: requestBody, error: requestParseError } = parseRequestBodySafe(response); + const toolCalls = normalizeToolCalls(response); + const observedToolCalls = toolCalls.map((toolCall) => toolCall.toolName); + const expectedToolCalls = metadata.toolPolicy?.expectedToolCalls ?? []; + const missingExpectedToolCalls = expectedToolCalls.filter((toolName) => !observedToolCalls.includes(toolName)); + const unexpectedToolCalls = + metadata.toolPolicy?.allowOtherTools === false + ? observedToolCalls.filter((toolName) => !expectedToolCalls.includes(toolName)) + : []; + const toolErrors = toolCalls.filter((toolCall) => toolCall.error); + + const classifications: EvalFailureClassification[] = []; + if (requestParseError) classifications.push('malformed-request'); + if (!getPromptText(requestBody, 'system')) classifications.push('missing-system-prompt'); + if (!getPromptText(requestBody, 'messages')) classifications.push('missing-user-prompt'); + if (missingExpectedToolCalls.length > 0) classifications.push('missing-expected-tool'); + if (unexpectedToolCalls.length > 0) classifications.push('unexpected-tool-call'); + if (toolErrors.length > 0) classifications.push('tool-error'); + if (toolCalls.some((toolCall) => !toolCall.hasResult)) classifications.push('no-tool-result'); + if (!response.text.trim()) classifications.push('empty-final-answer'); + + const manifest = { + schemaVersion: 1, + id, + metadata, + provider: { + model: requestBody?.model, + requestBodyCaptured: Boolean(response.request.body) + }, + prompts: { + system: getPromptText(requestBody, 'system'), + user: getPromptText(requestBody, 'messages') + }, + steps: response.steps.map((step, index) => ({ + index: index + 1, + text: step.text, + toolCalls: toolCalls.filter((toolCall) => toolCall.step === index + 1) + })), + finalAnswer: response.text, + diagnostics: { + classifications, + expectedToolCalls, + observedToolCalls, + missingExpectedToolCalls, + unexpectedToolCalls, + toolErrors + } + } satisfies EvalReplayManifest; + + return evalReplayManifestSchema.parse(manifest); +}; + +export const traceVercelAiResponse = ( + response: GenerateTextResponse, + expect: ExpectStatic, + metadata: EvalTraceMetadata = {} +) => { const traceFolder = ensureTraceFolderExistsExpect(expect); const humanTraceFile = path.join(traceFolder, 'human.txt'); const humanTrace = ` @@ -53,4 +173,12 @@ ${response.steps.map((step, index) => `Step: ${index + 1}\n\n${stepToHuman(step) const responseJson = path.join(traceFolder, 'response.json'); fs.writeFileSync(responseJson, JSON.stringify(response, null, 2)); + + const replayManifest = buildReplayManifest({ + id: expect.getState().currentTestName ?? 'unknown', + response, + metadata + }); + const replayJson = path.join(traceFolder, EVAL_REPLAY_FILE_NAME); + fs.writeFileSync(replayJson, JSON.stringify(replayManifest, null, 2)); };