From 3d9a8050d04c064f9991bc6cdb0cf299143ec6de Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Fri, 15 May 2026 13:32:16 -0700 Subject: [PATCH 1/7] robust test harness for evluating tmcp + claude code with the california schools data and official test cases --- .gitignore | 3 + env.example.list | 20 +- evals/README.md | 241 +++++++ evals/bird_mini | 1 + evals/cases/get-datasource-metadata.json | 12 + evals/cases/list-datasources.json | 12 + evals/cases/query-datasource.json | 12 + evals/grade-bird.ts | 652 +++++++++++++++++++ evals/grade-suite.ts | 282 +++++++++ evals/grade.ts | 137 ++++ evals/hooks/post-tool-langsmith.mjs | 195 ++++++ evals/hooks/pre-tool-langsmith.mjs | 59 ++ evals/hooks/stop-langsmith.mjs | 105 +++ evals/run-case.ts | 737 ++++++++++++++++++++++ evals/run-suite.ts | 338 ++++++++++ evals/scripts/precompute-bird-answers.py | 196 ++++++ evals/suites/bird-california-schools.json | 728 +++++++++++++++++++++ package.json | 5 + 18 files changed, 3734 insertions(+), 1 deletion(-) create mode 100644 evals/README.md create mode 160000 evals/bird_mini create mode 100644 evals/cases/get-datasource-metadata.json create mode 100644 evals/cases/list-datasources.json create mode 100644 evals/cases/query-datasource.json create mode 100644 evals/grade-bird.ts create mode 100644 evals/grade-suite.ts create mode 100644 evals/grade.ts create mode 100644 evals/hooks/post-tool-langsmith.mjs create mode 100644 evals/hooks/pre-tool-langsmith.mjs create mode 100644 evals/hooks/stop-langsmith.mjs create mode 100644 evals/run-case.ts create mode 100644 evals/run-suite.ts create mode 100644 evals/scripts/precompute-bird-answers.py create mode 100644 evals/suites/bird-california-schools.json diff --git a/.gitignore b/.gitignore index 4f616a65a..5f72d51e5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ node_modules/ junit/ coverage/ test-logs/ +evals/suite-runs/ +evals/runs/ +evals/grades/ config.json env.list manifest.json diff --git a/env.example.list b/env.example.list index 2bdb6c2ac..3ee28728a 100644 --- a/env.example.list +++ b/env.example.list @@ -1,3 +1,4 @@ +# ── MCP Server ──────────────────────────────────────────────────────────────── TRANSPORT=stdio SERVER=https://my-tableau-server.com SITE_NAME= @@ -12,4 +13,21 @@ DISABLE_QUERY_DATASOURCE_VALIDATION_REQUESTS= DISABLE_METADATA_API_REQUESTS= OAUTH_EMBEDDED_AUTHZ_SERVER= ADVERTISE_API_SCOPES= -OAUTH_DISABLE_SCOPES= \ No newline at end of file +OAUTH_DISABLE_SCOPES= + +# ── Eval Suite ──────────────────────────────────────────────────────────────── +# LUID of the published datasource to run the eval against. +# Find it in Tableau: open the datasource details page and copy the ID from the URL. +EVAL_DATASOURCE_LUID= + +# LangSmith — traces from each eval run are posted here. +# Get your API key at https://smith.langchain.com → Settings → API Keys. +LANGSMITH_API_KEY= +# Project name to post traces to (default: tableau-mcp-evals). +LANGSMITH_PROJECT=tableau-mcp-evals + +# OpenAI — used by the LLM judge for semantic correctness grading. +# Without this, semantic_match is skipped and the verdict relies on numeric_match only. +OPENAI_API_KEY= +# Model used by the LLM judge (default: gpt-4o-mini). +BIRD_GRADE_MODEL=gpt-4o-mini \ No newline at end of file diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 000000000..40e9b4578 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,241 @@ +# Tableau MCP Eval Harness + +This folder contains a local eval harness that runs Claude Code against the Tableau MCP server, +routes traces to LangSmith, and grades agent responses against known-correct answers. + +## How It Works + +``` +suite file (bird-california-schools.json) + → run-suite.ts + → run-case.ts (one per question) + → claude -p --mcp-config ... --settings ... + → tableau-mcp stdio server + → Claude Code hooks (PreToolUse / PostToolUse / Stop) + → LangSmith (traces posted in real time) + → pre-tool-times.jsonl, hook.jsonl (local timing records) + → agent-output.jsonl (full Claude stream) + → suite-summary.json (timing, tokens, tool counts per case) + → grade-bird.ts (one per run, grades the 4 signals) + → grades/YYYY-MM-DD//bird-result.json (per-case verdict) +``` + +Claude Code is the eval runtime — not a simulated agent. Every tool call goes to the real Tableau Cloud or Server APIs. LangSmith receives the full trace while the run is in progress. + +--- + +## Setup + +### 1. Build the MCP server + +```bash +npm run build +``` + +### 2. Set credentials + +```bash +# LangSmith +export LANGSMITH_API_KEY="ls__..." +export LANGSMITH_PROJECT="your_project_name" + +# Tableau (PAT auth) +export AUTH="pat" +export SERVER="https://your-site.online.tableau.com" +export SITE_NAME="your-site" +export PAT_NAME="your-pat-name" +export PAT_VALUE="your-pat-secret" + +# BIRD California Schools suite +export EVAL_DATASOURCE_LUID="" + +# LLM judge (for semantic grading) +export OPENAI_API_KEY="sk-..." +``` + +Or put these in a `.env` file at the repo root. + +--- + +## BIRD California Schools Suite + +The primary eval suite is 30 questions from the +[BIRD Mini-Dev benchmark](https://bird-bench.github.io/) scoped to the California Schools database. +A single published Tableau datasource joins the source tables (schools, frpm, satscores). + +The suite file lives at `evals/suites/bird-california-schools.json` and is committed to the repo. +It contains precomputed expected answers so no database access is required to run evals. + +### Run the full suite + +```bash +npm run eval:suite +``` + +### Run a subset + +```bash +npm run eval:suite -- --difficulty simple # 8 simple cases +npm run eval:suite -- --ids 5,11,12 # specific question IDs +``` + +### Grade a full suite run + +After `eval:suite` completes, grade every case at once and produce a single `suite-grade.json`: + +```bash +npm run eval:grade:suite # auto-discovers most recent suite run +npm run eval:grade:suite -- evals/suite-runs/YYYY-MM-DD/ # explicit +``` + +Output is written to `evals/grades/YYYY-MM-DD//suite-grade.json`. + +### Grade a single run + +```bash +npm run eval:grade:bird -- evals/runs/YYYY-MM-DD/ +``` + +Output is written to `evals/grades/YYYY-MM-DD//bird-result.json`. + +Override the LLM judge model (default: `gpt-4o-mini`) for either grader: + +```bash +BIRD_GRADE_MODEL=gpt-4o npm run eval:grade:suite +``` + +--- + +## Grading Signals + +Each run is graded on four signals. The verdict is driven by the outcome signals only; the structural signals are recorded for debugging. + +### Outcome signals (drive the verdict) + +| Signal | Method | What it checks | +|---|---|---| +| `numeric_match` | Code | Expected value or row count appears in Claude's final message (±1% tolerance for floats, substring match for strings) | +| `semantic_match` | LLM judge (GPT-4o-mini) | Claude's final message correctly answers the question vs. the gold summary | + +### Structural signals (informational) + +| Signal | Method | What it checks | +|---|---|---| +| `columns_match` | Tool call inspection | Expected VizQL field captions present in `query-datasource` call | +| `filters_match` | Tool call inspection | Expected filter field captions present in `query-datasource` call | + +The structural signals use subset matching — extra columns or filters are fine. They help diagnose +*why* an agent failed but don't block a correct answer from passing. + +### Verdicts + +| Verdict | Meaning | +|---|---| +| `pass` | Both `numeric_match` and `semantic_match` passed | +| `partial` | One of the two outcome signals passed (or one was unavailable) | +| `fail` | Both outcome signals failed | +| `error` | Claude Code exited with a non-zero code | +| `skip` | Both outcome signals were unavailable (e.g. no OpenAI key) | + +--- + +## Expected Answers + +The suite file ships with precomputed answers from the executed sql against the source of truth california schools table. If you wanted to regenerate it (there's probably no reason to) run the following file (requires the BIRD SQLite snapshot): + +```bash +python3 evals/scripts/precompute-bird-answers.py +``` + +This reads the gold SQL from `bird_mini/data/test_cases/mini_dev_sqlite.json`, executes it against +`bird_mini/data/dev_databases/california_schools/california_schools.sqlite`, and writes +`evals/suites/bird-california-schools.json` with: + +- `expected_value` — scalar result (for COUNT, MAX, MIN queries) +- `expected_row_count` — number of rows returned (for list queries) +- `expected_columns` — VizQL field captions that should appear in the query +- `expected_filter_fields` — filter field captions from the reference VDS query +- `ai_summarized_answer` — natural language gold answer (for the LLM judge) + +--- + +## Local Artifacts + +All output directories are organized by date (`YYYY-MM-DD`) so runs and grades from different days stay separate. + +``` +evals/ + runs/ + YYYY-MM-DD/ + / + run.json metadata, timing, exit code, question_id + agent-output.jsonl raw Claude Code stream JSON (all events) + hook.jsonl one record per tool call (name, input, timing) + pre-tool-times.jsonl PreToolUse timestamps keyed by tool_use_id + stop.json Stop hook data (transcript path, session info) + claude-settings.json per-run hook config (ephemeral) + mcp-config.json per-run MCP server config (ephemeral) + logs/ MCP server file logs + + suite-runs/ + YYYY-MM-DD/ + / + suite-summary.json aggregate: timing, tokens, tool counts per case + cases/ ephemeral per-case JSON files written before each run + + grades/ + YYYY-MM-DD/ + / + bird-result.json per-case grading output (4 signals + verdict) + / + suite-grade.json aggregate grading output for a full suite run +``` + +`evals/runs/`, `evals/suite-runs/`, and `evals/grades/` are all gitignored. + +--- + +## Ad Hoc Runs + +Run a single natural-language prompt without a case file: + +```bash +npm run eval:claude -- input "how many schools have SAT scores above 1200?" +``` + +Ad hoc runs are tagged `adhoc`, use the existing `grade.ts` grader (not `grade-bird.ts`), and +produce the same local artifacts. + +--- + +## LangSmith Trace Structure + +Each eval case produces one parent `chain` run in LangSmith containing all child spans for that +case. The parent's `start_time` and `end_time` are the wall clock times of the `claude -p` process. + +### Child spans + +| Span name | Type | Source | Timing | +|---|---|---|---| +| `initialization` | `chain` | `run-case.ts` (post-process) | `startedAt` → first stream event | +| `assistant-message-N` | `llm` | `run-case.ts` (post-process) | inferred from stream timestamps | +| `` | `tool` | `PostToolUse` hook (real time) | `PreToolUse ts` → `PostToolUse ts` | +| `claude-result-` | `chain` | `run-case.ts` (post-process) | pinned to `finishedAt` | + +### How span timing is calculated + +**Tool call spans** (``) are the most accurate. The `PreToolUse` hook fires the instant Claude Code dispatches the tool call and writes `{tool_use_id, ts}` to `pre-tool-times.jsonl`. The `PostToolUse` hook fires when the MCP server returns and uses the matching `ts` as `start_time`. This gives true MCP round-trip latency. + +**Assistant message spans** are inferred after Claude exits by parsing `agent-output.jsonl`. The inference works in three layers: + +1. **Real timestamps** — if the stream event carries a `timestamp` field, it is used directly. +2. **Hook anchors** — for any assistant event whose content contains a `tool_use` block, the `PreToolUse` timestamp for that `tool_use_id` is injected as the event's real end time, and the `PostToolUse` timestamp is injected as the start time of the next event. This anchors the timeline at every tool call boundary. +3. **Linear interpolation** — events between anchors are distributed proportionally by index between the surrounding anchor times. + +The `end_time` of each assistant message is the `start_time` of the next traceable event, so spans tile continuously rather than all ending at the same moment. + +**Initialization span** is a synthetic `chain` span posted after Claude exits. It covers the window from process launch to the first stream event — Claude Code startup, MCP server subprocess launch, and tool capability negotiation. This ensures the full wall clock time is accounted for in the langsmith trace waterfall. + +### Known gap + +The `claude-result-success` span is always posted with `start_time = end_time = finishedAt` (0.00s) because it represents the Claude Code process exit signal, not a timed operation. diff --git a/evals/bird_mini b/evals/bird_mini new file mode 160000 index 000000000..387f095bf --- /dev/null +++ b/evals/bird_mini @@ -0,0 +1 @@ +Subproject commit 387f095bf8e6fbf1692c6435fff57dcfd70c2dc0 diff --git a/evals/cases/get-datasource-metadata.json b/evals/cases/get-datasource-metadata.json new file mode 100644 index 000000000..81a757dbf --- /dev/null +++ b/evals/cases/get-datasource-metadata.json @@ -0,0 +1,12 @@ +{ + "id": "get-datasource-metadata", + "name": "Get Datasource Metadata", + "description": "Exercises metadata retrieval for a known datasource LUID provided by EVAL_DATASOURCE_LUID.", + "tags": ["datasource", "metadata", "real-server"], + "prompt": "Use the Tableau MCP server to get metadata for datasource LUID {{env.EVAL_DATASOURCE_LUID}}. Summarize the datasource name, fields, and any warnings returned by the tool. Do not query data rows.", + "expected_tools": ["get-datasource-metadata"], + "budget": { + "max_tool_calls": 5, + "max_wall_ms": 120000 + } +} diff --git a/evals/cases/list-datasources.json b/evals/cases/list-datasources.json new file mode 100644 index 000000000..26aadc48b --- /dev/null +++ b/evals/cases/list-datasources.json @@ -0,0 +1,12 @@ +{ + "id": "list-datasources", + "name": "List Datasources", + "description": "Smoke test for a real Tableau Cloud/Server connection. Claude Code should use the Tableau MCP server to list datasources on the configured site.", + "tags": ["smoke", "datasource", "real-server"], + "prompt": "Use the Tableau MCP server to list data sources available on the configured Tableau site. Return a concise summary with datasource names and IDs when available. Do not invent data if the server returns no results.", + "expected_tools": ["list-datasources"], + "budget": { + "max_tool_calls": 5, + "max_wall_ms": 120000 + } +} diff --git a/evals/cases/query-datasource.json b/evals/cases/query-datasource.json new file mode 100644 index 000000000..b06b51fd5 --- /dev/null +++ b/evals/cases/query-datasource.json @@ -0,0 +1,12 @@ +{ + "id": "query-datasource", + "name": "Query Datasource", + "description": "Runs a small, bounded query against a known datasource LUID provided by EVAL_DATASOURCE_LUID.", + "tags": ["datasource", "query", "real-server"], + "prompt": "Use the Tableau MCP server to query datasource LUID {{env.EVAL_DATASOURCE_LUID}}. First inspect or use known metadata if needed, then run this query exactly: {{env.EVAL_DATASOURCE_QUERY}}. Keep the row limit small and summarize the returned rows.", + "expected_tools": ["query-datasource"], + "budget": { + "max_tool_calls": 8, + "max_wall_ms": 180000 + } +} diff --git a/evals/grade-bird.ts b/evals/grade-bird.ts new file mode 100644 index 000000000..0f4713777 --- /dev/null +++ b/evals/grade-bird.ts @@ -0,0 +1,652 @@ +/** + * BIRD-specific grader for California Schools eval runs. + * + * Grades a single run directory using four signals: + * columns_match — required VizQL fields present in the query-datasource call + * filters_match — required filter fields present in the query-datasource call + * numeric_match — expected value / row count found in Claude's final message + * semantic_match — LLM judge comparing Claude's final message to the gold summary + * + * Usage: + * npx tsx evals/grade-bird.ts evals/runs/ + * + * Required environment: + * OPENAI_API_KEY (for LLM judge; set BIRD_GRADE_MODEL to override model) + */ + +/* eslint-disable no-console */ + +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import OpenAI from 'openai'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const GRADES_DIR = path.join(EVALS_DIR, 'grades'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} + +const runDirArg = process.argv[2]; +if (!runDirArg) { + console.error('Usage: npx tsx evals/grade-bird.ts '); + process.exit(1); +} + +const absRunDir = path.resolve(runDirArg); + +// ─── Types ─────────────────────────────────────────────────────────────────── + +type RunMeta = { + run_id: string; + case_id: string; + wall_ms?: number; + claude_exit_code?: number; + timed_out?: boolean; + metadata?: { + question_id?: number; + difficulty?: string; + suite_file?: string; + }; +}; + +type HookRecord = { + tool_name?: string; + normalized_tool_name?: string; + tool_input?: unknown; +}; + +type VizqlField = { + fieldCaption?: string; + fieldAlias?: string; + function?: string; + calculation?: string; + sortDirection?: string; + sortPriority?: number; +}; + +type VizqlFilter = { + field?: { + fieldCaption?: string; + calculation?: string; + }; + filterType?: string; +}; + +type VizqlQuery = { + fields?: Array; + filters?: Array; +}; + +type QueryDatasourceInput = { + datasource_luid?: string; + query?: VizqlQuery; + options?: unknown; +}; + +type BirdCase = { + question_id: number; + question: string; + difficulty: string; + answer_type: 'scalar' | 'list'; + expected_value: number | string | null; + expected_row_count: number | null; + ai_summarized_answer: string; + expected_columns: Array; + expected_filter_fields: Array; +}; + +type TokenUsage = { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; +}; + +type TokenTotals = { + input_tokens: number | null; + cache_creation_tokens: number | null; + cache_read_tokens: number | null; + output_tokens: number | null; + total_context_tokens: number | null; +}; + +type ClaudeStreamEvent = { + type?: string; + result?: string; + usage?: TokenUsage; + modelUsage?: TokenUsage; + message?: { + model?: string; + usage?: TokenUsage; + content?: Array<{ type?: string; text?: string }> | string; + }; +}; + +type LlmJudgeResult = { + correct: boolean; + score: number; + reason: string; +}; + +type BirdGradeResult = { + run_id: string; + question_id: number; + difficulty: string; + graded_at: string; + model: string | null; + wall_s: number | null; + tokens: TokenTotals; + tool_calls: number; + tools_used: Array; + signals: { + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + }; + details: { + expected_columns: Array; + actual_columns: Array; + missing_columns: Array; + expected_filter_fields: Array; + actual_filter_fields: Array; + missing_filter_fields: Array; + expected_value: number | string | null; + expected_row_count: number | null; + extracted_number: number | null; + final_message_preview: string; + llm_judge: LlmJudgeResult | null; + llm_judge_error: string | null; + }; + verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip'; +}; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +function readJsonl(filePath: string): Array { + if (!fs.existsSync(filePath)) return []; + return fs + .readFileSync(filePath, 'utf-8') + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + try { + return JSON.parse(line) as T; + } catch { + return null; + } + }) + .filter((v): v is T => v !== null); +} + +function readOptionalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T; + } catch { + return null; + } +} + +function normalizeToolName(toolName: string): string { + const parts = toolName.split('__'); + const raw = parts[parts.length - 1] || toolName; + return raw.replace(/^tableau_/, '').replace(/^tableau-/, ''); +} + +function parseStreamEvents(agentOutputJsonl: string): Array { + return agentOutputJsonl + .split('\n') + .filter((l) => l.trim()) + .map((l) => { + try { + return JSON.parse(l) as ClaudeStreamEvent; + } catch { + return null; + } + }) + .filter((e): e is ClaudeStreamEvent => e !== null); +} + +function sumUsage(events: Array): TokenTotals { + let input = 0; + let cacheCreate = 0; + let cacheRead = 0; + let output = 0; + let found = false; + + for (const event of events) { + const u = event.message?.usage ?? event.usage ?? event.modelUsage; + if (!u) continue; + input += u.input_tokens ?? 0; + cacheCreate += u.cache_creation_input_tokens ?? 0; + cacheRead += u.cache_read_input_tokens ?? 0; + output += u.output_tokens ?? 0; + found = true; + } + + if (!found) { + return { + input_tokens: null, + cache_creation_tokens: null, + cache_read_tokens: null, + output_tokens: null, + total_context_tokens: null, + }; + } + + return { + input_tokens: input, + cache_creation_tokens: cacheCreate, + cache_read_tokens: cacheRead, + output_tokens: output, + total_context_tokens: input + cacheCreate + cacheRead, + }; +} + +/** + * Extract token totals from the stream. + * Prefers the result event's aggregated usage when available (session-level total + * from Claude Code). Falls back to summing across all assistant events. + */ +function extractTokenTotals(events: Array): TokenTotals { + // If the result event carries an aggregate, use it — but it may lack cache breakdown. + // Sum across all assistant events instead, which have per-turn cache detail. + const assistantEvents = events.filter((e) => e.type === 'assistant'); + if (assistantEvents.length > 0) return sumUsage(assistantEvents); + + // Last resort: result event usage. + const resultEvents = events.filter((e) => e.type === 'result'); + if (resultEvents.length > 0) return sumUsage(resultEvents); + + return sumUsage([]); +} + +/** Extract the model name from the first assistant message in the stream. */ +function extractModelName(events: Array): string | null { + for (const event of events) { + if (event.type === 'assistant' && event.message?.model) { + return event.message.model; + } + } + return null; +} + +/** Extract the final assistant text from Claude's stream-json output. */ +function extractFinalMessage(agentOutputJsonl: string): string { + const events = parseStreamEvents(agentOutputJsonl); + + // Prefer the result event's text field + const resultEvents = events.filter((e) => e.type === 'result' && e.result); + if (resultEvents.length > 0) { + return resultEvents[resultEvents.length - 1].result ?? ''; + } + + // Fall back to the last assistant message's text content + const assistantEvents = events.filter((e) => e.type === 'assistant'); + for (let i = assistantEvents.length - 1; i >= 0; i--) { + const content = assistantEvents[i].message?.content; + if (Array.isArray(content)) { + const textBlocks = content.filter((b) => b.type === 'text' && b.text).map((b) => b.text ?? ''); + if (textBlocks.length > 0) return textBlocks.join('\n'); + } + if (typeof content === 'string' && content.trim()) return content; + } + + return ''; +} + +/** Extract all query-datasource tool inputs from hook.jsonl. */ +function extractQueryDatasourceInputs(hookRecords: Array): Array { + return hookRecords + .filter((r) => normalizeToolName(r.normalized_tool_name ?? r.tool_name ?? '') === 'query-datasource') + .map((r) => r.tool_input as QueryDatasourceInput) + .filter(Boolean); +} + +/** Collect field captions from a VizQL fields array. */ +function collectFieldCaptions(fields: Array | undefined): Array { + return (fields ?? []) + .map((f) => f.fieldCaption ?? '') + .filter(Boolean) + .map((c) => c.toLowerCase()); +} + +/** Collect filter field captions from a VizQL filters array. */ +function collectFilterCaptions(filters: Array | undefined): Array { + return (filters ?? []) + .map((f) => f.field?.fieldCaption ?? '') + .filter(Boolean) + .map((c) => c.toLowerCase()); +} + +/** + * Try to extract a number from the final message that matches the expected value. + * Returns the extracted number, or null if no suitable number was found. + */ +function extractMatchingNumber( + text: string, + expected: number | string | null, +): number | null { + if (expected === null) return null; + + // Normalize expected to a number if possible + let expectedNum: number | null = null; + if (typeof expected === 'number') { + expectedNum = expected; + } else { + const parsed = parseFloat(String(expected).replace(/,/g, '')); + if (!isNaN(parsed)) expectedNum = parsed; + } + if (expectedNum === null) return null; + + // Extract all numbers from the message (strip commas first) + const clean = text.replace(/,/g, ''); + const matches = clean.match(/-?\d+(?:\.\d+)?/g) ?? []; + const candidates = matches.map((m) => parseFloat(m)).filter((n) => !isNaN(n)); + + // Look for a match with tolerance + const isClose = (a: number, b: number): boolean => { + if (b === 0) return Math.abs(a) < 0.001; + return Math.abs(a - b) / Math.abs(b) <= 0.01 || Math.abs(a - b) <= 0.001; + }; + + // Also check percentage form (e.g. "90.5%" for 0.905) + const percentageCandidates = candidates.map((n) => n / 100); + + for (const candidate of [...candidates, ...percentageCandidates]) { + if (isClose(candidate, expectedNum)) return candidate; + } + return null; +} + +/** + * Check if a numeric answer (or row count) appears in the final message. + */ +function checkNumericMatch( + finalMessage: string, + birdCase: BirdCase, +): { matched: boolean; extracted: number | null } { + const target = + birdCase.answer_type === 'scalar' ? birdCase.expected_value : birdCase.expected_row_count; + + if (target === null) return { matched: false, extracted: null }; + + // For string scalars (school names, phone numbers), do a case-insensitive substring check + if (typeof target === 'string') { + const matched = finalMessage.toLowerCase().includes(target.toLowerCase()); + return { matched, extracted: null }; + } + + const extracted = extractMatchingNumber(finalMessage, target); + return { matched: extracted !== null, extracted }; +} + +// ─── LLM Judge ─────────────────────────────────────────────────────────────── + +async function runLlmJudge( + birdCase: BirdCase, + finalMessage: string, + openai: OpenAI, + model: string, +): Promise { + const targetValue = + birdCase.answer_type === 'scalar' + ? `Expected value: ${String(birdCase.expected_value)}` + : `Expected row count: ${String(birdCase.expected_row_count)}`; + + const prompt = [ + 'You are evaluating whether an AI agent correctly answered a data question about California schools.', + '', + `Question: ${birdCase.question}`, + '', + `Gold answer summary: ${birdCase.ai_summarized_answer}`, + `${targetValue}`, + '', + "Agent's response:", + finalMessage.slice(0, 3000), + '', + "Does the agent's response correctly answer the question?", + 'Consider:', + '- Numeric values must match (allow rounding/formatting differences, e.g. "90.5%" and "0.905" are the same)', + '- The agent must address the right entities (not a proxy or wrong dimension)', + '- Partial credit if the agent got the right approach but a slightly wrong number', + '', + 'Respond with JSON only:', + '{"correct": true|false, "score": 0.0-1.0, "reason": "one sentence"}', + ].join('\n'); + + const response = await openai.chat.completions.create({ + model, + messages: [{ role: 'user', content: prompt }], + response_format: { type: 'json_object' }, + temperature: 0, + }); + + const raw = response.choices[0]?.message?.content ?? '{}'; + const parsed = JSON.parse(raw) as Partial; + return { + correct: parsed.correct ?? false, + score: typeof parsed.score === 'number' ? parsed.score : parsed.correct ? 1.0 : 0.0, + reason: parsed.reason ?? '', + }; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main(): Promise { + const runMeta = readOptionalJson(path.join(absRunDir, 'run.json')); + if (!runMeta) { + console.error(`run.json not found in ${absRunDir}`); + process.exit(1); + } + + const questionId = runMeta.metadata?.question_id; + const suiteFilePath = runMeta.metadata?.suite_file; + + if (!questionId || !suiteFilePath) { + console.error( + 'run.json is missing metadata.question_id or metadata.suite_file.\n' + + 'This grader only works on runs produced by run-suite.ts.', + ); + process.exit(1); + } + + if (!fs.existsSync(suiteFilePath)) { + console.error(`Suite file not found: ${suiteFilePath}`); + process.exit(1); + } + + const suite = JSON.parse(fs.readFileSync(suiteFilePath, 'utf-8')) as Array; + const birdCase = suite.find((c) => c.question_id === questionId); + if (!birdCase) { + console.error(`Question ID ${questionId} not found in suite file.`); + process.exit(1); + } + + const hookRecords = readJsonl(path.join(absRunDir, 'hook.jsonl')); + const agentOutputRaw = fs.existsSync(path.join(absRunDir, 'agent-output.jsonl')) + ? fs.readFileSync(path.join(absRunDir, 'agent-output.jsonl'), 'utf-8') + : ''; + + const streamEvents = parseStreamEvents(agentOutputRaw); + const modelName = extractModelName(streamEvents); + const tokens = extractTokenTotals(streamEvents); + const finalMessage = extractFinalMessage(agentOutputRaw); + const queryInputs = extractQueryDatasourceInputs(hookRecords); + + // ── Signal 1 & 2: columns_match, filters_match ────────────────────────── + + let columnsMatch: boolean | null = null; + let filtersMatch: boolean | null = null; + let actualColumns: Array = []; + let actualFilterFields: Array = []; + let missingColumns: Array = []; + let missingFilterFields: Array = []; + + if (queryInputs.length > 0) { + // Union of all columns/filters across all query-datasource calls (agent may call it multiple times) + const allActualColumns = new Set(); + const allActualFilters = new Set(); + for (const input of queryInputs) { + for (const c of collectFieldCaptions(input.query?.fields)) allActualColumns.add(c); + for (const f of collectFilterCaptions(input.query?.filters)) allActualFilters.add(f); + } + actualColumns = [...allActualColumns]; + actualFilterFields = [...allActualFilters]; + + const expectedColsLower = birdCase.expected_columns.map((c) => c.toLowerCase()); + const expectedFiltersLower = birdCase.expected_filter_fields.map((f) => f.toLowerCase()); + + missingColumns = birdCase.expected_columns.filter( + (c) => !allActualColumns.has(c.toLowerCase()), + ); + missingFilterFields = birdCase.expected_filter_fields.filter( + (f) => !allActualFilters.has(f.toLowerCase()), + ); + + columnsMatch = missingColumns.length === 0 && expectedColsLower.length > 0; + filtersMatch = missingFilterFields.length === 0 && expectedFiltersLower.length > 0; + } + + // ── Signal 3: numeric_match ───────────────────────────────────────────── + + let numericMatch: boolean | null = null; + let extractedNumber: number | null = null; + if (finalMessage) { + const result = checkNumericMatch(finalMessage, birdCase); + numericMatch = result.matched; + extractedNumber = result.extracted; + } + + // ── Signal 4: semantic_match (LLM judge) ──────────────────────────────── + + let semanticMatch: number | null = null; + let llmJudge: LlmJudgeResult | null = null; + let llmJudgeError: string | null = null; + + if (!finalMessage) { + llmJudgeError = 'No final message found in agent output.'; + } else { + const openaiKey = process.env.OPENAI_API_KEY; + if (!openaiKey) { + llmJudgeError = 'OPENAI_API_KEY not set; skipping LLM judge.'; + console.warn('WARN: OPENAI_API_KEY not set — semantic_match will be null.'); + } else { + const openai = new OpenAI({ apiKey: openaiKey }); + const judgeModel = process.env.BIRD_GRADE_MODEL ?? 'gpt-4o-mini'; + try { + llmJudge = await runLlmJudge(birdCase, finalMessage, openai, judgeModel); + semanticMatch = llmJudge.score; + } catch (error: unknown) { + llmJudgeError = error instanceof Error ? error.message : String(error); + console.error(`LLM judge error: ${llmJudgeError}`); + } + } + } + + // ── Verdict ────────────────────────────────────────────────────────────── + + function deriveVerdict(): BirdGradeResult['verdict'] { + if (runMeta?.claude_exit_code != null && runMeta.claude_exit_code !== 0) return 'error'; + // columns_match / filters_match are diagnostic only — they do not drive the verdict. + // Verdict is determined solely by whether the agent produced the correct answer. + if (semanticMatch === null && numericMatch === null) return 'skip'; + const semanticOk = semanticMatch != null && semanticMatch >= 0.8; + const numericOk = numericMatch === true; + if (semanticOk && numericOk) return 'pass'; + if (semanticOk || numericOk) return 'partial'; + return 'fail'; + } + + // ── Tool summary ───────────────────────────────────────────────────────── + + const toolsUsed = [ + ...new Set( + hookRecords + .map((r) => normalizeToolName(r.normalized_tool_name ?? r.tool_name ?? '')) + .filter(Boolean), + ), + ]; + + // ── Assemble result ────────────────────────────────────────────────────── + + const birdResult: BirdGradeResult = { + run_id: runMeta.run_id, + question_id: questionId, + difficulty: runMeta.metadata?.difficulty ?? birdCase.difficulty ?? 'unknown', + graded_at: new Date().toISOString(), + model: modelName, + wall_s: runMeta.wall_ms != null ? Math.round(runMeta.wall_ms / 1000) : null, + tokens, + tool_calls: hookRecords.length, + tools_used: toolsUsed, + signals: { + numeric_match: numericMatch, + semantic_match: semanticMatch, + columns_match: columnsMatch, + filters_match: filtersMatch, + }, + details: { + expected_columns: birdCase.expected_columns, + actual_columns: actualColumns, + missing_columns: missingColumns, + expected_filter_fields: birdCase.expected_filter_fields, + actual_filter_fields: actualFilterFields, + missing_filter_fields: missingFilterFields, + expected_value: + birdCase.answer_type === 'scalar' ? birdCase.expected_value : null, + expected_row_count: birdCase.expected_row_count, + extracted_number: extractedNumber, + final_message_preview: finalMessage.slice(0, 500), + llm_judge: llmJudge, + llm_judge_error: llmJudgeError, + }, + verdict: deriveVerdict(), + }; + + const runId = path.basename(absRunDir); + const gradeDir = path.join(GRADES_DIR, dateSlug(), runId); + fs.mkdirSync(gradeDir, { recursive: true }); + const resultPath = path.join(gradeDir, 'bird-result.json'); + fs.writeFileSync(resultPath, JSON.stringify(birdResult, null, 2)); + + // ── Print summary ───────────────────────────────────────────────────────── + + const v = birdResult.verdict.toUpperCase(); + const s = birdResult.signals; + console.log(`\nGrade: Q${questionId} (${birdResult.difficulty})`); + console.log(`Verdict: ${v}`); + console.log(`numeric_match: ${s.numeric_match === null ? 'n/a' : s.numeric_match ? 'YES' : 'NO'}`); + console.log( + `semantic_match: ${s.semantic_match === null ? 'n/a' : s.semantic_match.toFixed(2)}` + + (llmJudge ? ` — ${llmJudge.reason}` : llmJudgeError ? ` (${llmJudgeError})` : ''), + ); + console.log( + `columns_match: ${s.columns_match === null ? 'n/a' : s.columns_match ? 'YES' : `NO (missing: ${missingColumns.join(', ')})`}`, + ); + console.log( + `filters_match: ${s.filters_match === null ? 'n/a' : s.filters_match ? 'YES' : `NO (missing: ${missingFilterFields.join(', ')})`}`, + ); + const t = birdResult.tokens; + console.log(`Model: ${birdResult.model ?? 'n/a'}`); + console.log(`Wall time: ${birdResult.wall_s != null ? `${birdResult.wall_s}s` : 'n/a'}`); + console.log(`Tokens (total context): ${t.total_context_tokens ?? 'n/a'}`); + console.log(` input: ${t.input_tokens ?? 'n/a'}`); + console.log(` cache_creation: ${t.cache_creation_tokens ?? 'n/a'}`); + console.log(` cache_read: ${t.cache_read_tokens ?? 'n/a'}`); + console.log(` output: ${t.output_tokens ?? 'n/a'}`); + console.log(`Tool calls: ${birdResult.tool_calls} (${toolsUsed.join(', ') || 'none'})`); + console.log(`Result: ${resultPath}`); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/evals/grade-suite.ts b/evals/grade-suite.ts new file mode 100644 index 000000000..12c53aaea --- /dev/null +++ b/evals/grade-suite.ts @@ -0,0 +1,282 @@ +/** + * Batch grader for a completed BIRD suite run. + * + * Grades every case in a suite run and writes a single suite-grade.json + * to evals/grades/YYYY-MM-DD//. + * + * Usage: + * npx tsx evals/grade-suite.ts evals/suite-runs/YYYY-MM-DD/ + * npx tsx evals/grade-suite.ts # auto-discovers most recent suite run + * + * Optional environment: + * OPENAI_API_KEY — enables semantic_match (LLM judge); verdict degrades to numeric-only without it + * BIRD_GRADE_MODEL — override LLM judge model (default: gpt-4o-mini) + */ + +/* eslint-disable no-console */ + +import { execFileSync } from 'child_process'; +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const SUITE_RUNS_DIR = path.join(EVALS_DIR, 'suite-runs'); +const GRADES_DIR = path.join(EVALS_DIR, 'grades'); +const GRADE_BIRD_SCRIPT = path.join(EVALS_DIR, 'grade-bird.ts'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} + +// ─── Types ──────────────────────────────────────────────────────────────────── + +type SuiteSummary = { + suite_run_id: string; + suite_file: string; + started_at: string; + finished_at: string; + total_wall_ms: number; + cases: Array<{ + question_id: number; + difficulty: string; + run_id: string; + run_dir: string; + exit_code: number | null; + wall_ms: number | null; + timed_out: boolean; + tool_calls: number; + tools_used: Array; + tokens: { + input: number | null; + output: number | null; + cache_read: number | null; + cache_creation: number | null; + }; + error: string | null; + }>; + aggregate: { + total_cases: number; + completed: number; + errored: number; + timed_out: number; + avg_wall_ms: number | null; + total_input_tokens: number; + total_output_tokens: number; + }; +}; + +type BirdResult = { + run_id: string; + question_id: number; + difficulty: string; + graded_at: string; + model: string | null; + wall_s: number | null; + tokens: { + input_tokens: number | null; + output_tokens: number | null; + cache_creation_tokens: number | null; + cache_read_tokens: number | null; + total_context_tokens: number | null; + }; + tool_calls: number; + tools_used: Array; + signals: { + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + }; + verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip'; +}; + +type CaseGrade = { + question_id: number; + difficulty: string; + run_id: string; + verdict: BirdResult['verdict'] | 'grading_error'; + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + model: string | null; + wall_s: number | null; + tool_calls: number; + tools_used: Array; + tokens: BirdResult['tokens'] | null; + grade_file: string | null; + grading_error: string | null; +}; + +// ─── Discovery ──────────────────────────────────────────────────────────────── + +function findMostRecentSuiteRunDir(): string { + const allSummaries: Array<{ mtime: number; dir: string }> = []; + + for (const entry of fs.readdirSync(SUITE_RUNS_DIR, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const sub = path.join(SUITE_RUNS_DIR, entry.name); + + // Support both flat (legacy) and date-nested layouts + const candidates = [sub, ...fs.readdirSync(sub, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => path.join(sub, e.name))]; + + for (const dir of candidates) { + const summaryPath = path.join(dir, 'suite-summary.json'); + if (fs.existsSync(summaryPath)) { + allSummaries.push({ mtime: fs.statSync(summaryPath).mtimeMs, dir }); + } + } + } + + if (allSummaries.length === 0) { + console.error(`No suite runs found under ${SUITE_RUNS_DIR}`); + process.exit(1); + } + + allSummaries.sort((a, b) => b.mtime - a.mtime); + return allSummaries[0].dir; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main(): Promise { + const suiteRunDirArg = process.argv[2]; + const suiteRunDir = suiteRunDirArg + ? path.resolve(suiteRunDirArg) + : findMostRecentSuiteRunDir(); + + const summaryPath = path.join(suiteRunDir, 'suite-summary.json'); + if (!fs.existsSync(summaryPath)) { + console.error(`suite-summary.json not found in: ${suiteRunDir}`); + process.exit(1); + } + + const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf-8')) as SuiteSummary; + const { suite_run_id, cases } = summary; + const today = dateSlug(); + + console.log(`\nGrading suite: ${suite_run_id}`); + console.log(`Cases: ${cases.length}`); + console.log(`Run dir: ${suiteRunDir}\n`); + + const caseGrades: Array = []; + + for (let i = 0; i < cases.length; i++) { + const c = cases[i]; + const label = `Q${c.question_id} (${c.difficulty}) [${i + 1}/${cases.length}]`; + process.stdout.write(` ${label.padEnd(32)}`); + + // Derive where grade-bird.ts will write its output + const gradeFile = path.join(GRADES_DIR, today, c.run_id, 'bird-result.json'); + + let gradeResult: BirdResult | null = null; + let gradingError: string | null = null; + + try { + execFileSync('npx', ['tsx', GRADE_BIRD_SCRIPT, c.run_dir], { + env: process.env, + cwd: REPO_ROOT, + stdio: 'pipe', + }); + if (fs.existsSync(gradeFile)) { + gradeResult = JSON.parse(fs.readFileSync(gradeFile, 'utf-8')) as BirdResult; + } else { + gradingError = 'bird-result.json not found after grading'; + } + } catch (err: unknown) { + const e = err as { message?: string; stderr?: Buffer }; + gradingError = e.stderr?.toString().trim() || e.message || 'unknown grading error'; + } + + const verdict = gradeResult?.verdict ?? 'grading_error'; + const verdictLabel = verdict === 'pass' ? '✓ PASS' + : verdict === 'partial' ? '~ PARTIAL' + : verdict === 'fail' ? '✗ FAIL' + : verdict === 'error' ? '! ERROR' + : verdict === 'skip' ? '- SKIP' + : '? GRADING_ERROR'; + + process.stdout.write(`${verdictLabel}\n`); + + caseGrades.push({ + question_id: c.question_id, + difficulty: c.difficulty, + run_id: c.run_id, + verdict, + numeric_match: gradeResult?.signals.numeric_match ?? null, + semantic_match: gradeResult?.signals.semantic_match ?? null, + columns_match: gradeResult?.signals.columns_match ?? null, + filters_match: gradeResult?.signals.filters_match ?? null, + model: gradeResult?.model ?? null, + wall_s: gradeResult?.wall_s ?? null, + tool_calls: gradeResult?.tool_calls ?? c.tool_calls, + tools_used: gradeResult?.tools_used ?? c.tools_used, + tokens: gradeResult?.tokens ?? null, + grade_file: gradeResult ? gradeFile : null, + grading_error: gradingError, + }); + } + + // ── Aggregate stats ────────────────────────────────────────────────────────── + + const counts = { + pass: caseGrades.filter((g) => g.verdict === 'pass').length, + partial: caseGrades.filter((g) => g.verdict === 'partial').length, + fail: caseGrades.filter((g) => g.verdict === 'fail').length, + error: caseGrades.filter((g) => g.verdict === 'error').length, + skip: caseGrades.filter((g) => g.verdict === 'skip').length, + grading_error: caseGrades.filter((g) => g.verdict === 'grading_error').length, + }; + const total = caseGrades.length; + const passRate = total > 0 ? counts.pass / total : 0; + + const suiteGrade = { + suite_run_id, + suite_file: summary.suite_file, + suite_started_at: summary.started_at, + suite_finished_at: summary.finished_at, + graded_at: new Date().toISOString(), + summary: { + total, + pass: counts.pass, + partial: counts.partial, + fail: counts.fail, + error: counts.error, + skip: counts.skip, + grading_error: counts.grading_error, + pass_rate: Math.round(passRate * 1000) / 1000, + }, + cases: caseGrades, + }; + + const gradeOutDir = path.join(GRADES_DIR, today, suite_run_id); + fs.mkdirSync(gradeOutDir, { recursive: true }); + const suiteGradePath = path.join(gradeOutDir, 'suite-grade.json'); + fs.writeFileSync(suiteGradePath, JSON.stringify(suiteGrade, null, 2)); + + // ── Print summary ───────────────────────────────────────────────────────── + + console.log('\n═══════════════════════════════════════'); + console.log(`Suite grade: ${suite_run_id}`); + console.log( + ` Pass rate: ${counts.pass}/${total} (${(passRate * 100).toFixed(1)}%)`, + ); + console.log( + ` pass=${counts.pass} partial=${counts.partial} fail=${counts.fail}` + + ` error=${counts.error} skip=${counts.skip}` + + (counts.grading_error > 0 ? ` grading_error=${counts.grading_error}` : ''), + ); + console.log(` Grade file: ${suiteGradePath}`); +} + +main().catch((err: unknown) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/evals/grade.ts b/evals/grade.ts new file mode 100644 index 000000000..05bbe8df1 --- /dev/null +++ b/evals/grade.ts @@ -0,0 +1,137 @@ +/** + * Local grader for Claude Code + LangSmith eval runs. + * + * Usage: + * npx tsx evals/grade.ts evals/runs/ + */ + +/* eslint-disable no-console */ + +import * as fs from 'fs'; +import * as path from 'path'; + +const runDir = process.argv[2]; +if (!runDir) { + console.error('Usage: npx tsx evals/grade.ts '); + process.exit(1); +} + +type RunMeta = { + run_id: string; + case_id: string; + expected_tools?: Array; + budget: { + max_tool_calls: number; + max_wall_ms: number; + }; + wall_ms?: number; + claude_exit_code?: number; + timed_out?: boolean; + langsmith?: { + project: string; + endpoint: string; + parent_run_id: string; + }; +}; + +type HookRecord = { + tool_name?: string; + normalized_tool_name?: string; + langsmith_run_id?: string; +}; + +type ChildPost = { + ok?: boolean; + status?: number; + tool_name?: string; +}; + +const absRunDir = path.resolve(runDir); +const runMeta = JSON.parse(fs.readFileSync(path.join(absRunDir, 'run.json'), 'utf-8')) as RunMeta; +const hookRecords = readJsonl(path.join(absRunDir, 'hook.jsonl')); +const childPosts = readJsonl(path.join(absRunDir, 'langsmith-child-runs.jsonl')); +const stop = readOptionalJson(path.join(absRunDir, 'stop.json')); + +const observedTools = hookRecords + .map((record) => normalizeToolName(record.normalized_tool_name ?? record.tool_name ?? '')) + .filter(Boolean); +const expectedTools = runMeta.expected_tools ?? []; +const missingTools = expectedTools.filter( + (tool) => !observedTools.includes(normalizeToolName(tool)), +); +const failedLangSmithPosts = childPosts.filter((post) => post.ok === false); + +type Outcome = 'pass' | 'fail' | 'timeout' | 'budget_exceeded' | 'error'; + +function deriveOutcome(): Outcome { + if (runMeta.timed_out) return 'timeout'; + if (runMeta.claude_exit_code != null && runMeta.claude_exit_code !== 0) return 'error'; + if (hookRecords.length > runMeta.budget.max_tool_calls) return 'budget_exceeded'; + if (missingTools.length > 0 || failedLangSmithPosts.length > 0) return 'fail'; + return 'pass'; +} + +const result = { + run_id: runMeta.run_id, + case_id: runMeta.case_id, + graded_at: new Date().toISOString(), + outcome: deriveOutcome(), + langsmith: runMeta.langsmith ?? null, + expected_tools: expectedTools, + observed_tools: observedTools, + missing_tools: missingTools, + tool_calls: hookRecords.length, + langsmith_child_posts: { + total: childPosts.length, + failed: failedLangSmithPosts.length, + statuses: childPosts.map((post) => post.status).filter((status) => status != null), + }, + wall_ms: runMeta.wall_ms ?? null, + budget: runMeta.budget, + stop, +}; + +const resultPath = path.join(absRunDir, 'result.json'); +fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); + +console.log(`\nGrade: ${runMeta.run_id}`); +console.log(`Outcome: ${result.outcome.toUpperCase()}`); +console.log(`Tool calls: ${result.tool_calls} (budget: ${runMeta.budget.max_tool_calls})`); +console.log(`Expected: ${expectedTools.length ? expectedTools.join(', ') : 'n/a'}`); +console.log(`Observed: ${observedTools.length ? observedTools.join(', ') : 'none'}`); +console.log(`Missing: ${missingTools.length ? missingTools.join(', ') : 'none'}`); +console.log( + `LangSmith: ${childPosts.length - failedLangSmithPosts.length}/${childPosts.length} child posts ok`, +); +console.log(`Result: ${resultPath}`); + +function readJsonl(filePath: string): Array { + if (!fs.existsSync(filePath)) return []; + return fs + .readFileSync(filePath, 'utf-8') + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + try { + return JSON.parse(line) as T; + } catch { + return null; + } + }) + .filter((value): value is T => value !== null); +} + +function readOptionalJson(filePath: string): unknown { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, 'utf-8')); + } catch { + return null; + } +} + +function normalizeToolName(toolName: string): string { + const parts = toolName.split('__'); + const raw = parts[parts.length - 1] || toolName; + return raw.replace(/^tableau_/, '').replace(/^tableau-/, ''); +} diff --git a/evals/hooks/post-tool-langsmith.mjs b/evals/hooks/post-tool-langsmith.mjs new file mode 100644 index 000000000..fe155d191 --- /dev/null +++ b/evals/hooks/post-tool-langsmith.mjs @@ -0,0 +1,195 @@ +#!/usr/bin/env node +/** + * Claude Code PostToolUse hook. + * + * Reads the hook envelope from stdin and creates a child LangSmith tool run + * under LANGSMITH_PARENT_RUN_ID. The runner sets all required environment + * variables; no secrets are written to disk by this hook. + */ + +/* eslint-disable @typescript-eslint/explicit-function-return-type */ + +import { randomUUID } from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +const input = await readStdin(); +const envelope = parseJson(input); + +if (!envelope) { + process.exit(0); +} + +const apiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY; +const parentRunId = process.env.LANGSMITH_PARENT_RUN_ID; +const project = process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; +const endpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; +const runId = process.env.TMCP_EVAL_RUN_ID; +const runDir = process.env.TMCP_EVAL_RUN_DIR; +const childRunsLog = process.env.TMCP_EVAL_CHILD_RUNS_LOG; +const hookLog = process.env.TMCP_EVAL_HOOK_LOG; + +if (!apiKey || !parentRunId) { + appendJsonl(hookLog, { + ts: new Date().toISOString(), + warning: 'LANGSMITH_API_KEY/LANGCHAIN_API_KEY or LANGSMITH_PARENT_RUN_ID is not set', + tool_name: envelope.tool_name ?? null, + }); + process.exit(0); +} + +const now = new Date().toISOString(); +const toolName = String(envelope.tool_name ?? 'unknown-tool'); +const childRunId = randomUUID(); +const isFailure = envelope.hook_event_name === 'PostToolUseFailure'; + +// Estimate tool start time. The hook fires at completion so `now` is the end. +// Use the previous hook record's timestamp as the start (end of the last tool call +// or the run's started_at for the first call). This gives the waterfall real width. +const startTime = estimateToolStartTime(); +const toolResponse = envelope.tool_response ?? envelope.tool_output ?? envelope.error ?? null; + +const langSmithRun = { + id: childRunId, + name: normalizeToolName(toolName), + run_type: 'tool', + inputs: truncateValue({ + tool_name: toolName, + tool_input: envelope.tool_input ?? null, + }), + outputs: truncateValue({ + tool_response: toolResponse, + }), + start_time: startTime, + end_time: now, + parent_run_id: parentRunId, + session_name: project, + tags: ['tmcp-eval', 'claude-code', 'mcp-tool', normalizeToolName(toolName)], + error: isFailure ? stringifyError(toolResponse) : undefined, + extra: { + run_id: runId, + hook_event_name: envelope.hook_event_name ?? null, + session_id: envelope.session_id ?? null, + transcript_path: envelope.transcript_path ?? null, + tool_use_id: envelope.tool_use_id ?? null, + raw_tool_name: toolName, + }, +}; + +appendJsonl(hookLog, { + ts: now, + start_time: startTime, + run_id: runId, + langsmith_run_id: childRunId, + langsmith_parent_run_id: parentRunId, + hook_event_name: envelope.hook_event_name ?? null, + tool_name: toolName, + normalized_tool_name: normalizeToolName(toolName), + tool_use_id: envelope.tool_use_id ?? null, + tool_input: envelope.tool_input ?? null, +}); + +try { + const response = await fetch(`${endpoint.replace(/\/$/, '')}/runs`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': apiKey, + }, + body: JSON.stringify(langSmithRun), + }); + + const responseBody = await response.text(); + appendJsonl(childRunsLog, { + ts: now, + child_run_id: childRunId, + parent_run_id: parentRunId, + tool_name: toolName, + status: response.status, + ok: response.ok, + response: responseBody ? safeJsonOrText(responseBody) : null, + }); + + if (!response.ok) { + process.exit(0); + } +} catch (error) { + appendJsonl(childRunsLog, { + ts: now, + child_run_id: childRunId, + parent_run_id: parentRunId, + tool_name: toolName, + ok: false, + error: error instanceof Error ? error.message : String(error), + }); +} + +function estimateToolStartTime() { + // Prefer the PreToolUse timestamp for this exact tool_use_id — true dispatch time. + const toolUseId = envelope.tool_use_id; + if (runDir && toolUseId) { + const preToolTimesPath = path.join(runDir, 'pre-tool-times.jsonl'); + if (fs.existsSync(preToolTimesPath)) { + const lines = fs.readFileSync(preToolTimesPath, 'utf-8').trim().split('\n').filter(Boolean); + for (let i = lines.length - 1; i >= 0; i--) { + const record = parseJson(lines[i]); + if (record?.tool_use_id === toolUseId && record?.ts) return record.ts; + } + } + } + // Fall back to now (0ms duration) if PreToolUse record is missing. + return now; +} + +function readStdin() { + return new Promise((resolve) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => resolve(data)); + }); +} + +function parseJson(value) { + try { + return JSON.parse(value); + } catch { + return null; + } +} + +function appendJsonl(filePath, value) { + if (!filePath) return; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); +} + +function normalizeToolName(toolNameValue) { + const parts = toolNameValue.split('__'); + return parts[parts.length - 1] || toolNameValue; +} + +function truncateValue(value) { + const maxChars = Number(process.env.LANGSMITH_MAX_PAYLOAD_CHARS ?? 200_000); + const serialized = JSON.stringify(value); + if (serialized.length <= maxChars) return value; + return { + truncated: true, + preview: serialized.slice(0, maxChars), + }; +} + +function stringifyError(value) { + if (!value) return 'Claude Code reported tool failure'; + return typeof value === 'string' ? value : JSON.stringify(truncateValue(value)); +} + +function safeJsonOrText(value) { + try { + return JSON.parse(value); + } catch { + return value; + } +} diff --git a/evals/hooks/pre-tool-langsmith.mjs b/evals/hooks/pre-tool-langsmith.mjs new file mode 100644 index 000000000..00f6707c5 --- /dev/null +++ b/evals/hooks/pre-tool-langsmith.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node +/** + * Claude Code PreToolUse hook. + * + * Records the exact timestamp when each tool call is dispatched so that + * the PostToolUse hook can compute true tool execution latency rather than + * approximating it from the previous hook record. + * + * Writes one record per tool call to pre-tool-times.jsonl in the run directory. + */ + +/* eslint-disable @typescript-eslint/explicit-function-return-type */ + +import fs from 'fs'; +import path from 'path'; + +const input = await readStdin(); +const envelope = parseJson(input); + +if (!envelope) { + process.exit(0); +} + +const now = new Date().toISOString(); +const runDir = process.env.TMCP_EVAL_RUN_DIR; + +if (runDir && envelope.tool_use_id) { + const preToolTimesPath = path.join(runDir, 'pre-tool-times.jsonl'); + appendJsonl(preToolTimesPath, { + ts: now, + tool_use_id: envelope.tool_use_id, + tool_name: envelope.tool_name ?? null, + }); +} + +function readStdin() { + return new Promise((resolve) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => resolve(data)); + }); +} + +function parseJson(value) { + try { + return JSON.parse(value); + } catch { + return null; + } +} + +function appendJsonl(filePath, value) { + if (!filePath) return; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); +} diff --git a/evals/hooks/stop-langsmith.mjs b/evals/hooks/stop-langsmith.mjs new file mode 100644 index 000000000..c915d2bdd --- /dev/null +++ b/evals/hooks/stop-langsmith.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +/** + * Claude Code Stop hook. + * + * Captures token usage from the Claude transcript and patches the parent + * LangSmith run with stop-hook metadata. The runner also patches the parent + * run after process exit with final stdout/status. + */ + +/* eslint-disable @typescript-eslint/explicit-function-return-type */ + +import fs from 'fs'; +import path from 'path'; + +const input = await readStdin(); +const envelope = parseJson(input) ?? {}; +const runDir = process.env.TMCP_EVAL_RUN_DIR; +if (!runDir) { + process.stderr.write('[stop-hook] TMCP_EVAL_RUN_DIR is not set — cannot write stop.json\n'); + process.exit(0); +} +const apiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY; +const parentRunId = process.env.LANGSMITH_PARENT_RUN_ID; +const endpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; + +const stopData = { + run_id: process.env.TMCP_EVAL_RUN_ID, + ts: new Date().toISOString(), + transcript_path: envelope.transcript_path ?? null, + stop_hook_active: envelope.stop_hook_active ?? null, + last_assistant_message: envelope.last_assistant_message ?? null, + usage: readUsage(envelope.transcript_path), +}; + +fs.mkdirSync(runDir, { recursive: true }); +fs.writeFileSync(path.join(runDir, 'stop.json'), `${JSON.stringify(stopData, null, 2)}\n`); + +if (apiKey && parentRunId) { + try { + await fetch(`${endpoint.replace(/\/$/, '')}/runs/${parentRunId}`, { + method: 'PATCH', + headers: { + 'content-type': 'application/json', + 'x-api-key': apiKey, + }, + body: JSON.stringify({ + extra: { + stop_hook: stopData, + }, + }), + }); + } catch { + // Hook failures should never fail the Claude Code session. + } +} + +function readUsage(transcriptPath) { + const usage = { + input_tokens: null, + output_tokens: null, + cache_read_input_tokens: null, + cache_creation_input_tokens: null, + }; + + if (!transcriptPath) return usage; + + try { + const lines = fs.readFileSync(transcriptPath, 'utf-8').trim().split('\n'); + for (const line of lines) { + const entry = parseJson(line); + const entryUsage = entry?.message?.usage ?? entry?.usage; + if (!entryUsage) continue; + + usage.input_tokens = (usage.input_tokens ?? 0) + (entryUsage.input_tokens ?? 0); + usage.output_tokens = (usage.output_tokens ?? 0) + (entryUsage.output_tokens ?? 0); + usage.cache_read_input_tokens = + (usage.cache_read_input_tokens ?? 0) + (entryUsage.cache_read_input_tokens ?? 0); + usage.cache_creation_input_tokens = + (usage.cache_creation_input_tokens ?? 0) + (entryUsage.cache_creation_input_tokens ?? 0); + } + } catch { + return usage; + } + + return usage; +} + +function readStdin() { + return new Promise((resolve) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => resolve(data)); + }); +} + +function parseJson(value) { + try { + return JSON.parse(value); + } catch { + return null; + } +} diff --git a/evals/run-case.ts b/evals/run-case.ts new file mode 100644 index 000000000..55a7c12f5 --- /dev/null +++ b/evals/run-case.ts @@ -0,0 +1,737 @@ +/** + * Claude Code + LangSmith eval runner for the Tableau MCP server. + * + * Usage: + * npx tsx evals/run-case.ts evals/cases/list-datasources.json + * npx tsx evals/run-case.ts input "sales by region" + * + * Required environment: + * LANGSMITH_API_KEY or LANGCHAIN_API_KEY + * SERVER plus one supported Tableau auth configuration, for example: + * AUTH=pat SITE_NAME= PAT_NAME= PAT_VALUE= + */ + +/* eslint-disable no-console */ + +import { execFileSync } from 'child_process'; +import { randomUUID } from 'crypto'; +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const RUNS_DIR = path.join(EVALS_DIR, 'runs'); +const HOOKS_DIR = path.join(EVALS_DIR, 'hooks'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} +const MCP_SERVER_ENTRY = path.join(REPO_ROOT, 'build', 'index.js'); + +type EvalCase = { + id: string; + name?: string; + description?: string; + prompt: string; + expected_tools?: Array; + tags?: Array; + metadata?: Record; + budget?: { + max_tool_calls?: number; + max_wall_ms?: number; + }; +}; + +type LangSmithRun = { + id: string; + name: string; + run_type: 'chain' | 'tool' | 'llm'; + inputs?: Record; + outputs?: Record; + start_time?: string; + end_time?: string; + parent_run_id?: string; + session_name?: string; + tags?: Array; + extra?: Record; + error?: string; +}; + +const args = process.argv.slice(2); +const positionalArgs = args.filter((arg) => !arg.startsWith('--')); +const runIdArg = getArgValue('--run-id'); + +if (positionalArgs.length === 0) { + console.error( + 'Usage: npx tsx evals/run-case.ts [--run-id ]\n' + + ' or: npx tsx evals/run-case.ts input "" [--run-id ]', + ); + process.exit(1); +} + +const evalCase = loadEvalCase(positionalArgs); +const runId = runIdArg ?? `${evalCase.id}-${Date.now()}-${randomUUID().slice(0, 8)}`; +const runDir = path.join(RUNS_DIR, dateSlug(), runId); +const logsDir = path.join(runDir, 'logs'); +const hookLog = path.join(runDir, 'hook.jsonl'); +const childRunsLog = path.join(runDir, 'langsmith-child-runs.jsonl'); +const messageRunsLog = path.join(runDir, 'langsmith-message-runs.jsonl'); +const settingsPath = path.join(runDir, 'claude-settings.json'); +const mcpConfigPath = path.join(runDir, 'mcp-config.json'); + +const langSmithApiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY; +const langSmithProject = process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; +const langSmithEndpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; +const budget = { + max_tool_calls: evalCase.budget?.max_tool_calls ?? 20, + max_wall_ms: evalCase.budget?.max_wall_ms ?? 5 * 60 * 1000, +}; + +preflight(); +fs.mkdirSync(logsDir, { recursive: true }); + +const prompt = expandTemplate(evalCase.prompt); +const parentRunId = randomUUID(); +const startedAt = new Date().toISOString(); + +const runMeta = { + run_id: runId, + case_id: evalCase.id, + name: evalCase.name ?? evalCase.id, + description: evalCase.description ?? null, + started_at: startedAt, + prompt, + expected_tools: evalCase.expected_tools ?? [], + tags: evalCase.tags ?? [], + metadata: evalCase.metadata ?? {}, + budget, + langsmith: { + project: langSmithProject, + endpoint: langSmithEndpoint, + parent_run_id: parentRunId, + }, + git_sha: tryExec('git', ['rev-parse', 'HEAD']), + git_branch: tryExec('git', ['rev-parse', '--abbrev-ref', 'HEAD']), +}; + +fs.writeFileSync(path.join(runDir, 'run.json'), JSON.stringify(runMeta, null, 2)); +writeClaudeSettings(); +writeMcpConfig(); + +console.log(`\nRun ID: ${runId}`); +console.log(`Run dir: ${runDir}`); +console.log(`LangSmith project: ${langSmithProject}`); + +createLangSmithRun({ + id: parentRunId, + name: evalCase.name ?? evalCase.id, + run_type: 'chain', + inputs: { prompt, case: evalCase }, + start_time: startedAt, + session_name: langSmithProject, + tags: ['tmcp-eval', 'claude-code', ...(evalCase.tags ?? [])], + extra: { + run_id: runId, + case_id: evalCase.id, + git_sha: runMeta.git_sha, + git_branch: runMeta.git_branch, + expected_tools: evalCase.expected_tools ?? [], + }, +}); + +const startMs = Date.now(); +let claudeExitCode = 0; +let agentOutput = ''; +let errorMessage: string | undefined; + +try { + const output = execFileSync( + 'claude', + [ + '--settings', + settingsPath, + '--mcp-config', + mcpConfigPath, + '--strict-mcp-config', + '--allowedTools', + 'ToolSearch,mcp__tableau__*', + '--output-format', + 'stream-json', + '--verbose', + '-p', + prompt, + ], + { + env: { + ...process.env, + LANGSMITH_API_KEY: langSmithApiKey, + LANGSMITH_PROJECT: langSmithProject, + LANGSMITH_ENDPOINT: langSmithEndpoint, + LANGSMITH_PARENT_RUN_ID: parentRunId, + TMCP_EVAL_RUN_ID: runId, + TMCP_EVAL_RUN_DIR: runDir, + TMCP_EVAL_HOOK_LOG: hookLog, + TMCP_EVAL_CHILD_RUNS_LOG: childRunsLog, + }, + cwd: REPO_ROOT, + timeout: budget.max_wall_ms + 10_000, + maxBuffer: 25 * 1024 * 1024, + }, + ); + agentOutput = output.toString(); +} catch (error: unknown) { + const execError = error as { + status?: number; + stdout?: Buffer; + stderr?: Buffer; + message?: string; + }; + claudeExitCode = execError.status ?? 1; + agentOutput = execError.stdout?.toString() ?? ''; + errorMessage = [execError.message, execError.stderr?.toString()].filter(Boolean).join('\n'); + console.error(`claude exited with code ${claudeExitCode}`); + if (errorMessage) console.error(errorMessage); +} + +const finishedAt = new Date().toISOString(); +const wallMs = Date.now() - startMs; +fs.writeFileSync(path.join(runDir, 'agent-output.jsonl'), agentOutput); +const messageTracePosts = postClaudeStreamEvents(agentOutput, parentRunId); +fs.writeFileSync( + path.join(runDir, 'langsmith-message-runs.json'), + JSON.stringify(messageTracePosts, null, 2), +); + +const finishedMeta = { + ...runMeta, + finished_at: finishedAt, + wall_ms: wallMs, + claude_exit_code: claudeExitCode, + timed_out: wallMs >= budget.max_wall_ms, + error: errorMessage ?? null, +}; +fs.writeFileSync(path.join(runDir, 'run.json'), JSON.stringify(finishedMeta, null, 2)); + +const stopPath = path.join(runDir, 'stop.json'); +const stopData = fs.existsSync(stopPath) + ? (JSON.parse(fs.readFileSync(stopPath, 'utf-8')) as Record) + : {}; + +const finalLangSmithUpdate = tryUpdateLangSmithRun(parentRunId, { + outputs: { + success: claudeExitCode === 0, + agent_output_jsonl: truncate(agentOutput), + message_trace_posts: messageTracePosts, + stop: stopData, + }, + end_time: finishedAt, + error: errorMessage, + extra: { + ...runMeta.langsmith, + wall_ms: wallMs, + claude_exit_code: claudeExitCode, + timed_out: wallMs >= budget.max_wall_ms, + message_trace_posts: messageTracePosts, + stop: stopData, + }, +}); +fs.writeFileSync( + path.join(runDir, 'langsmith-final-update.json'), + JSON.stringify(finalLangSmithUpdate, null, 2), +); + +console.log(`\nRun complete in ${wallMs}ms (exit ${claudeExitCode})`); +console.log(`Parent LangSmith run: ${parentRunId}`); +if (!finalLangSmithUpdate.ok) { + console.warn(`Final LangSmith update failed: ${finalLangSmithUpdate.error}`); +} +console.log(`Run dir: ${runDir}`); +console.log(`To grade locally: npx tsx evals/grade.ts ${runDir}`); + +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + +function loadEvalCase(positional: Array): EvalCase { + const [modeOrPath, ...rest] = positional; + if (modeOrPath === 'input') { + const message = rest.join(' ').trim(); + if (!message) { + throw new Error('Ad hoc input mode requires a message, for example: input "sales by region"'); + } + const slug = slugify(message); + return { + id: `adhoc-${slug}`, + name: `Ad Hoc: ${message.slice(0, 80)}`, + description: 'Ad hoc Claude Code session against the Tableau MCP server.', + prompt: message, + tags: ['adhoc', 'real-server'], + metadata: { + source: 'cli-input', + }, + budget: { + max_tool_calls: 20, + max_wall_ms: 5 * 60 * 1000, + }, + }; + } + + return JSON.parse(fs.readFileSync(path.resolve(modeOrPath), 'utf-8')) as EvalCase; +} + +function slugify(value: string): string { + const slug = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 48); + return slug || 'input'; +} + +function preflight(): void { + if (!langSmithApiKey) { + throw new Error('LANGSMITH_API_KEY or LANGCHAIN_API_KEY must be set.'); + } + if (!fs.existsSync(MCP_SERVER_ENTRY)) { + throw new Error(`MCP server build not found at ${MCP_SERVER_ENTRY}. Run npm run build first.`); + } + if (!process.env.SERVER && process.env.AUTH !== 'oauth') { + throw new Error('SERVER must be set for real Tableau Cloud/Server evals.'); + } + const auth = process.env.AUTH ?? 'pat'; + if (auth === 'pat' && (!process.env.PAT_NAME || !process.env.PAT_VALUE)) { + throw new Error('PAT_NAME and PAT_VALUE must be set when AUTH is "pat" or unset.'); + } +} + +function writeClaudeSettings(): void { + const preToolHook = path.join(HOOKS_DIR, 'pre-tool-langsmith.mjs'); + const postToolHook = path.join(HOOKS_DIR, 'post-tool-langsmith.mjs'); + const stopHook = path.join(HOOKS_DIR, 'stop-langsmith.mjs'); + const claudeSettings = { + hooks: { + PreToolUse: [ + { + matcher: '', + hooks: [{ type: 'command', command: `node ${JSON.stringify(preToolHook)}` }], + }, + ], + PostToolUse: [ + { + matcher: '', + hooks: [{ type: 'command', command: `node ${JSON.stringify(postToolHook)}` }], + }, + ], + PostToolUseFailure: [ + { + matcher: '', + hooks: [{ type: 'command', command: `node ${JSON.stringify(postToolHook)}` }], + }, + ], + Stop: [ + { + hooks: [{ type: 'command', command: `node ${JSON.stringify(stopHook)}` }], + }, + ], + }, + }; + fs.writeFileSync(settingsPath, JSON.stringify(claudeSettings, null, 2)); +} + +function writeMcpConfig(): void { + const mcpConfig = { + mcpServers: { + tableau: { + command: 'node', + args: [MCP_SERVER_ENTRY], + env: { + TRANSPORT: 'stdio', + TMCP_EVAL_RUN_ID: runId, + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }, + }; + fs.writeFileSync(mcpConfigPath, JSON.stringify(mcpConfig, null, 2)); +} + +function expandTemplate(value: string): string { + return value.replace(/\{\{env\.([A-Z0-9_]+)\}\}/g, (_match, name: string) => { + const envValue = process.env[name]; + if (!envValue) { + throw new Error(`Prompt references {{env.${name}}}, but ${name} is not set.`); + } + return envValue; + }); +} + +function createLangSmithRun(run: LangSmithRun): void { + requestLangSmith('/runs', 'POST', run); +} + +function tryCreateLangSmithRun(run: LangSmithRun): { ok: true } | { ok: false; error: string } { + try { + createLangSmithRun(run); + return { ok: true }; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +function updateLangSmithRun(runIdToUpdate: string, body: Partial): void { + requestLangSmith(`/runs/${runIdToUpdate}`, 'PATCH', body); +} + +function tryUpdateLangSmithRun( + runIdToUpdate: string, + body: Partial, +): { ok: true } | { ok: false; error: string } { + try { + updateLangSmithRun(runIdToUpdate, body); + return { ok: true }; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +function requestLangSmith(pathSuffix: string, method: 'POST' | 'PATCH', body: unknown): void { + const endpoint = `${langSmithEndpoint.replace(/\/$/, '')}${pathSuffix}`; + const response = fetchSync(endpoint, method, body); + if (response.status < 200 || response.status >= 300) { + throw new Error( + `LangSmith ${method} ${pathSuffix} failed: ${response.status} ${response.body}`, + ); + } +} + +function fetchSync( + url: string, + method: 'POST' | 'PATCH', + body: unknown, +): { status: number; body: string } { + const script = ` +const url = process.argv[1]; +const method = process.argv[2]; +const body = JSON.parse(process.argv[3]); +fetch(url, { + method, + headers: { + 'content-type': 'application/json', + 'x-api-key': process.env.LANGSMITH_API_KEY, + }, + body: JSON.stringify(body), +}).then(async (response) => { + console.log(JSON.stringify({ status: response.status, body: await response.text() })); +}).catch((error) => { + console.error(error); + process.exit(1); +}); +`; + const output = execFileSync(process.execPath, ['-e', script, url, method, JSON.stringify(body)], { + env: { ...process.env, LANGSMITH_API_KEY: langSmithApiKey }, + maxBuffer: 10 * 1024 * 1024, + }).toString(); + return JSON.parse(output) as { status: number; body: string }; +} + +function tryExec(command: string, commandArgs: Array): string | null { + try { + return execFileSync(command, commandArgs, { cwd: REPO_ROOT }).toString().trim(); + } catch { + return null; + } +} + +type ClaudeStreamEvent = { + type?: string; + subtype?: string; + message?: { + id?: string; + model?: string; + role?: string; + content?: unknown; + usage?: unknown; + }; + result?: string; + usage?: unknown; + modelUsage?: unknown; + uuid?: string; + session_id?: string; + timestamp?: string; + [key: string]: unknown; +}; + +/** + * Build a map of stream event index → real timestamp using PreToolUse and PostToolUse + * hook records as anchors. + * + * For each assistant event that contains a tool_use content block: + * - The event's own time is set to the PreToolUse dispatch timestamp (tool call start) + * - The immediately following event's time is set to the PostToolUse timestamp (tool call end) + * + * These anchors are passed into inferEventTimes so interpolation only fills the gaps + * between real data points rather than spreading linearly across the whole session. + */ +function buildToolTimeAnchors(events: Array): Map { + const anchors = new Map(); + + const preToolPath = path.join(runDir, 'pre-tool-times.jsonl'); + const preToolRecords = fs.existsSync(preToolPath) + ? parseJsonl<{ ts: string; tool_use_id: string }>( + fs.readFileSync(preToolPath, 'utf-8'), + ) + : []; + const dispatchTimes = new Map(); + for (const r of preToolRecords) { + if (r.tool_use_id && r.ts) dispatchTimes.set(r.tool_use_id, new Date(r.ts).getTime()); + } + + const hookRecords = fs.existsSync(hookLog) + ? parseJsonl<{ ts: string; tool_use_id?: string }>(fs.readFileSync(hookLog, 'utf-8')) + : []; + const completeTimes = new Map(); + for (const r of hookRecords) { + if (r.tool_use_id && r.ts) completeTimes.set(r.tool_use_id, new Date(r.ts).getTime()); + } + + if (dispatchTimes.size === 0 && completeTimes.size === 0) return anchors; + + for (let i = 0; i < events.length; i++) { + const event = events[i]; + if (event.type !== 'assistant' || !Array.isArray(event.message?.content)) continue; + + const content = event.message.content as Array<{ type?: string; id?: string }>; + const toolUseBlock = content.find((b) => b.type === 'tool_use' && b.id); + if (!toolUseBlock?.id) continue; + + const dispatchMs = dispatchTimes.get(toolUseBlock.id); + if (dispatchMs != null) anchors.set(i, dispatchMs); + + const completeMs = completeTimes.get(toolUseBlock.id); + if (completeMs != null && i + 1 < events.length) anchors.set(i + 1, completeMs); + } + + return anchors; +} + +function postClaudeStreamEvents( + streamJsonl: string, + parentRunIdToUse: string, +): { + total_events: number; + posted: number; + failed: number; +} { + const events = parseJsonl(streamJsonl); + const toolTimeAnchors = buildToolTimeAnchors(events); + const eventTimes = inferEventTimes(events, toolTimeAnchors); + const traceableEvents = events + .map((event, streamIndex) => ({ event, streamIndex })) + .filter(({ event }) => event.type === 'assistant' || event.type === 'result'); + + let posted = 0; + let failed = 0; + + // Post a synthetic initialization span covering Claude Code startup time — + // the gap from when the process was launched to the first stream event. + const firstEventTime = + traceableEvents.length > 0 + ? (eventTimes[traceableEvents[0].streamIndex] ?? startedAt) + : startedAt; + if (firstEventTime > startedAt) { + const initResult = tryCreateLangSmithRun({ + id: randomUUID(), + name: 'initialization', + run_type: 'chain', + inputs: { prompt }, + outputs: {}, + start_time: startedAt, + end_time: firstEventTime, + parent_run_id: parentRunIdToUse, + session_name: langSmithProject, + tags: ['tmcp-eval', 'claude-code', 'initialization'], + extra: { run_id: runId, description: 'Claude Code startup and MCP server initialization' }, + }); + if (initResult.ok) posted += 1; + else failed += 1; + } + + traceableEvents.forEach(({ event, streamIndex }, index) => { + const childRunId = randomUUID(); + const isAssistant = event.type === 'assistant'; + const name = isAssistant + ? `assistant-message-${index + 1}` + : `claude-result-${event.subtype ?? 'final'}`; + const eventTime = eventTimes[streamIndex] ?? finishedAt; + const nextTraceable = traceableEvents[index + 1]; + const endTime = nextTraceable + ? (eventTimes[nextTraceable.streamIndex] ?? finishedAt) + : finishedAt; + const postResult = tryCreateLangSmithRun({ + id: childRunId, + name, + run_type: isAssistant ? 'llm' : 'chain', + inputs: truncateJsonValue({ + prompt, + event_index: index, + stream_index: streamIndex, + session_id: event.session_id ?? null, + }) as Record, + outputs: truncateJsonValue({ + message: event.message ?? null, + result: event.result ?? null, + usage: event.usage ?? event.message?.usage ?? null, + model_usage: event.modelUsage ?? null, + }) as Record, + start_time: eventTime, + end_time: endTime, + parent_run_id: parentRunIdToUse, + session_name: langSmithProject, + tags: ['tmcp-eval', 'claude-code', isAssistant ? 'assistant-message' : 'claude-result'], + extra: { + run_id: runId, + event_type: event.type ?? null, + event_subtype: event.subtype ?? null, + event_uuid: event.uuid ?? null, + stream_index: streamIndex, + inferred_timestamp: event.timestamp ? false : true, + message_id: event.message?.id ?? null, + model: event.message?.model ?? null, + }, + }); + + if (postResult.ok) { + posted += 1; + } else { + failed += 1; + } + + appendJsonl(messageRunsLog, { + ts: new Date().toISOString(), + child_run_id: childRunId, + parent_run_id: parentRunIdToUse, + name, + stream_index: streamIndex, + event_time: eventTime, + ok: postResult.ok, + error: postResult.ok ? null : postResult.error, + }); + }); + + return { + total_events: traceableEvents.length, + posted, + failed, + }; +} + +function inferEventTimes( + events: Array, + anchors: Map = new Map(), +): Array { + if (events.length === 0) return []; + + const startMs = new Date(startedAt).getTime(); + const finishMs = new Date(finishedAt).getTime(); + const explicitTimes = events.map((event, index) => { + // Real hook-derived timestamps take highest priority. + const anchor = anchors.get(index); + if (anchor != null && Number.isFinite(anchor)) return anchor; + if (event.timestamp) return new Date(event.timestamp).getTime(); + if (event.type === 'result') return finishMs; + return null; + }); + + const inferredTimes = events.map((_event, index) => { + const explicitTime = explicitTimes[index]; + if (explicitTime != null && Number.isFinite(explicitTime)) return explicitTime; + + const previousIndex = findPreviousExplicitTimeIndex(explicitTimes, index); + const nextIndex = findNextExplicitTimeIndex(explicitTimes, index); + + if (previousIndex != null && nextIndex != null) { + const previousTime = explicitTimes[previousIndex] ?? startMs; + const nextTime = explicitTimes[nextIndex] ?? finishMs; + const fraction = (index - previousIndex) / (nextIndex - previousIndex); + return Math.round(previousTime + (nextTime - previousTime) * fraction); + } + + if (previousIndex != null) { + return (explicitTimes[previousIndex] ?? startMs) + (index - previousIndex) * 10; + } + + if (nextIndex != null) { + return (explicitTimes[nextIndex] ?? finishMs) - (nextIndex - index) * 10; + } + + const fraction = events.length === 1 ? 0 : index / (events.length - 1); + return Math.round(startMs + (finishMs - startMs) * fraction); + }); + + for (let index = 1; index < inferredTimes.length; index += 1) { + if (inferredTimes[index] <= inferredTimes[index - 1]) { + inferredTimes[index] = inferredTimes[index - 1] + 1; + } + } + + return inferredTimes.map((time) => new Date(time).toISOString()); +} + +function findPreviousExplicitTimeIndex(times: Array, index: number): number | null { + for (let cursor = index - 1; cursor >= 0; cursor -= 1) { + if (times[cursor] != null && Number.isFinite(times[cursor])) return cursor; + } + return null; +} + +function findNextExplicitTimeIndex(times: Array, index: number): number | null { + for (let cursor = index + 1; cursor < times.length; cursor += 1) { + if (times[cursor] != null && Number.isFinite(times[cursor])) return cursor; + } + return null; +} + +function parseJsonl(jsonl: string): Array { + return jsonl + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + try { + return JSON.parse(line) as T; + } catch { + return null; + } + }) + .filter((event): event is T => event !== null); +} + +function appendJsonl(filePath: string, value: unknown): void { + fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); +} + +function truncate(value: string): string { + const maxChars = Number(process.env.LANGSMITH_MAX_PAYLOAD_CHARS ?? 200_000); + return value.length > maxChars ? `${value.slice(0, maxChars)}\n...[truncated]` : value; +} + +function truncateJsonValue(value: unknown): unknown { + const maxChars = Number(process.env.LANGSMITH_MAX_PAYLOAD_CHARS ?? 200_000); + const serialized = JSON.stringify(value); + if (serialized.length <= maxChars) return value; + return { + truncated: true, + preview: serialized.slice(0, maxChars), + }; +} diff --git a/evals/run-suite.ts b/evals/run-suite.ts new file mode 100644 index 000000000..e398b3c80 --- /dev/null +++ b/evals/run-suite.ts @@ -0,0 +1,338 @@ +/** + * Suite runner for the BIRD California Schools eval. + * + * Runs all 30 cases (or a filtered subset) sequentially against the live + * Tableau MCP server. Each case is executed via run-case.ts so trace routing, + * hooks, and LangSmith posting are identical to ad-hoc runs. + * + * Usage: + * npx tsx evals/run-suite.ts + * npx tsx evals/run-suite.ts --suite evals/suites/bird-california-schools.json + * npx tsx evals/run-suite.ts --difficulty simple + * npx tsx evals/run-suite.ts --ids 5,11,12 + * + * Required environment: same as run-case.ts (Tableau auth + LANGSMITH_API_KEY). + * Set EVAL_DATASOURCE_LUID to the LUID of the published California Schools datasource. + */ + +/* eslint-disable no-console */ + +import { execFileSync } from 'child_process'; +import { randomUUID } from 'crypto'; +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const RUNS_DIR = path.join(EVALS_DIR, 'runs'); +const SUITE_RUNS_DIR = path.join(EVALS_DIR, 'suite-runs'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} +const DEFAULT_SUITE = path.join(EVALS_DIR, 'suites', 'bird-california-schools.json'); + +const args = process.argv.slice(2); + +const suiteFile = getArgValue('--suite') ?? DEFAULT_SUITE; +const difficultyFilter = getArgValue('--difficulty'); +const idsFilter = getArgValue('--ids') + ?.split(',') + .map((s) => parseInt(s.trim(), 10)) + .filter(Number.isFinite); + +type BirdCase = { + question_id: number; + question: string; + evidence: string; + difficulty: string; + answer_type: 'scalar' | 'list'; + expected_value: number | string | null; + expected_row_count: number | null; + ai_summarized_answer: string; + expected_columns: Array; + expected_filter_fields: Array; + prompt: string; + expected_tools: Array; + budget: { max_wall_ms: number }; +}; + +type EvalCaseFile = { + id: string; + name: string; + description: string; + prompt: string; + expected_tools: Array; + tags: Array; + metadata: Record; + budget: { max_wall_ms: number }; +}; + +type StopData = { + usage?: { + input_tokens: number | null; + output_tokens: number | null; + cache_read_input_tokens: number | null; + cache_creation_input_tokens: number | null; + }; +}; + +type HookRecord = { + tool_name?: string; + normalized_tool_name?: string; +}; + +type CaseRunResult = { + question_id: number; + difficulty: string; + run_id: string; + run_dir: string; + exit_code: number | null; + wall_ms: number | null; + timed_out: boolean; + tool_calls: number; + tools_used: Array; + tokens: { + input: number | null; + output: number | null; + cache_read: number | null; + cache_creation: number | null; + }; + error: string | null; +}; + +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + +function readJsonlFile(filePath: string): Array { + if (!fs.existsSync(filePath)) return []; + return fs + .readFileSync(filePath, 'utf-8') + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + try { + return JSON.parse(line) as T; + } catch { + return null; + } + }) + .filter((v): v is T => v !== null); +} + +function readOptionalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T; + } catch { + return null; + } +} + +function buildEvalCaseFile(birdCase: BirdCase, absoluteSuitePath: string): EvalCaseFile { + return { + id: `bird-ca-schools-${birdCase.question_id}`, + name: `BIRD California Schools Q${birdCase.question_id}`, + description: birdCase.question, + prompt: birdCase.prompt, + expected_tools: birdCase.expected_tools, + tags: ['bird', 'california-schools', birdCase.difficulty], + metadata: { + question_id: birdCase.question_id, + difficulty: birdCase.difficulty, + suite: 'bird-california-schools', + suite_file: absoluteSuitePath, + }, + budget: birdCase.budget, + }; +} + +function collectRunMetrics(runDir: string): Omit { + const runMeta = readOptionalJson<{ + run_id: string; + wall_ms?: number; + claude_exit_code?: number; + timed_out?: boolean; + error?: string | null; + budget: { max_wall_ms: number }; + }>(path.join(runDir, 'run.json')); + + const hookRecords = readJsonlFile(path.join(runDir, 'hook.jsonl')); + const stop = readOptionalJson(path.join(runDir, 'stop.json')); + const runId = path.basename(runDir); + + const tools = hookRecords + .map((r) => { + const raw = r.normalized_tool_name ?? r.tool_name ?? ''; + const parts = raw.split('__'); + return parts[parts.length - 1] || raw; + }) + .filter(Boolean); + + return { + run_id: runMeta?.run_id ?? runId, + run_dir: runDir, + exit_code: runMeta?.claude_exit_code ?? null, + wall_ms: runMeta?.wall_ms ?? null, + timed_out: runMeta?.timed_out ?? false, + tool_calls: hookRecords.length, + tools_used: [...new Set(tools)], + tokens: { + input: stop?.usage?.input_tokens ?? null, + output: stop?.usage?.output_tokens ?? null, + cache_read: stop?.usage?.cache_read_input_tokens ?? null, + cache_creation: stop?.usage?.cache_creation_input_tokens ?? null, + }, + error: runMeta?.error ?? null, + }; +} + +function main(): void { + if (!fs.existsSync(suiteFile)) { + console.error(`Suite file not found: ${suiteFile}`); + process.exit(1); + } + + if (!process.env.EVAL_DATASOURCE_LUID) { + console.error('EVAL_DATASOURCE_LUID must be set to the LUID of the published datasource to evaluate against.'); + process.exit(1); + } + + const allCases = JSON.parse(fs.readFileSync(suiteFile, 'utf-8')) as Array; + + let cases = allCases; + if (difficultyFilter) { + cases = cases.filter((c) => c.difficulty === difficultyFilter); + console.log(`Filtered to difficulty="${difficultyFilter}": ${cases.length} cases`); + } + if (idsFilter?.length) { + cases = cases.filter((c) => idsFilter.includes(c.question_id)); + console.log(`Filtered to ids=${idsFilter.join(',')}: ${cases.length} cases`); + } + + if (cases.length === 0) { + console.error('No cases matched the filters.'); + process.exit(1); + } + + const today = dateSlug(); + const suiteRunId = `bird-ca-schools-${Date.now()}-${randomUUID().slice(0, 8)}`; + const suiteRunDir = path.join(SUITE_RUNS_DIR, today, suiteRunId); + const casesDir = path.join(suiteRunDir, 'cases'); + fs.mkdirSync(casesDir, { recursive: true }); + fs.mkdirSync(path.join(RUNS_DIR, today), { recursive: true }); + + const absoluteSuitePath = path.resolve(suiteFile); + const startedAt = new Date().toISOString(); + + console.log(`\nSuite run: ${suiteRunId}`); + console.log(`Suite: ${suiteFile}`); + console.log(`Cases: ${cases.length}`); + console.log(`Run dir: ${suiteRunDir}\n`); + + const runCaseScript = path.join(EVALS_DIR, 'run-case.ts'); + const results: Array = []; + + for (let i = 0; i < cases.length; i++) { + const birdCase = cases[i]; + const caseLabel = `Q${birdCase.question_id} (${birdCase.difficulty}) [${i + 1}/${cases.length}]`; + console.log(`\n─── ${caseLabel} ───`); + console.log(` ${birdCase.question.slice(0, 80)}${birdCase.question.length > 80 ? '...' : ''}`); + + const evalCaseFile = buildEvalCaseFile(birdCase, absoluteSuitePath); + const caseFilePath = path.join(casesDir, `case-${birdCase.question_id}.json`); + fs.writeFileSync(caseFilePath, JSON.stringify(evalCaseFile, null, 2)); + + const runId = `${evalCaseFile.id}-${Date.now()}-${randomUUID().slice(0, 8)}`; + const runDir = path.join(RUNS_DIR, today, runId); + + const budgetSec = Math.round(birdCase.budget.max_wall_ms / 1000); + process.stdout.write(` Running Claude (up to ${budgetSec}s)...`); + + let spawnError: string | null = null; + try { + execFileSync( + 'npx', + ['tsx', runCaseScript, caseFilePath, '--run-id', runId], + { + env: process.env, + cwd: REPO_ROOT, + timeout: birdCase.budget.max_wall_ms + 30_000, + stdio: 'pipe', + }, + ); + } catch (error: unknown) { + const e = error as { message?: string }; + spawnError = e.message ?? 'unknown spawn error'; + } + process.stdout.write(' done\n'); + + const metrics = collectRunMetrics(runDir); + results.push({ + question_id: birdCase.question_id, + difficulty: birdCase.difficulty, + ...metrics, + error: metrics.error ?? spawnError, + }); + + const status = + metrics.exit_code === 0 ? 'OK' : metrics.timed_out ? 'TIMEOUT' : `EXIT ${metrics.exit_code}`; + console.log( + ` ${status} | ${metrics.wall_ms != null ? `${Math.round(metrics.wall_ms / 1000)}s` : '?s'} | ` + + `${metrics.tool_calls} tool calls | ` + + `tokens: ${metrics.tokens.input ?? '?'} in / ${metrics.tokens.output ?? '?'} out`, + ); + } + + const finishedAt = new Date().toISOString(); + const totalWallMs = results.reduce((sum, r) => sum + (r.wall_ms ?? 0), 0); + const completed = results.filter((r) => r.exit_code === 0).length; + const errored = results.filter((r) => r.exit_code !== 0 && !r.timed_out).length; + const timedOut = results.filter((r) => r.timed_out).length; + const totalInput = results.reduce((s, r) => s + (r.tokens.input ?? 0), 0); + const totalOutput = results.reduce((s, r) => s + (r.tokens.output ?? 0), 0); + const avgWall = + results.filter((r) => r.wall_ms != null).length > 0 + ? Math.round(totalWallMs / results.filter((r) => r.wall_ms != null).length) + : null; + + const summary = { + suite_run_id: suiteRunId, + suite_file: absoluteSuitePath, + started_at: startedAt, + finished_at: finishedAt, + total_wall_ms: totalWallMs, + cases: results, + aggregate: { + total_cases: results.length, + completed, + errored, + timed_out: timedOut, + avg_wall_ms: avgWall, + total_input_tokens: totalInput, + total_output_tokens: totalOutput, + }, + }; + + const summaryPath = path.join(suiteRunDir, 'suite-summary.json'); + fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2)); + + console.log('\n═══════════════════════════════════════'); + console.log(`Suite complete: ${suiteRunId}`); + console.log(` ${completed}/${results.length} OK | ${errored} error | ${timedOut} timeout`); + console.log(` Total wall time: ${(totalWallMs / 1000).toFixed(1)}s`); + console.log(` Total tokens: ${totalInput} in / ${totalOutput} out`); + console.log(` Summary: ${summaryPath}`); + console.log(`\nTo grade cases:`); + for (const r of results) { + console.log(` npx tsx evals/grade-bird.ts ${r.run_dir}`); + } +} + +main(); diff --git a/evals/scripts/precompute-bird-answers.py b/evals/scripts/precompute-bird-answers.py new file mode 100644 index 000000000..357651e1d --- /dev/null +++ b/evals/scripts/precompute-bird-answers.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +One-time script to precompute expected answers for the BIRD California Schools eval suite. + +Reads the BIRD SQLite snapshot and VDS case file, executes gold SQL queries, and +writes evals/suites/bird-california-schools.json with all expected answers baked in. +The output file is committed to the repo so eval runners never need SQLite access. + +Requires: + bird_mini/data/dev_databases/california_schools/california_schools.sqlite + bird_mini/data/test_cases/mini_dev_sqlite.json + bird_mini/data/test_cases/mini_dev_postgresql_vds.json + +Usage: + python3 evals/scripts/precompute-bird-answers.py +""" + +import json +import sqlite3 +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent.parent +SQLITE_PATH = ( + REPO_ROOT + / "bird_mini" + / "data" + / "dev_databases" + / "california_schools" + / "california_schools.sqlite" +) +SQLITE_CASES_PATH = REPO_ROOT / "bird_mini" / "data" / "test_cases" / "mini_dev_sqlite.json" +VDS_CASES_PATH = ( + REPO_ROOT / "bird_mini" / "data" / "test_cases" / "mini_dev_postgresql_vds.json" +) +OUTPUT_PATH = REPO_ROOT / "evals" / "suites" / "bird-california-schools.json" + +DEFAULT_BUDGET = {"max_wall_ms": 180000} + + +DATASOURCE_LUID_PLACEHOLDER = "{{env.EVAL_DATASOURCE_LUID}}" + + +def build_prompt(question: str, evidence: str) -> str: + if not evidence: + return f"Using datasource {DATASOURCE_LUID_PLACEHOLDER}: {question}" + separator = " " if question.rstrip().endswith(("?", "!", ".")) else ". " + return f"Using datasource {DATASOURCE_LUID_PLACEHOLDER}: {question}{separator}{evidence}" + + +def run_sql(conn: sqlite3.Connection, sql: str): + """Execute SQL and return (rows, column_names, error).""" + try: + cursor = conn.execute(sql) + rows = cursor.fetchall() + columns = [d[0] for d in cursor.description] if cursor.description else [] + return rows, columns, None + except Exception as exc: + return None, None, str(exc) + + +def classify_result(rows, columns): + """ + Determine answer_type, expected_value, and expected_row_count. + + Scalar: result is exactly 1 row × 1 column — the cell value is the answer. + List: everything else — row count is the meaningful fact. + """ + if not rows and not columns: + return "list", None, 0 + + if len(rows) == 1 and len(columns) == 1: + raw = rows[0][0] + if isinstance(raw, float): + value = raw + elif isinstance(raw, int): + value = raw + else: + # String scalar (e.g. a year returned as text) + try: + value = int(raw) + except (TypeError, ValueError): + try: + value = float(raw) + except (TypeError, ValueError): + value = raw + return "scalar", value, 1 + + return "list", None, len(rows) + + +def extract_vds_columns(vds_query: dict) -> list[str]: + """Field captions from VDS_QUERY.fields (the SELECT clause).""" + captions = [] + for field in vds_query.get("fields", []): + caption = field.get("fieldCaption") + if caption: + captions.append(caption) + return captions + + +def extract_vds_filter_fields(vds_query: dict) -> list[str]: + """Field captions from VDS_QUERY.filters (the WHERE clause).""" + captions = [] + for f in vds_query.get("filters", []): + caption = f.get("field", {}).get("fieldCaption") + if caption: + captions.append(caption) + return captions + + +def main(): + for path in [SQLITE_PATH, SQLITE_CASES_PATH, VDS_CASES_PATH]: + if not path.exists(): + print(f"ERROR: required file not found: {path}", file=sys.stderr) + sys.exit(1) + + print(f"Loading SQLite cases from {SQLITE_CASES_PATH.relative_to(REPO_ROOT)}") + with open(SQLITE_CASES_PATH) as f: + sqlite_cases = { + c["question_id"]: c + for c in json.load(f) + if c["db_id"] == "california_schools" + } + + print(f"Loading VDS cases from {VDS_CASES_PATH.relative_to(REPO_ROOT)}") + with open(VDS_CASES_PATH) as f: + vds_cases = { + c["question_id"]: c + for c in json.load(f) + if c["db_id"] == "california_schools" + } + + print(f"Connecting to {SQLITE_PATH.relative_to(REPO_ROOT)}") + conn = sqlite3.connect(str(SQLITE_PATH)) + + suite = [] + errors = [] + + for qid, sqlite_case in sorted(sqlite_cases.items()): + vds_case = vds_cases.get(qid, {}) + question = sqlite_case["question"] + evidence = sqlite_case.get("evidence", "") + difficulty = sqlite_case.get("difficulty", "simple") + sql = sqlite_case["SQL"] + ai_summary = vds_case.get("ai_summarized_answer", "") + vds_query = vds_case.get("VDS_QUERY", {}) + + rows, columns, error = run_sql(conn, sql) + + if error: + print(f" Q{qid} ({difficulty}): SQL ERROR — {error}") + errors.append({"question_id": qid, "error": error}) + answer_type, expected_value, expected_row_count = "list", None, None + else: + answer_type, expected_value, expected_row_count = classify_result(rows, columns) + print( + f" Q{qid} ({difficulty}): {answer_type}, " + f"value={expected_value}, rows={expected_row_count}" + ) + + expected_columns = extract_vds_columns(vds_query) + expected_filter_fields = extract_vds_filter_fields(vds_query) + + suite.append( + { + "question_id": qid, + "question": question, + "evidence": evidence, + "difficulty": difficulty, + "answer_type": answer_type, + "expected_value": expected_value, + "expected_row_count": expected_row_count, + "ai_summarized_answer": ai_summary, + "expected_columns": expected_columns, + "expected_filter_fields": expected_filter_fields, + "prompt": build_prompt(question, evidence), + "expected_tools": ["query-datasource"], + "budget": DEFAULT_BUDGET, + } + ) + + conn.close() + + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(OUTPUT_PATH, "w") as f: + json.dump(suite, f, indent=2) + f.write("\n") + + print(f"\nWrote {len(suite)} cases to {OUTPUT_PATH.relative_to(REPO_ROOT)}") + if errors: + print(f"WARNING: {len(errors)} cases had SQL errors: {[e['question_id'] for e in errors]}") + + +if __name__ == "__main__": + main() diff --git a/evals/suites/bird-california-schools.json b/evals/suites/bird-california-schools.json new file mode 100644 index 000000000..f6167b4ed --- /dev/null +++ b/evals/suites/bird-california-schools.json @@ -0,0 +1,728 @@ +[ + { + "question_id": 5, + "question": "How many schools with an average score in Math greater than 400 in the SAT test are exclusively virtual?", + "evidence": "Exclusively virtual refers to Virtual = 'F'", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": 4, + "expected_row_count": 1, + "ai_summarized_answer": "There are 4 schools that are exclusively virtual and have an average Math SAT score greater than 400.", + "expected_columns": [ + "School" + ], + "expected_filter_fields": [ + "Virtual", + "Avgscrmath" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: How many schools with an average score in Math greater than 400 in the SAT test are exclusively virtual? Exclusively virtual refers to Virtual = 'F'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 11, + "question": "Please list the codes of the schools with a total enrollment of over 500.", + "evidence": "Total enrollment can be represented by `Enrollment (K-12)` + `Enrollment (Ages 5-17)`", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 7806, + "ai_summarized_answer": "The query returns 7,806 school codes with total enrollment over 500, including codes like 01100170109835, 01100170112607, 01100170124172, and 7,803 more.", + "expected_columns": [ + "Cdscode" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Please list the codes of the schools with a total enrollment of over 500. Total enrollment can be represented by `Enrollment (K-12)` + `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 12, + "question": "Among the schools with an SAT excellence rate of over 0.3, what is the highest eligible free rate for students aged 5-17?", + "evidence": "Excellence rate = NumGE1500 / NumTstTakr; Eligible free rates for students aged 5-17 = `Free Meal Count (Ages 5-17)` / `Enrollment (Ages 5-17)`", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": 0.9049079754601227, + "expected_row_count": 1, + "ai_summarized_answer": "The highest eligible free rate for students aged 5-17 among schools with an SAT excellence rate over 0.3 is approximately 0.905 or 90.5%.", + "expected_columns": [ + "Eligible Free Rate Max" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Among the schools with an SAT excellence rate of over 0.3, what is the highest eligible free rate for students aged 5-17? Excellence rate = NumGE1500 / NumTstTakr; Eligible free rates for students aged 5-17 = `Free Meal Count (Ages 5-17)` / `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 17, + "question": "Rank schools by their average score in Writing where the score is greater than 499, showing their charter numbers.", + "evidence": "Valid charter number means the number is not null", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 58, + "ai_summarized_answer": "The results show 58 schools ranked by writing scores above 499, with the top school (charter 0210) scoring 630 and rank 1, charter 0890 scoring 593 at rank 2, charter 0290 scoring 582 at rank 3, and 55 more schools ranked down to charter 1400 with a score of 501 at rank 58.", + "expected_columns": [ + "Cdscode", + "Charternum", + "Avgscrwrite" + ], + "expected_filter_fields": [ + "Avgscrwrite", + "Charternum" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Rank schools by their average score in Writing where the score is greater than 499, showing their charter numbers. Valid charter number means the number is not null", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 23, + "question": "List the names of schools with more than 30 difference in enrollements between K-12 and ages 5-17? Please also give the full street adress of the schools.", + "evidence": "Diffrence in enrollement = `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1239, + "ai_summarized_answer": "The data shows 1,239 schools with enrollment differences greater than 30, including Alameda County Community at 313 West Winton Avenue, California School for the Deaf-Fremont at 39350 Gallaudet Drive, Alameda High at 2201 Encinal Avenue, and 1,236 more schools.", + "expected_columns": [ + "Cdscode", + "School Name", + "Street" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: List the names of schools with more than 30 difference in enrollements between K-12 and ages 5-17? Please also give the full street adress of the schools. Diffrence in enrollement = `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 24, + "question": "Give the names of the schools with the percent eligible for free meals in K-12 is more than 0.1 and test takers whose test score is greater than or equal to 1500?", + "evidence": "Percent eligible for free meals = Free Meal Count (K-12) / Total (Enrollment (K-12)", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1165, + "ai_summarized_answer": "The results list 1,167 schools meeting both criteria, including FAME Public Charter, Envision Academy for Arts & Technology, Alameda Science and Technology Institute, Alameda High, Alternatives in Action, and 1,162 more.", + "expected_columns": [ + "School Name" + ], + "expected_filter_fields": [ + "Numge1500" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Give the names of the schools with the percent eligible for free meals in K-12 is more than 0.1 and test takers whose test score is greater than or equal to 1500? Percent eligible for free meals = Free Meal Count (K-12) / Total (Enrollment (K-12)", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 25, + "question": "Name schools in Riverside which the average of average math score for SAT is grater than 400, what is the funding type of these schools?", + "evidence": "Average of average math = sum(average math scores) / count(schools).", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 6, + "ai_summarized_answer": "Six schools in Riverside meet the criteria: Arlington High, John W. North High, Martin Luther King Jr. High, Polytechnic High, and Ramona High all have no charter funding (None), while River Springs Charter is directly funded.", + "expected_columns": [ + "Cds", + "School Name", + "Charter Funding Type" + ], + "expected_filter_fields": [ + "District Name", + "Avgscrmath" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Name schools in Riverside which the average of average math score for SAT is grater than 400, what is the funding type of these schools? Average of average math = sum(average math scores) / count(schools).", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 26, + "question": "State the names and full communication address of high schools in Monterey which has more than 800 free or reduced price meals for ages 15-17?", + "evidence": "Full communication address should include Street, City, State and zip code if any.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 5, + "ai_summarized_answer": "Five high schools meet the criteria: Alisal High at 777 Williams Road, Salinas, CA 93905-1907; Everett Alvarez High at 1900 Independence Boulevard, Salinas, CA 93906-5300; North Salinas High at 55 Kip Drive, Salinas, CA 93906-2908; Salinas High at 726 South Main Street, Salinas, CA 93901-3243; and Soledad High at 425 Gabilan Drive, Soledad, CA 93960-3207.", + "expected_columns": [ + "School Name", + "Street", + "City", + "State", + "Zip" + ], + "expected_filter_fields": [ + "County", + "Free Meal Count (Ages 5-17)", + "School Type" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: State the names and full communication address of high schools in Monterey which has more than 800 free or reduced price meals for ages 15-17? Full communication address should include Street, City, State and zip code if any.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 27, + "question": "What is the average score in writing for the schools that were opened after 1991 or closed before 2000? List the school names along with the score. Also, list the communication number of the schools if there is any.", + "evidence": "Communication number refers to phone number.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 8574, + "ai_summarized_answer": "The query returns 8,574 schools with their writing scores and phone numbers. Many schools have null writing scores. Examples include FAME Public Charter with a score of 505.0 and no phone number, Envision Academy for Arts & Technology with 395.0 and phone (510) 596-8901, and 8,572 more schools with varying scores and phone availability.", + "expected_columns": [ + "School", + "Avgscrwrite" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the average score in writing for the schools that were opened after 1991 or closed before 2000? List the school names along with the score. Also, list the communication number of the schools if there is any. Communication number refers to phone number.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 28, + "question": "Consider the average difference between K-12 enrollment and 15-17 enrollment of schools that are locally funded, list the names and DOC type of schools which has a difference above this average.", + "evidence": "Difference between K-12 enrollment and 15-17 enrollment can be computed by `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 57, + "ai_summarized_answer": "The results show 57 locally funded schools with enrollment differences above the average. Examples include Mountain Oaks (DOC 00), Castle Rock (DOC 00), Clovis Online Charter (DOC 54), Washington Elementary (DOC 52), West Park Charter Academy (DOC 52), and 52 more schools with various DOC types.", + "expected_columns": [ + "School", + "Doc" + ], + "expected_filter_fields": [ + "Fundingtype" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Consider the average difference between K-12 enrollment and 15-17 enrollment of schools that are locally funded, list the names and DOC type of schools which has a difference above this average. Difference between K-12 enrollment and 15-17 enrollment can be computed by `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 31, + "question": "What is the eligible free rate of the 10th and 11th schools with the highest enrolment for students in grades 1 through 12?", + "evidence": "K-12 refers to students in grades 1 through 12; Eligible free rate for K-12 = `Free Meal Count (K-12)` / `Enrollment (K-12)`", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 2, + "ai_summarized_answer": "The eligible free rates for the 10th and 11th schools with the highest enrollment are approximately 0.134 (13.4%) and 0.291 (29.1%) respectively.", + "expected_columns": [ + "Cds", + "Percent (%) Eligible Free (K-12)" + ], + "expected_filter_fields": [ + "Cds" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the eligible free rate of the 10th and 11th schools with the highest enrolment for students in grades 1 through 12? K-12 refers to students in grades 1 through 12; Eligible free rate for K-12 = `Free Meal Count (K-12)` / `Enrollment (K-12)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 32, + "question": "What is the eligible free or reduced price meal rate for the top 5 schools in grades 1-12 with the highest free or reduced price meal count of the schools with the ownership code 66?", + "evidence": "grades 1-12 means K-12; Eligible free or reduced price meal rate for K-12 = `FRPM Count (K-12)` / `Enrollment (K-12)`", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 5, + "ai_summarized_answer": "The top 5 schools with ownership code 66 have the following FRPM rates: Paramount High at 91.8%, Calexico High at 99.9%, Bell Senior High at 89.6%, Anaheim High at 89.6%, and Bell Gardens High at 91.4%.", + "expected_columns": [ + "School Name", + "FRPM Rate" + ], + "expected_filter_fields": [ + "Soc", + "Cdscode" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the eligible free or reduced price meal rate for the top 5 schools in grades 1-12 with the highest free or reduced price meal count of the schools with the ownership code 66? grades 1-12 means K-12; Eligible free or reduced price meal rate for K-12 = `FRPM Count (K-12)` / `Enrollment (K-12)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 36, + "question": "Under whose administration is the school with the highest number of students scoring 1500 or more on the SAT? Indicate their full names.", + "evidence": "full name means first name, last name; There are at most 3 administrators for each school; SAT Scores are greater or equal to 1500 refers to NumGE1500", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The school with the highest number of students scoring 1500 or more on the SAT is administered by Michelle King. No additional administrators are listed for her school.", + "expected_columns": [ + "administrator's first name", + "administrator's last name", + "Admfname2", + "Admlname2", + "Admfname3", + "Admlname3", + "Numge1500" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Under whose administration is the school with the highest number of students scoring 1500 or more on the SAT? Indicate their full names. full name means first name, last name; There are at most 3 administrators for each school; SAT Scores are greater or equal to 1500 refers to NumGE1500", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 37, + "question": "What is the complete address of the school with the lowest excellence rate? Indicate the Street, City, Zip and State.", + "evidence": "Execellence Rate = NumGE1500 / NumTstTakr; complete address has Street, City, State, Zip code", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The school with the lowest excellence rate is located at 2125 Jefferson Avenue, Berkeley, CA 94703-1414.", + "expected_columns": [ + "School", + "Street", + "City", + "State", + "Zip" + ], + "expected_filter_fields": [ + "Street" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the complete address of the school with the lowest excellence rate? Indicate the Street, City, Zip and State. Execellence Rate = NumGE1500 / NumTstTakr; complete address has Street, City, State, Zip code", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 39, + "question": "What is the average number of test takers from Fresno schools that opened between 1/1/1980 and 12/31/1980?", + "evidence": "between 1/1/1980 and 12/31/1980 means the year = 1980", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": 137.88888888888889, + "expected_row_count": 1, + "ai_summarized_answer": "The average number of test takers from Fresno schools that opened in 1980 is approximately 137.89.", + "expected_columns": [ + "Numtsttakr" + ], + "expected_filter_fields": [ + "County", + "Opendate" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the average number of test takers from Fresno schools that opened between 1/1/1980 and 12/31/1980? between 1/1/1980 and 12/31/1980 means the year = 1980", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 40, + "question": "What is the telephone number for the school with the lowest average score in reading in Fresno Unified?", + "evidence": "Fresno Unified is a name of district;", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": "(559) 248-5100", + "expected_row_count": 1, + "ai_summarized_answer": "The phone number for the Fresno Unified school with the lowest average reading score is (559) 248-5100.", + "expected_columns": [ + "Phone", + "Avgscrread" + ], + "expected_filter_fields": [ + "District", + "Avgscrread" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the telephone number for the school with the lowest average score in reading in Fresno Unified? Fresno Unified is a name of district;", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 41, + "question": "List the names of virtual schools that are among the top 5 in their respective counties based on average reading scores.", + "evidence": "Exclusively virtual refers to Virtual = 'F'; respective counties means PARTITION BY County", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 34, + "ai_summarized_answer": "The results show 34 virtual schools that rank in the top 5 within their respective counties. Examples include Academy of Arts and Sciences: Fresno, Dunlap Leadership Academy, Insight School of California, California Virtual Academy @ Kings, National University Academy, Armona, and 29 more schools.", + "expected_columns": [], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: List the names of virtual schools that are among the top 5 in their respective counties based on average reading scores. Exclusively virtual refers to Virtual = 'F'; respective counties means PARTITION BY County", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 45, + "question": "What is the average writing score of each of the schools managed by Ricci Ulrich? List the schools and the corresponding average writing scores.", + "evidence": "Usually, administrators manage the school stuff.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "There is one school managed by Ricci Ulrich: Buchanan High with an average writing score of 507.", + "expected_columns": [ + "School", + "Avgscrwrite" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the average writing score of each of the schools managed by Ricci Ulrich? List the schools and the corresponding average writing scores. Usually, administrators manage the school stuff.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 46, + "question": "Which state special schools have the highest number of enrollees from grades 1 through 12?", + "evidence": "State Special Schools refers to DOC = 31; Grades 1 through 12 means K-12", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": "California School for the Deaf-Fremont", + "expected_row_count": 1, + "ai_summarized_answer": "The state special school with the highest K-12 enrollment is California School for the Deaf-Fremont.", + "expected_columns": [ + "School", + "Enrollment (K-12)" + ], + "expected_filter_fields": [ + "Doc" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Which state special schools have the highest number of enrollees from grades 1 through 12? State Special Schools refers to DOC = 31; Grades 1 through 12 means K-12", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 47, + "question": "What is the monthly average number of schools that opened in Alameda County under the jurisdiction of the Elementary School District in 1980?", + "evidence": "Elementary School District refers to DOC = 52; Monthly average number of schools that opened in 1980 = count(schools that opened in 1980) / 12", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": 1.4166666666666667, + "expected_row_count": 1, + "ai_summarized_answer": "The monthly average number of Elementary School District schools that opened in Alameda County in 1980 is approximately 1.42 schools per month.", + "expected_columns": [ + "Monthly Average Schools Opened" + ], + "expected_filter_fields": [ + "Doc", + "County", + "Opendate" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the monthly average number of schools that opened in Alameda County under the jurisdiction of the Elementary School District in 1980? Elementary School District refers to DOC = 52; Monthly average number of schools that opened in 1980 = count(schools that opened in 1980) / 12", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 48, + "question": "What is the ratio of merged Unified School District schools in Orange County to merged Elementary School District schools?", + "evidence": "Elementary School District refers to DOC = 52; Unified School District refers to DOC = 54.", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": 0.5714285714285714, + "expected_row_count": 1, + "ai_summarized_answer": "The ratio of merged Unified School District schools to merged Elementary School District schools in Orange County is approximately 0.571, meaning there are about 0.57 unified schools for every elementary school.", + "expected_columns": [ + "Ratio Unified to Elementary" + ], + "expected_filter_fields": [ + "Statustype", + "County" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the ratio of merged Unified School District schools in Orange County to merged Elementary School District schools? Elementary School District refers to DOC = 52; Unified School District refers to DOC = 54.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 50, + "question": "What is the postal street address for the school with the 7th highest Math average? Indicate the school's name.", + "evidence": "Postal street and mailing street are synonyms.", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The query returns the top 6 schools by math average. The 7th highest would be Saratoga High at 20300 Herriman Avenue (based on the ordering shown).", + "expected_columns": [ + "School", + "Mailstreet", + "Avgscrmath" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the postal street address for the school with the 7th highest Math average? Indicate the school's name. Postal street and mailing street are synonyms.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 62, + "question": "What is the total number of non-chartered schools in the county of Los Angeles with a percent (%) of eligible free meals for grades 1 through 12 that is less than 0.18%?", + "evidence": "non-chartered schools refer to schools whose Charter = 0; K-12 means grades 1 through 12; percent of eligible free rate for K-12 = `Free Meal Count (K-12)` * 100 / `Enrollment (K-12)`", + "difficulty": "challenging", + "answer_type": "scalar", + "expected_value": 1, + "expected_row_count": 1, + "ai_summarized_answer": "There is 1 non-chartered school in Los Angeles County with an eligible free meal rate less than 0.18%.", + "expected_columns": [ + "School" + ], + "expected_filter_fields": [ + "County", + "Charter" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the total number of non-chartered schools in the county of Los Angeles with a percent (%) of eligible free meals for grades 1 through 12 that is less than 0.18%? non-chartered schools refer to schools whose Charter = 0; K-12 means grades 1 through 12; percent of eligible free rate for K-12 = `Free Meal Count (K-12)` * 100 / `Enrollment (K-12)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 72, + "question": "How many students from the ages of 5 to 17 are enrolled at the State Special School school in Fremont for the 2014-2015 academic year?", + "evidence": "State Special School means EdOpsCode = 'SSS'", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 2, + "ai_summarized_answer": "There are two State Special Schools in Fremont for the 2014-2015 academic year with enrollments of 40.0 and 335.0 students aged 5-17.", + "expected_columns": [ + "Enrollment (Ages 5-17)" + ], + "expected_filter_fields": [ + "Edopscode", + "City", + "Academic Year" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: How many students from the ages of 5 to 17 are enrolled at the State Special School school in Fremont for the 2014-2015 academic year? State Special School means EdOpsCode = 'SSS'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 77, + "question": "Which schools served a grade span of Kindergarten to 9th grade in the county of Los Angeles and what is its Percent (%) Eligible FRPM (Ages 5-17)?", + "evidence": "Percent (%) Eligible FRPM (Ages 5-17) can be acquired by `FRPM Count (Ages 5-17)` / `Enrollment (Ages 5-17)` * 100", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 2, + "ai_summarized_answer": "Two schools in Los Angeles County serve grades K-9: White Oak Elementary with a 3.76% eligible FRPM rate and The Accelerated with a 97.64% eligible FRPM rate.", + "expected_columns": [ + "School", + "Percent Eligible FRPM (Ages 5-17)" + ], + "expected_filter_fields": [ + "County", + "Gsserved" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Which schools served a grade span of Kindergarten to 9th grade in the county of Los Angeles and what is its Percent (%) Eligible FRPM (Ages 5-17)? Percent (%) Eligible FRPM (Ages 5-17) can be acquired by `FRPM Count (Ages 5-17)` / `Enrollment (Ages 5-17)` * 100", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 79, + "question": "Between San Diego and Santa Barbara, which county offers the most number of schools that does not offer physical building? Indicate the amount.", + "evidence": "'Does not offer physical building' means Virtual = F in the database.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "San Diego County offers the most virtual schools (those without physical buildings) with a count of 8 schools.", + "expected_columns": [ + "County", + "Virtual" + ], + "expected_filter_fields": [ + "County", + "Virtual" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Between San Diego and Santa Barbara, which county offers the most number of schools that does not offer physical building? Indicate the amount. 'Does not offer physical building' means Virtual = F in the database.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 82, + "question": "What is the grade span offered in the school with the highest longitude?", + "evidence": "the highest longitude refers to the school with the maximum absolute longitude value.", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": "K-8", + "expected_row_count": 1, + "ai_summarized_answer": "The school with the highest absolute longitude value has a null grade span (no grade span information available). The next highest is K-8 with an absolute longitude of 124.28481", + "expected_columns": [ + "Gsoffered", + "Max absolute longitude" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the grade span offered in the school with the highest longitude? the highest longitude refers to the school with the maximum absolute longitude value.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 83, + "question": "Of the schools that offers a magnet program serving a grade span of Kindergarten to 8th grade, how many offers Multiple Provision Types? List the number of cities that offers a Kindergarten to 8th grade span and indicate how many schools are there serving such grade span for each city.", + "evidence": "Kindergarten to 8th grade refers to K-8; 'Offers a magnet program' means Magnet = 1; Multiple Provision Types refers to `NSLP Provision Status` = 'Multiple Provision Types'", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "There is 1 city (Adelanto) with a K-8 magnet school offering Multiple Provision Types, and it has 1 such school.", + "expected_columns": [ + "City", + "Cdscode" + ], + "expected_filter_fields": [ + "Magnet", + "Gsoffered", + "NSLP Provision Status" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Of the schools that offers a magnet program serving a grade span of Kindergarten to 8th grade, how many offers Multiple Provision Types? List the number of cities that offers a Kindergarten to 8th grade span and indicate how many schools are there serving such grade span for each city. Kindergarten to 8th grade refers to K-8; 'Offers a magnet program' means Magnet = 1; Multiple Provision Types refers to `NSLP Provision Status` = 'Multiple Provision Types'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 85, + "question": "What is the Percent (%) Eligible Free (K-12) in the school administered by an administrator whose first name is Alusine. List the district code of the school.", + "evidence": "Percent (%) Eligible Free (K-12) = `Free Meal Count (K-12)` / `Enrollment (K-12)` * 100%", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The school administered by Alusine has a percent eligible free rate of approximately 70.15% for K-12 students, and the district code is 64857.", + "expected_columns": [ + "District Code", + "Percent Eligible Free (K-12)" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the Percent (%) Eligible Free (K-12) in the school administered by an administrator whose first name is Alusine. List the district code of the school. Percent (%) Eligible Free (K-12) = `Free Meal Count (K-12)` / `Enrollment (K-12)` * 100%", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 87, + "question": "What are the valid e-mail addresses of the administrator of the school located in the San Bernardino county, City of San Bernardino City Unified that opened between 1/1/2009 to 12/31/2010 whose school types are public Intermediate/Middle Schools and Unified Schools?", + "evidence": "Intermediate/Middle Schools refers to SOC = 62; Unified School refers to DOC = 54; years between 2009 and 2010 can refer to 'between 1/1/2009 to 12/31/2010'", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The administrator email addresses for the qualifying school are a.lucero@realjourney.org and j.hernandez@realjourney.org.", + "expected_columns": [ + "Admemail1", + "Admemail2", + "Admemail3" + ], + "expected_filter_fields": [ + "County", + "City", + "Doc", + "Soc", + "Opendate" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What are the valid e-mail addresses of the administrator of the school located in the San Bernardino county, City of San Bernardino City Unified that opened between 1/1/2009 to 12/31/2010 whose school types are public Intermediate/Middle Schools and Unified Schools? Intermediate/Middle Schools refers to SOC = 62; Unified School refers to DOC = 54; years between 2009 and 2010 can refer to 'between 1/1/2009 to 12/31/2010'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + } +] diff --git a/package.json b/package.json index 49816b403..30f50aeb5 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,11 @@ "test": "vitest --config ./vitest.config.ts", "test:e2e": "vitest --config ./vitest.config.e2e.ts", "test:eval": "vitest --config ./vitest.config.eval.ts", + "eval:claude": "tsx evals/run-case.ts", + "eval:grade": "tsx evals/grade.ts", + "eval:suite": "tsx evals/run-suite.ts", + "eval:grade:bird": "tsx evals/grade-bird.ts", + "eval:grade:suite": "tsx evals/grade-suite.ts", "test:oauth:embedded": "vitest --config ./vitest.config.oauth.embedded.ts", "test:oauth:tableau": "npx playwright test", "coverage": "vitest run --config ./vitest.config.ts --coverage", From f25b8dbe8f3159f8b413a925652d39833755b8b1 Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Fri, 15 May 2026 14:53:40 -0700 Subject: [PATCH 2/7] version bump --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 14e71054b..10cb50a7c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@tableau/mcp-server", - "version": "2.2.4", + "version": "2.2.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@tableau/mcp-server", - "version": "2.2.4", + "version": "2.2.5", "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/sdk": "^1.26.0", diff --git a/package.json b/package.json index 30f50aeb5..6b10c423b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@tableau/mcp-server", "description": "Helping agents see and understand data.", - "version": "2.2.4", + "version": "2.2.5", "repository": { "type": "git", "url": "git+https://github.com/tableau/tableau-mcp.git" From 1544a53442368ff4ab90028757a46e0fc2418f5a Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Fri, 15 May 2026 15:27:32 -0700 Subject: [PATCH 3/7] docs update --- docs/docs/evaluation/_category_.json | 8 + docs/docs/evaluation/claude-code-harness.md | 219 ++++++++++++++++++++ docs/package-lock.json | 33 ++- 3 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 docs/docs/evaluation/_category_.json create mode 100644 docs/docs/evaluation/claude-code-harness.md diff --git a/docs/docs/evaluation/_category_.json b/docs/docs/evaluation/_category_.json new file mode 100644 index 000000000..215419487 --- /dev/null +++ b/docs/docs/evaluation/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Evaluation", + "position": 8, + "link": { + "type": "generated-index", + "description": "Tools and benchmarks for evaluating the accuracy of Claude Code against the Tableau MCP server." + } +} diff --git a/docs/docs/evaluation/claude-code-harness.md b/docs/docs/evaluation/claude-code-harness.md new file mode 100644 index 000000000..fc23bfcb0 --- /dev/null +++ b/docs/docs/evaluation/claude-code-harness.md @@ -0,0 +1,219 @@ +--- +sidebar_position: 1 +--- + +# Claude Code Eval Harness + +The Tableau MCP eval harness runs [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) as a live agent against the Tableau MCP server and grades its responses against known-correct answers. It is designed to measure query accuracy — not simulated tool calls, with real API round-trips to a live Tableau Cloud or Server site. + +The primary benchmark is 30 questions from the [BIRD Mini-Dev](https://bird-bench.github.io/) dataset scoped to the California Schools database. + +--- + +## Prerequisites + +### Required + +- **Claude Code** installed and available on your `PATH` (`claude --version` should work) +- **A published California Schools datasource** on your Tableau Cloud or Server site. This is a single published datasource that joins the three California Schools source tables (`schools`, `frpm`, `satscores`). You will need its LUID. A .tdsx for this data source is provided at within the project at `evals/bird_mini/data/tableau_datasources/california_schools`. +- **Tableau credentials** — any supported auth method (PAT, OAuth, direct trust). See [Authentication](/docs/configuration/mcp-config/authentication) for setup. + +### Optional + +- **LangSmith account** — for trace visualization. Without it, the harness still runs and grades locally; traces are simply not posted. +- **OpenAI API key** — for semantic grading (LLM judge). Without it, the harness falls back to numeric-only grading. + +--- + +## Environment Variables + +Copy `env.example.list` to `.env` at the repo root and fill in the values below. + +### Required + +| Variable | Description | +|---|---| +| `SERVER` | Your Tableau Cloud or Server URL (e.g. `https://10ax.online.tableau.com`) | +| `SITE_NAME` | Your Tableau site name | +| `PAT_NAME` | Personal Access Token name | +| `PAT_VALUE` | Personal Access Token secret | +| `EVAL_DATASOURCE_LUID` | LUID of the published California Schools datasource. Find it in Tableau by opening the datasource and copying the ID from the URL. | + +### Optional — LangSmith + +| Variable | Default | Description | +|---|---|---| +| `LANGSMITH_API_KEY` | — | Your LangSmith API key. Get one at [smith.langchain.com](https://smith.langchain.com) → Settings → API Keys. If not set, traces are not posted. | +| `LANGSMITH_PROJECT` | `tableau-mcp-evals` | LangSmith project to post traces to. | + +### Optional — Grading + +| Variable | Default | Description | +|---|---|---| +| `OPENAI_API_KEY` | — | OpenAI API key for the LLM judge. If not set, `semantic_match` is skipped and the verdict is based on `numeric_match` only. | +| `BIRD_GRADE_MODEL` | `gpt-4o-mini` | Model used by the LLM judge. Override with e.g. `gpt-4o` for higher-quality grading. | + +--- + +## Running the Harness + +### Run the full BIRD suite (30 questions) + +```bash +npm run eval:suite +``` + +### Run a filtered subset + +```bash +npm run eval:suite -- --difficulty simple # only the 8 "simple" questions +npm run eval:suite -- --difficulty moderate # only "moderate" questions +npm run eval:suite -- --ids 5,11,12 # specific question IDs +``` + +Difficulty levels map to the BIRD benchmark's own classifications: `simple`, `moderate`, and `challenging`. + +### Run a single question by ID + +```bash +npm run eval:suite -- --ids 5 +``` + +### Run an ad hoc custom question + +Pass any natural language prompt directly, without a suite file: + +```bash +npm run eval:claude -- input "How many schools have an average SAT math score above 400?" +``` + +Ad hoc runs use the same Claude Code + Tableau MCP setup as suite runs and produce the same local artifacts. They are graded by `grade.ts` (tool coverage only), not `grade-bird.ts`, because there is no precomputed expected answer to compare against. + +--- + +## Grading + +### Grade a full suite run + +After `eval:suite` completes, grade every case at once: + +```bash +npm run eval:grade:suite # auto-discovers most recent +npm run eval:grade:suite -- evals/suite-runs/YYYY-MM-DD/ # explicit path +``` + +This writes a single `suite-grade.json` to `evals/grades/YYYY-MM-DD//`. + +### Grade a single run + +```bash +npm run eval:grade:bird -- evals/runs/YYYY-MM-DD/ +``` + +Output goes to `evals/grades/YYYY-MM-DD//bird-result.json`. + +--- + +## Grading Signals and Metrics + +Each run is evaluated on four signals. The **verdict** is determined by the two outcome signals only. The structural signals are recorded for debugging purposes and do not affect the verdict. + +### Outcome signals (drive the verdict) + +| Signal | Method | Description | +|---|---|---| +| `numeric_match` | Code | The expected numeric value or row count appears in Claude's final message. Integer matches are exact; float matches allow ±1% tolerance. String answers (e.g. school names) use case-insensitive substring matching. | +| `semantic_match` | LLM judge | GPT-4o-mini (or `BIRD_GRADE_MODEL`) scores Claude's final message against the gold answer summary on a 0–1 scale. A score ≥ 0.8 passes. Requires `OPENAI_API_KEY`. | + +### Structural signals (informational) + +| Signal | Method | Description | +|---|---|---| +| `columns_match` | Tool call inspection | The expected VizQL field captions are present in the `query-datasource` call. Extra columns are allowed; only missing required columns count against this signal. | +| `filters_match` | Tool call inspection | The expected filter field captions are present in the `query-datasource` call. Same subset-matching logic as `columns_match`. | + +### Verdicts + +| Verdict | Meaning | +|---|---| +| `pass` | Both `numeric_match` and `semantic_match` passed | +| `partial` | One of the two outcome signals passed (or one was unavailable) | +| `fail` | Both outcome signals failed | +| `error` | Claude Code exited with a non-zero code | +| `skip` | Both outcome signals were unavailable (e.g. no `OPENAI_API_KEY` and no numeric result to match) | + +--- + +## Metrics Captured Per Run + +Each case run records the following, available in `bird-result.json` (individual) or `suite-grade.json` (aggregated): + +| Metric | Source | Notes | +|---|---|---| +| `wall_s` | Process timing | Seconds from `claude -p` invocation to process exit | +| `tool_calls` | `hook.jsonl` | Total number of tool calls made | +| `tools_used` | `hook.jsonl` | Deduplicated list of tool names called | +| `model` | Claude stream | Model name reported by the Claude stream | +| `tokens.input_tokens` | Claude stream | Summed across all assistant messages | +| `tokens.output_tokens` | Claude stream | Summed across all assistant messages | +| `tokens.cache_creation_tokens` | Claude stream | Prompt cache write tokens | +| `tokens.cache_read_tokens` | Claude stream | Prompt cache read tokens | +| `tokens.total_context_tokens` | Computed | `input + cache_creation + cache_read` | + +--- + +## Output Directories + +All output is written locally and gitignored. Directories are organized by date. + +``` +evals/ + runs/ + YYYY-MM-DD/ + / one folder per case run + run.json metadata, timing, exit code + agent-output.jsonl full Claude Code stream + hook.jsonl one record per tool call + pre-tool-times.jsonl tool dispatch timestamps + stop.json stop hook data + logs/ MCP server logs + + suite-runs/ + YYYY-MM-DD/ + / + suite-summary.json aggregate timing and token counts for all cases in the run + + grades/ + YYYY-MM-DD/ + / + bird-result.json per-case grading output + / + suite-grade.json aggregate grading output for a full suite run +``` + +--- + +## LangSmith Integration + +When `LANGSMITH_API_KEY` is set, each case run posts a full trace to LangSmith in real time via Claude Code hooks. Each trace contains: + +- One parent span covering the full Claude Code process (wall time) +- A synthetic `initialization` span for Claude startup and MCP server negotiation +- One `assistant-message-N` span per LLM turn (timing inferred from the Claude stream) +- One tool span per tool call with accurate start/end times from `PreToolUse` and `PostToolUse` hooks + +Traces are posted to the project set by `LANGSMITH_PROJECT` (default: `tableau-mcp-evals`). + +--- + +## About the BIRD Dataset + +The BIRD (BIg Bench for laRge-scale Database Grounded Text-to-SQL Evaluation) Mini-Dev benchmark is a standard text-to-SQL evaluation dataset. The 30 California Schools questions used here were selected because the underlying database can be represented as a single Tableau published datasource that joins three source tables. + +The suite file at `evals/suites/bird-california-schools.json` ships with precomputed expected answers (row counts, scalar values, and gold answer summaries) so no database access is required to run grading. To regenerate it from the raw BIRD SQLite snapshot: + +```bash +python3 evals/scripts/precompute-bird-answers.py +``` + +This requires the California Schools SQLite file at `evals/bird_mini/data/dev_databases/california_schools/california_schools.sqlite`. diff --git a/docs/package-lock.json b/docs/package-lock.json index ab1d1e1f2..0eb697fc8 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -233,6 +233,7 @@ "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.38.0.tgz", "integrity": "sha512-PTAFMJOpVtJweExEYYgdmSCC6n4V/R+ctDL3fRQy77ulZM/p+zMLIQC9c7HCQE1zqpauvVck3f2zYSejaUTtrw==", "license": "MIT", + "peer": true, "dependencies": { "@algolia/client-common": "5.38.0", "@algolia/requester-browser-xhr": "5.38.0", @@ -380,6 +381,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.4.tgz", "integrity": "sha512-2BCOP7TN8M+gVDj7/ht3hsaO/B/n5oDbiAyyvnRlNOs+u1o+JWNYTQrmpuNp1/Wq2gcFrI01JAW+paEKDMx/CA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.3", @@ -2214,6 +2216,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -2236,6 +2239,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -2345,6 +2349,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -2766,6 +2771,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -3629,6 +3635,7 @@ "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.9.0.tgz", "integrity": "sha512-PP+iDJg+lj4cn/7GbbmiguaQ8OX08YxnzQ17KqRC4ufJm11jdyXD33wA7vVtbeG/BkkgkiB/K7YyPHCPwmfVhg==", "license": "MIT", + "peer": true, "dependencies": { "@docusaurus/core": "3.9.0", "@docusaurus/logger": "3.9.0", @@ -4381,6 +4388,7 @@ "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-3.1.1.tgz", "integrity": "sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==", "license": "MIT", + "peer": true, "dependencies": { "@types/mdx": "^2.0.0" }, @@ -4708,6 +4716,7 @@ "resolved": "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz", "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.21.3", "@svgr/babel-preset": "8.1.0", @@ -5338,6 +5347,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.12.tgz", "integrity": "sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -5677,6 +5687,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5762,6 +5773,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -5807,6 +5819,7 @@ "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.38.0.tgz", "integrity": "sha512-8VJKIzheeI9cjuVJhU1hYEVetOTe7LvA+CujAI7yqvYsPtZfVEvv1pg9AeFNtHBg/ZoSLGU5LPijhcY5l3Ea9g==", "license": "MIT", + "peer": true, "dependencies": { "@algolia/abtesting": "1.4.0", "@algolia/client-abtesting": "5.38.0", @@ -6270,6 +6283,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.3", "caniuse-lite": "^1.0.30001741", @@ -6569,6 +6583,7 @@ "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-11.0.3.tgz", "integrity": "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@chevrotain/cst-dts-gen": "11.0.3", "@chevrotain/gast": "11.0.3", @@ -7267,6 +7282,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -7586,6 +7602,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.1.tgz", "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -8007,6 +8024,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -9190,6 +9208,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -13825,6 +13844,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -14385,6 +14405,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -15288,6 +15309,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -16098,6 +16120,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.1.1.tgz", "integrity": "sha512-w8nqGImo45dmMIfljjMwOGtbmC/mk4CMYhWIicdSflH91J9TyCyczcPFXJzrZ/ZXcgGRFeP6BU0BEJTw6tZdfQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -16107,6 +16130,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.1.1.tgz", "integrity": "sha512-Dlq/5LAZgF0Gaz6yiqZCf6VCcZs1ghAJyrsu84Q/GT0gV+mCxbfmKNoGRKBYMJ8IEdGPqu49YWXD02GCknEDkw==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.26.0" }, @@ -16162,6 +16186,7 @@ "resolved": "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-6.0.0.tgz", "integrity": "sha512-YMMxTUQV/QFSnbgrP3tjDzLHRg7vsbMn8e9HAa8o/1iXoiomo48b7sk/kkmWEuWNDPJVlKSJRB6Y2fHqdJk+SQ==", "license": "MIT", + "peer": true, "dependencies": { "@types/react": "*" }, @@ -16190,6 +16215,7 @@ "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz", "integrity": "sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.12.13", "history": "^4.9.0", @@ -18010,7 +18036,8 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "license": "0BSD", + "peer": true }, "node_modules/type-fest": { "version": "2.19.0", @@ -18073,6 +18100,7 @@ "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -18420,6 +18448,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -18676,6 +18705,7 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.101.3.tgz", "integrity": "sha512-7b0dTKR3Ed//AD/6kkx/o7duS8H3f1a4w3BYpIriX4BzIhjkn4teo05cptsxvLesHFKK5KObnadmCHBwGc+51A==", "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -19262,6 +19292,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.11.tgz", "integrity": "sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } From 18137bfb2089d2763793b164b15fd306d95ac9f5 Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Fri, 15 May 2026 16:03:15 -0700 Subject: [PATCH 4/7] linting fixes --- evals/grade-bird.ts | 20 +++++++++++--------- evals/grade-suite.ts | 36 +++++++++++++++++++++--------------- evals/run-case.ts | 4 +--- evals/run-suite.ts | 26 +++++++++++++------------- 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/evals/grade-bird.ts b/evals/grade-bird.ts index 0f4713777..91d531cba 100644 --- a/evals/grade-bird.ts +++ b/evals/grade-bird.ts @@ -293,7 +293,9 @@ function extractFinalMessage(agentOutputJsonl: string): string { for (let i = assistantEvents.length - 1; i >= 0; i--) { const content = assistantEvents[i].message?.content; if (Array.isArray(content)) { - const textBlocks = content.filter((b) => b.type === 'text' && b.text).map((b) => b.text ?? ''); + const textBlocks = content + .filter((b) => b.type === 'text' && b.text) + .map((b) => b.text ?? ''); if (textBlocks.length > 0) return textBlocks.join('\n'); } if (typeof content === 'string' && content.trim()) return content; @@ -305,7 +307,9 @@ function extractFinalMessage(agentOutputJsonl: string): string { /** Extract all query-datasource tool inputs from hook.jsonl. */ function extractQueryDatasourceInputs(hookRecords: Array): Array { return hookRecords - .filter((r) => normalizeToolName(r.normalized_tool_name ?? r.tool_name ?? '') === 'query-datasource') + .filter( + (r) => normalizeToolName(r.normalized_tool_name ?? r.tool_name ?? '') === 'query-datasource', + ) .map((r) => r.tool_input as QueryDatasourceInput) .filter(Boolean); } @@ -330,10 +334,7 @@ function collectFilterCaptions(filters: Array | undefined): Array { expected_filter_fields: birdCase.expected_filter_fields, actual_filter_fields: actualFilterFields, missing_filter_fields: missingFilterFields, - expected_value: - birdCase.answer_type === 'scalar' ? birdCase.expected_value : null, + expected_value: birdCase.answer_type === 'scalar' ? birdCase.expected_value : null, expected_row_count: birdCase.expected_row_count, extracted_number: extractedNumber, final_message_preview: finalMessage.slice(0, 500), @@ -623,7 +623,9 @@ async function main(): Promise { const s = birdResult.signals; console.log(`\nGrade: Q${questionId} (${birdResult.difficulty})`); console.log(`Verdict: ${v}`); - console.log(`numeric_match: ${s.numeric_match === null ? 'n/a' : s.numeric_match ? 'YES' : 'NO'}`); + console.log( + `numeric_match: ${s.numeric_match === null ? 'n/a' : s.numeric_match ? 'YES' : 'NO'}`, + ); console.log( `semantic_match: ${s.semantic_match === null ? 'n/a' : s.semantic_match.toFixed(2)}` + (llmJudge ? ` — ${llmJudge.reason}` : llmJudgeError ? ` (${llmJudgeError})` : ''), diff --git a/evals/grade-suite.ts b/evals/grade-suite.ts index 12c53aaea..6dd030b0b 100644 --- a/evals/grade-suite.ts +++ b/evals/grade-suite.ts @@ -123,9 +123,13 @@ function findMostRecentSuiteRunDir(): string { const sub = path.join(SUITE_RUNS_DIR, entry.name); // Support both flat (legacy) and date-nested layouts - const candidates = [sub, ...fs.readdirSync(sub, { withFileTypes: true }) - .filter((e) => e.isDirectory()) - .map((e) => path.join(sub, e.name))]; + const candidates = [ + sub, + ...fs + .readdirSync(sub, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => path.join(sub, e.name)), + ]; for (const dir of candidates) { const summaryPath = path.join(dir, 'suite-summary.json'); @@ -148,9 +152,7 @@ function findMostRecentSuiteRunDir(): string { async function main(): Promise { const suiteRunDirArg = process.argv[2]; - const suiteRunDir = suiteRunDirArg - ? path.resolve(suiteRunDirArg) - : findMostRecentSuiteRunDir(); + const suiteRunDir = suiteRunDirArg ? path.resolve(suiteRunDirArg) : findMostRecentSuiteRunDir(); const summaryPath = path.join(suiteRunDir, 'suite-summary.json'); if (!fs.existsSync(summaryPath)) { @@ -196,12 +198,18 @@ async function main(): Promise { } const verdict = gradeResult?.verdict ?? 'grading_error'; - const verdictLabel = verdict === 'pass' ? '✓ PASS' - : verdict === 'partial' ? '~ PARTIAL' - : verdict === 'fail' ? '✗ FAIL' - : verdict === 'error' ? '! ERROR' - : verdict === 'skip' ? '- SKIP' - : '? GRADING_ERROR'; + const verdictLabel = + verdict === 'pass' + ? '✓ PASS' + : verdict === 'partial' + ? '~ PARTIAL' + : verdict === 'fail' + ? '✗ FAIL' + : verdict === 'error' + ? '! ERROR' + : verdict === 'skip' + ? '- SKIP' + : '? GRADING_ERROR'; process.stdout.write(`${verdictLabel}\n`); @@ -265,9 +273,7 @@ async function main(): Promise { console.log('\n═══════════════════════════════════════'); console.log(`Suite grade: ${suite_run_id}`); - console.log( - ` Pass rate: ${counts.pass}/${total} (${(passRate * 100).toFixed(1)}%)`, - ); + console.log(` Pass rate: ${counts.pass}/${total} (${(passRate * 100).toFixed(1)}%)`); console.log( ` pass=${counts.pass} partial=${counts.partial} fail=${counts.fail}` + ` error=${counts.error} skip=${counts.skip}` + diff --git a/evals/run-case.ts b/evals/run-case.ts index 55a7c12f5..68961cf8b 100644 --- a/evals/run-case.ts +++ b/evals/run-case.ts @@ -489,9 +489,7 @@ function buildToolTimeAnchors(events: Array): Map( - fs.readFileSync(preToolPath, 'utf-8'), - ) + ? parseJsonl<{ ts: string; tool_use_id: string }>(fs.readFileSync(preToolPath, 'utf-8')) : []; const dispatchTimes = new Map(); for (const r of preToolRecords) { diff --git a/evals/run-suite.ts b/evals/run-suite.ts index e398b3c80..4fb8cdc9f 100644 --- a/evals/run-suite.ts +++ b/evals/run-suite.ts @@ -200,7 +200,9 @@ function main(): void { } if (!process.env.EVAL_DATASOURCE_LUID) { - console.error('EVAL_DATASOURCE_LUID must be set to the LUID of the published datasource to evaluate against.'); + console.error( + 'EVAL_DATASOURCE_LUID must be set to the LUID of the published datasource to evaluate against.', + ); process.exit(1); } @@ -243,7 +245,9 @@ function main(): void { const birdCase = cases[i]; const caseLabel = `Q${birdCase.question_id} (${birdCase.difficulty}) [${i + 1}/${cases.length}]`; console.log(`\n─── ${caseLabel} ───`); - console.log(` ${birdCase.question.slice(0, 80)}${birdCase.question.length > 80 ? '...' : ''}`); + console.log( + ` ${birdCase.question.slice(0, 80)}${birdCase.question.length > 80 ? '...' : ''}`, + ); const evalCaseFile = buildEvalCaseFile(birdCase, absoluteSuitePath); const caseFilePath = path.join(casesDir, `case-${birdCase.question_id}.json`); @@ -257,16 +261,12 @@ function main(): void { let spawnError: string | null = null; try { - execFileSync( - 'npx', - ['tsx', runCaseScript, caseFilePath, '--run-id', runId], - { - env: process.env, - cwd: REPO_ROOT, - timeout: birdCase.budget.max_wall_ms + 30_000, - stdio: 'pipe', - }, - ); + execFileSync('npx', ['tsx', runCaseScript, caseFilePath, '--run-id', runId], { + env: process.env, + cwd: REPO_ROOT, + timeout: birdCase.budget.max_wall_ms + 30_000, + stdio: 'pipe', + }); } catch (error: unknown) { const e = error as { message?: string }; spawnError = e.message ?? 'unknown spawn error'; @@ -329,7 +329,7 @@ function main(): void { console.log(` Total wall time: ${(totalWallMs / 1000).toFixed(1)}s`); console.log(` Total tokens: ${totalInput} in / ${totalOutput} out`); console.log(` Summary: ${summaryPath}`); - console.log(`\nTo grade cases:`); + console.log('\nTo grade cases:'); for (const r of results) { console.log(` npx tsx evals/grade-bird.ts ${r.run_dir}`); } From 7baa93dc097ec48ed9ef94f6ddfa55d20c645017 Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Mon, 20 Jul 2026 14:50:11 -0700 Subject: [PATCH 5/7] case run fixes --- evals/run-case.ts | 2 +- evals/run-suite.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/run-case.ts b/evals/run-case.ts index 68961cf8b..ae532081e 100644 --- a/evals/run-case.ts +++ b/evals/run-case.ts @@ -250,7 +250,7 @@ if (!finalLangSmithUpdate.ok) { console.warn(`Final LangSmith update failed: ${finalLangSmithUpdate.error}`); } console.log(`Run dir: ${runDir}`); -console.log(`To grade locally: npx tsx evals/grade.ts ${runDir}`); +console.log(`To grade locally: npx tsx ${path.join(EVALS_DIR, 'grade.ts')} ${runDir}`); function getArgValue(flag: string): string | undefined { const index = args.indexOf(flag); diff --git a/evals/run-suite.ts b/evals/run-suite.ts index 4fb8cdc9f..ad7ea277a 100644 --- a/evals/run-suite.ts +++ b/evals/run-suite.ts @@ -331,7 +331,7 @@ function main(): void { console.log(` Summary: ${summaryPath}`); console.log('\nTo grade cases:'); for (const r of results) { - console.log(` npx tsx evals/grade-bird.ts ${r.run_dir}`); + console.log(` npx tsx ${path.join(EVALS_DIR, 'grade-bird.ts')} ${r.run_dir}`); } } From 6ac90cdba9a87c8c9920c084edd45967b791926e Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Tue, 21 Jul 2026 21:13:06 -0700 Subject: [PATCH 6/7] make runner and grader coding agent agnostic --- .gitignore | 1 + evals/README.md | 279 +++++------ evals/adapters/claude-code.ts | 161 +++++++ evals/adapters/codex.ts | 195 ++++++++ evals/adapters/cursor.ts | 168 +++++++ evals/adapters/index.ts | 35 ++ evals/adapters/run-headless.ts | 70 +++ evals/adapters/types.ts | 115 +++++ evals/grade-bird.ts | 668 +++++++++++--------------- evals/grade-suite.ts | 84 +++- evals/grade.ts | 192 ++++---- evals/hooks/post-tool-langsmith.mjs | 195 -------- evals/hooks/pre-tool-langsmith.mjs | 59 --- evals/hooks/stop-langsmith.mjs | 105 ----- evals/langsmith-reader.ts | 284 ++++++++++++ evals/model-normalize.ts | 45 ++ evals/pricing.ts | 68 +++ evals/report.ts | 320 +++++++++++++ evals/run-case.ts | 694 +++++----------------------- evals/run-suite.ts | 133 ++---- package-lock.json | 100 +++- package.json | 3 + 22 files changed, 2295 insertions(+), 1679 deletions(-) create mode 100644 evals/adapters/claude-code.ts create mode 100644 evals/adapters/codex.ts create mode 100644 evals/adapters/cursor.ts create mode 100644 evals/adapters/index.ts create mode 100644 evals/adapters/run-headless.ts create mode 100644 evals/adapters/types.ts delete mode 100644 evals/hooks/post-tool-langsmith.mjs delete mode 100644 evals/hooks/pre-tool-langsmith.mjs delete mode 100644 evals/hooks/stop-langsmith.mjs create mode 100644 evals/langsmith-reader.ts create mode 100644 evals/model-normalize.ts create mode 100644 evals/pricing.ts create mode 100644 evals/report.ts diff --git a/.gitignore b/.gitignore index 5f72d51e5..70430886b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ test-logs/ evals/suite-runs/ evals/runs/ evals/grades/ +evals/reports/ config.json env.list manifest.json diff --git a/evals/README.md b/evals/README.md index 40e9b4578..82c1e5dd6 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,7 +1,9 @@ # Tableau MCP Eval Harness -This folder contains a local eval harness that runs Claude Code against the Tableau MCP server, -routes traces to LangSmith, and grades agent responses against known-correct answers. +This folder contains a local eval harness that runs a **coding agent** (Claude Code, +Cursor, or Codex) against the Tableau MCP server, traces the run to LangSmith via the +official coding-agent tracing plugin for that agent, and grades responses against +known-correct answers — sourcing every metric from the LangSmith trace. ## How It Works @@ -9,34 +11,62 @@ routes traces to LangSmith, and grades agent responses against known-correct ans suite file (bird-california-schools.json) → run-suite.ts → run-case.ts (one per question) - → claude -p --mcp-config ... --settings ... - → tableau-mcp stdio server - → Claude Code hooks (PreToolUse / PostToolUse / Stop) - → LangSmith (traces posted in real time) - → pre-tool-times.jsonl, hook.jsonl (local timing records) - → agent-output.jsonl (full Claude stream) - → suite-summary.json (timing, tokens, tool counts per case) - → grade-bird.ts (one per run, grades the 4 signals) - → grades/YYYY-MM-DD//bird-result.json (per-case verdict) + → AgentAdapter (claude-code | cursor | codex) builds the CLI invocation + → claude / cursor-agent / codex exec ── with MCP config for the tableau server + → tableau-mcp stdio server → real Tableau Cloud/Server APIs + → LangSmith coding-agent plugin posts the full trace + (stamped with eval_run_id + suite_run_id + harness + model metadata) + → run.json (run metadata + eval_run_id correlation) + → agent-output.jsonl (raw agent stream — debugging only, never a grading input) + → grade-suite.ts → grade-bird.ts (one per run) + → fetch trace from LangSmith by eval_run_id ← single source of truth + → derive signals + latency/cost/token/tool metrics + → semantic judge runs headless via GRADER_HARNESS + → grades/YYYY-MM-DD//bird-result.json + → report.ts → longitudinal quality report across cohorts (harness × model) ``` -Claude Code is the eval runtime — not a simulated agent. Every tool call goes to the real Tableau Cloud or Server APIs. LangSmith receives the full trace while the run is in progress. +The coding agent is the eval runtime — not a simulated agent. Every tool call goes to +the real Tableau APIs. Tracing and all grading signals come from the LangSmith trace; +there is no local-artifact fallback for grading. --- -## Setup +## Prerequisites -### 1. Build the MCP server +### 1. Install the coding-agent CLIs you plan to use + +- **Claude Code** — `claude` on PATH +- **Cursor** — `cursor-agent` on PATH +- **Codex** — `codex` on PATH + +### 2. Install the LangSmith coding-agent tracing plugin for each harness + +Grading reads the trace back from LangSmith, so the corresponding plugin **must** be +installed and configured for whichever harness you run. See LangSmith's coding-agent +tracing docs: . +The adapters enable tracing via `TRACE_TO_LANGSMITH=true` plus the harness-specific +`*_LANGSMITH_*` env vars and stamp the `eval_run_id` correlation metadata. + +> If no trace with a matching `eval_run_id` appears within the grader's poll window, +> the case is recorded as `grading_error` (no local fallback). This almost always +> means the plugin is not installed/configured for that harness. + +### 3. Build the MCP server ```bash npm run build ``` -### 2. Set credentials +--- + +## Configuration + +Set these in a `.env` file at the repo root (loaded automatically): ```bash -# LangSmith -export LANGSMITH_API_KEY="ls__..." +# LangSmith (shared project; runs are filtered by eval_run_id metadata) +export LANGSMITH_API_KEY="lsv2_..." export LANGSMITH_PROJECT="your_project_name" # Tableau (PAT auth) @@ -49,46 +79,47 @@ export PAT_VALUE="your-pat-secret" # BIRD California Schools suite export EVAL_DATASOURCE_LUID="" -# LLM judge (for semantic grading) -export OPENAI_API_KEY="sk-..." +# Agent under test +AGENT_HARNESS=claude-code # claude-code | cursor | codex +# AGENT_MODEL=claude-sonnet-4-5 # optional; omit to use the CLI default + +# Grader / semantic judge (runs headless via its own harness) +GRADER_HARNESS=claude-code # claude-code | cursor | codex +# GRADER_MODEL=claude-sonnet-4-5 + +TRACE_TO_LANGSMITH=true ``` -Or put these in a `.env` file at the repo root. +`AGENT_HARNESS`/`AGENT_MODEL` can also be overridden per invocation with +`--agent-harness` / `--agent-model`. --- ## BIRD California Schools Suite The primary eval suite is 30 questions from the -[BIRD Mini-Dev benchmark](https://bird-bench.github.io/) scoped to the California Schools database. -A single published Tableau datasource joins the source tables (schools, frpm, satscores). - -The suite file lives at `evals/suites/bird-california-schools.json` and is committed to the repo. -It contains precomputed expected answers so no database access is required to run evals. +[BIRD Mini-Dev benchmark](https://bird-bench.github.io/) scoped to the California +Schools database, joined into a single published Tableau datasource. The suite file at +`evals/suites/bird-california-schools.json` ships with precomputed expected answers, so +no database access is required to run evals. ### Run the full suite ```bash npm run eval:suite +npm run eval:suite -- --difficulty simple # subset by difficulty +npm run eval:suite -- --ids 5,11,12 # specific question IDs +npm run eval:suite -- --agent-harness codex --agent-model gpt-5.6-codex ``` -### Run a subset - -```bash -npm run eval:suite -- --difficulty simple # 8 simple cases -npm run eval:suite -- --ids 5,11,12 # specific question IDs -``` - -### Grade a full suite run - -After `eval:suite` completes, grade every case at once and produce a single `suite-grade.json`: +### Grade a suite run (sources all metrics from LangSmith) ```bash -npm run eval:grade:suite # auto-discovers most recent suite run -npm run eval:grade:suite -- evals/suite-runs/YYYY-MM-DD/ # explicit +npm run eval:grade:suite # most recent suite run +npm run eval:grade:suite -- evals/suite-runs/YYYY-MM-DD/ ``` -Output is written to `evals/grades/YYYY-MM-DD//suite-grade.json`. +Output: `evals/grades/YYYY-MM-DD//suite-grade.json`. ### Grade a single run @@ -96,146 +127,120 @@ Output is written to `evals/grades/YYYY-MM-DD//suite-grade.json`. npm run eval:grade:bird -- evals/runs/YYYY-MM-DD/ ``` -Output is written to `evals/grades/YYYY-MM-DD//bird-result.json`. +Output: `evals/grades/YYYY-MM-DD//bird-result.json`. + +The semantic judge runs headless through `GRADER_HARNESS`/`GRADER_MODEL` (temperature 0 +where the CLI supports it). Trace poll behavior is tunable with +`GRADE_TRACE_TIMEOUT_MS` (default 60000) and `GRADE_TRACE_POLL_MS` (default 5000). -Override the LLM judge model (default: `gpt-4o-mini`) for either grader: +### Ad hoc runs ```bash -BIRD_GRADE_MODEL=gpt-4o npm run eval:grade:suite +npm run eval:run -- input "how many schools have SAT scores above 1200?" +npm run eval:grade -- evals/runs/YYYY-MM-DD/ # tool-coverage grading (trace-sourced) ``` --- -## Grading Signals +## Longitudinal Report -Each run is graded on four signals. The verdict is driven by the outcome signals only; the structural signals are recorded for debugging. +Roll up every graded case into cohorts by (harness × normalized model) and over time: -### Outcome signals (drive the verdict) +```bash +npm run eval:report +npm run eval:report -- --since 2026-07-01 +npm run eval:report -- --harness codex --model gpt-5.6-codex +``` -| Signal | Method | What it checks | -|---|---|---| -| `numeric_match` | Code | Expected value or row count appears in Claude's final message (±1% tolerance for floats, substring match for strings) | -| `semantic_match` | LLM judge (GPT-4o-mini) | Claude's final message correctly answers the question vs. the gold summary | +Output under `evals/reports//`: `longitudinal.json`, `longitudinal.csv`, +`summary.md`. Per-case metrics: **accuracy** (verdict-derived), **latency** +(`wall_s`, `ttft_s`), **cost** (USD; LangSmith-reported or estimated from +`pricing.ts`), **tool-call count**, **error count**, token totals, and the four +quality signals. + +--- -### Structural signals (informational) +## Grading Signals | Signal | Method | What it checks | |---|---|---| -| `columns_match` | Tool call inspection | Expected VizQL field captions present in `query-datasource` call | -| `filters_match` | Tool call inspection | Expected filter field captions present in `query-datasource` call | +| `numeric_match` | Code | Expected value / row count appears in the agent's final message (±1% for floats, substring for strings) | +| `semantic_match` | LLM judge via `GRADER_HARNESS` | Final message correctly answers the question vs. the gold summary | +| `columns_match` | Trace tool-call inspection | Expected VizQL field captions present in a `query-datasource` call | +| `filters_match` | Trace tool-call inspection | Expected filter field captions present in a `query-datasource` call | -The structural signals use subset matching — extra columns or filters are fine. They help diagnose -*why* an agent failed but don't block a correct answer from passing. +`numeric_match` and `semantic_match` drive the verdict; `columns_match` / `filters_match` +are diagnostic (subset matching — extra columns/filters are fine). ### Verdicts | Verdict | Meaning | |---|---| | `pass` | Both `numeric_match` and `semantic_match` passed | -| `partial` | One of the two outcome signals passed (or one was unavailable) | +| `partial` | One outcome signal passed (or one was unavailable) | | `fail` | Both outcome signals failed | -| `error` | Claude Code exited with a non-zero code | -| `skip` | Both outcome signals were unavailable (e.g. no OpenAI key) | +| `error` | The agent exited non-zero, or the trace shows an error and no answer | +| `skip` | Both outcome signals were unavailable | +| `grading_error` | No matching LangSmith trace was found within the poll window | --- -## Expected Answers - -The suite file ships with precomputed answers from the executed sql against the source of truth california schools table. If you wanted to regenerate it (there's probably no reason to) run the following file (requires the BIRD SQLite snapshot): - -```bash -python3 evals/scripts/precompute-bird-answers.py -``` - -This reads the gold SQL from `bird_mini/data/test_cases/mini_dev_sqlite.json`, executes it against -`bird_mini/data/dev_databases/california_schools/california_schools.sqlite`, and writes -`evals/suites/bird-california-schools.json` with: +## Architecture -- `expected_value` — scalar result (for COUNT, MAX, MIN queries) -- `expected_row_count` — number of rows returned (for list queries) -- `expected_columns` — VizQL field captions that should appear in the query -- `expected_filter_fields` — filter field captions from the reference VDS query -- `ai_summarized_answer` — natural language gold answer (for the LLM judge) +| Module | Responsibility | +|---|---| +| `adapters/types.ts` | `AgentAdapter` interface + shared eval-metadata builder | +| `adapters/claude-code.ts`, `cursor.ts`, `codex.ts` | Per-harness CLI invocation, MCP config, plugin tracing env | +| `adapters/index.ts` | Harness registry + `resolveHarness` | +| `adapters/run-headless.ts` | One-shot headless prompt (no MCP) for the judge + JSON extraction | +| `langsmith-reader.ts` | Fetch a trace by `eval_run_id`; normalize tools/tokens/cost/timings | +| `model-normalize.ts` | Canonical model ids so the same model lines up across harnesses | +| `pricing.ts` | Per-model price map for cost fallback when LangSmith has no cost | +| `run-case.ts` | Run one case through the selected adapter; write `run.json` | +| `run-suite.ts` | Fan out cases; pass harness/model/suite-run-id through | +| `grade-bird.ts` | Trace-sourced BIRD grader + headless semantic judge | +| `grade-suite.ts` | Batch grade a suite run; aggregate quality/latency/cost | +| `grade.ts` | Ad hoc tool-coverage grader (trace-sourced) | +| `report.ts` | Longitudinal cohort report (JSON/CSV/Markdown) | --- ## Local Artifacts -All output directories are organized by date (`YYYY-MM-DD`) so runs and grades from different days stay separate. +Output directories are organized by date (`YYYY-MM-DD`). ``` evals/ - runs/ - YYYY-MM-DD/ - / - run.json metadata, timing, exit code, question_id - agent-output.jsonl raw Claude Code stream JSON (all events) - hook.jsonl one record per tool call (name, input, timing) - pre-tool-times.jsonl PreToolUse timestamps keyed by tool_use_id - stop.json Stop hook data (transcript path, session info) - claude-settings.json per-run hook config (ephemeral) - mcp-config.json per-run MCP server config (ephemeral) - logs/ MCP server file logs - - suite-runs/ - YYYY-MM-DD/ - / - suite-summary.json aggregate: timing, tokens, tool counts per case - cases/ ephemeral per-case JSON files written before each run - - grades/ - YYYY-MM-DD/ - / - bird-result.json per-case grading output (4 signals + verdict) - / - suite-grade.json aggregate grading output for a full suite run + runs/YYYY-MM-DD// + run.json metadata, eval_run_id, harness/model, timing, exit code + agent-output.jsonl raw agent stream (debugging only; NOT a grading input) + mcp-config.json / codex-home/ / cursor-workspace/ per-run MCP config (ephemeral) + judge/ headless judge working dir (ephemeral) + logs/ MCP server file logs + suite-runs/YYYY-MM-DD// + suite-summary.json run-time metadata (wall/exit/timeout); metrics deferred to grading + cases/ ephemeral per-case JSON files + grades/YYYY-MM-DD/ + /bird-result.json per-case verdict + full metric set + /suite-grade.json aggregate grading output for a suite run + reports// longitudinal.json / longitudinal.csv / summary.md ``` -`evals/runs/`, `evals/suite-runs/`, and `evals/grades/` are all gitignored. +`evals/runs/`, `evals/suite-runs/`, `evals/grades/`, and `evals/reports/` are gitignored. --- -## Ad Hoc Runs +## Regenerating Expected Answers -Run a single natural-language prompt without a case file: +The suite ships with precomputed answers. To regenerate (rarely needed; requires the +BIRD SQLite snapshot): ```bash -npm run eval:claude -- input "how many schools have SAT scores above 1200?" +python3 evals/scripts/precompute-bird-answers.py ``` -Ad hoc runs are tagged `adhoc`, use the existing `grade.ts` grader (not `grade-bird.ts`), and -produce the same local artifacts. - ---- - -## LangSmith Trace Structure - -Each eval case produces one parent `chain` run in LangSmith containing all child spans for that -case. The parent's `start_time` and `end_time` are the wall clock times of the `claude -p` process. - -### Child spans - -| Span name | Type | Source | Timing | -|---|---|---|---| -| `initialization` | `chain` | `run-case.ts` (post-process) | `startedAt` → first stream event | -| `assistant-message-N` | `llm` | `run-case.ts` (post-process) | inferred from stream timestamps | -| `` | `tool` | `PostToolUse` hook (real time) | `PreToolUse ts` → `PostToolUse ts` | -| `claude-result-` | `chain` | `run-case.ts` (post-process) | pinned to `finishedAt` | - -### How span timing is calculated - -**Tool call spans** (``) are the most accurate. The `PreToolUse` hook fires the instant Claude Code dispatches the tool call and writes `{tool_use_id, ts}` to `pre-tool-times.jsonl`. The `PostToolUse` hook fires when the MCP server returns and uses the matching `ts` as `start_time`. This gives true MCP round-trip latency. - -**Assistant message spans** are inferred after Claude exits by parsing `agent-output.jsonl`. The inference works in three layers: - -1. **Real timestamps** — if the stream event carries a `timestamp` field, it is used directly. -2. **Hook anchors** — for any assistant event whose content contains a `tool_use` block, the `PreToolUse` timestamp for that `tool_use_id` is injected as the event's real end time, and the `PostToolUse` timestamp is injected as the start time of the next event. This anchors the timeline at every tool call boundary. -3. **Linear interpolation** — events between anchors are distributed proportionally by index between the surrounding anchor times. - -The `end_time` of each assistant message is the `start_time` of the next traceable event, so spans tile continuously rather than all ending at the same moment. - -**Initialization span** is a synthetic `chain` span posted after Claude exits. It covers the window from process launch to the first stream event — Claude Code startup, MCP server subprocess launch, and tool capability negotiation. This ensures the full wall clock time is accounted for in the langsmith trace waterfall. - -### Known gap - -The `claude-result-success` span is always posted with `start_time = end_time = finishedAt` (0.00s) because it represents the Claude Code process exit signal, not a timed operation. +This executes the gold SQL from `bird_mini/data/test_cases/mini_dev_sqlite.json` against +`bird_mini/data/dev_databases/california_schools/california_schools.sqlite` and writes +`expected_value`, `expected_row_count`, `expected_columns`, `expected_filter_fields`, and +`ai_summarized_answer` into `evals/suites/bird-california-schools.json`. +``` diff --git a/evals/adapters/claude-code.ts b/evals/adapters/claude-code.ts new file mode 100644 index 000000000..d9d9c789b --- /dev/null +++ b/evals/adapters/claude-code.ts @@ -0,0 +1,161 @@ +/** + * Claude Code adapter. + * + * Invocation (agent-under-test): `claude -p [--model M] --mcp-config + * --strict-mcp-config --allowedTools "ToolSearch,mcp__tableau__*" + * --output-format stream-json --verbose` + * Tracing: the LangSmith Claude Code plugin, enabled via TRACE_TO_LANGSMITH + + * CC_LANGSMITH_* env; correlation carried in CC_LANGSMITH_METADATA. + * Determinism (judge): CLAUDE_CODE_TEMPERATURE (verified against installed CLI docs; + * startup-only env var). + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +import { + AgentAdapter, + AgentInvocation, + buildEvalMetadata, + HeadlessContext, + RunContext, +} from './types.js'; + +function mcpConfigPath(runDir: string): string { + return path.join(runDir, 'mcp-config.json'); +} + +function baseTraceEnv(langsmith: { + apiKey: string; + project: string; + endpoint: string; +}): Record { + return { + TRACE_TO_LANGSMITH: 'true', + CC_LANGSMITH_API_KEY: langsmith.apiKey, + CC_LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_API_KEY: langsmith.apiKey, + LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_ENDPOINT: langsmith.endpoint, + }; +} + +export const claudeCodeAdapter: AgentAdapter = { + harness: 'claude-code', + + resolveModel(requested) { + return requested?.trim() ?? ''; + }, + + writeConfig(ctx: RunContext) { + const logsDir = path.join(ctx.runDir, 'logs'); + fs.mkdirSync(logsDir, { recursive: true }); + const mcpConfig = { + mcpServers: { + tableau: { + command: 'node', + args: [ctx.mcpServerEntry], + env: { + ...ctx.mcpServerEnv, + TRANSPORT: 'stdio', + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }, + }; + fs.writeFileSync(mcpConfigPath(ctx.runDir), JSON.stringify(mcpConfig, null, 2)); + }, + + buildInvocation(ctx: RunContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + suiteRunId: ctx.suiteRunId, + harness: 'claude-code', + model: ctx.model, + questionId: ctx.questionId, + }); + const args = [ + '-p', + ctx.prompt, + '--mcp-config', + mcpConfigPath(ctx.runDir), + '--strict-mcp-config', + '--allowedTools', + 'ToolSearch,mcp__tableau__*', + '--output-format', + 'stream-json', + '--verbose', + ]; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'claude', + args, + env: { + ...baseTraceEnv(ctx.langsmith), + CC_LANGSMITH_METADATA: JSON.stringify(metadata), + }, + cwd: process.cwd(), + timeoutMs: ctx.budget.maxWallMs + 10_000, + }; + }, + + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + harness: 'claude-code', + model: ctx.model, + role: ctx.role, + }); + const args = ['-p', ctx.prompt, '--output-format', 'json']; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'claude', + args, + env: { + ...baseTraceEnv(ctx.langsmith), + CC_LANGSMITH_METADATA: JSON.stringify(metadata), + CLAUDE_CODE_TEMPERATURE: String(ctx.temperature), + }, + cwd: process.cwd(), + timeoutMs: ctx.timeoutMs, + }; + }, + + extractFinalText(stdout: string): string { + // Headless judge uses --output-format json → a single JSON object with `.result`. + const trimmed = stdout.trim(); + try { + const obj = JSON.parse(trimmed) as { result?: string; text?: string }; + if (typeof obj.result === 'string') return obj.result; + if (typeof obj.text === 'string') return obj.text; + } catch { + // Fall through to stream-json line scan. + } + // stream-json fallback: last assistant text / result event. + const lines = trimmed.split('\n').filter((l) => l.trim()); + for (let i = lines.length - 1; i >= 0; i--) { + try { + const ev = JSON.parse(lines[i]) as { + type?: string; + result?: string; + message?: { content?: Array<{ type?: string; text?: string }> | string }; + }; + if (ev.type === 'result' && typeof ev.result === 'string') return ev.result; + const content = ev.message?.content; + if (Array.isArray(content)) { + const text = content + .filter((b) => b.type === 'text' && b.text) + .map((b) => b.text) + .join('\n'); + if (text) return text; + } else if (typeof content === 'string' && content.trim()) { + return content; + } + } catch { + continue; + } + } + return trimmed; + }, +}; diff --git a/evals/adapters/codex.ts b/evals/adapters/codex.ts new file mode 100644 index 000000000..8ae9490a3 --- /dev/null +++ b/evals/adapters/codex.ts @@ -0,0 +1,195 @@ +/** + * OpenAI Codex CLI adapter. + * + * Invocation (agent-under-test): `codex exec --json -m -C + * --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox ` + * MCP: configured via `[mcp_servers.tableau]` in an isolated `$CODEX_HOME/config.toml`. + * Tracing: the LangSmith Codex plugin, enabled via TRACE_TO_LANGSMITH + + * LANGSMITH_CODEX_* env (falls back to LANGSMITH_*); plugin hooks live in config.toml. + * Determinism (judge): `-c temperature=0` (reasoning models may ignore it; JSON output + * is the primary stability mechanism). + * + * We set CODEX_HOME to a per-run directory so the tableau MCP config and plugin + * settings never touch the user's global ~/.codex. The exact plugin-metadata wiring + * must be verified against the installed Codex LangSmith plugin. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +import { + AgentAdapter, + AgentInvocation, + buildEvalMetadata, + HeadlessContext, + RunContext, +} from './types.js'; + +function codexHome(runDir: string): string { + return path.join(runDir, 'codex-home'); +} + +function tomlString(value: string): string { + return JSON.stringify(value); // TOML basic strings share JSON escaping for our inputs. +} + +function tomlStringArray(values: Array): string { + return `[${values.map(tomlString).join(', ')}]`; +} + +/** Render a minimal config.toml with an optional MCP server + env table. */ +function renderConfigToml(opts: { mcp?: { entry: string; env: Record } }): string { + const lines: Array = []; + if (opts.mcp) { + lines.push('[mcp_servers.tableau]'); + lines.push(`command = ${tomlString('node')}`); + lines.push(`args = ${tomlStringArray([opts.mcp.entry])}`); + lines.push(''); + lines.push('[mcp_servers.tableau.env]'); + for (const [k, v] of Object.entries(opts.mcp.env)) { + lines.push(`${k} = ${tomlString(v)}`); + } + lines.push(''); + } + return `${lines.join('\n')}\n`; +} + +function baseTraceEnv( + runDir: string, + langsmith: { apiKey: string; project: string; endpoint: string }, + metadata: Record, +): Record { + const home = codexHome(runDir); + fs.mkdirSync(home, { recursive: true }); + // Best-effort correlation channel (verify against installed Codex plugin). + fs.writeFileSync( + path.join(home, 'langsmith.json'), + JSON.stringify({ enabled: true, project: langsmith.project, metadata }, null, 2), + ); + return { + CODEX_HOME: home, + TRACE_TO_LANGSMITH: 'true', + LANGSMITH_CODEX_API_KEY: langsmith.apiKey, + LANGSMITH_CODEX_PROJECT: langsmith.project, + LANGSMITH_CODEX_ENDPOINT: langsmith.endpoint, + LANGSMITH_API_KEY: langsmith.apiKey, + LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_ENDPOINT: langsmith.endpoint, + LANGSMITH_CODEX_METADATA: JSON.stringify(metadata), + }; +} + +export const codexAdapter: AgentAdapter = { + harness: 'codex', + + resolveModel(requested) { + return requested?.trim() ?? ''; + }, + + writeConfig(ctx: RunContext) { + const home = codexHome(ctx.runDir); + const logsDir = path.join(ctx.runDir, 'logs'); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(logsDir, { recursive: true }); + const toml = renderConfigToml({ + mcp: { + entry: ctx.mcpServerEntry, + env: { + ...ctx.mcpServerEnv, + TRANSPORT: 'stdio', + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }); + fs.writeFileSync(path.join(home, 'config.toml'), toml); + }, + + buildInvocation(ctx: RunContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + suiteRunId: ctx.suiteRunId, + harness: 'codex', + model: ctx.model, + questionId: ctx.questionId, + }); + const args = [ + 'exec', + '--json', + '--skip-git-repo-check', + '--dangerously-bypass-approvals-and-sandbox', + '-C', + process.cwd(), + ]; + if (ctx.model) args.push('-m', ctx.model); + args.push(ctx.prompt); + return { + command: 'codex', + args, + env: baseTraceEnv(ctx.runDir, ctx.langsmith, metadata), + cwd: process.cwd(), + timeoutMs: ctx.budget.maxWallMs + 10_000, + }; + }, + + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation { + const home = codexHome(ctx.runDir); + fs.mkdirSync(home, { recursive: true }); + // Empty config (no MCP servers) for the judge. + fs.writeFileSync(path.join(home, 'config.toml'), renderConfigToml({})); + const metadata = buildEvalMetadata({ + runId: ctx.runId, + harness: 'codex', + model: ctx.model, + role: ctx.role, + }); + const args = [ + 'exec', + '--json', + '--skip-git-repo-check', + '--dangerously-bypass-approvals-and-sandbox', + '-c', + `temperature=${ctx.temperature}`, + '-C', + ctx.runDir, + ]; + if (ctx.model) args.push('-m', ctx.model); + args.push(ctx.prompt); + return { + command: 'codex', + args, + env: baseTraceEnv(ctx.runDir, ctx.langsmith, metadata), + cwd: ctx.runDir, + timeoutMs: ctx.timeoutMs, + }; + }, + + extractFinalText(stdout: string): string { + const lines = stdout + .trim() + .split('\n') + .filter((l) => l.trim()); + let lastText = ''; + for (const line of lines) { + let ev: Record; + try { + ev = JSON.parse(line) as Record; + } catch { + continue; + } + // Tolerate several codex event shapes across versions. + const item = ev.item as { text?: string; item_type?: string; type?: string } | undefined; + const msg = ev.msg as { type?: string; message?: string } | undefined; + if (item && typeof item.text === 'string' && item.text.trim()) { + lastText = item.text; + } else if (msg && typeof msg.message === 'string' && msg.message.trim()) { + lastText = msg.message; + } else if (typeof ev.message === 'string' && (ev.message as string).trim()) { + lastText = ev.message as string; + } else if (typeof ev.text === 'string' && (ev.text as string).trim()) { + lastText = ev.text as string; + } + } + return lastText || stdout.trim(); + }, +}; diff --git a/evals/adapters/cursor.ts b/evals/adapters/cursor.ts new file mode 100644 index 000000000..28f45d291 --- /dev/null +++ b/evals/adapters/cursor.ts @@ -0,0 +1,168 @@ +/** + * Cursor CLI (`cursor-agent`) adapter. + * + * Invocation (agent-under-test): `cursor-agent -p [--model M] + * --output-format stream-json --force --approve-mcps --trust --workspace ` + * MCP: cursor-agent discovers `.cursor/mcp.json` in the workspace dir; the adapter + * writes an isolated workspace per run. + * Tracing: the LangSmith Cursor plugin, enabled via TRACE_TO_LANGSMITH + + * LANGSMITH_CURSOR_* env (falls back to LANGSMITH_*). Requires Node >= 22.13. + * Determinism (judge): cursor-agent exposes NO temperature flag — the judge relies on + * JSON output only. + * + * NOTE: `cursor-agent` has no `--mcp-config` flag and no per-tool allowlist, so we + * isolate via a per-run workspace. The custom-metadata propagation mechanism for the + * Cursor plugin is set via env here and must be verified against the installed plugin. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +import { + AgentAdapter, + AgentInvocation, + buildEvalMetadata, + HeadlessContext, + RunContext, +} from './types.js'; + +function workspaceDir(runDir: string): string { + return path.join(runDir, 'cursor-workspace'); +} + +function baseTraceEnv( + langsmith: { apiKey: string; project: string; endpoint: string }, + metadata: Record, +): Record { + return { + TRACE_TO_LANGSMITH: 'true', + LANGSMITH_CURSOR_API_KEY: langsmith.apiKey, + LANGSMITH_CURSOR_PROJECT: langsmith.project, + LANGSMITH_CURSOR_ENDPOINT: langsmith.endpoint, + LANGSMITH_API_KEY: langsmith.apiKey, + LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_ENDPOINT: langsmith.endpoint, + // Best-effort correlation channel (verify against installed Cursor plugin). + LANGSMITH_CURSOR_METADATA: JSON.stringify(metadata), + }; +} + +export const cursorAdapter: AgentAdapter = { + harness: 'cursor', + + resolveModel(requested) { + return requested?.trim() ?? ''; + }, + + writeConfig(ctx: RunContext) { + const ws = workspaceDir(ctx.runDir); + const cursorDir = path.join(ws, '.cursor'); + const logsDir = path.join(ctx.runDir, 'logs'); + fs.mkdirSync(cursorDir, { recursive: true }); + fs.mkdirSync(logsDir, { recursive: true }); + const mcpConfig = { + mcpServers: { + tableau: { + command: 'node', + args: [ctx.mcpServerEntry], + env: { + ...ctx.mcpServerEnv, + TRANSPORT: 'stdio', + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }, + }; + fs.writeFileSync(path.join(cursorDir, 'mcp.json'), JSON.stringify(mcpConfig, null, 2)); + }, + + buildInvocation(ctx: RunContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + suiteRunId: ctx.suiteRunId, + harness: 'cursor', + model: ctx.model, + questionId: ctx.questionId, + }); + const ws = workspaceDir(ctx.runDir); + const args = [ + '-p', + ctx.prompt, + '--output-format', + 'stream-json', + '--force', + '--approve-mcps', + '--trust', + '--workspace', + ws, + ]; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'cursor-agent', + args, + env: baseTraceEnv(ctx.langsmith, metadata), + cwd: process.cwd(), + timeoutMs: ctx.budget.maxWallMs + 10_000, + }; + }, + + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + harness: 'cursor', + model: ctx.model, + role: ctx.role, + }); + // No MCP tools for the judge; ask/read-only mode keeps it from taking actions. + const args = ['-p', ctx.prompt, '--output-format', 'json', '--mode', 'ask']; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'cursor-agent', + args, + env: baseTraceEnv(ctx.langsmith, metadata), + cwd: process.cwd(), + timeoutMs: ctx.timeoutMs, + }; + }, + + extractFinalText(stdout: string): string { + const trimmed = stdout.trim(); + // `--output-format json` → single object; try direct parse first. + try { + const obj = JSON.parse(trimmed) as { + result?: string; + text?: string; + message?: { content?: Array<{ type?: string; text?: string }> | string }; + }; + if (typeof obj.result === 'string') return obj.result; + if (typeof obj.text === 'string') return obj.text; + } catch { + // Fall through to JSONL scan. + } + const lines = trimmed.split('\n').filter((l) => l.trim()); + for (let i = lines.length - 1; i >= 0; i--) { + try { + const ev = JSON.parse(lines[i]) as { + type?: string; + result?: string; + message?: { content?: Array<{ type?: string; text?: string }> | string }; + }; + if (ev.type === 'result' && typeof ev.result === 'string') return ev.result; + const content = ev.message?.content; + if (Array.isArray(content)) { + const text = content + .filter((b) => b.type === 'text' && b.text) + .map((b) => b.text) + .join('\n'); + if (text) return text; + } else if (typeof content === 'string' && content.trim()) { + return content; + } + } catch { + continue; + } + } + return trimmed; + }, +}; diff --git a/evals/adapters/index.ts b/evals/adapters/index.ts new file mode 100644 index 000000000..750ec6081 --- /dev/null +++ b/evals/adapters/index.ts @@ -0,0 +1,35 @@ +/** Adapter registry: resolve an AgentAdapter by harness id. */ + +import { claudeCodeAdapter } from './claude-code.js'; +import { codexAdapter } from './codex.js'; +import { cursorAdapter } from './cursor.js'; +import { AGENT_HARNESSES, AgentAdapter, AgentHarness, isAgentHarness } from './types.js'; + +const REGISTRY: Record = { + 'claude-code': claudeCodeAdapter, + cursor: cursorAdapter, + codex: codexAdapter, +}; + +export function getAdapter(harness: AgentHarness): AgentAdapter { + return REGISTRY[harness]; +} + +/** + * Resolve the harness from an explicit value or `process.env`, defaulting to + * claude-code. Throws on an unrecognized value. + */ +export function resolveHarness(value: string | undefined, envVar: string): AgentHarness { + const raw = value ?? process.env[envVar]; + if (raw == null || raw.trim() === '') return 'claude-code'; + const normalized = raw.trim(); + if (!isAgentHarness(normalized)) { + throw new Error( + `Invalid ${envVar}="${normalized}". Expected one of: ${AGENT_HARNESSES.join(', ')}.`, + ); + } + return normalized; +} + +export * from './types.js'; +export { claudeCodeAdapter, codexAdapter, cursorAdapter }; diff --git a/evals/adapters/run-headless.ts b/evals/adapters/run-headless.ts new file mode 100644 index 000000000..0ff54bc07 --- /dev/null +++ b/evals/adapters/run-headless.ts @@ -0,0 +1,70 @@ +/** + * Headless one-shot helper: send a single prompt to a coding-agent CLI (no MCP + * tools) and return the final assistant text. Used by the grader's semantic judge. + */ + +import { execFileSync } from 'child_process'; + +import { AgentAdapter, HeadlessContext } from './types.js'; + +export type HeadlessRunResult = { + text: string; + exitCode: number; + stdout: string; + stderr: string; +}; + +export function runHeadless(adapter: AgentAdapter, ctx: HeadlessContext): HeadlessRunResult { + const invocation = adapter.buildHeadlessInvocation(ctx); + let stdout = ''; + let stderr = ''; + let exitCode = 0; + try { + stdout = execFileSync(invocation.command, invocation.args, { + env: { ...process.env, ...invocation.env }, + cwd: invocation.cwd, + timeout: invocation.timeoutMs, + maxBuffer: 25 * 1024 * 1024, + }).toString(); + } catch (error: unknown) { + const e = error as { status?: number; stdout?: Buffer; stderr?: Buffer; message?: string }; + exitCode = e.status ?? 1; + stdout = e.stdout?.toString() ?? ''; + stderr = [e.message, e.stderr?.toString()].filter(Boolean).join('\n'); + } + return { text: adapter.extractFinalText(stdout), exitCode, stdout, stderr }; +} + +/** + * Extract the first balanced JSON object from a text blob. Coding-agent CLIs often + * wrap the judge's JSON in prose/markdown fences, so we scan for `{ ... }`. + */ +export function extractJsonObject(text: string): T | null { + // Prefer fenced ```json blocks. + const fenceMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/i); + const candidates: Array = []; + if (fenceMatch?.[1]) candidates.push(fenceMatch[1].trim()); + candidates.push(text.trim()); + + for (const candidate of candidates) { + const start = candidate.indexOf('{'); + if (start === -1) continue; + let depth = 0; + for (let i = start; i < candidate.length; i++) { + const ch = candidate[i]; + if (ch === '{') depth += 1; + else if (ch === '}') { + depth -= 1; + if (depth === 0) { + const slice = candidate.slice(start, i + 1); + try { + return JSON.parse(slice) as T; + } catch { + break; + } + } + } + } + } + return null; +} diff --git a/evals/adapters/types.ts b/evals/adapters/types.ts new file mode 100644 index 000000000..653ea0eb1 --- /dev/null +++ b/evals/adapters/types.ts @@ -0,0 +1,115 @@ +/** + * Agent harness abstraction for the Tableau MCP eval harness. + * + * An `AgentAdapter` encapsulates everything agent-specific about invoking a + * coding-agent CLI: the command + argv (including the model flag), the MCP + * server config in the agent's own format, and the environment that enables the + * official LangSmith coding-agent tracing plugin. Adapters do NOT post traces or + * compute grades — the plugin traces automatically and the grader reads the trace + * back from LangSmith (single source of truth). + */ + +export type AgentHarness = 'claude-code' | 'cursor' | 'codex'; + +export const AGENT_HARNESSES: ReadonlyArray = ['claude-code', 'cursor', 'codex']; + +export function isAgentHarness(value: string | undefined): value is AgentHarness { + return value != null && (AGENT_HARNESSES as ReadonlyArray).includes(value); +} + +/** A fully-formed CLI invocation the runner can hand to `execFileSync`. */ +export type AgentInvocation = { + command: string; + args: Array; + env: Record; + cwd: string; + timeoutMs: number; +}; + +/** Result of spawning an agent CLI (persisted verbatim for debugging only). */ +export type AgentResult = { + exitCode: number; + stdout: string; + stderr: string; +}; + +/** LangSmith connection + correlation shared by every invocation. */ +export type LangSmithConfig = { + apiKey: string; + project: string; + endpoint: string; +}; + +/** Context for a full agent-under-test run (with MCP tools attached). */ +export type RunContext = { + runId: string; + suiteRunId: string | null; + runDir: string; + prompt: string; + /** Resolved model id ('' means "let the CLI use its default"). */ + model: string; + mcpServerEntry: string; + mcpServerEnv: Record; + questionId: number | null; + langsmith: LangSmithConfig; + budget: { maxWallMs: number; maxToolCalls: number }; +}; + +/** Context for a headless one-shot prompt (no MCP tools) — e.g. the grader judge. */ +export type HeadlessContext = { + runId: string; + runDir: string; + prompt: string; + model: string; + /** 0 for deterministic judging; applied only where the CLI supports it. */ + temperature: number; + /** Attribution role for trace metadata, e.g. 'judge'. */ + role: string; + langsmith: LangSmithConfig; + timeoutMs: number; +}; + +export interface AgentAdapter { + readonly harness: AgentHarness; + + /** Resolve the model to use: the requested id, or the adapter's default (''=CLI default). */ + resolveModel(requested: string | undefined): string; + + /** + * Write any per-run config files the CLI needs (MCP server config in the + * agent's own format). Called before `buildInvocation`. + */ + writeConfig(ctx: RunContext): void; + + /** Build the CLI invocation for a full agent-under-test run (MCP tools attached). */ + buildInvocation(ctx: RunContext): AgentInvocation; + + /** + * Build the CLI invocation for a headless one-shot prompt with no MCP tools, + * requesting structured/JSON-friendly output. Used by the grader's judge. + */ + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation; + + /** Extract the final assistant text from this agent's raw stdout. */ + extractFinalText(stdout: string): string; +} + +/** Build the `coding-agent` custom metadata blob shared across adapters. */ +export function buildEvalMetadata(ctx: { + runId: string; + suiteRunId?: string | null; + harness: AgentHarness; + model: string; + questionId?: number | null; + role?: string; +}): Record { + const meta: Record = { + eval_run_id: ctx.runId, + harness: ctx.harness, + }; + if (ctx.suiteRunId) meta.suite_run_id = ctx.suiteRunId; + if (ctx.model) meta.model = ctx.model; + if (ctx.questionId != null) meta.question_id = ctx.questionId; + if (ctx.role) meta.eval_role = ctx.role; + return meta; +} diff --git a/evals/grade-bird.ts b/evals/grade-bird.ts index 91d531cba..145359861 100644 --- a/evals/grade-bird.ts +++ b/evals/grade-bird.ts @@ -1,26 +1,34 @@ /** - * BIRD-specific grader for California Schools eval runs. + * BIRD-specific grader for California Schools eval runs — trace-sourced. * - * Grades a single run directory using four signals: - * columns_match — required VizQL fields present in the query-datasource call - * filters_match — required filter fields present in the query-datasource call - * numeric_match — expected value / row count found in Claude's final message - * semantic_match — LLM judge comparing Claude's final message to the gold summary + * All per-case signals are derived from the coding-agent trace in LangSmith + * (fetched by `eval_run_id`), not from local artifacts. Signals: + * columns_match — required VizQL fields present in a query-datasource tool call + * filters_match — required filter fields present in a query-datasource tool call + * numeric_match — expected value / row count found in the agent's final message + * semantic_match — LLM judge (run via GRADER_HARNESS/GRADER_MODEL, headless) * * Usage: - * npx tsx evals/grade-bird.ts evals/runs/ + * npx tsx evals/grade-bird.ts evals/runs// * * Required environment: - * OPENAI_API_KEY (for LLM judge; set BIRD_GRADE_MODEL to override model) + * LANGSMITH_API_KEY (or LANGCHAIN_API_KEY), LANGSMITH_PROJECT (fallback for older runs) + * Optional: + * GRADER_HARNESS=claude-code|cursor|codex (default claude-code), GRADER_MODEL= + * GRADE_TRACE_TIMEOUT_MS (default 60000), GRADE_TRACE_POLL_MS (default 5000) */ /* eslint-disable no-console */ import dotenv from 'dotenv'; import * as fs from 'fs'; -import OpenAI from 'openai'; import * as path from 'path'; +import { getAdapter, HeadlessContext, resolveHarness } from './adapters/index.js'; +import { extractJsonObject, runHeadless } from './adapters/run-headless.js'; +import { fetchTraceSummary, findVizqlQuery, makeClient, TraceSummary } from './langsmith-reader.js'; +import { normalizeModel } from './model-normalize.js'; + const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); dotenv.config({ path: path.join(REPO_ROOT, '.env') }); @@ -37,7 +45,6 @@ if (!runDirArg) { console.error('Usage: npx tsx evals/grade-bird.ts '); process.exit(1); } - const absRunDir = path.resolve(runDirArg); // ─── Types ─────────────────────────────────────────────────────────────────── @@ -45,8 +52,12 @@ const absRunDir = path.resolve(runDirArg); type RunMeta = { run_id: string; case_id: string; + eval_run_id?: string; + harness?: string; + model?: string | null; + langsmith_project?: string; wall_ms?: number; - claude_exit_code?: number; + agent_exit_code?: number; timed_out?: boolean; metadata?: { question_id?: number; @@ -55,39 +66,9 @@ type RunMeta = { }; }; -type HookRecord = { - tool_name?: string; - normalized_tool_name?: string; - tool_input?: unknown; -}; - -type VizqlField = { - fieldCaption?: string; - fieldAlias?: string; - function?: string; - calculation?: string; - sortDirection?: string; - sortPriority?: number; -}; - -type VizqlFilter = { - field?: { - fieldCaption?: string; - calculation?: string; - }; - filterType?: string; -}; - -type VizqlQuery = { - fields?: Array; - filters?: Array; -}; - -type QueryDatasourceInput = { - datasource_luid?: string; - query?: VizqlQuery; - options?: unknown; -}; +type VizqlField = { fieldCaption?: string }; +type VizqlFilter = { field?: { fieldCaption?: string } }; +type VizqlQuery = { fields?: Array; filters?: Array }; type BirdCase = { question_id: number; @@ -101,55 +82,44 @@ type BirdCase = { expected_filter_fields: Array; }; -type TokenUsage = { - input_tokens?: number; - output_tokens?: number; - cache_read_input_tokens?: number; - cache_creation_input_tokens?: number; -}; - -type TokenTotals = { - input_tokens: number | null; - cache_creation_tokens: number | null; - cache_read_tokens: number | null; - output_tokens: number | null; - total_context_tokens: number | null; -}; - -type ClaudeStreamEvent = { - type?: string; - result?: string; - usage?: TokenUsage; - modelUsage?: TokenUsage; - message?: { - model?: string; - usage?: TokenUsage; - content?: Array<{ type?: string; text?: string }> | string; - }; -}; - -type LlmJudgeResult = { - correct: boolean; - score: number; - reason: string; -}; +type LlmJudgeResult = { correct: boolean; score: number; reason: string }; type BirdGradeResult = { run_id: string; + eval_run_id: string; question_id: number; difficulty: string; graded_at: string; + harness: string | null; model: string | null; + model_normalized: string | null; + grader_harness: string; + grader_model: string | null; + // Latency / cost / volume metrics (from trace). wall_s: number | null; - tokens: TokenTotals; + ttft_s: number | null; + cost_usd: number | null; + cost_source: TraceSummary['costSource'] | 'n/a'; + tokens: { + input_tokens: number | null; + output_tokens: number | null; + cache_read_tokens: number | null; + cache_creation_tokens: number | null; + total_tokens: number | null; + }; tool_calls: number; tools_used: Array; + llm_calls: number; + subagent_count: number; + error_count: number; + // Quality signals. signals: { numeric_match: boolean | null; semantic_match: number | null; columns_match: boolean | null; filters_match: boolean | null; }; + accuracy: number | null; details: { expected_columns: Array; actual_columns: Array; @@ -163,28 +133,13 @@ type BirdGradeResult = { final_message_preview: string; llm_judge: LlmJudgeResult | null; llm_judge_error: string | null; + trace_error: string | null; }; - verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip'; + verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip' | 'grading_error'; }; // ─── Helpers ───────────────────────────────────────────────────────────────── -function readJsonl(filePath: string): Array { - if (!fs.existsSync(filePath)) return []; - return fs - .readFileSync(filePath, 'utf-8') - .split('\n') - .filter((line) => line.trim()) - .map((line) => { - try { - return JSON.parse(line) as T; - } catch { - return null; - } - }) - .filter((v): v is T => v !== null); -} - function readOptionalJson(filePath: string): T | null { if (!fs.existsSync(filePath)) return null; try { @@ -194,127 +149,6 @@ function readOptionalJson(filePath: string): T | null { } } -function normalizeToolName(toolName: string): string { - const parts = toolName.split('__'); - const raw = parts[parts.length - 1] || toolName; - return raw.replace(/^tableau_/, '').replace(/^tableau-/, ''); -} - -function parseStreamEvents(agentOutputJsonl: string): Array { - return agentOutputJsonl - .split('\n') - .filter((l) => l.trim()) - .map((l) => { - try { - return JSON.parse(l) as ClaudeStreamEvent; - } catch { - return null; - } - }) - .filter((e): e is ClaudeStreamEvent => e !== null); -} - -function sumUsage(events: Array): TokenTotals { - let input = 0; - let cacheCreate = 0; - let cacheRead = 0; - let output = 0; - let found = false; - - for (const event of events) { - const u = event.message?.usage ?? event.usage ?? event.modelUsage; - if (!u) continue; - input += u.input_tokens ?? 0; - cacheCreate += u.cache_creation_input_tokens ?? 0; - cacheRead += u.cache_read_input_tokens ?? 0; - output += u.output_tokens ?? 0; - found = true; - } - - if (!found) { - return { - input_tokens: null, - cache_creation_tokens: null, - cache_read_tokens: null, - output_tokens: null, - total_context_tokens: null, - }; - } - - return { - input_tokens: input, - cache_creation_tokens: cacheCreate, - cache_read_tokens: cacheRead, - output_tokens: output, - total_context_tokens: input + cacheCreate + cacheRead, - }; -} - -/** - * Extract token totals from the stream. - * Prefers the result event's aggregated usage when available (session-level total - * from Claude Code). Falls back to summing across all assistant events. - */ -function extractTokenTotals(events: Array): TokenTotals { - // If the result event carries an aggregate, use it — but it may lack cache breakdown. - // Sum across all assistant events instead, which have per-turn cache detail. - const assistantEvents = events.filter((e) => e.type === 'assistant'); - if (assistantEvents.length > 0) return sumUsage(assistantEvents); - - // Last resort: result event usage. - const resultEvents = events.filter((e) => e.type === 'result'); - if (resultEvents.length > 0) return sumUsage(resultEvents); - - return sumUsage([]); -} - -/** Extract the model name from the first assistant message in the stream. */ -function extractModelName(events: Array): string | null { - for (const event of events) { - if (event.type === 'assistant' && event.message?.model) { - return event.message.model; - } - } - return null; -} - -/** Extract the final assistant text from Claude's stream-json output. */ -function extractFinalMessage(agentOutputJsonl: string): string { - const events = parseStreamEvents(agentOutputJsonl); - - // Prefer the result event's text field - const resultEvents = events.filter((e) => e.type === 'result' && e.result); - if (resultEvents.length > 0) { - return resultEvents[resultEvents.length - 1].result ?? ''; - } - - // Fall back to the last assistant message's text content - const assistantEvents = events.filter((e) => e.type === 'assistant'); - for (let i = assistantEvents.length - 1; i >= 0; i--) { - const content = assistantEvents[i].message?.content; - if (Array.isArray(content)) { - const textBlocks = content - .filter((b) => b.type === 'text' && b.text) - .map((b) => b.text ?? ''); - if (textBlocks.length > 0) return textBlocks.join('\n'); - } - if (typeof content === 'string' && content.trim()) return content; - } - - return ''; -} - -/** Extract all query-datasource tool inputs from hook.jsonl. */ -function extractQueryDatasourceInputs(hookRecords: Array): Array { - return hookRecords - .filter( - (r) => normalizeToolName(r.normalized_tool_name ?? r.tool_name ?? '') === 'query-datasource', - ) - .map((r) => r.tool_input as QueryDatasourceInput) - .filter(Boolean); -} - -/** Collect field captions from a VizQL fields array. */ function collectFieldCaptions(fields: Array | undefined): Array { return (fields ?? []) .map((f) => f.fieldCaption ?? '') @@ -322,7 +156,6 @@ function collectFieldCaptions(fields: Array | undefined): Array c.toLowerCase()); } -/** Collect filter field captions from a VizQL filters array. */ function collectFilterCaptions(filters: Array | undefined): Array { return (filters ?? []) .map((f) => f.field?.fieldCaption ?? '') @@ -330,14 +163,8 @@ function collectFilterCaptions(filters: Array | undefined): Array c.toLowerCase()); } -/** - * Try to extract a number from the final message that matches the expected value. - * Returns the extracted number, or null if no suitable number was found. - */ function extractMatchingNumber(text: string, expected: number | string | null): number | null { if (expected === null) return null; - - // Normalize expected to a number if possible let expectedNum: number | null = null; if (typeof expected === 'number') { expectedNum = expected; @@ -347,56 +174,50 @@ function extractMatchingNumber(text: string, expected: number | string | null): } if (expectedNum === null) return null; - // Extract all numbers from the message (strip commas first) const clean = text.replace(/,/g, ''); const matches = clean.match(/-?\d+(?:\.\d+)?/g) ?? []; const candidates = matches.map((m) => parseFloat(m)).filter((n) => !isNaN(n)); - // Look for a match with tolerance const isClose = (a: number, b: number): boolean => { if (b === 0) return Math.abs(a) < 0.001; return Math.abs(a - b) / Math.abs(b) <= 0.01 || Math.abs(a - b) <= 0.001; }; - - // Also check percentage form (e.g. "90.5%" for 0.905) const percentageCandidates = candidates.map((n) => n / 100); - for (const candidate of [...candidates, ...percentageCandidates]) { if (isClose(candidate, expectedNum)) return candidate; } return null; } -/** - * Check if a numeric answer (or row count) appears in the final message. - */ function checkNumericMatch( finalMessage: string, birdCase: BirdCase, ): { matched: boolean; extracted: number | null } { const target = birdCase.answer_type === 'scalar' ? birdCase.expected_value : birdCase.expected_row_count; - if (target === null) return { matched: false, extracted: null }; - - // For string scalars (school names, phone numbers), do a case-insensitive substring check if (typeof target === 'string') { - const matched = finalMessage.toLowerCase().includes(target.toLowerCase()); - return { matched, extracted: null }; + return { matched: finalMessage.toLowerCase().includes(target.toLowerCase()), extracted: null }; } - const extracted = extractMatchingNumber(finalMessage, target); return { matched: extracted !== null, extracted }; } -// ─── LLM Judge ─────────────────────────────────────────────────────────────── +// ─── LLM Judge (via GRADER_HARNESS) ──────────────────────────────────────────── -async function runLlmJudge( +function runJudge( birdCase: BirdCase, finalMessage: string, - openai: OpenAI, - model: string, -): Promise { +): { + result: LlmJudgeResult | null; + error: string | null; + harness: string; + model: string | null; +} { + const graderHarness = resolveHarness(undefined, 'GRADER_HARNESS'); + const adapter = getAdapter(graderHarness); + const graderModel = adapter.resolveModel(process.env.GRADER_MODEL); + const targetValue = birdCase.answer_type === 'scalar' ? `Expected value: ${String(birdCase.expected_value)}` @@ -419,28 +240,68 @@ async function runLlmJudge( '- The agent must address the right entities (not a proxy or wrong dimension)', '- Partial credit if the agent got the right approach but a slightly wrong number', '', - 'Respond with JSON only:', + 'Respond with a single JSON object and nothing else:', '{"correct": true|false, "score": 0.0-1.0, "reason": "one sentence"}', ].join('\n'); - const response = await openai.chat.completions.create({ - model, - messages: [{ role: 'user', content: prompt }], - response_format: { type: 'json_object' }, + const judgeDir = path.join(absRunDir, 'judge'); + fs.mkdirSync(judgeDir, { recursive: true }); + + const ctx: HeadlessContext = { + runId: `${path.basename(absRunDir)}-judge`, + runDir: judgeDir, + prompt, + model: graderModel, temperature: 0, - }); + role: 'judge', + langsmith: { + apiKey: process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY ?? '', + project: process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals', + endpoint: process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com', + }, + timeoutMs: Number(process.env.GRADE_JUDGE_TIMEOUT_MS ?? 120_000), + }; - const raw = response.choices[0]?.message?.content ?? '{}'; - const parsed = JSON.parse(raw) as Partial; + const outcome = runHeadless(adapter, ctx); + if (outcome.exitCode !== 0 && !outcome.text) { + return { + result: null, + error: `judge exited ${outcome.exitCode}: ${outcome.stderr}`, + harness: graderHarness, + model: graderModel || null, + }; + } + const parsed = extractJsonObject>(outcome.text); + if (!parsed) { + return { + result: null, + error: `could not parse judge JSON from output: ${outcome.text.slice(0, 200)}`, + harness: graderHarness, + model: graderModel || null, + }; + } return { - correct: parsed.correct ?? false, - score: typeof parsed.score === 'number' ? parsed.score : parsed.correct ? 1.0 : 0.0, - reason: parsed.reason ?? '', + result: { + correct: parsed.correct ?? false, + score: typeof parsed.score === 'number' ? parsed.score : parsed.correct ? 1.0 : 0.0, + reason: parsed.reason ?? '', + }, + error: null, + harness: graderHarness, + model: graderModel || null, }; } // ─── Main ───────────────────────────────────────────────────────────────────── +function writeResult(result: BirdGradeResult): string { + const gradeDir = path.join(GRADES_DIR, dateSlug(), result.run_id); + fs.mkdirSync(gradeDir, { recursive: true }); + const resultPath = path.join(gradeDir, 'bird-result.json'); + fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); + return resultPath; +} + async function main(): Promise { const runMeta = readOptionalJson(path.join(absRunDir, 'run.json')); if (!runMeta) { @@ -450,7 +311,6 @@ async function main(): Promise { const questionId = runMeta.metadata?.question_id; const suiteFilePath = runMeta.metadata?.suite_file; - if (!questionId || !suiteFilePath) { console.error( 'run.json is missing metadata.question_id or metadata.suite_file.\n' + @@ -458,7 +318,6 @@ async function main(): Promise { ); process.exit(1); } - if (!fs.existsSync(suiteFilePath)) { console.error(`Suite file not found: ${suiteFilePath}`); process.exit(1); @@ -471,180 +330,203 @@ async function main(): Promise { process.exit(1); } - const hookRecords = readJsonl(path.join(absRunDir, 'hook.jsonl')); - const agentOutputRaw = fs.existsSync(path.join(absRunDir, 'agent-output.jsonl')) - ? fs.readFileSync(path.join(absRunDir, 'agent-output.jsonl'), 'utf-8') - : ''; + const runId = runMeta.run_id ?? path.basename(absRunDir); + const evalRunId = runMeta.eval_run_id ?? runId; + const projectName = + runMeta.langsmith_project ?? process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; + const difficulty = runMeta.metadata?.difficulty ?? birdCase.difficulty ?? 'unknown'; - const streamEvents = parseStreamEvents(agentOutputRaw); - const modelName = extractModelName(streamEvents); - const tokens = extractTokenTotals(streamEvents); - const finalMessage = extractFinalMessage(agentOutputRaw); - const queryInputs = extractQueryDatasourceInputs(hookRecords); + const baseResult: BirdGradeResult = { + run_id: runId, + eval_run_id: evalRunId, + question_id: questionId, + difficulty, + graded_at: new Date().toISOString(), + harness: runMeta.harness ?? null, + model: runMeta.model ?? null, + model_normalized: null, + grader_harness: resolveHarness(undefined, 'GRADER_HARNESS'), + grader_model: null, + wall_s: runMeta.wall_ms != null ? Math.round(runMeta.wall_ms / 1000) : null, + ttft_s: null, + cost_usd: null, + cost_source: 'n/a', + tokens: { + input_tokens: null, + output_tokens: null, + cache_read_tokens: null, + cache_creation_tokens: null, + total_tokens: null, + }, + tool_calls: 0, + tools_used: [], + llm_calls: 0, + subagent_count: 0, + error_count: 0, + signals: { + numeric_match: null, + semantic_match: null, + columns_match: null, + filters_match: null, + }, + accuracy: null, + details: { + expected_columns: birdCase.expected_columns, + actual_columns: [], + missing_columns: [], + expected_filter_fields: birdCase.expected_filter_fields, + actual_filter_fields: [], + missing_filter_fields: [], + expected_value: birdCase.answer_type === 'scalar' ? birdCase.expected_value : null, + expected_row_count: birdCase.expected_row_count, + extracted_number: null, + final_message_preview: '', + llm_judge: null, + llm_judge_error: null, + trace_error: null, + }, + verdict: 'grading_error', + }; - // ── Signal 1 & 2: columns_match, filters_match ────────────────────────── + // ── Fetch trace (single source of truth) ───────────────────────────────── + const client = makeClient(); + const summary = await fetchTraceSummary(client, { + projectName, + evalRunId, + pollIntervalMs: Number(process.env.GRADE_TRACE_POLL_MS ?? 5_000), + timeoutMs: Number(process.env.GRADE_TRACE_TIMEOUT_MS ?? 60_000), + }); - let columnsMatch: boolean | null = null; - let filtersMatch: boolean | null = null; - let actualColumns: Array = []; - let actualFilterFields: Array = []; - let missingColumns: Array = []; - let missingFilterFields: Array = []; + if (!summary) { + baseResult.details.trace_error = `No LangSmith trace found for eval_run_id=${evalRunId} in project "${projectName}" within the timeout. Ensure the coding-agent LangSmith plugin is installed and configured for the ${runMeta.harness ?? 'agent'} harness.`; + baseResult.verdict = 'grading_error'; + const p = writeResult(baseResult); + console.error(`\nGrade: Q${questionId} — GRADING_ERROR (no trace)`); + console.error(baseResult.details.trace_error); + console.error(`Result: ${p}`); + return; + } - if (queryInputs.length > 0) { - // Union of all columns/filters across all query-datasource calls (agent may call it multiple times) + // ── Metrics from trace ──────────────────────────────────────────────────── + baseResult.model = summary.model ?? runMeta.model ?? null; + baseResult.model_normalized = normalizeModel(baseResult.model); + baseResult.ttft_s = summary.ttftMs != null ? Math.round(summary.ttftMs / 100) / 10 : null; + if (summary.wallMs != null) baseResult.wall_s = Math.round(summary.wallMs / 1000); + baseResult.cost_usd = summary.costUsd; + baseResult.cost_source = summary.costSource; + baseResult.tokens = { + input_tokens: summary.tokens.input, + output_tokens: summary.tokens.output, + cache_read_tokens: summary.tokens.cacheRead, + cache_creation_tokens: summary.tokens.cacheWrite, + total_tokens: summary.tokens.total, + }; + baseResult.tool_calls = summary.toolCalls.length; + baseResult.tools_used = [...new Set(summary.toolCalls.map((t) => t.normalizedName))]; + baseResult.llm_calls = summary.llmRunCount; + baseResult.subagent_count = summary.subagentCount; + baseResult.error_count = summary.errorCount; + + const finalMessage = summary.finalText; + baseResult.details.final_message_preview = finalMessage.slice(0, 500); + + // ── Signals 1 & 2: columns_match / filters_match ───────────────────────── + const queryCalls = summary.toolCalls.filter((t) => t.normalizedName === 'query-datasource'); + if (queryCalls.length > 0) { const allActualColumns = new Set(); const allActualFilters = new Set(); - for (const input of queryInputs) { - for (const c of collectFieldCaptions(input.query?.fields)) allActualColumns.add(c); - for (const f of collectFilterCaptions(input.query?.filters)) allActualFilters.add(f); + for (const call of queryCalls) { + const query = findVizqlQuery(call.inputs) as VizqlQuery | null; + for (const c of collectFieldCaptions(query?.fields)) allActualColumns.add(c); + for (const f of collectFilterCaptions(query?.filters)) allActualFilters.add(f); } - actualColumns = [...allActualColumns]; - actualFilterFields = [...allActualFilters]; - - const expectedColsLower = birdCase.expected_columns.map((c) => c.toLowerCase()); - const expectedFiltersLower = birdCase.expected_filter_fields.map((f) => f.toLowerCase()); - - missingColumns = birdCase.expected_columns.filter( + baseResult.details.actual_columns = [...allActualColumns]; + baseResult.details.actual_filter_fields = [...allActualFilters]; + baseResult.details.missing_columns = birdCase.expected_columns.filter( (c) => !allActualColumns.has(c.toLowerCase()), ); - missingFilterFields = birdCase.expected_filter_fields.filter( + baseResult.details.missing_filter_fields = birdCase.expected_filter_fields.filter( (f) => !allActualFilters.has(f.toLowerCase()), ); - - columnsMatch = missingColumns.length === 0 && expectedColsLower.length > 0; - filtersMatch = missingFilterFields.length === 0 && expectedFiltersLower.length > 0; + baseResult.signals.columns_match = + baseResult.details.missing_columns.length === 0 && birdCase.expected_columns.length > 0; + baseResult.signals.filters_match = + baseResult.details.missing_filter_fields.length === 0 && + birdCase.expected_filter_fields.length > 0; } - // ── Signal 3: numeric_match ───────────────────────────────────────────── - - let numericMatch: boolean | null = null; - let extractedNumber: number | null = null; + // ── Signal 3: numeric_match ─────────────────────────────────────────────── if (finalMessage) { - const result = checkNumericMatch(finalMessage, birdCase); - numericMatch = result.matched; - extractedNumber = result.extracted; + const numeric = checkNumericMatch(finalMessage, birdCase); + baseResult.signals.numeric_match = numeric.matched; + baseResult.details.extracted_number = numeric.extracted; } - // ── Signal 4: semantic_match (LLM judge) ──────────────────────────────── - - let semanticMatch: number | null = null; - let llmJudge: LlmJudgeResult | null = null; - let llmJudgeError: string | null = null; - + // ── Signal 4: semantic_match (judge via GRADER_HARNESS) ────────────────── if (!finalMessage) { - llmJudgeError = 'No final message found in agent output.'; + baseResult.details.llm_judge_error = 'No final message found in trace outputs.'; } else { - const openaiKey = process.env.OPENAI_API_KEY; - if (!openaiKey) { - llmJudgeError = 'OPENAI_API_KEY not set; skipping LLM judge.'; - console.warn('WARN: OPENAI_API_KEY not set — semantic_match will be null.'); - } else { - const openai = new OpenAI({ apiKey: openaiKey }); - const judgeModel = process.env.BIRD_GRADE_MODEL ?? 'gpt-4o-mini'; - try { - llmJudge = await runLlmJudge(birdCase, finalMessage, openai, judgeModel); - semanticMatch = llmJudge.score; - } catch (error: unknown) { - llmJudgeError = error instanceof Error ? error.message : String(error); - console.error(`LLM judge error: ${llmJudgeError}`); - } - } + const judge = runJudge(birdCase, finalMessage); + baseResult.grader_harness = judge.harness; + baseResult.grader_model = judge.model; + baseResult.details.llm_judge = judge.result; + baseResult.details.llm_judge_error = judge.error; + baseResult.signals.semantic_match = judge.result?.score ?? null; } - // ── Verdict ────────────────────────────────────────────────────────────── - - function deriveVerdict(): BirdGradeResult['verdict'] { - if (runMeta?.claude_exit_code != null && runMeta.claude_exit_code !== 0) return 'error'; - // columns_match / filters_match are diagnostic only — they do not drive the verdict. - // Verdict is determined solely by whether the agent produced the correct answer. - if (semanticMatch === null && numericMatch === null) return 'skip'; - const semanticOk = semanticMatch != null && semanticMatch >= 0.8; - const numericOk = numericMatch === true; + // ── Verdict + accuracy ──────────────────────────────────────────────────── + const s = baseResult.signals; + baseResult.verdict = (() => { + if (runMeta.agent_exit_code != null && runMeta.agent_exit_code !== 0) return 'error'; + if (summary.hadError && s.semantic_match === null && s.numeric_match === null) return 'error'; + if (s.semantic_match === null && s.numeric_match === null) return 'skip'; + const semanticOk = s.semantic_match != null && s.semantic_match >= 0.8; + const numericOk = s.numeric_match === true; if (semanticOk && numericOk) return 'pass'; if (semanticOk || numericOk) return 'partial'; return 'fail'; - } - - // ── Tool summary ───────────────────────────────────────────────────────── - - const toolsUsed = [ - ...new Set( - hookRecords - .map((r) => normalizeToolName(r.normalized_tool_name ?? r.tool_name ?? '')) - .filter(Boolean), - ), - ]; - - // ── Assemble result ────────────────────────────────────────────────────── - - const birdResult: BirdGradeResult = { - run_id: runMeta.run_id, - question_id: questionId, - difficulty: runMeta.metadata?.difficulty ?? birdCase.difficulty ?? 'unknown', - graded_at: new Date().toISOString(), - model: modelName, - wall_s: runMeta.wall_ms != null ? Math.round(runMeta.wall_ms / 1000) : null, - tokens, - tool_calls: hookRecords.length, - tools_used: toolsUsed, - signals: { - numeric_match: numericMatch, - semantic_match: semanticMatch, - columns_match: columnsMatch, - filters_match: filtersMatch, - }, - details: { - expected_columns: birdCase.expected_columns, - actual_columns: actualColumns, - missing_columns: missingColumns, - expected_filter_fields: birdCase.expected_filter_fields, - actual_filter_fields: actualFilterFields, - missing_filter_fields: missingFilterFields, - expected_value: birdCase.answer_type === 'scalar' ? birdCase.expected_value : null, - expected_row_count: birdCase.expected_row_count, - extracted_number: extractedNumber, - final_message_preview: finalMessage.slice(0, 500), - llm_judge: llmJudge, - llm_judge_error: llmJudgeError, - }, - verdict: deriveVerdict(), - }; - - const runId = path.basename(absRunDir); - const gradeDir = path.join(GRADES_DIR, dateSlug(), runId); - fs.mkdirSync(gradeDir, { recursive: true }); - const resultPath = path.join(gradeDir, 'bird-result.json'); - fs.writeFileSync(resultPath, JSON.stringify(birdResult, null, 2)); + })(); + baseResult.accuracy = + baseResult.verdict === 'pass' + ? 1 + : baseResult.verdict === 'partial' + ? 0.5 + : baseResult.verdict === 'fail' + ? 0 + : null; + + const resultPath = writeResult(baseResult); // ── Print summary ───────────────────────────────────────────────────────── - - const v = birdResult.verdict.toUpperCase(); - const s = birdResult.signals; - console.log(`\nGrade: Q${questionId} (${birdResult.difficulty})`); - console.log(`Verdict: ${v}`); + console.log(`\nGrade: Q${questionId} (${difficulty}) — ${baseResult.verdict.toUpperCase()}`); + console.log(`Harness/model: ${baseResult.harness ?? '?'} / ${baseResult.model ?? 'n/a'}`); console.log( `numeric_match: ${s.numeric_match === null ? 'n/a' : s.numeric_match ? 'YES' : 'NO'}`, ); console.log( `semantic_match: ${s.semantic_match === null ? 'n/a' : s.semantic_match.toFixed(2)}` + - (llmJudge ? ` — ${llmJudge.reason}` : llmJudgeError ? ` (${llmJudgeError})` : ''), + (baseResult.details.llm_judge + ? ` — ${baseResult.details.llm_judge.reason}` + : baseResult.details.llm_judge_error + ? ` (${baseResult.details.llm_judge_error})` + : ''), + ); + console.log( + `columns_match: ${s.columns_match === null ? 'n/a' : s.columns_match ? 'YES' : `NO (missing: ${baseResult.details.missing_columns.join(', ')})`}`, + ); + console.log( + `filters_match: ${s.filters_match === null ? 'n/a' : s.filters_match ? 'YES' : `NO (missing: ${baseResult.details.missing_filter_fields.join(', ')})`}`, + ); + console.log(`Wall / TTFT: ${baseResult.wall_s ?? '?'}s / ${baseResult.ttft_s ?? '?'}s`); + console.log( + `Cost: ${baseResult.cost_usd != null ? `$${baseResult.cost_usd.toFixed(4)} (${baseResult.cost_source})` : 'n/a'}`, ); console.log( - `columns_match: ${s.columns_match === null ? 'n/a' : s.columns_match ? 'YES' : `NO (missing: ${missingColumns.join(', ')})`}`, + `Tokens (total): ${baseResult.tokens.total_tokens ?? 'n/a'} | LLM calls: ${baseResult.llm_calls} | errors: ${baseResult.error_count}`, ); console.log( - `filters_match: ${s.filters_match === null ? 'n/a' : s.filters_match ? 'YES' : `NO (missing: ${missingFilterFields.join(', ')})`}`, + `Tool calls: ${baseResult.tool_calls} (${baseResult.tools_used.join(', ') || 'none'})`, ); - const t = birdResult.tokens; - console.log(`Model: ${birdResult.model ?? 'n/a'}`); - console.log(`Wall time: ${birdResult.wall_s != null ? `${birdResult.wall_s}s` : 'n/a'}`); - console.log(`Tokens (total context): ${t.total_context_tokens ?? 'n/a'}`); - console.log(` input: ${t.input_tokens ?? 'n/a'}`); - console.log(` cache_creation: ${t.cache_creation_tokens ?? 'n/a'}`); - console.log(` cache_read: ${t.cache_read_tokens ?? 'n/a'}`); - console.log(` output: ${t.output_tokens ?? 'n/a'}`); - console.log(`Tool calls: ${birdResult.tool_calls} (${toolsUsed.join(', ') || 'none'})`); console.log(`Result: ${resultPath}`); } diff --git a/evals/grade-suite.ts b/evals/grade-suite.ts index 6dd030b0b..da959c592 100644 --- a/evals/grade-suite.ts +++ b/evals/grade-suite.ts @@ -41,6 +41,8 @@ type SuiteSummary = { started_at: string; finished_at: string; total_wall_ms: number; + harness?: string | null; + model?: string | null; cases: Array<{ question_id: number; difficulty: string; @@ -49,65 +51,63 @@ type SuiteSummary = { exit_code: number | null; wall_ms: number | null; timed_out: boolean; - tool_calls: number; - tools_used: Array; - tokens: { - input: number | null; - output: number | null; - cache_read: number | null; - cache_creation: number | null; - }; error: string | null; }>; - aggregate: { - total_cases: number; - completed: number; - errored: number; - timed_out: number; - avg_wall_ms: number | null; - total_input_tokens: number; - total_output_tokens: number; - }; }; type BirdResult = { run_id: string; + eval_run_id: string; question_id: number; difficulty: string; graded_at: string; + harness: string | null; model: string | null; + model_normalized: string | null; wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; tokens: { input_tokens: number | null; output_tokens: number | null; cache_creation_tokens: number | null; cache_read_tokens: number | null; - total_context_tokens: number | null; + total_tokens: number | null; }; tool_calls: number; tools_used: Array; + llm_calls: number; + error_count: number; signals: { numeric_match: boolean | null; semantic_match: number | null; columns_match: boolean | null; filters_match: boolean | null; }; - verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip'; + accuracy: number | null; + verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip' | 'grading_error'; }; type CaseGrade = { question_id: number; difficulty: string; run_id: string; - verdict: BirdResult['verdict'] | 'grading_error'; + verdict: BirdResult['verdict']; numeric_match: boolean | null; semantic_match: number | null; columns_match: boolean | null; filters_match: boolean | null; + accuracy: number | null; + harness: string | null; model: string | null; + model_normalized: string | null; wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; tool_calls: number; tools_used: Array; + llm_calls: number | null; + error_count: number | null; tokens: BirdResult['tokens'] | null; grade_file: string | null; grading_error: string | null; @@ -222,10 +222,17 @@ async function main(): Promise { semantic_match: gradeResult?.signals.semantic_match ?? null, columns_match: gradeResult?.signals.columns_match ?? null, filters_match: gradeResult?.signals.filters_match ?? null, + accuracy: gradeResult?.accuracy ?? null, + harness: gradeResult?.harness ?? null, model: gradeResult?.model ?? null, + model_normalized: gradeResult?.model_normalized ?? null, wall_s: gradeResult?.wall_s ?? null, - tool_calls: gradeResult?.tool_calls ?? c.tool_calls, - tools_used: gradeResult?.tools_used ?? c.tools_used, + ttft_s: gradeResult?.ttft_s ?? null, + cost_usd: gradeResult?.cost_usd ?? null, + tool_calls: gradeResult?.tool_calls ?? 0, + tools_used: gradeResult?.tools_used ?? [], + llm_calls: gradeResult?.llm_calls ?? null, + error_count: gradeResult?.error_count ?? null, tokens: gradeResult?.tokens ?? null, grade_file: gradeResult ? gradeFile : null, grading_error: gradingError, @@ -245,12 +252,25 @@ async function main(): Promise { const total = caseGrades.length; const passRate = total > 0 ? counts.pass / total : 0; + const mean = (values: Array): number | null => + values.length > 0 ? values.reduce((a, b) => a + b, 0) / values.length : null; + const sum = (values: Array): number => values.reduce((a, b) => a + b, 0); + + const accuracyValues = caseGrades.map((g) => g.accuracy).filter((v): v is number => v != null); + const wallValues = caseGrades.map((g) => g.wall_s).filter((v): v is number => v != null); + const ttftValues = caseGrades.map((g) => g.ttft_s).filter((v): v is number => v != null); + const costValues = caseGrades.map((g) => g.cost_usd).filter((v): v is number => v != null); + const toolCallValues = caseGrades.map((g) => g.tool_calls); + const errorValues = caseGrades.map((g) => g.error_count).filter((v): v is number => v != null); + const suiteGrade = { suite_run_id, suite_file: summary.suite_file, suite_started_at: summary.started_at, suite_finished_at: summary.finished_at, graded_at: new Date().toISOString(), + harness: summary.harness ?? caseGrades.find((g) => g.harness)?.harness ?? null, + model: caseGrades.find((g) => g.model_normalized)?.model_normalized ?? summary.model ?? null, summary: { total, pass: counts.pass, @@ -261,6 +281,19 @@ async function main(): Promise { grading_error: counts.grading_error, pass_rate: Math.round(passRate * 1000) / 1000, }, + metrics: { + mean_accuracy: accuracyValues.length + ? Math.round((mean(accuracyValues) ?? 0) * 1000) / 1000 + : null, + mean_wall_s: wallValues.length ? Math.round((mean(wallValues) ?? 0) * 10) / 10 : null, + mean_ttft_s: ttftValues.length ? Math.round((mean(ttftValues) ?? 0) * 10) / 10 : null, + total_cost_usd: costValues.length ? Math.round(sum(costValues) * 1e4) / 1e4 : null, + mean_cost_usd: costValues.length ? Math.round((mean(costValues) ?? 0) * 1e6) / 1e6 : null, + mean_tool_calls: toolCallValues.length + ? Math.round((mean(toolCallValues) ?? 0) * 10) / 10 + : null, + total_errors: errorValues.length ? sum(errorValues) : null, + }, cases: caseGrades, }; @@ -273,12 +306,19 @@ async function main(): Promise { console.log('\n═══════════════════════════════════════'); console.log(`Suite grade: ${suite_run_id}`); + console.log(` Harness/model: ${suiteGrade.harness ?? '?'} / ${suiteGrade.model ?? '?'}`); console.log(` Pass rate: ${counts.pass}/${total} (${(passRate * 100).toFixed(1)}%)`); console.log( ` pass=${counts.pass} partial=${counts.partial} fail=${counts.fail}` + ` error=${counts.error} skip=${counts.skip}` + (counts.grading_error > 0 ? ` grading_error=${counts.grading_error}` : ''), ); + const m = suiteGrade.metrics; + console.log( + ` accuracy=${m.mean_accuracy ?? '?'} mean_wall=${m.mean_wall_s ?? '?'}s ` + + `mean_ttft=${m.mean_ttft_s ?? '?'}s total_cost=${m.total_cost_usd != null ? `$${m.total_cost_usd}` : '?'} ` + + `mean_tools=${m.mean_tool_calls ?? '?'} errors=${m.total_errors ?? '?'}`, + ); console.log(` Grade file: ${suiteGradePath}`); } diff --git a/evals/grade.ts b/evals/grade.ts index 05bbe8df1..6926118c9 100644 --- a/evals/grade.ts +++ b/evals/grade.ts @@ -1,15 +1,25 @@ /** - * Local grader for Claude Code + LangSmith eval runs. + * Ad hoc grader for coding-agent + LangSmith eval runs. + * + * Tool coverage is sourced from the LangSmith trace (by `eval_run_id`), not local + * artifacts. Grades tool coverage + budget/timeout/exit only; use grade-bird.ts for + * answer-quality grading. * * Usage: - * npx tsx evals/grade.ts evals/runs/ + * npx tsx evals/grade.ts evals/runs// */ /* eslint-disable no-console */ +import dotenv from 'dotenv'; import * as fs from 'fs'; import * as path from 'path'; +import { fetchTraceSummary, makeClient, normalizeToolName } from './langsmith-reader.js'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + const runDir = process.argv[2]; if (!runDir) { console.error('Usage: npx tsx evals/grade.ts '); @@ -19,119 +29,89 @@ if (!runDir) { type RunMeta = { run_id: string; case_id: string; + eval_run_id?: string; + harness?: string; + model?: string | null; + langsmith_project?: string; expected_tools?: Array; - budget: { - max_tool_calls: number; - max_wall_ms: number; - }; + budget: { max_tool_calls: number; max_wall_ms: number }; wall_ms?: number; - claude_exit_code?: number; + agent_exit_code?: number; timed_out?: boolean; - langsmith?: { - project: string; - endpoint: string; - parent_run_id: string; - }; -}; - -type HookRecord = { - tool_name?: string; - normalized_tool_name?: string; - langsmith_run_id?: string; }; -type ChildPost = { - ok?: boolean; - status?: number; - tool_name?: string; -}; +type Outcome = 'pass' | 'fail' | 'timeout' | 'budget_exceeded' | 'error' | 'grading_error'; const absRunDir = path.resolve(runDir); const runMeta = JSON.parse(fs.readFileSync(path.join(absRunDir, 'run.json'), 'utf-8')) as RunMeta; -const hookRecords = readJsonl(path.join(absRunDir, 'hook.jsonl')); -const childPosts = readJsonl(path.join(absRunDir, 'langsmith-child-runs.jsonl')); -const stop = readOptionalJson(path.join(absRunDir, 'stop.json')); - -const observedTools = hookRecords - .map((record) => normalizeToolName(record.normalized_tool_name ?? record.tool_name ?? '')) - .filter(Boolean); -const expectedTools = runMeta.expected_tools ?? []; -const missingTools = expectedTools.filter( - (tool) => !observedTools.includes(normalizeToolName(tool)), -); -const failedLangSmithPosts = childPosts.filter((post) => post.ok === false); - -type Outcome = 'pass' | 'fail' | 'timeout' | 'budget_exceeded' | 'error'; - -function deriveOutcome(): Outcome { - if (runMeta.timed_out) return 'timeout'; - if (runMeta.claude_exit_code != null && runMeta.claude_exit_code !== 0) return 'error'; - if (hookRecords.length > runMeta.budget.max_tool_calls) return 'budget_exceeded'; - if (missingTools.length > 0 || failedLangSmithPosts.length > 0) return 'fail'; - return 'pass'; -} - -const result = { - run_id: runMeta.run_id, - case_id: runMeta.case_id, - graded_at: new Date().toISOString(), - outcome: deriveOutcome(), - langsmith: runMeta.langsmith ?? null, - expected_tools: expectedTools, - observed_tools: observedTools, - missing_tools: missingTools, - tool_calls: hookRecords.length, - langsmith_child_posts: { - total: childPosts.length, - failed: failedLangSmithPosts.length, - statuses: childPosts.map((post) => post.status).filter((status) => status != null), - }, - wall_ms: runMeta.wall_ms ?? null, - budget: runMeta.budget, - stop, -}; -const resultPath = path.join(absRunDir, 'result.json'); -fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); - -console.log(`\nGrade: ${runMeta.run_id}`); -console.log(`Outcome: ${result.outcome.toUpperCase()}`); -console.log(`Tool calls: ${result.tool_calls} (budget: ${runMeta.budget.max_tool_calls})`); -console.log(`Expected: ${expectedTools.length ? expectedTools.join(', ') : 'n/a'}`); -console.log(`Observed: ${observedTools.length ? observedTools.join(', ') : 'none'}`); -console.log(`Missing: ${missingTools.length ? missingTools.join(', ') : 'none'}`); -console.log( - `LangSmith: ${childPosts.length - failedLangSmithPosts.length}/${childPosts.length} child posts ok`, -); -console.log(`Result: ${resultPath}`); - -function readJsonl(filePath: string): Array { - if (!fs.existsSync(filePath)) return []; - return fs - .readFileSync(filePath, 'utf-8') - .split('\n') - .filter((line) => line.trim()) - .map((line) => { - try { - return JSON.parse(line) as T; - } catch { - return null; - } - }) - .filter((value): value is T => value !== null); -} +async function main(): Promise { + const evalRunId = runMeta.eval_run_id ?? runMeta.run_id; + const projectName = + runMeta.langsmith_project ?? process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; + + const client = makeClient(); + const summary = await fetchTraceSummary(client, { + projectName, + evalRunId, + pollIntervalMs: Number(process.env.GRADE_TRACE_POLL_MS ?? 5_000), + timeoutMs: Number(process.env.GRADE_TRACE_TIMEOUT_MS ?? 60_000), + }); + + const expectedTools = runMeta.expected_tools ?? []; + const observedTools = summary ? [...new Set(summary.toolCalls.map((t) => t.normalizedName))] : []; + const toolCalls = summary?.toolCalls.length ?? 0; + const missingTools = expectedTools.filter( + (tool) => !observedTools.includes(normalizeToolName(tool)), + ); + + const outcome: Outcome = (() => { + if (!summary) return 'grading_error'; + if (runMeta.timed_out) return 'timeout'; + if (runMeta.agent_exit_code != null && runMeta.agent_exit_code !== 0) return 'error'; + if (toolCalls > runMeta.budget.max_tool_calls) return 'budget_exceeded'; + if (missingTools.length > 0) return 'fail'; + return 'pass'; + })(); + + const result = { + run_id: runMeta.run_id, + case_id: runMeta.case_id, + eval_run_id: evalRunId, + graded_at: new Date().toISOString(), + outcome, + harness: runMeta.harness ?? null, + model: summary?.model ?? runMeta.model ?? null, + langsmith_project: projectName, + trace_id: summary?.traceId ?? null, + expected_tools: expectedTools, + observed_tools: observedTools, + missing_tools: missingTools, + tool_calls: toolCalls, + llm_calls: summary?.llmRunCount ?? null, + cost_usd: summary?.costUsd ?? null, + wall_ms: runMeta.wall_ms ?? summary?.wallMs ?? null, + budget: runMeta.budget, + trace_error: summary + ? null + : `No LangSmith trace found for eval_run_id=${evalRunId} in project "${projectName}".`, + }; -function readOptionalJson(filePath: string): unknown { - if (!fs.existsSync(filePath)) return null; - try { - return JSON.parse(fs.readFileSync(filePath, 'utf-8')); - } catch { - return null; - } + const resultPath = path.join(absRunDir, 'result.json'); + fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); + + console.log(`\nGrade: ${runMeta.run_id}`); + console.log(`Outcome: ${result.outcome.toUpperCase()}`); + console.log(`Harness: ${result.harness ?? 'n/a'} / ${result.model ?? 'n/a'}`); + console.log(`Tool calls: ${result.tool_calls} (budget: ${runMeta.budget.max_tool_calls})`); + console.log(`Expected: ${expectedTools.length ? expectedTools.join(', ') : 'n/a'}`); + console.log(`Observed: ${observedTools.length ? observedTools.join(', ') : 'none'}`); + console.log(`Missing: ${missingTools.length ? missingTools.join(', ') : 'none'}`); + if (result.trace_error) console.log(`Trace: ${result.trace_error}`); + console.log(`Result: ${resultPath}`); } -function normalizeToolName(toolName: string): string { - const parts = toolName.split('__'); - const raw = parts[parts.length - 1] || toolName; - return raw.replace(/^tableau_/, '').replace(/^tableau-/, ''); -} +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/evals/hooks/post-tool-langsmith.mjs b/evals/hooks/post-tool-langsmith.mjs deleted file mode 100644 index fe155d191..000000000 --- a/evals/hooks/post-tool-langsmith.mjs +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env node -/** - * Claude Code PostToolUse hook. - * - * Reads the hook envelope from stdin and creates a child LangSmith tool run - * under LANGSMITH_PARENT_RUN_ID. The runner sets all required environment - * variables; no secrets are written to disk by this hook. - */ - -/* eslint-disable @typescript-eslint/explicit-function-return-type */ - -import { randomUUID } from 'crypto'; -import fs from 'fs'; -import path from 'path'; - -const input = await readStdin(); -const envelope = parseJson(input); - -if (!envelope) { - process.exit(0); -} - -const apiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY; -const parentRunId = process.env.LANGSMITH_PARENT_RUN_ID; -const project = process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; -const endpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; -const runId = process.env.TMCP_EVAL_RUN_ID; -const runDir = process.env.TMCP_EVAL_RUN_DIR; -const childRunsLog = process.env.TMCP_EVAL_CHILD_RUNS_LOG; -const hookLog = process.env.TMCP_EVAL_HOOK_LOG; - -if (!apiKey || !parentRunId) { - appendJsonl(hookLog, { - ts: new Date().toISOString(), - warning: 'LANGSMITH_API_KEY/LANGCHAIN_API_KEY or LANGSMITH_PARENT_RUN_ID is not set', - tool_name: envelope.tool_name ?? null, - }); - process.exit(0); -} - -const now = new Date().toISOString(); -const toolName = String(envelope.tool_name ?? 'unknown-tool'); -const childRunId = randomUUID(); -const isFailure = envelope.hook_event_name === 'PostToolUseFailure'; - -// Estimate tool start time. The hook fires at completion so `now` is the end. -// Use the previous hook record's timestamp as the start (end of the last tool call -// or the run's started_at for the first call). This gives the waterfall real width. -const startTime = estimateToolStartTime(); -const toolResponse = envelope.tool_response ?? envelope.tool_output ?? envelope.error ?? null; - -const langSmithRun = { - id: childRunId, - name: normalizeToolName(toolName), - run_type: 'tool', - inputs: truncateValue({ - tool_name: toolName, - tool_input: envelope.tool_input ?? null, - }), - outputs: truncateValue({ - tool_response: toolResponse, - }), - start_time: startTime, - end_time: now, - parent_run_id: parentRunId, - session_name: project, - tags: ['tmcp-eval', 'claude-code', 'mcp-tool', normalizeToolName(toolName)], - error: isFailure ? stringifyError(toolResponse) : undefined, - extra: { - run_id: runId, - hook_event_name: envelope.hook_event_name ?? null, - session_id: envelope.session_id ?? null, - transcript_path: envelope.transcript_path ?? null, - tool_use_id: envelope.tool_use_id ?? null, - raw_tool_name: toolName, - }, -}; - -appendJsonl(hookLog, { - ts: now, - start_time: startTime, - run_id: runId, - langsmith_run_id: childRunId, - langsmith_parent_run_id: parentRunId, - hook_event_name: envelope.hook_event_name ?? null, - tool_name: toolName, - normalized_tool_name: normalizeToolName(toolName), - tool_use_id: envelope.tool_use_id ?? null, - tool_input: envelope.tool_input ?? null, -}); - -try { - const response = await fetch(`${endpoint.replace(/\/$/, '')}/runs`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-api-key': apiKey, - }, - body: JSON.stringify(langSmithRun), - }); - - const responseBody = await response.text(); - appendJsonl(childRunsLog, { - ts: now, - child_run_id: childRunId, - parent_run_id: parentRunId, - tool_name: toolName, - status: response.status, - ok: response.ok, - response: responseBody ? safeJsonOrText(responseBody) : null, - }); - - if (!response.ok) { - process.exit(0); - } -} catch (error) { - appendJsonl(childRunsLog, { - ts: now, - child_run_id: childRunId, - parent_run_id: parentRunId, - tool_name: toolName, - ok: false, - error: error instanceof Error ? error.message : String(error), - }); -} - -function estimateToolStartTime() { - // Prefer the PreToolUse timestamp for this exact tool_use_id — true dispatch time. - const toolUseId = envelope.tool_use_id; - if (runDir && toolUseId) { - const preToolTimesPath = path.join(runDir, 'pre-tool-times.jsonl'); - if (fs.existsSync(preToolTimesPath)) { - const lines = fs.readFileSync(preToolTimesPath, 'utf-8').trim().split('\n').filter(Boolean); - for (let i = lines.length - 1; i >= 0; i--) { - const record = parseJson(lines[i]); - if (record?.tool_use_id === toolUseId && record?.ts) return record.ts; - } - } - } - // Fall back to now (0ms duration) if PreToolUse record is missing. - return now; -} - -function readStdin() { - return new Promise((resolve) => { - let data = ''; - process.stdin.setEncoding('utf8'); - process.stdin.on('data', (chunk) => { - data += chunk; - }); - process.stdin.on('end', () => resolve(data)); - }); -} - -function parseJson(value) { - try { - return JSON.parse(value); - } catch { - return null; - } -} - -function appendJsonl(filePath, value) { - if (!filePath) return; - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); -} - -function normalizeToolName(toolNameValue) { - const parts = toolNameValue.split('__'); - return parts[parts.length - 1] || toolNameValue; -} - -function truncateValue(value) { - const maxChars = Number(process.env.LANGSMITH_MAX_PAYLOAD_CHARS ?? 200_000); - const serialized = JSON.stringify(value); - if (serialized.length <= maxChars) return value; - return { - truncated: true, - preview: serialized.slice(0, maxChars), - }; -} - -function stringifyError(value) { - if (!value) return 'Claude Code reported tool failure'; - return typeof value === 'string' ? value : JSON.stringify(truncateValue(value)); -} - -function safeJsonOrText(value) { - try { - return JSON.parse(value); - } catch { - return value; - } -} diff --git a/evals/hooks/pre-tool-langsmith.mjs b/evals/hooks/pre-tool-langsmith.mjs deleted file mode 100644 index 00f6707c5..000000000 --- a/evals/hooks/pre-tool-langsmith.mjs +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env node -/** - * Claude Code PreToolUse hook. - * - * Records the exact timestamp when each tool call is dispatched so that - * the PostToolUse hook can compute true tool execution latency rather than - * approximating it from the previous hook record. - * - * Writes one record per tool call to pre-tool-times.jsonl in the run directory. - */ - -/* eslint-disable @typescript-eslint/explicit-function-return-type */ - -import fs from 'fs'; -import path from 'path'; - -const input = await readStdin(); -const envelope = parseJson(input); - -if (!envelope) { - process.exit(0); -} - -const now = new Date().toISOString(); -const runDir = process.env.TMCP_EVAL_RUN_DIR; - -if (runDir && envelope.tool_use_id) { - const preToolTimesPath = path.join(runDir, 'pre-tool-times.jsonl'); - appendJsonl(preToolTimesPath, { - ts: now, - tool_use_id: envelope.tool_use_id, - tool_name: envelope.tool_name ?? null, - }); -} - -function readStdin() { - return new Promise((resolve) => { - let data = ''; - process.stdin.setEncoding('utf8'); - process.stdin.on('data', (chunk) => { - data += chunk; - }); - process.stdin.on('end', () => resolve(data)); - }); -} - -function parseJson(value) { - try { - return JSON.parse(value); - } catch { - return null; - } -} - -function appendJsonl(filePath, value) { - if (!filePath) return; - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); -} diff --git a/evals/hooks/stop-langsmith.mjs b/evals/hooks/stop-langsmith.mjs deleted file mode 100644 index c915d2bdd..000000000 --- a/evals/hooks/stop-langsmith.mjs +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env node -/** - * Claude Code Stop hook. - * - * Captures token usage from the Claude transcript and patches the parent - * LangSmith run with stop-hook metadata. The runner also patches the parent - * run after process exit with final stdout/status. - */ - -/* eslint-disable @typescript-eslint/explicit-function-return-type */ - -import fs from 'fs'; -import path from 'path'; - -const input = await readStdin(); -const envelope = parseJson(input) ?? {}; -const runDir = process.env.TMCP_EVAL_RUN_DIR; -if (!runDir) { - process.stderr.write('[stop-hook] TMCP_EVAL_RUN_DIR is not set — cannot write stop.json\n'); - process.exit(0); -} -const apiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY; -const parentRunId = process.env.LANGSMITH_PARENT_RUN_ID; -const endpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; - -const stopData = { - run_id: process.env.TMCP_EVAL_RUN_ID, - ts: new Date().toISOString(), - transcript_path: envelope.transcript_path ?? null, - stop_hook_active: envelope.stop_hook_active ?? null, - last_assistant_message: envelope.last_assistant_message ?? null, - usage: readUsage(envelope.transcript_path), -}; - -fs.mkdirSync(runDir, { recursive: true }); -fs.writeFileSync(path.join(runDir, 'stop.json'), `${JSON.stringify(stopData, null, 2)}\n`); - -if (apiKey && parentRunId) { - try { - await fetch(`${endpoint.replace(/\/$/, '')}/runs/${parentRunId}`, { - method: 'PATCH', - headers: { - 'content-type': 'application/json', - 'x-api-key': apiKey, - }, - body: JSON.stringify({ - extra: { - stop_hook: stopData, - }, - }), - }); - } catch { - // Hook failures should never fail the Claude Code session. - } -} - -function readUsage(transcriptPath) { - const usage = { - input_tokens: null, - output_tokens: null, - cache_read_input_tokens: null, - cache_creation_input_tokens: null, - }; - - if (!transcriptPath) return usage; - - try { - const lines = fs.readFileSync(transcriptPath, 'utf-8').trim().split('\n'); - for (const line of lines) { - const entry = parseJson(line); - const entryUsage = entry?.message?.usage ?? entry?.usage; - if (!entryUsage) continue; - - usage.input_tokens = (usage.input_tokens ?? 0) + (entryUsage.input_tokens ?? 0); - usage.output_tokens = (usage.output_tokens ?? 0) + (entryUsage.output_tokens ?? 0); - usage.cache_read_input_tokens = - (usage.cache_read_input_tokens ?? 0) + (entryUsage.cache_read_input_tokens ?? 0); - usage.cache_creation_input_tokens = - (usage.cache_creation_input_tokens ?? 0) + (entryUsage.cache_creation_input_tokens ?? 0); - } - } catch { - return usage; - } - - return usage; -} - -function readStdin() { - return new Promise((resolve) => { - let data = ''; - process.stdin.setEncoding('utf8'); - process.stdin.on('data', (chunk) => { - data += chunk; - }); - process.stdin.on('end', () => resolve(data)); - }); -} - -function parseJson(value) { - try { - return JSON.parse(value); - } catch { - return null; - } -} diff --git a/evals/langsmith-reader.ts b/evals/langsmith-reader.ts new file mode 100644 index 000000000..ca40ef0b4 --- /dev/null +++ b/evals/langsmith-reader.ts @@ -0,0 +1,284 @@ +/** + * LangSmith trace reader. + * + * The graders source all per-case metrics from the coding-agent trace in LangSmith + * (single source of truth). This module fetches the root run for an eval by its + * `eval_run_id` metadata (written by the runner into the plugin's custom metadata), + * pulls the child runs of that trace, and normalizes them into a `TraceSummary`. + */ + +import { Client, Run } from 'langsmith'; + +import { estimateCostUsd } from './pricing.js'; + +export type TraceToolCall = { + name: string; + normalizedName: string; + inputs: unknown; + outputs: unknown; + error: string | null; + startMs: number | null; + endMs: number | null; +}; + +export type TraceTokens = { + input: number | null; + output: number | null; + cacheRead: number | null; + cacheWrite: number | null; + total: number | null; +}; + +export type TraceSummary = { + traceId: string; + rootRunId: string; + model: string | null; + finalText: string; + toolCalls: Array; + tokens: TraceTokens; + costUsd: number | null; + costSource: 'langsmith' | 'estimated' | 'unavailable'; + wallMs: number | null; + ttftMs: number | null; + llmRunCount: number; + subagentCount: number; + errorCount: number; + hadError: boolean; +}; + +type KVMap = Record; + +export function makeClient(): Client { + return new Client({ + apiKey: process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY, + apiUrl: process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com', + }); +} + +export function normalizeToolName(toolName: string): string { + const parts = toolName.split('__'); + const raw = parts[parts.length - 1] || toolName; + return raw.replace(/^tableau_/, '').replace(/^tableau-/, ''); +} + +function toMs(value: number | string | undefined): number | null { + if (value == null) return null; + if (typeof value === 'number') return value; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function getMetadata(run: Run): KVMap { + const extra = (run.extra ?? {}) as KVMap; + return (extra.metadata as KVMap) ?? {}; +} + +/** Deeply search an object for the first `{ fields?, filters? }` VizQL-shaped query. */ +export function findVizqlQuery(value: unknown): { fields?: unknown; filters?: unknown } | null { + if (value == null || typeof value !== 'object') return null; + const obj = value as KVMap; + if ('fields' in obj || 'filters' in obj) { + return obj as { fields?: unknown; filters?: unknown }; + } + for (const v of Object.values(obj)) { + if (typeof v === 'object' && v !== null) { + const found = findVizqlQuery(v); + if (found) return found; + } + } + return null; +} + +/** Extract the assistant final text from a run's outputs, tolerating several shapes. */ +export function extractTextFromOutputs(outputs: KVMap | undefined): string { + if (!outputs) return ''; + const visit = (value: unknown, depth: number): string => { + if (depth > 6 || value == null) return ''; + if (typeof value === 'string') return value; + if (Array.isArray(value)) { + // Message arrays: prefer the last element with text content. + for (let i = value.length - 1; i >= 0; i--) { + const text = visit(value[i], depth + 1); + if (text.trim()) return text; + } + return ''; + } + if (typeof value === 'object') { + const obj = value as KVMap; + for (const key of ['output', 'result', 'text', 'content', 'message', 'messages', 'choices']) { + if (key in obj) { + const text = visit(obj[key], depth + 1); + if (text.trim()) return text; + } + } + } + return ''; + }; + return visit(outputs, 0).trim(); +} + +function extractServerCost(run: Run): number | null { + const anyRun = run as unknown as KVMap; + for (const key of ['total_cost', 'totalCost']) { + const v = anyRun[key]; + if (typeof v === 'number') return v; + if (typeof v === 'string' && v.trim() && Number.isFinite(Number(v))) return Number(v); + } + return null; +} + +function accumulateTokens(runs: Array): TraceTokens { + let input = 0; + let output = 0; + let cacheRead = 0; + let cacheWrite = 0; + let found = false; + + for (const run of runs) { + if (run.run_type !== 'llm') continue; + const prompt = run.prompt_tokens ?? 0; + const completion = run.completion_tokens ?? 0; + if (run.prompt_tokens != null || run.completion_tokens != null) found = true; + input += prompt; + output += completion; + + // Cache detail lives on usage_metadata (coding-agent-v1 preserves it). + const meta = getMetadata(run); + const usage = (meta.usage_metadata ?? (run.extra as KVMap)?.usage_metadata) as + | KVMap + | undefined; + if (usage) { + const inputDetails = usage.input_token_details as KVMap | undefined; + if (inputDetails) { + const cr = Number(inputDetails.cache_read ?? 0); + const cc = Number(inputDetails.cache_creation ?? inputDetails.cache_write ?? 0); + if (Number.isFinite(cr)) cacheRead += cr; + if (Number.isFinite(cc)) cacheWrite += cc; + found = true; + } + } + } + + if (!found) { + return { input: null, output: null, cacheRead: null, cacheWrite: null, total: null }; + } + return { + input, + output, + cacheRead, + cacheWrite, + total: input + output + cacheRead + cacheWrite, + }; +} + +async function collect(iterable: AsyncIterable): Promise> { + const out: Array = []; + for await (const item of iterable) out.push(item); + return out; +} + +async function findRootRun( + client: Client, + projectName: string, + evalRunId: string, +): Promise { + const filter = `has(metadata, '{"eval_run_id": "${evalRunId}"}')`; + const runs = await collect(client.listRuns({ projectName, filter, isRoot: true, limit: 1 })); + return runs[0] ?? null; +} + +export type FetchOptions = { + projectName: string; + evalRunId: string; + pollIntervalMs?: number; + timeoutMs?: number; +}; + +/** + * Fetch and summarize the trace for an eval run, polling to absorb ingestion latency. + * Returns null if no matching trace appears within the timeout (grader → grading_error). + */ +export async function fetchTraceSummary( + client: Client, + opts: FetchOptions, +): Promise { + const pollIntervalMs = opts.pollIntervalMs ?? 5_000; + const timeoutMs = opts.timeoutMs ?? 60_000; + const deadline = Date.now() + timeoutMs; + + let root: Run | null = null; + for (;;) { + root = await findRootRun(client, opts.projectName, opts.evalRunId); + if (root || Date.now() >= deadline) break; + await new Promise((r) => setTimeout(r, pollIntervalMs)); + } + if (!root || !root.trace_id) return null; + + const allRuns = await collect(client.listRuns({ traceId: root.trace_id })); + + const toolRuns = allRuns.filter((r) => r.run_type === 'tool'); + const llmRuns = allRuns.filter((r) => r.run_type === 'llm'); + const subagentRuns = allRuns.filter( + (r) => r.run_type === 'chain' && String(getMetadata(r).ls_run_type ?? '') === 'subagent', + ); + + const toolCalls: Array = toolRuns.map((r) => ({ + name: r.name, + normalizedName: normalizeToolName(r.name), + inputs: r.inputs, + outputs: r.outputs, + error: r.error ?? null, + startMs: toMs(r.start_time), + endMs: toMs(r.end_time), + })); + + const tokens = accumulateTokens(allRuns); + const rootMeta = getMetadata(root); + const model = + (rootMeta.ls_model_name as string | undefined) ?? + llmRuns.map((r) => getMetadata(r).ls_model_name as string | undefined).find(Boolean) ?? + null; + + const rootStart = toMs(root.start_time); + const rootEnd = toMs(root.end_time); + const wallMs = rootStart != null && rootEnd != null ? rootEnd - rootStart : null; + const firstLlmStart = llmRuns + .map((r) => toMs(r.start_time)) + .filter((v): v is number => v != null) + .sort((a, b) => a - b)[0]; + const ttftMs = rootStart != null && firstLlmStart != null ? firstLlmStart - rootStart : null; + + const errorRuns = allRuns.filter((r) => r.error != null && r.error !== ''); + + let costUsd = extractServerCost(root); + let costSource: TraceSummary['costSource'] = costUsd != null ? 'langsmith' : 'unavailable'; + if (costUsd == null) { + const estimate = estimateCostUsd(model, { + input: tokens.input, + output: tokens.output, + cacheRead: tokens.cacheRead, + cacheWrite: tokens.cacheWrite, + }); + if (estimate != null) { + costUsd = estimate; + costSource = 'estimated'; + } + } + + return { + traceId: root.trace_id, + rootRunId: root.id, + model, + finalText: extractTextFromOutputs(root.outputs as KVMap | undefined), + toolCalls, + tokens, + costUsd, + costSource, + wallMs, + ttftMs, + llmRunCount: llmRuns.length, + subagentCount: subagentRuns.length, + errorCount: errorRuns.length, + hadError: (root.error != null && root.error !== '') || errorRuns.length > 0, + }; +} diff --git a/evals/model-normalize.ts b/evals/model-normalize.ts new file mode 100644 index 000000000..580c529ee --- /dev/null +++ b/evals/model-normalize.ts @@ -0,0 +1,45 @@ +/** + * Model id normalization. + * + * Different harnesses report the same underlying model with different ids + * (Cursor `sonnet-4.5`, Claude `claude-sonnet-4-5-20250101`, LangSmith + * `ls_model_name`, etc.). Reports group on a single canonical id so the same model + * run through different harnesses lines up. Both the raw id and the normalized id + * are recorded on each grade. + */ + +/** Ordered rules: the first regex that matches maps to the canonical id. */ +const RULES: Array<{ pattern: RegExp; canonical: string }> = [ + // Anthropic Claude family + { pattern: /opus-4[.\- ]?8/i, canonical: 'claude-opus-4.8' }, + { pattern: /opus-4[.\- ]?7/i, canonical: 'claude-opus-4.7' }, + { pattern: /opus-4[.\- ]?6/i, canonical: 'claude-opus-4.6' }, + { pattern: /opus-4/i, canonical: 'claude-opus-4' }, + { pattern: /sonnet-4[.\- ]?6|sonnet-4-6/i, canonical: 'claude-sonnet-4.6' }, + { pattern: /sonnet-4[.\- ]?5|sonnet-4-5/i, canonical: 'claude-sonnet-4.5' }, + { pattern: /sonnet-4/i, canonical: 'claude-sonnet-4' }, + { pattern: /haiku-4[.\- ]?5|haiku-4-5/i, canonical: 'claude-haiku-4.5' }, + { pattern: /claude.*3[.\- ]?7.*sonnet|3-7-sonnet/i, canonical: 'claude-3.7-sonnet' }, + // OpenAI / Codex family + { pattern: /gpt-5[.\- ]?6.*codex|gpt-5-6-codex/i, canonical: 'gpt-5.6-codex' }, + { pattern: /gpt-5.*codex/i, canonical: 'gpt-5-codex' }, + { pattern: /gpt-5[.\- ]?6/i, canonical: 'gpt-5.6' }, + { pattern: /gpt-5/i, canonical: 'gpt-5' }, + { pattern: /gpt-4o[.\- ]?mini/i, canonical: 'gpt-4o-mini' }, + { pattern: /gpt-4o/i, canonical: 'gpt-4o' }, + { pattern: /o4[.\- ]?mini/i, canonical: 'o4-mini' }, + { pattern: /o3/i, canonical: 'o3' }, + // Google Gemini + { pattern: /gemini-3[.\- ]?1.*pro/i, canonical: 'gemini-3.1-pro' }, + { pattern: /gemini.*pro/i, canonical: 'gemini-pro' }, +]; + +export function normalizeModel(raw: string | null | undefined): string { + if (!raw) return 'unknown'; + const trimmed = raw.trim(); + for (const rule of RULES) { + if (rule.pattern.test(trimmed)) return rule.canonical; + } + // Fall back to a lightly-cleaned raw id (strip trailing date stamps). + return trimmed.replace(/-\d{8}$/, '').toLowerCase(); +} diff --git a/evals/pricing.ts b/evals/pricing.ts new file mode 100644 index 000000000..a9dad39d2 --- /dev/null +++ b/evals/pricing.ts @@ -0,0 +1,68 @@ +/** + * Per-model pricing map (USD per 1M tokens), used only as a fallback when LangSmith + * does not report a computed cost for a trace. Keyed by canonical (normalized) model + * id from `model-normalize.ts`. Keep this current; prices drift. + * + * Rates are approximate list prices and intended for relative comparison in reports, + * not billing. Cache-read is typically a fraction of input; cache-write a small + * premium. Where a model's exact rates are unknown, omit it — cost will be null. + */ + +import { normalizeModel } from './model-normalize.js'; + +export type ModelPricePer1M = { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; +}; + +// USD per 1,000,000 tokens. +const PRICES: Record = { + 'claude-opus-4.8': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-opus-4.7': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-opus-4.6': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-opus-4': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-sonnet-4.6': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'claude-sonnet-4.5': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'claude-sonnet-4': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'claude-haiku-4.5': { input: 1, output: 5, cacheRead: 0.1, cacheWrite: 1.25 }, + 'claude-3.7-sonnet': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'gpt-5.6-codex': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-5-codex': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-5.6': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-5': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-4o': { input: 2.5, output: 10, cacheRead: 1.25, cacheWrite: 2.5 }, + 'gpt-4o-mini': { input: 0.15, output: 0.6, cacheRead: 0.075, cacheWrite: 0.15 }, +}; + +export type TokenCounts = { + input: number | null; + output: number | null; + cacheRead: number | null; + cacheWrite: number | null; +}; + +/** + * Estimate cost in USD from token counts. Returns null if the model has no pricing + * entry (so callers can distinguish "free" from "unknown"). + */ +export function estimateCostUsd( + model: string | null | undefined, + tokens: TokenCounts, +): number | null { + const canonical = normalizeModel(model); + const price = PRICES[canonical]; + if (!price) return null; + const input = tokens.input ?? 0; + const output = tokens.output ?? 0; + const cacheRead = tokens.cacheRead ?? 0; + const cacheWrite = tokens.cacheWrite ?? 0; + const cost = + (input * price.input + + output * price.output + + cacheRead * price.cacheRead + + cacheWrite * price.cacheWrite) / + 1_000_000; + return Math.round(cost * 1e6) / 1e6; +} diff --git a/evals/report.ts b/evals/report.ts new file mode 100644 index 000000000..dc8558d38 --- /dev/null +++ b/evals/report.ts @@ -0,0 +1,320 @@ +/** + * Longitudinal quality report for the Tableau MCP eval suite. + * + * Scans every graded case (bird-result.json under evals/grades/) and rolls the + * per-case metrics up into cohorts by (harness, normalized model) and over time, + * emitting JSON + CSV + a Markdown summary under `evals/reports//`. + * + * Per-case metrics reported: accuracy (verdict-derived), latency (wall_s, ttft_s), + * cost ($), tool-call count, error count, token totals, and the four quality signals + * (numeric/semantic/columns/filters match). + * + * Usage: + * npx tsx evals/report.ts # scan all grades + * npx tsx evals/report.ts --since 2026-07-01 + * npx tsx evals/report.ts --harness codex --model gpt-5.6-codex + */ + +/* eslint-disable no-console */ + +import * as fs from 'fs'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const GRADES_DIR = path.join(EVALS_DIR, 'grades'); +const REPORTS_DIR = path.join(EVALS_DIR, 'reports'); + +const args = process.argv.slice(2); +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} +const sinceFilter = getArgValue('--since'); +const harnessFilter = getArgValue('--harness'); +const modelFilter = getArgValue('--model'); + +type BirdResult = { + run_id: string; + eval_run_id?: string; + question_id: number; + difficulty: string; + graded_at: string; + harness: string | null; + model: string | null; + model_normalized: string | null; + wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; + tokens: { + input_tokens: number | null; + output_tokens: number | null; + total_tokens: number | null; + }; + tool_calls: number; + llm_calls?: number; + error_count?: number; + signals: { + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + }; + accuracy: number | null; + verdict: string; +}; + +type CohortStats = { + cohort: string; + harness: string; + model: string; + n: number; + pass: number; + partial: number; + fail: number; + error: number; + skip: number; + grading_error: number; + pass_rate: number | null; + mean_accuracy: number | null; + mean_wall_s: number | null; + mean_ttft_s: number | null; + total_cost_usd: number | null; + mean_cost_usd: number | null; + mean_tool_calls: number | null; + total_tokens: number | null; + total_errors: number | null; + numeric_match_rate: number | null; + semantic_match_rate: number | null; + columns_match_rate: number | null; + filters_match_rate: number | null; + first_graded_at: string; + last_graded_at: string; +}; + +// ─── Collection ───────────────────────────────────────────────────────────── + +function walkBirdResults(dir: string): Array { + const out: Array = []; + if (!fs.existsSync(dir)) return out; + const stack = [dir]; + while (stack.length) { + const current = stack.pop() as string; + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) { + stack.push(full); + } else if (entry.name === 'bird-result.json') { + try { + out.push(JSON.parse(fs.readFileSync(full, 'utf-8')) as BirdResult); + } catch { + // Skip malformed files. + } + } + } + } + return out; +} + +function mean(values: Array): number | null { + return values.length ? values.reduce((a, b) => a + b, 0) / values.length : null; +} +function sum(values: Array): number { + return values.reduce((a, b) => a + b, 0); +} +function round(value: number | null, places: number): number | null { + if (value == null) return null; + const f = 10 ** places; + return Math.round(value * f) / f; +} +function rate(values: Array): number | null { + const defined = values.filter((v): v is boolean => v != null); + return defined.length ? defined.filter(Boolean).length / defined.length : null; +} + +function computeCohort(cohortKey: string, results: Array): CohortStats { + const [harness, model] = cohortKey.split('||'); + const verdicts = results.map((r) => r.verdict); + const countVerdict = (v: string): number => verdicts.filter((x) => x === v).length; + const graded = results.filter((r) => r.verdict !== 'grading_error').length; + + const gradedAt = results.map((r) => r.graded_at).sort(); + + return { + cohort: cohortKey, + harness, + model, + n: results.length, + pass: countVerdict('pass'), + partial: countVerdict('partial'), + fail: countVerdict('fail'), + error: countVerdict('error'), + skip: countVerdict('skip'), + grading_error: countVerdict('grading_error'), + pass_rate: graded > 0 ? round(countVerdict('pass') / graded, 3) : null, + mean_accuracy: round( + mean(results.map((r) => r.accuracy).filter((v): v is number => v != null)), + 3, + ), + mean_wall_s: round(mean(results.map((r) => r.wall_s).filter((v): v is number => v != null)), 1), + mean_ttft_s: round(mean(results.map((r) => r.ttft_s).filter((v): v is number => v != null)), 1), + total_cost_usd: round(sum(results.map((r) => r.cost_usd ?? 0)), 4), + mean_cost_usd: round( + mean(results.map((r) => r.cost_usd).filter((v): v is number => v != null)), + 6, + ), + mean_tool_calls: round(mean(results.map((r) => r.tool_calls)), 1), + total_tokens: round(sum(results.map((r) => r.tokens?.total_tokens ?? 0)), 0), + total_errors: round(sum(results.map((r) => r.error_count ?? 0)), 0), + numeric_match_rate: round(rate(results.map((r) => r.signals.numeric_match)), 3), + semantic_match_rate: round( + mean(results.map((r) => r.signals.semantic_match).filter((v): v is number => v != null)), + 3, + ), + columns_match_rate: round(rate(results.map((r) => r.signals.columns_match)), 3), + filters_match_rate: round(rate(results.map((r) => r.signals.filters_match)), 3), + first_graded_at: gradedAt[0] ?? '', + last_graded_at: gradedAt[gradedAt.length - 1] ?? '', + }; +} + +// ─── CSV / Markdown ─────────────────────────────────────────────────────────── + +const COHORT_COLUMNS: Array = [ + 'harness', + 'model', + 'n', + 'pass_rate', + 'mean_accuracy', + 'mean_wall_s', + 'mean_ttft_s', + 'total_cost_usd', + 'mean_cost_usd', + 'mean_tool_calls', + 'total_errors', + 'numeric_match_rate', + 'semantic_match_rate', + 'columns_match_rate', + 'filters_match_rate', + 'first_graded_at', + 'last_graded_at', +]; + +function toCsv(cohorts: Array): string { + const header = COHORT_COLUMNS.join(','); + const rows = cohorts.map((c) => + COHORT_COLUMNS.map((col) => { + const value = c[col]; + const str = value == null ? '' : String(value); + return /[",\n]/.test(str) ? `"${str.replace(/"/g, '""')}"` : str; + }).join(','), + ); + return [header, ...rows].join('\n'); +} + +function toMarkdown(cohorts: Array, totalCases: number): string { + const lines: Array = []; + lines.push('# Tableau MCP Eval — Longitudinal Report'); + lines.push(''); + lines.push(`Generated: ${new Date().toISOString()}`); + lines.push(`Graded cases scanned: ${totalCases}`); + if (sinceFilter) lines.push(`Since: ${sinceFilter}`); + if (harnessFilter) lines.push(`Harness filter: ${harnessFilter}`); + if (modelFilter) lines.push(`Model filter: ${modelFilter}`); + lines.push(''); + lines.push('## Cohorts (harness × normalized model)'); + lines.push(''); + lines.push( + '| Harness | Model | N | Pass rate | Accuracy | Wall (s) | TTFT (s) | Total $ | $/case | Tools/case | Errors | Semantic | Cols | Filters |', + ); + lines.push( + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ); + for (const c of cohorts) { + lines.push( + `| ${c.harness} | ${c.model} | ${c.n} | ${fmt(c.pass_rate)} | ${fmt(c.mean_accuracy)} | ` + + `${fmt(c.mean_wall_s)} | ${fmt(c.mean_ttft_s)} | ${fmtMoney(c.total_cost_usd)} | ${fmtMoney(c.mean_cost_usd)} | ` + + `${fmt(c.mean_tool_calls)} | ${c.total_errors ?? ''} | ${fmt(c.semantic_match_rate)} | ` + + `${fmt(c.columns_match_rate)} | ${fmt(c.filters_match_rate)} |`, + ); + } + lines.push(''); + return lines.join('\n'); +} + +function fmt(value: number | null): string { + return value == null ? '—' : String(value); +} +function fmtMoney(value: number | null): string { + return value == null ? '—' : `$${value}`; +} + +// ─── Main ─────────────────────────────────────────────────────────────────── + +function main(): void { + let results = walkBirdResults(GRADES_DIR); + + if (sinceFilter) results = results.filter((r) => r.graded_at >= sinceFilter); + if (harnessFilter) results = results.filter((r) => (r.harness ?? '') === harnessFilter); + if (modelFilter) { + results = results.filter((r) => (r.model_normalized ?? r.model ?? '') === modelFilter); + } + + if (results.length === 0) { + console.error( + `No graded cases found under ${GRADES_DIR}` + + (sinceFilter || harnessFilter || modelFilter ? ' matching the given filters.' : '.'), + ); + process.exit(1); + } + + const cohortMap = new Map>(); + for (const r of results) { + const key = `${r.harness ?? 'unknown'}||${r.model_normalized ?? r.model ?? 'unknown'}`; + const bucket = cohortMap.get(key) ?? []; + bucket.push(r); + cohortMap.set(key, bucket); + } + + const cohorts = [...cohortMap.entries()] + .map(([key, rs]) => computeCohort(key, rs)) + .sort((a, b) => (b.pass_rate ?? -1) - (a.pass_rate ?? -1)); + + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const outDir = path.join(REPORTS_DIR, stamp); + fs.mkdirSync(outDir, { recursive: true }); + + const jsonReport = { + generated_at: new Date().toISOString(), + filters: { + since: sinceFilter ?? null, + harness: harnessFilter ?? null, + model: modelFilter ?? null, + }, + total_cases: results.length, + cohorts, + }; + + const jsonPath = path.join(outDir, 'longitudinal.json'); + const csvPath = path.join(outDir, 'longitudinal.csv'); + const mdPath = path.join(outDir, 'summary.md'); + fs.writeFileSync(jsonPath, JSON.stringify(jsonReport, null, 2)); + fs.writeFileSync(csvPath, toCsv(cohorts)); + fs.writeFileSync(mdPath, toMarkdown(cohorts, results.length)); + + console.log( + `\nLongitudinal report over ${results.length} graded cases, ${cohorts.length} cohorts:\n`, + ); + for (const c of cohorts) { + console.log( + ` ${c.harness} / ${c.model}: n=${c.n} pass_rate=${fmt(c.pass_rate)} ` + + `acc=${fmt(c.mean_accuracy)} wall=${fmt(c.mean_wall_s)}s $=${fmtMoney(c.total_cost_usd)} ` + + `tools/case=${fmt(c.mean_tool_calls)} errors=${c.total_errors ?? 0}`, + ); + } + console.log(`\n JSON: ${jsonPath}`); + console.log(` CSV: ${csvPath}`); + console.log(` MD: ${mdPath}`); +} + +main(); diff --git a/evals/run-case.ts b/evals/run-case.ts index ae532081e..f37c3fd47 100644 --- a/evals/run-case.ts +++ b/evals/run-case.ts @@ -1,14 +1,22 @@ /** - * Claude Code + LangSmith eval runner for the Tableau MCP server. + * Multi-agent eval runner for the Tableau MCP server. + * + * Drives Claude Code, Cursor, or Codex (selected by AGENT_HARNESS) against the + * Tableau MCP server for a single case. Tracing is handled by the official LangSmith + * coding-agent plugin for the selected agent (enabled via env in the adapter); this + * runner only spawns the CLI, stamps the `eval_run_id` correlation metadata, and + * records run metadata. Grading reads the trace back from LangSmith. * * Usage: - * npx tsx evals/run-case.ts evals/cases/list-datasources.json - * npx tsx evals/run-case.ts input "sales by region" + * npx tsx evals/run-case.ts evals/cases/list-datasources.json [--run-id ] + * npx tsx evals/run-case.ts input "sales by region" [--run-id ] * * Required environment: - * LANGSMITH_API_KEY or LANGCHAIN_API_KEY - * SERVER plus one supported Tableau auth configuration, for example: - * AUTH=pat SITE_NAME= PAT_NAME= PAT_VALUE= + * LANGSMITH_API_KEY (or LANGCHAIN_API_KEY), LANGSMITH_PROJECT + * SERVER plus a supported Tableau auth config, e.g. + * AUTH=pat SITE_NAME= PAT_NAME= PAT_VALUE= + * Optional: + * AGENT_HARNESS=claude-code|cursor|codex (default claude-code), AGENT_MODEL= */ /* eslint-disable no-console */ @@ -19,18 +27,19 @@ import dotenv from 'dotenv'; import * as fs from 'fs'; import * as path from 'path'; +import { AgentInvocation, getAdapter, resolveHarness, RunContext } from './adapters/index.js'; + const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); dotenv.config({ path: path.join(REPO_ROOT, '.env') }); const EVALS_DIR = path.join(REPO_ROOT, 'evals'); const RUNS_DIR = path.join(EVALS_DIR, 'runs'); -const HOOKS_DIR = path.join(EVALS_DIR, 'hooks'); +const MCP_SERVER_ENTRY = path.join(REPO_ROOT, 'build', 'index.js'); function dateSlug(): string { const d = new Date(); return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; } -const MCP_SERVER_ENTRY = path.join(REPO_ROOT, 'build', 'index.js'); type EvalCase = { id: string; @@ -46,216 +55,138 @@ type EvalCase = { }; }; -type LangSmithRun = { - id: string; - name: string; - run_type: 'chain' | 'tool' | 'llm'; - inputs?: Record; - outputs?: Record; - start_time?: string; - end_time?: string; - parent_run_id?: string; - session_name?: string; - tags?: Array; - extra?: Record; - error?: string; -}; - const args = process.argv.slice(2); const positionalArgs = args.filter((arg) => !arg.startsWith('--')); -const runIdArg = getArgValue('--run-id'); + +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} if (positionalArgs.length === 0) { console.error( - 'Usage: npx tsx evals/run-case.ts [--run-id ]\n' + + 'Usage: npx tsx evals/run-case.ts [--run-id ] [--agent-harness ] [--agent-model ]\n' + ' or: npx tsx evals/run-case.ts input "" [--run-id ]', ); process.exit(1); } +const runIdArg = getArgValue('--run-id'); +const suiteRunId = getArgValue('--suite-run-id') ?? null; + +const harness = resolveHarness(getArgValue('--agent-harness'), 'AGENT_HARNESS'); +const adapter = getAdapter(harness); +const model = adapter.resolveModel(getArgValue('--agent-model') ?? process.env.AGENT_MODEL); + +const langSmithApiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY ?? ''; +const langSmithProject = process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; +const langSmithEndpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; + const evalCase = loadEvalCase(positionalArgs); const runId = runIdArg ?? `${evalCase.id}-${Date.now()}-${randomUUID().slice(0, 8)}`; const runDir = path.join(RUNS_DIR, dateSlug(), runId); -const logsDir = path.join(runDir, 'logs'); -const hookLog = path.join(runDir, 'hook.jsonl'); -const childRunsLog = path.join(runDir, 'langsmith-child-runs.jsonl'); -const messageRunsLog = path.join(runDir, 'langsmith-message-runs.jsonl'); -const settingsPath = path.join(runDir, 'claude-settings.json'); -const mcpConfigPath = path.join(runDir, 'mcp-config.json'); - -const langSmithApiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY; -const langSmithProject = process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; -const langSmithEndpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; + const budget = { - max_tool_calls: evalCase.budget?.max_tool_calls ?? 20, - max_wall_ms: evalCase.budget?.max_wall_ms ?? 5 * 60 * 1000, + maxToolCalls: evalCase.budget?.max_tool_calls ?? 20, + maxWallMs: evalCase.budget?.max_wall_ms ?? 5 * 60 * 1000, }; preflight(); -fs.mkdirSync(logsDir, { recursive: true }); +fs.mkdirSync(runDir, { recursive: true }); const prompt = expandTemplate(evalCase.prompt); -const parentRunId = randomUUID(); const startedAt = new Date().toISOString(); +const questionId = + typeof evalCase.metadata?.question_id === 'number' + ? (evalCase.metadata.question_id as number) + : null; + +const runContext: RunContext = { + runId, + suiteRunId, + runDir, + prompt, + model, + mcpServerEntry: MCP_SERVER_ENTRY, + mcpServerEnv: tableauServerEnv(), + questionId, + langsmith: { apiKey: langSmithApiKey, project: langSmithProject, endpoint: langSmithEndpoint }, + budget, +}; + const runMeta = { run_id: runId, + suite_run_id: suiteRunId, case_id: evalCase.id, name: evalCase.name ?? evalCase.id, description: evalCase.description ?? null, started_at: startedAt, prompt, + harness, + model: model || null, + eval_run_id: runId, + langsmith_project: langSmithProject, expected_tools: evalCase.expected_tools ?? [], tags: evalCase.tags ?? [], metadata: evalCase.metadata ?? {}, budget, - langsmith: { - project: langSmithProject, - endpoint: langSmithEndpoint, - parent_run_id: parentRunId, - }, git_sha: tryExec('git', ['rev-parse', 'HEAD']), git_branch: tryExec('git', ['rev-parse', '--abbrev-ref', 'HEAD']), }; +adapter.writeConfig(runContext); fs.writeFileSync(path.join(runDir, 'run.json'), JSON.stringify(runMeta, null, 2)); -writeClaudeSettings(); -writeMcpConfig(); -console.log(`\nRun ID: ${runId}`); -console.log(`Run dir: ${runDir}`); +console.log(`\nRun ID: ${runId}`); +console.log(`Harness / model: ${harness} / ${model || '(cli default)'}`); console.log(`LangSmith project: ${langSmithProject}`); +console.log(`Run dir: ${runDir}`); -createLangSmithRun({ - id: parentRunId, - name: evalCase.name ?? evalCase.id, - run_type: 'chain', - inputs: { prompt, case: evalCase }, - start_time: startedAt, - session_name: langSmithProject, - tags: ['tmcp-eval', 'claude-code', ...(evalCase.tags ?? [])], - extra: { - run_id: runId, - case_id: evalCase.id, - git_sha: runMeta.git_sha, - git_branch: runMeta.git_branch, - expected_tools: evalCase.expected_tools ?? [], - }, -}); +const invocation: AgentInvocation = adapter.buildInvocation(runContext); const startMs = Date.now(); -let claudeExitCode = 0; -let agentOutput = ''; +let exitCode = 0; +let stdout = ''; let errorMessage: string | undefined; try { - const output = execFileSync( - 'claude', - [ - '--settings', - settingsPath, - '--mcp-config', - mcpConfigPath, - '--strict-mcp-config', - '--allowedTools', - 'ToolSearch,mcp__tableau__*', - '--output-format', - 'stream-json', - '--verbose', - '-p', - prompt, - ], - { - env: { - ...process.env, - LANGSMITH_API_KEY: langSmithApiKey, - LANGSMITH_PROJECT: langSmithProject, - LANGSMITH_ENDPOINT: langSmithEndpoint, - LANGSMITH_PARENT_RUN_ID: parentRunId, - TMCP_EVAL_RUN_ID: runId, - TMCP_EVAL_RUN_DIR: runDir, - TMCP_EVAL_HOOK_LOG: hookLog, - TMCP_EVAL_CHILD_RUNS_LOG: childRunsLog, - }, - cwd: REPO_ROOT, - timeout: budget.max_wall_ms + 10_000, - maxBuffer: 25 * 1024 * 1024, - }, - ); - agentOutput = output.toString(); + stdout = execFileSync(invocation.command, invocation.args, { + env: { ...process.env, ...invocation.env }, + cwd: invocation.cwd, + timeout: invocation.timeoutMs, + maxBuffer: 25 * 1024 * 1024, + }).toString(); } catch (error: unknown) { - const execError = error as { - status?: number; - stdout?: Buffer; - stderr?: Buffer; - message?: string; - }; - claudeExitCode = execError.status ?? 1; - agentOutput = execError.stdout?.toString() ?? ''; - errorMessage = [execError.message, execError.stderr?.toString()].filter(Boolean).join('\n'); - console.error(`claude exited with code ${claudeExitCode}`); + const e = error as { status?: number; stdout?: Buffer; stderr?: Buffer; message?: string }; + exitCode = e.status ?? 1; + stdout = e.stdout?.toString() ?? ''; + errorMessage = [e.message, e.stderr?.toString()].filter(Boolean).join('\n'); + console.error(`${invocation.command} exited with code ${exitCode}`); if (errorMessage) console.error(errorMessage); } const finishedAt = new Date().toISOString(); const wallMs = Date.now() - startMs; -fs.writeFileSync(path.join(runDir, 'agent-output.jsonl'), agentOutput); -const messageTracePosts = postClaudeStreamEvents(agentOutput, parentRunId); -fs.writeFileSync( - path.join(runDir, 'langsmith-message-runs.json'), - JSON.stringify(messageTracePosts, null, 2), -); + +// Raw agent stdout is persisted for human debugging only — never a grading input. +fs.writeFileSync(path.join(runDir, 'agent-output.jsonl'), stdout); const finishedMeta = { ...runMeta, finished_at: finishedAt, wall_ms: wallMs, - claude_exit_code: claudeExitCode, - timed_out: wallMs >= budget.max_wall_ms, + agent_exit_code: exitCode, + timed_out: wallMs >= budget.maxWallMs, error: errorMessage ?? null, }; fs.writeFileSync(path.join(runDir, 'run.json'), JSON.stringify(finishedMeta, null, 2)); -const stopPath = path.join(runDir, 'stop.json'); -const stopData = fs.existsSync(stopPath) - ? (JSON.parse(fs.readFileSync(stopPath, 'utf-8')) as Record) - : {}; - -const finalLangSmithUpdate = tryUpdateLangSmithRun(parentRunId, { - outputs: { - success: claudeExitCode === 0, - agent_output_jsonl: truncate(agentOutput), - message_trace_posts: messageTracePosts, - stop: stopData, - }, - end_time: finishedAt, - error: errorMessage, - extra: { - ...runMeta.langsmith, - wall_ms: wallMs, - claude_exit_code: claudeExitCode, - timed_out: wallMs >= budget.max_wall_ms, - message_trace_posts: messageTracePosts, - stop: stopData, - }, -}); -fs.writeFileSync( - path.join(runDir, 'langsmith-final-update.json'), - JSON.stringify(finalLangSmithUpdate, null, 2), -); - -console.log(`\nRun complete in ${wallMs}ms (exit ${claudeExitCode})`); -console.log(`Parent LangSmith run: ${parentRunId}`); -if (!finalLangSmithUpdate.ok) { - console.warn(`Final LangSmith update failed: ${finalLangSmithUpdate.error}`); -} +console.log(`\nRun complete in ${wallMs}ms (exit ${exitCode})`); console.log(`Run dir: ${runDir}`); console.log(`To grade locally: npx tsx ${path.join(EVALS_DIR, 'grade.ts')} ${runDir}`); -function getArgValue(flag: string): string | undefined { - const index = args.indexOf(flag); - return index === -1 ? undefined : args[index + 1]; -} +// ─── Helpers ──────────────────────────────────────────────────────────────── function loadEvalCase(positional: Array): EvalCase { const [modeOrPath, ...rest] = positional; @@ -264,23 +195,16 @@ function loadEvalCase(positional: Array): EvalCase { if (!message) { throw new Error('Ad hoc input mode requires a message, for example: input "sales by region"'); } - const slug = slugify(message); return { - id: `adhoc-${slug}`, + id: `adhoc-${slugify(message)}`, name: `Ad Hoc: ${message.slice(0, 80)}`, - description: 'Ad hoc Claude Code session against the Tableau MCP server.', + description: 'Ad hoc coding-agent session against the Tableau MCP server.', prompt: message, tags: ['adhoc', 'real-server'], - metadata: { - source: 'cli-input', - }, - budget: { - max_tool_calls: 20, - max_wall_ms: 5 * 60 * 1000, - }, + metadata: { source: 'cli-input' }, + budget: { max_tool_calls: 20, max_wall_ms: 5 * 60 * 1000 }, }; } - return JSON.parse(fs.readFileSync(path.resolve(modeOrPath), 'utf-8')) as EvalCase; } @@ -293,6 +217,37 @@ function slugify(value: string): string { return slug || 'input'; } +/** Environment passed through to the Tableau MCP stdio server subprocess. */ +function tableauServerEnv(): Record { + const keys = [ + 'SERVER', + 'SITE_NAME', + 'AUTH', + 'PAT_NAME', + 'PAT_VALUE', + 'JWT', + 'USERNAME', + 'PASSWORD', + 'CONNECTED_APP_CLIENT_ID', + 'CONNECTED_APP_SECRET_ID', + 'CONNECTED_APP_SECRET_VALUE', + 'DATASOURCE_CREDENTIALS', + 'DEFAULT_LOG_LEVEL', + 'INCLUDE_TOOLS', + 'EXCLUDE_TOOLS', + 'MAX_RESULT_LIMIT', + 'DISABLE_QUERY_DATASOURCE_FILTER_VALIDATION', + 'DISABLE_METADATA_API_REQUESTS', + 'FLOW_TOOLS_ENABLED', + ]; + const env: Record = {}; + for (const key of keys) { + const value = process.env[key]; + if (value != null && value !== '') env[key] = value; + } + return env; +} + function preflight(): void { if (!langSmithApiKey) { throw new Error('LANGSMITH_API_KEY or LANGCHAIN_API_KEY must be set.'); @@ -309,58 +264,6 @@ function preflight(): void { } } -function writeClaudeSettings(): void { - const preToolHook = path.join(HOOKS_DIR, 'pre-tool-langsmith.mjs'); - const postToolHook = path.join(HOOKS_DIR, 'post-tool-langsmith.mjs'); - const stopHook = path.join(HOOKS_DIR, 'stop-langsmith.mjs'); - const claudeSettings = { - hooks: { - PreToolUse: [ - { - matcher: '', - hooks: [{ type: 'command', command: `node ${JSON.stringify(preToolHook)}` }], - }, - ], - PostToolUse: [ - { - matcher: '', - hooks: [{ type: 'command', command: `node ${JSON.stringify(postToolHook)}` }], - }, - ], - PostToolUseFailure: [ - { - matcher: '', - hooks: [{ type: 'command', command: `node ${JSON.stringify(postToolHook)}` }], - }, - ], - Stop: [ - { - hooks: [{ type: 'command', command: `node ${JSON.stringify(stopHook)}` }], - }, - ], - }, - }; - fs.writeFileSync(settingsPath, JSON.stringify(claudeSettings, null, 2)); -} - -function writeMcpConfig(): void { - const mcpConfig = { - mcpServers: { - tableau: { - command: 'node', - args: [MCP_SERVER_ENTRY], - env: { - TRANSPORT: 'stdio', - TMCP_EVAL_RUN_ID: runId, - FILE_LOGGER_DIRECTORY: logsDir, - ENABLED_LOGGERS: 'fileLogger', - }, - }, - }, - }; - fs.writeFileSync(mcpConfigPath, JSON.stringify(mcpConfig, null, 2)); -} - function expandTemplate(value: string): string { return value.replace(/\{\{env\.([A-Z0-9_]+)\}\}/g, (_match, name: string) => { const envValue = process.env[name]; @@ -371,81 +274,6 @@ function expandTemplate(value: string): string { }); } -function createLangSmithRun(run: LangSmithRun): void { - requestLangSmith('/runs', 'POST', run); -} - -function tryCreateLangSmithRun(run: LangSmithRun): { ok: true } | { ok: false; error: string } { - try { - createLangSmithRun(run); - return { ok: true }; - } catch (error) { - return { - ok: false, - error: error instanceof Error ? error.message : String(error), - }; - } -} - -function updateLangSmithRun(runIdToUpdate: string, body: Partial): void { - requestLangSmith(`/runs/${runIdToUpdate}`, 'PATCH', body); -} - -function tryUpdateLangSmithRun( - runIdToUpdate: string, - body: Partial, -): { ok: true } | { ok: false; error: string } { - try { - updateLangSmithRun(runIdToUpdate, body); - return { ok: true }; - } catch (error) { - return { - ok: false, - error: error instanceof Error ? error.message : String(error), - }; - } -} - -function requestLangSmith(pathSuffix: string, method: 'POST' | 'PATCH', body: unknown): void { - const endpoint = `${langSmithEndpoint.replace(/\/$/, '')}${pathSuffix}`; - const response = fetchSync(endpoint, method, body); - if (response.status < 200 || response.status >= 300) { - throw new Error( - `LangSmith ${method} ${pathSuffix} failed: ${response.status} ${response.body}`, - ); - } -} - -function fetchSync( - url: string, - method: 'POST' | 'PATCH', - body: unknown, -): { status: number; body: string } { - const script = ` -const url = process.argv[1]; -const method = process.argv[2]; -const body = JSON.parse(process.argv[3]); -fetch(url, { - method, - headers: { - 'content-type': 'application/json', - 'x-api-key': process.env.LANGSMITH_API_KEY, - }, - body: JSON.stringify(body), -}).then(async (response) => { - console.log(JSON.stringify({ status: response.status, body: await response.text() })); -}).catch((error) => { - console.error(error); - process.exit(1); -}); -`; - const output = execFileSync(process.execPath, ['-e', script, url, method, JSON.stringify(body)], { - env: { ...process.env, LANGSMITH_API_KEY: langSmithApiKey }, - maxBuffer: 10 * 1024 * 1024, - }).toString(); - return JSON.parse(output) as { status: number; body: string }; -} - function tryExec(command: string, commandArgs: Array): string | null { try { return execFileSync(command, commandArgs, { cwd: REPO_ROOT }).toString().trim(); @@ -453,283 +281,3 @@ function tryExec(command: string, commandArgs: Array): string | null { return null; } } - -type ClaudeStreamEvent = { - type?: string; - subtype?: string; - message?: { - id?: string; - model?: string; - role?: string; - content?: unknown; - usage?: unknown; - }; - result?: string; - usage?: unknown; - modelUsage?: unknown; - uuid?: string; - session_id?: string; - timestamp?: string; - [key: string]: unknown; -}; - -/** - * Build a map of stream event index → real timestamp using PreToolUse and PostToolUse - * hook records as anchors. - * - * For each assistant event that contains a tool_use content block: - * - The event's own time is set to the PreToolUse dispatch timestamp (tool call start) - * - The immediately following event's time is set to the PostToolUse timestamp (tool call end) - * - * These anchors are passed into inferEventTimes so interpolation only fills the gaps - * between real data points rather than spreading linearly across the whole session. - */ -function buildToolTimeAnchors(events: Array): Map { - const anchors = new Map(); - - const preToolPath = path.join(runDir, 'pre-tool-times.jsonl'); - const preToolRecords = fs.existsSync(preToolPath) - ? parseJsonl<{ ts: string; tool_use_id: string }>(fs.readFileSync(preToolPath, 'utf-8')) - : []; - const dispatchTimes = new Map(); - for (const r of preToolRecords) { - if (r.tool_use_id && r.ts) dispatchTimes.set(r.tool_use_id, new Date(r.ts).getTime()); - } - - const hookRecords = fs.existsSync(hookLog) - ? parseJsonl<{ ts: string; tool_use_id?: string }>(fs.readFileSync(hookLog, 'utf-8')) - : []; - const completeTimes = new Map(); - for (const r of hookRecords) { - if (r.tool_use_id && r.ts) completeTimes.set(r.tool_use_id, new Date(r.ts).getTime()); - } - - if (dispatchTimes.size === 0 && completeTimes.size === 0) return anchors; - - for (let i = 0; i < events.length; i++) { - const event = events[i]; - if (event.type !== 'assistant' || !Array.isArray(event.message?.content)) continue; - - const content = event.message.content as Array<{ type?: string; id?: string }>; - const toolUseBlock = content.find((b) => b.type === 'tool_use' && b.id); - if (!toolUseBlock?.id) continue; - - const dispatchMs = dispatchTimes.get(toolUseBlock.id); - if (dispatchMs != null) anchors.set(i, dispatchMs); - - const completeMs = completeTimes.get(toolUseBlock.id); - if (completeMs != null && i + 1 < events.length) anchors.set(i + 1, completeMs); - } - - return anchors; -} - -function postClaudeStreamEvents( - streamJsonl: string, - parentRunIdToUse: string, -): { - total_events: number; - posted: number; - failed: number; -} { - const events = parseJsonl(streamJsonl); - const toolTimeAnchors = buildToolTimeAnchors(events); - const eventTimes = inferEventTimes(events, toolTimeAnchors); - const traceableEvents = events - .map((event, streamIndex) => ({ event, streamIndex })) - .filter(({ event }) => event.type === 'assistant' || event.type === 'result'); - - let posted = 0; - let failed = 0; - - // Post a synthetic initialization span covering Claude Code startup time — - // the gap from when the process was launched to the first stream event. - const firstEventTime = - traceableEvents.length > 0 - ? (eventTimes[traceableEvents[0].streamIndex] ?? startedAt) - : startedAt; - if (firstEventTime > startedAt) { - const initResult = tryCreateLangSmithRun({ - id: randomUUID(), - name: 'initialization', - run_type: 'chain', - inputs: { prompt }, - outputs: {}, - start_time: startedAt, - end_time: firstEventTime, - parent_run_id: parentRunIdToUse, - session_name: langSmithProject, - tags: ['tmcp-eval', 'claude-code', 'initialization'], - extra: { run_id: runId, description: 'Claude Code startup and MCP server initialization' }, - }); - if (initResult.ok) posted += 1; - else failed += 1; - } - - traceableEvents.forEach(({ event, streamIndex }, index) => { - const childRunId = randomUUID(); - const isAssistant = event.type === 'assistant'; - const name = isAssistant - ? `assistant-message-${index + 1}` - : `claude-result-${event.subtype ?? 'final'}`; - const eventTime = eventTimes[streamIndex] ?? finishedAt; - const nextTraceable = traceableEvents[index + 1]; - const endTime = nextTraceable - ? (eventTimes[nextTraceable.streamIndex] ?? finishedAt) - : finishedAt; - const postResult = tryCreateLangSmithRun({ - id: childRunId, - name, - run_type: isAssistant ? 'llm' : 'chain', - inputs: truncateJsonValue({ - prompt, - event_index: index, - stream_index: streamIndex, - session_id: event.session_id ?? null, - }) as Record, - outputs: truncateJsonValue({ - message: event.message ?? null, - result: event.result ?? null, - usage: event.usage ?? event.message?.usage ?? null, - model_usage: event.modelUsage ?? null, - }) as Record, - start_time: eventTime, - end_time: endTime, - parent_run_id: parentRunIdToUse, - session_name: langSmithProject, - tags: ['tmcp-eval', 'claude-code', isAssistant ? 'assistant-message' : 'claude-result'], - extra: { - run_id: runId, - event_type: event.type ?? null, - event_subtype: event.subtype ?? null, - event_uuid: event.uuid ?? null, - stream_index: streamIndex, - inferred_timestamp: event.timestamp ? false : true, - message_id: event.message?.id ?? null, - model: event.message?.model ?? null, - }, - }); - - if (postResult.ok) { - posted += 1; - } else { - failed += 1; - } - - appendJsonl(messageRunsLog, { - ts: new Date().toISOString(), - child_run_id: childRunId, - parent_run_id: parentRunIdToUse, - name, - stream_index: streamIndex, - event_time: eventTime, - ok: postResult.ok, - error: postResult.ok ? null : postResult.error, - }); - }); - - return { - total_events: traceableEvents.length, - posted, - failed, - }; -} - -function inferEventTimes( - events: Array, - anchors: Map = new Map(), -): Array { - if (events.length === 0) return []; - - const startMs = new Date(startedAt).getTime(); - const finishMs = new Date(finishedAt).getTime(); - const explicitTimes = events.map((event, index) => { - // Real hook-derived timestamps take highest priority. - const anchor = anchors.get(index); - if (anchor != null && Number.isFinite(anchor)) return anchor; - if (event.timestamp) return new Date(event.timestamp).getTime(); - if (event.type === 'result') return finishMs; - return null; - }); - - const inferredTimes = events.map((_event, index) => { - const explicitTime = explicitTimes[index]; - if (explicitTime != null && Number.isFinite(explicitTime)) return explicitTime; - - const previousIndex = findPreviousExplicitTimeIndex(explicitTimes, index); - const nextIndex = findNextExplicitTimeIndex(explicitTimes, index); - - if (previousIndex != null && nextIndex != null) { - const previousTime = explicitTimes[previousIndex] ?? startMs; - const nextTime = explicitTimes[nextIndex] ?? finishMs; - const fraction = (index - previousIndex) / (nextIndex - previousIndex); - return Math.round(previousTime + (nextTime - previousTime) * fraction); - } - - if (previousIndex != null) { - return (explicitTimes[previousIndex] ?? startMs) + (index - previousIndex) * 10; - } - - if (nextIndex != null) { - return (explicitTimes[nextIndex] ?? finishMs) - (nextIndex - index) * 10; - } - - const fraction = events.length === 1 ? 0 : index / (events.length - 1); - return Math.round(startMs + (finishMs - startMs) * fraction); - }); - - for (let index = 1; index < inferredTimes.length; index += 1) { - if (inferredTimes[index] <= inferredTimes[index - 1]) { - inferredTimes[index] = inferredTimes[index - 1] + 1; - } - } - - return inferredTimes.map((time) => new Date(time).toISOString()); -} - -function findPreviousExplicitTimeIndex(times: Array, index: number): number | null { - for (let cursor = index - 1; cursor >= 0; cursor -= 1) { - if (times[cursor] != null && Number.isFinite(times[cursor])) return cursor; - } - return null; -} - -function findNextExplicitTimeIndex(times: Array, index: number): number | null { - for (let cursor = index + 1; cursor < times.length; cursor += 1) { - if (times[cursor] != null && Number.isFinite(times[cursor])) return cursor; - } - return null; -} - -function parseJsonl(jsonl: string): Array { - return jsonl - .split('\n') - .filter((line) => line.trim()) - .map((line) => { - try { - return JSON.parse(line) as T; - } catch { - return null; - } - }) - .filter((event): event is T => event !== null); -} - -function appendJsonl(filePath: string, value: unknown): void { - fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); -} - -function truncate(value: string): string { - const maxChars = Number(process.env.LANGSMITH_MAX_PAYLOAD_CHARS ?? 200_000); - return value.length > maxChars ? `${value.slice(0, maxChars)}\n...[truncated]` : value; -} - -function truncateJsonValue(value: unknown): unknown { - const maxChars = Number(process.env.LANGSMITH_MAX_PAYLOAD_CHARS ?? 200_000); - const serialized = JSON.stringify(value); - if (serialized.length <= maxChars) return value; - return { - truncated: true, - preview: serialized.slice(0, maxChars), - }; -} diff --git a/evals/run-suite.ts b/evals/run-suite.ts index ad7ea277a..bd67ef1eb 100644 --- a/evals/run-suite.ts +++ b/evals/run-suite.ts @@ -2,14 +2,17 @@ * Suite runner for the BIRD California Schools eval. * * Runs all 30 cases (or a filtered subset) sequentially against the live - * Tableau MCP server. Each case is executed via run-case.ts so trace routing, - * hooks, and LangSmith posting are identical to ad-hoc runs. + * Tableau MCP server. Each case is executed via run-case.ts, so agent selection + * (AGENT_HARNESS/AGENT_MODEL) and LangSmith plugin tracing are identical to ad-hoc + * runs. Per-case quality/latency/cost/token metrics are NOT collected here — they are + * sourced from the LangSmith trace at grading time (grade-suite.ts). * * Usage: * npx tsx evals/run-suite.ts * npx tsx evals/run-suite.ts --suite evals/suites/bird-california-schools.json * npx tsx evals/run-suite.ts --difficulty simple * npx tsx evals/run-suite.ts --ids 5,11,12 + * npx tsx evals/run-suite.ts --agent-harness codex --agent-model gpt-5.6-codex * * Required environment: same as run-case.ts (Tableau auth + LANGSMITH_API_KEY). * Set EVAL_DATASOURCE_LUID to the LUID of the published California Schools datasource. @@ -23,6 +26,8 @@ import dotenv from 'dotenv'; import * as fs from 'fs'; import * as path from 'path'; +import { getAdapter, resolveHarness } from './adapters/index.js'; + const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); dotenv.config({ path: path.join(REPO_ROOT, '.env') }); @@ -38,6 +43,11 @@ const DEFAULT_SUITE = path.join(EVALS_DIR, 'suites', 'bird-california-schools.js const args = process.argv.slice(2); +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + const suiteFile = getArgValue('--suite') ?? DEFAULT_SUITE; const difficultyFilter = getArgValue('--difficulty'); const idsFilter = getArgValue('--ids') @@ -45,6 +55,11 @@ const idsFilter = getArgValue('--ids') .map((s) => parseInt(s.trim(), 10)) .filter(Number.isFinite); +const harness = resolveHarness(getArgValue('--agent-harness'), 'AGENT_HARNESS'); +const model = getAdapter(harness).resolveModel( + getArgValue('--agent-model') ?? process.env.AGENT_MODEL, +); + type BirdCase = { question_id: number; question: string; @@ -72,60 +87,18 @@ type EvalCaseFile = { budget: { max_wall_ms: number }; }; -type StopData = { - usage?: { - input_tokens: number | null; - output_tokens: number | null; - cache_read_input_tokens: number | null; - cache_creation_input_tokens: number | null; - }; -}; - -type HookRecord = { - tool_name?: string; - normalized_tool_name?: string; -}; - type CaseRunResult = { question_id: number; difficulty: string; run_id: string; run_dir: string; + eval_run_id: string; exit_code: number | null; wall_ms: number | null; timed_out: boolean; - tool_calls: number; - tools_used: Array; - tokens: { - input: number | null; - output: number | null; - cache_read: number | null; - cache_creation: number | null; - }; error: string | null; }; -function getArgValue(flag: string): string | undefined { - const index = args.indexOf(flag); - return index === -1 ? undefined : args[index + 1]; -} - -function readJsonlFile(filePath: string): Array { - if (!fs.existsSync(filePath)) return []; - return fs - .readFileSync(filePath, 'utf-8') - .split('\n') - .filter((line) => line.trim()) - .map((line) => { - try { - return JSON.parse(line) as T; - } catch { - return null; - } - }) - .filter((v): v is T => v !== null); -} - function readOptionalJson(filePath: string): T | null { if (!fs.existsSync(filePath)) return null; try { @@ -156,39 +129,20 @@ function buildEvalCaseFile(birdCase: BirdCase, absoluteSuitePath: string): EvalC function collectRunMetrics(runDir: string): Omit { const runMeta = readOptionalJson<{ run_id: string; + eval_run_id?: string; wall_ms?: number; - claude_exit_code?: number; + agent_exit_code?: number; timed_out?: boolean; error?: string | null; - budget: { max_wall_ms: number }; }>(path.join(runDir, 'run.json')); - - const hookRecords = readJsonlFile(path.join(runDir, 'hook.jsonl')); - const stop = readOptionalJson(path.join(runDir, 'stop.json')); const runId = path.basename(runDir); - - const tools = hookRecords - .map((r) => { - const raw = r.normalized_tool_name ?? r.tool_name ?? ''; - const parts = raw.split('__'); - return parts[parts.length - 1] || raw; - }) - .filter(Boolean); - return { run_id: runMeta?.run_id ?? runId, run_dir: runDir, - exit_code: runMeta?.claude_exit_code ?? null, + eval_run_id: runMeta?.eval_run_id ?? runMeta?.run_id ?? runId, + exit_code: runMeta?.agent_exit_code ?? null, wall_ms: runMeta?.wall_ms ?? null, timed_out: runMeta?.timed_out ?? false, - tool_calls: hookRecords.length, - tools_used: [...new Set(tools)], - tokens: { - input: stop?.usage?.input_tokens ?? null, - output: stop?.usage?.output_tokens ?? null, - cache_read: stop?.usage?.cache_read_input_tokens ?? null, - cache_creation: stop?.usage?.cache_creation_input_tokens ?? null, - }, error: runMeta?.error ?? null, }; } @@ -233,10 +187,11 @@ function main(): void { const absoluteSuitePath = path.resolve(suiteFile); const startedAt = new Date().toISOString(); - console.log(`\nSuite run: ${suiteRunId}`); - console.log(`Suite: ${suiteFile}`); - console.log(`Cases: ${cases.length}`); - console.log(`Run dir: ${suiteRunDir}\n`); + console.log(`\nSuite run: ${suiteRunId}`); + console.log(`Suite: ${suiteFile}`); + console.log(`Cases: ${cases.length}`); + console.log(`Harness / model: ${harness} / ${model || '(cli default)'}`); + console.log(`Run dir: ${suiteRunDir}\n`); const runCaseScript = path.join(EVALS_DIR, 'run-case.ts'); const results: Array = []; @@ -257,11 +212,24 @@ function main(): void { const runDir = path.join(RUNS_DIR, today, runId); const budgetSec = Math.round(birdCase.budget.max_wall_ms / 1000); - process.stdout.write(` Running Claude (up to ${budgetSec}s)...`); + process.stdout.write(` Running ${harness} (up to ${budgetSec}s)...`); + + const runCaseArgs = [ + 'tsx', + runCaseScript, + caseFilePath, + '--run-id', + runId, + '--suite-run-id', + suiteRunId, + '--agent-harness', + harness, + ]; + if (model) runCaseArgs.push('--agent-model', model); let spawnError: string | null = null; try { - execFileSync('npx', ['tsx', runCaseScript, caseFilePath, '--run-id', runId], { + execFileSync('npx', runCaseArgs, { env: process.env, cwd: REPO_ROOT, timeout: birdCase.budget.max_wall_ms + 30_000, @@ -284,9 +252,7 @@ function main(): void { const status = metrics.exit_code === 0 ? 'OK' : metrics.timed_out ? 'TIMEOUT' : `EXIT ${metrics.exit_code}`; console.log( - ` ${status} | ${metrics.wall_ms != null ? `${Math.round(metrics.wall_ms / 1000)}s` : '?s'} | ` + - `${metrics.tool_calls} tool calls | ` + - `tokens: ${metrics.tokens.input ?? '?'} in / ${metrics.tokens.output ?? '?'} out`, + ` ${status} | ${metrics.wall_ms != null ? `${Math.round(metrics.wall_ms / 1000)}s` : '?s'}`, ); } @@ -295,8 +261,6 @@ function main(): void { const completed = results.filter((r) => r.exit_code === 0).length; const errored = results.filter((r) => r.exit_code !== 0 && !r.timed_out).length; const timedOut = results.filter((r) => r.timed_out).length; - const totalInput = results.reduce((s, r) => s + (r.tokens.input ?? 0), 0); - const totalOutput = results.reduce((s, r) => s + (r.tokens.output ?? 0), 0); const avgWall = results.filter((r) => r.wall_ms != null).length > 0 ? Math.round(totalWallMs / results.filter((r) => r.wall_ms != null).length) @@ -307,6 +271,8 @@ function main(): void { suite_file: absoluteSuitePath, started_at: startedAt, finished_at: finishedAt, + harness, + model: model || null, total_wall_ms: totalWallMs, cases: results, aggregate: { @@ -315,8 +281,6 @@ function main(): void { errored, timed_out: timedOut, avg_wall_ms: avgWall, - total_input_tokens: totalInput, - total_output_tokens: totalOutput, }, }; @@ -327,12 +291,9 @@ function main(): void { console.log(`Suite complete: ${suiteRunId}`); console.log(` ${completed}/${results.length} OK | ${errored} error | ${timedOut} timeout`); console.log(` Total wall time: ${(totalWallMs / 1000).toFixed(1)}s`); - console.log(` Total tokens: ${totalInput} in / ${totalOutput} out`); console.log(` Summary: ${summaryPath}`); - console.log('\nTo grade cases:'); - for (const r of results) { - console.log(` npx tsx ${path.join(EVALS_DIR, 'grade-bird.ts')} ${r.run_dir}`); - } + console.log('\nTo grade the whole suite (sources metrics from LangSmith traces):'); + console.log(` npx tsx ${path.join(EVALS_DIR, 'grade-suite.ts')} ${suiteRunDir}`); } main(); diff --git a/package-lock.json b/package-lock.json index 10cb50a7c..e0cf71a48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,6 +46,7 @@ "eslint-config-prettier": "^10.1.2", "eslint-plugin-prettier": "^5.2.6", "eslint-plugin-simple-import-sort": "^12.1.1", + "langsmith": "^0.8.4", "npm-run-all": "^4.1.5", "openai": "^5.23.2", "prettier": "^3.5.3", @@ -1816,6 +1817,7 @@ "integrity": "sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -2009,6 +2011,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.33.1.tgz", "integrity": "sha512-qwxv6dq682yVvgKKp2qWwLgRbscDAYktPptK4JPojCwwi3R9cwrvIxS4lvBpzmcqzR4bdn54Z0IG1uHFskW4dA==", "dev": true, + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.33.1", "@typescript-eslint/types": "8.33.1", @@ -2389,6 +2392,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "dev": true, + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3359,6 +3363,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.28.0.tgz", "integrity": "sha512-ocgh41VhRlf9+fVpe7QKzwLj9c92fDiqOj8Y3Sd4/ZmVA4Btx4PlUYPq4pp9JDyupkf1upbEXecxL2mwNV7jPQ==", "dev": true, + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -3419,6 +3424,7 @@ "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.5.tgz", "integrity": "sha512-zc1UmCpNltmVY34vuLRV61r1K27sWuX39E+uyUnY8xS2Bex88VV9cugG+UZbRSRGtGyFboj+D8JODyme1plMpw==", "dev": true, + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -3596,6 +3602,13 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3629,6 +3642,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3915,7 +3929,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=4.0" }, @@ -4333,6 +4346,7 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.7.tgz", "integrity": "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -5020,6 +5034,40 @@ "json-buffer": "3.0.1" } }, + "node_modules/langsmith": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.4.tgz", + "integrity": "sha512-e2zdhPUV/mLwVv+Gde/0gLGnCOE/zvZ3gGNh/rvkzrxsDz7qgUT4CJVUmJE2GwQ1A3FPtWKzy08oo//VV1nBFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5640,6 +5688,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -5670,6 +5728,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -5911,6 +5999,7 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "dev": true, + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -5961,8 +6050,7 @@ "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "peer": true + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "node_modules/punycode": { "version": "2.3.1", @@ -7025,6 +7113,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -7717,6 +7806,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7824,6 +7914,7 @@ "integrity": "sha512-qTl3VF7BvOupTR85Zc561sPEgxyUSNSvTQ9fit7DEMP7yPgvvIGm5Zfa1dOM+kOwWGNviK9uFM9ra77+OjK7lQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -7939,6 +8030,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -7951,6 +8043,7 @@ "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.2.tgz", "integrity": "sha512-fyNn/Rp016Bt5qvY0OQvIUCwW2vnaEBLxP42PmKbNIoasSYjML+8xyeADOPvBe+Xfl/ubIw4og7Lt9jflRsCNw==", "dev": true, + "peer": true, "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.2", @@ -8303,6 +8396,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 6b10c423b..b3d922627 100644 --- a/package.json +++ b/package.json @@ -47,11 +47,13 @@ "test": "vitest --config ./vitest.config.ts", "test:e2e": "vitest --config ./vitest.config.e2e.ts", "test:eval": "vitest --config ./vitest.config.eval.ts", + "eval:run": "tsx evals/run-case.ts", "eval:claude": "tsx evals/run-case.ts", "eval:grade": "tsx evals/grade.ts", "eval:suite": "tsx evals/run-suite.ts", "eval:grade:bird": "tsx evals/grade-bird.ts", "eval:grade:suite": "tsx evals/grade-suite.ts", + "eval:report": "tsx evals/report.ts", "test:oauth:embedded": "vitest --config ./vitest.config.oauth.embedded.ts", "test:oauth:tableau": "npx playwright test", "coverage": "vitest run --config ./vitest.config.ts --coverage", @@ -95,6 +97,7 @@ "eslint-config-prettier": "^10.1.2", "eslint-plugin-prettier": "^5.2.6", "eslint-plugin-simple-import-sort": "^12.1.1", + "langsmith": "^0.8.4", "npm-run-all": "^4.1.5", "openai": "^5.23.2", "prettier": "^3.5.3", From 75884a8f3d11cf3a23c4372fab0e00cc8eac1fa2 Mon Sep 17 00:00:00 2001 From: Joe Constantino Date: Tue, 21 Jul 2026 22:11:26 -0700 Subject: [PATCH 7/7] addtl sample questions --- team_context/eval-questions.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 team_context/eval-questions.md diff --git a/team_context/eval-questions.md b/team_context/eval-questions.md new file mode 100644 index 000000000..ca23a9229 --- /dev/null +++ b/team_context/eval-questions.md @@ -0,0 +1,13 @@ +1. How many tableau data sources do I have access to? +2. What are the workbooks on my tableau site? +3. pull me some insights from the +4. pull me some insights from the +5. Show me the dashboard +6. what does the tell me +7. generate some insights for a metric based on the +8. let's analyze this workbook: +9. what's the most popular content on my site? +10. Do we have any data about +11. Who should I reach out to to get more information about +12. what does the dashboard tell me? +13. \ No newline at end of file