diff --git a/.gitignore b/.gitignore index 4f616a65a..70430886b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ node_modules/ junit/ coverage/ test-logs/ +evals/suite-runs/ +evals/runs/ +evals/grades/ +evals/reports/ config.json env.list manifest.json diff --git a/docs/docs/evaluation/_category_.json b/docs/docs/evaluation/_category_.json new file mode 100644 index 000000000..215419487 --- /dev/null +++ b/docs/docs/evaluation/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Evaluation", + "position": 8, + "link": { + "type": "generated-index", + "description": "Tools and benchmarks for evaluating the accuracy of Claude Code against the Tableau MCP server." + } +} diff --git a/docs/docs/evaluation/claude-code-harness.md b/docs/docs/evaluation/claude-code-harness.md new file mode 100644 index 000000000..fc23bfcb0 --- /dev/null +++ b/docs/docs/evaluation/claude-code-harness.md @@ -0,0 +1,219 @@ +--- +sidebar_position: 1 +--- + +# Claude Code Eval Harness + +The Tableau MCP eval harness runs [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) as a live agent against the Tableau MCP server and grades its responses against known-correct answers. It is designed to measure query accuracy — not simulated tool calls, with real API round-trips to a live Tableau Cloud or Server site. + +The primary benchmark is 30 questions from the [BIRD Mini-Dev](https://bird-bench.github.io/) dataset scoped to the California Schools database. + +--- + +## Prerequisites + +### Required + +- **Claude Code** installed and available on your `PATH` (`claude --version` should work) +- **A published California Schools datasource** on your Tableau Cloud or Server site. This is a single published datasource that joins the three California Schools source tables (`schools`, `frpm`, `satscores`). You will need its LUID. A .tdsx for this data source is provided at within the project at `evals/bird_mini/data/tableau_datasources/california_schools`. +- **Tableau credentials** — any supported auth method (PAT, OAuth, direct trust). See [Authentication](/docs/configuration/mcp-config/authentication) for setup. + +### Optional + +- **LangSmith account** — for trace visualization. Without it, the harness still runs and grades locally; traces are simply not posted. +- **OpenAI API key** — for semantic grading (LLM judge). Without it, the harness falls back to numeric-only grading. + +--- + +## Environment Variables + +Copy `env.example.list` to `.env` at the repo root and fill in the values below. + +### Required + +| Variable | Description | +|---|---| +| `SERVER` | Your Tableau Cloud or Server URL (e.g. `https://10ax.online.tableau.com`) | +| `SITE_NAME` | Your Tableau site name | +| `PAT_NAME` | Personal Access Token name | +| `PAT_VALUE` | Personal Access Token secret | +| `EVAL_DATASOURCE_LUID` | LUID of the published California Schools datasource. Find it in Tableau by opening the datasource and copying the ID from the URL. | + +### Optional — LangSmith + +| Variable | Default | Description | +|---|---|---| +| `LANGSMITH_API_KEY` | — | Your LangSmith API key. Get one at [smith.langchain.com](https://smith.langchain.com) → Settings → API Keys. If not set, traces are not posted. | +| `LANGSMITH_PROJECT` | `tableau-mcp-evals` | LangSmith project to post traces to. | + +### Optional — Grading + +| Variable | Default | Description | +|---|---|---| +| `OPENAI_API_KEY` | — | OpenAI API key for the LLM judge. If not set, `semantic_match` is skipped and the verdict is based on `numeric_match` only. | +| `BIRD_GRADE_MODEL` | `gpt-4o-mini` | Model used by the LLM judge. Override with e.g. `gpt-4o` for higher-quality grading. | + +--- + +## Running the Harness + +### Run the full BIRD suite (30 questions) + +```bash +npm run eval:suite +``` + +### Run a filtered subset + +```bash +npm run eval:suite -- --difficulty simple # only the 8 "simple" questions +npm run eval:suite -- --difficulty moderate # only "moderate" questions +npm run eval:suite -- --ids 5,11,12 # specific question IDs +``` + +Difficulty levels map to the BIRD benchmark's own classifications: `simple`, `moderate`, and `challenging`. + +### Run a single question by ID + +```bash +npm run eval:suite -- --ids 5 +``` + +### Run an ad hoc custom question + +Pass any natural language prompt directly, without a suite file: + +```bash +npm run eval:claude -- input "How many schools have an average SAT math score above 400?" +``` + +Ad hoc runs use the same Claude Code + Tableau MCP setup as suite runs and produce the same local artifacts. They are graded by `grade.ts` (tool coverage only), not `grade-bird.ts`, because there is no precomputed expected answer to compare against. + +--- + +## Grading + +### Grade a full suite run + +After `eval:suite` completes, grade every case at once: + +```bash +npm run eval:grade:suite # auto-discovers most recent +npm run eval:grade:suite -- evals/suite-runs/YYYY-MM-DD/ # explicit path +``` + +This writes a single `suite-grade.json` to `evals/grades/YYYY-MM-DD//`. + +### Grade a single run + +```bash +npm run eval:grade:bird -- evals/runs/YYYY-MM-DD/ +``` + +Output goes to `evals/grades/YYYY-MM-DD//bird-result.json`. + +--- + +## Grading Signals and Metrics + +Each run is evaluated on four signals. The **verdict** is determined by the two outcome signals only. The structural signals are recorded for debugging purposes and do not affect the verdict. + +### Outcome signals (drive the verdict) + +| Signal | Method | Description | +|---|---|---| +| `numeric_match` | Code | The expected numeric value or row count appears in Claude's final message. Integer matches are exact; float matches allow ±1% tolerance. String answers (e.g. school names) use case-insensitive substring matching. | +| `semantic_match` | LLM judge | GPT-4o-mini (or `BIRD_GRADE_MODEL`) scores Claude's final message against the gold answer summary on a 0–1 scale. A score ≥ 0.8 passes. Requires `OPENAI_API_KEY`. | + +### Structural signals (informational) + +| Signal | Method | Description | +|---|---|---| +| `columns_match` | Tool call inspection | The expected VizQL field captions are present in the `query-datasource` call. Extra columns are allowed; only missing required columns count against this signal. | +| `filters_match` | Tool call inspection | The expected filter field captions are present in the `query-datasource` call. Same subset-matching logic as `columns_match`. | + +### Verdicts + +| Verdict | Meaning | +|---|---| +| `pass` | Both `numeric_match` and `semantic_match` passed | +| `partial` | One of the two outcome signals passed (or one was unavailable) | +| `fail` | Both outcome signals failed | +| `error` | Claude Code exited with a non-zero code | +| `skip` | Both outcome signals were unavailable (e.g. no `OPENAI_API_KEY` and no numeric result to match) | + +--- + +## Metrics Captured Per Run + +Each case run records the following, available in `bird-result.json` (individual) or `suite-grade.json` (aggregated): + +| Metric | Source | Notes | +|---|---|---| +| `wall_s` | Process timing | Seconds from `claude -p` invocation to process exit | +| `tool_calls` | `hook.jsonl` | Total number of tool calls made | +| `tools_used` | `hook.jsonl` | Deduplicated list of tool names called | +| `model` | Claude stream | Model name reported by the Claude stream | +| `tokens.input_tokens` | Claude stream | Summed across all assistant messages | +| `tokens.output_tokens` | Claude stream | Summed across all assistant messages | +| `tokens.cache_creation_tokens` | Claude stream | Prompt cache write tokens | +| `tokens.cache_read_tokens` | Claude stream | Prompt cache read tokens | +| `tokens.total_context_tokens` | Computed | `input + cache_creation + cache_read` | + +--- + +## Output Directories + +All output is written locally and gitignored. Directories are organized by date. + +``` +evals/ + runs/ + YYYY-MM-DD/ + / one folder per case run + run.json metadata, timing, exit code + agent-output.jsonl full Claude Code stream + hook.jsonl one record per tool call + pre-tool-times.jsonl tool dispatch timestamps + stop.json stop hook data + logs/ MCP server logs + + suite-runs/ + YYYY-MM-DD/ + / + suite-summary.json aggregate timing and token counts for all cases in the run + + grades/ + YYYY-MM-DD/ + / + bird-result.json per-case grading output + / + suite-grade.json aggregate grading output for a full suite run +``` + +--- + +## LangSmith Integration + +When `LANGSMITH_API_KEY` is set, each case run posts a full trace to LangSmith in real time via Claude Code hooks. Each trace contains: + +- One parent span covering the full Claude Code process (wall time) +- A synthetic `initialization` span for Claude startup and MCP server negotiation +- One `assistant-message-N` span per LLM turn (timing inferred from the Claude stream) +- One tool span per tool call with accurate start/end times from `PreToolUse` and `PostToolUse` hooks + +Traces are posted to the project set by `LANGSMITH_PROJECT` (default: `tableau-mcp-evals`). + +--- + +## About the BIRD Dataset + +The BIRD (BIg Bench for laRge-scale Database Grounded Text-to-SQL Evaluation) Mini-Dev benchmark is a standard text-to-SQL evaluation dataset. The 30 California Schools questions used here were selected because the underlying database can be represented as a single Tableau published datasource that joins three source tables. + +The suite file at `evals/suites/bird-california-schools.json` ships with precomputed expected answers (row counts, scalar values, and gold answer summaries) so no database access is required to run grading. To regenerate it from the raw BIRD SQLite snapshot: + +```bash +python3 evals/scripts/precompute-bird-answers.py +``` + +This requires the California Schools SQLite file at `evals/bird_mini/data/dev_databases/california_schools/california_schools.sqlite`. diff --git a/docs/package-lock.json b/docs/package-lock.json index ab1d1e1f2..0eb697fc8 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -233,6 +233,7 @@ "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.38.0.tgz", "integrity": "sha512-PTAFMJOpVtJweExEYYgdmSCC6n4V/R+ctDL3fRQy77ulZM/p+zMLIQC9c7HCQE1zqpauvVck3f2zYSejaUTtrw==", "license": "MIT", + "peer": true, "dependencies": { "@algolia/client-common": "5.38.0", "@algolia/requester-browser-xhr": "5.38.0", @@ -380,6 +381,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.4.tgz", "integrity": "sha512-2BCOP7TN8M+gVDj7/ht3hsaO/B/n5oDbiAyyvnRlNOs+u1o+JWNYTQrmpuNp1/Wq2gcFrI01JAW+paEKDMx/CA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.3", @@ -2214,6 +2216,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -2236,6 +2239,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -2345,6 +2349,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -2766,6 +2771,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -3629,6 +3635,7 @@ "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.9.0.tgz", "integrity": "sha512-PP+iDJg+lj4cn/7GbbmiguaQ8OX08YxnzQ17KqRC4ufJm11jdyXD33wA7vVtbeG/BkkgkiB/K7YyPHCPwmfVhg==", "license": "MIT", + "peer": true, "dependencies": { "@docusaurus/core": "3.9.0", "@docusaurus/logger": "3.9.0", @@ -4381,6 +4388,7 @@ "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-3.1.1.tgz", "integrity": "sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==", "license": "MIT", + "peer": true, "dependencies": { "@types/mdx": "^2.0.0" }, @@ -4708,6 +4716,7 @@ "resolved": "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz", "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.21.3", "@svgr/babel-preset": "8.1.0", @@ -5338,6 +5347,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.12.tgz", "integrity": "sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -5677,6 +5687,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5762,6 +5773,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -5807,6 +5819,7 @@ "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.38.0.tgz", "integrity": "sha512-8VJKIzheeI9cjuVJhU1hYEVetOTe7LvA+CujAI7yqvYsPtZfVEvv1pg9AeFNtHBg/ZoSLGU5LPijhcY5l3Ea9g==", "license": "MIT", + "peer": true, "dependencies": { "@algolia/abtesting": "1.4.0", "@algolia/client-abtesting": "5.38.0", @@ -6270,6 +6283,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.3", "caniuse-lite": "^1.0.30001741", @@ -6569,6 +6583,7 @@ "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-11.0.3.tgz", "integrity": "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@chevrotain/cst-dts-gen": "11.0.3", "@chevrotain/gast": "11.0.3", @@ -7267,6 +7282,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -7586,6 +7602,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.1.tgz", "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -8007,6 +8024,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -9190,6 +9208,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -13825,6 +13844,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -14385,6 +14405,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -15288,6 +15309,7 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -16098,6 +16120,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.1.1.tgz", "integrity": "sha512-w8nqGImo45dmMIfljjMwOGtbmC/mk4CMYhWIicdSflH91J9TyCyczcPFXJzrZ/ZXcgGRFeP6BU0BEJTw6tZdfQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -16107,6 +16130,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.1.1.tgz", "integrity": "sha512-Dlq/5LAZgF0Gaz6yiqZCf6VCcZs1ghAJyrsu84Q/GT0gV+mCxbfmKNoGRKBYMJ8IEdGPqu49YWXD02GCknEDkw==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.26.0" }, @@ -16162,6 +16186,7 @@ "resolved": "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-6.0.0.tgz", "integrity": "sha512-YMMxTUQV/QFSnbgrP3tjDzLHRg7vsbMn8e9HAa8o/1iXoiomo48b7sk/kkmWEuWNDPJVlKSJRB6Y2fHqdJk+SQ==", "license": "MIT", + "peer": true, "dependencies": { "@types/react": "*" }, @@ -16190,6 +16215,7 @@ "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz", "integrity": "sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.12.13", "history": "^4.9.0", @@ -18010,7 +18036,8 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "license": "0BSD", + "peer": true }, "node_modules/type-fest": { "version": "2.19.0", @@ -18073,6 +18100,7 @@ "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -18420,6 +18448,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -18676,6 +18705,7 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.101.3.tgz", "integrity": "sha512-7b0dTKR3Ed//AD/6kkx/o7duS8H3f1a4w3BYpIriX4BzIhjkn4teo05cptsxvLesHFKK5KObnadmCHBwGc+51A==", "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -19262,6 +19292,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.11.tgz", "integrity": "sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/env.example.list b/env.example.list index 2bdb6c2ac..3ee28728a 100644 --- a/env.example.list +++ b/env.example.list @@ -1,3 +1,4 @@ +# ── MCP Server ──────────────────────────────────────────────────────────────── TRANSPORT=stdio SERVER=https://my-tableau-server.com SITE_NAME= @@ -12,4 +13,21 @@ DISABLE_QUERY_DATASOURCE_VALIDATION_REQUESTS= DISABLE_METADATA_API_REQUESTS= OAUTH_EMBEDDED_AUTHZ_SERVER= ADVERTISE_API_SCOPES= -OAUTH_DISABLE_SCOPES= \ No newline at end of file +OAUTH_DISABLE_SCOPES= + +# ── Eval Suite ──────────────────────────────────────────────────────────────── +# LUID of the published datasource to run the eval against. +# Find it in Tableau: open the datasource details page and copy the ID from the URL. +EVAL_DATASOURCE_LUID= + +# LangSmith — traces from each eval run are posted here. +# Get your API key at https://smith.langchain.com → Settings → API Keys. +LANGSMITH_API_KEY= +# Project name to post traces to (default: tableau-mcp-evals). +LANGSMITH_PROJECT=tableau-mcp-evals + +# OpenAI — used by the LLM judge for semantic correctness grading. +# Without this, semantic_match is skipped and the verdict relies on numeric_match only. +OPENAI_API_KEY= +# Model used by the LLM judge (default: gpt-4o-mini). +BIRD_GRADE_MODEL=gpt-4o-mini \ No newline at end of file diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 000000000..82c1e5dd6 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,246 @@ +# Tableau MCP Eval Harness + +This folder contains a local eval harness that runs a **coding agent** (Claude Code, +Cursor, or Codex) against the Tableau MCP server, traces the run to LangSmith via the +official coding-agent tracing plugin for that agent, and grades responses against +known-correct answers — sourcing every metric from the LangSmith trace. + +## How It Works + +``` +suite file (bird-california-schools.json) + → run-suite.ts + → run-case.ts (one per question) + → AgentAdapter (claude-code | cursor | codex) builds the CLI invocation + → claude / cursor-agent / codex exec ── with MCP config for the tableau server + → tableau-mcp stdio server → real Tableau Cloud/Server APIs + → LangSmith coding-agent plugin posts the full trace + (stamped with eval_run_id + suite_run_id + harness + model metadata) + → run.json (run metadata + eval_run_id correlation) + → agent-output.jsonl (raw agent stream — debugging only, never a grading input) + → grade-suite.ts → grade-bird.ts (one per run) + → fetch trace from LangSmith by eval_run_id ← single source of truth + → derive signals + latency/cost/token/tool metrics + → semantic judge runs headless via GRADER_HARNESS + → grades/YYYY-MM-DD//bird-result.json + → report.ts → longitudinal quality report across cohorts (harness × model) +``` + +The coding agent is the eval runtime — not a simulated agent. Every tool call goes to +the real Tableau APIs. Tracing and all grading signals come from the LangSmith trace; +there is no local-artifact fallback for grading. + +--- + +## Prerequisites + +### 1. Install the coding-agent CLIs you plan to use + +- **Claude Code** — `claude` on PATH +- **Cursor** — `cursor-agent` on PATH +- **Codex** — `codex` on PATH + +### 2. Install the LangSmith coding-agent tracing plugin for each harness + +Grading reads the trace back from LangSmith, so the corresponding plugin **must** be +installed and configured for whichever harness you run. See LangSmith's coding-agent +tracing docs: . +The adapters enable tracing via `TRACE_TO_LANGSMITH=true` plus the harness-specific +`*_LANGSMITH_*` env vars and stamp the `eval_run_id` correlation metadata. + +> If no trace with a matching `eval_run_id` appears within the grader's poll window, +> the case is recorded as `grading_error` (no local fallback). This almost always +> means the plugin is not installed/configured for that harness. + +### 3. Build the MCP server + +```bash +npm run build +``` + +--- + +## Configuration + +Set these in a `.env` file at the repo root (loaded automatically): + +```bash +# LangSmith (shared project; runs are filtered by eval_run_id metadata) +export LANGSMITH_API_KEY="lsv2_..." +export LANGSMITH_PROJECT="your_project_name" + +# Tableau (PAT auth) +export AUTH="pat" +export SERVER="https://your-site.online.tableau.com" +export SITE_NAME="your-site" +export PAT_NAME="your-pat-name" +export PAT_VALUE="your-pat-secret" + +# BIRD California Schools suite +export EVAL_DATASOURCE_LUID="" + +# Agent under test +AGENT_HARNESS=claude-code # claude-code | cursor | codex +# AGENT_MODEL=claude-sonnet-4-5 # optional; omit to use the CLI default + +# Grader / semantic judge (runs headless via its own harness) +GRADER_HARNESS=claude-code # claude-code | cursor | codex +# GRADER_MODEL=claude-sonnet-4-5 + +TRACE_TO_LANGSMITH=true +``` + +`AGENT_HARNESS`/`AGENT_MODEL` can also be overridden per invocation with +`--agent-harness` / `--agent-model`. + +--- + +## BIRD California Schools Suite + +The primary eval suite is 30 questions from the +[BIRD Mini-Dev benchmark](https://bird-bench.github.io/) scoped to the California +Schools database, joined into a single published Tableau datasource. The suite file at +`evals/suites/bird-california-schools.json` ships with precomputed expected answers, so +no database access is required to run evals. + +### Run the full suite + +```bash +npm run eval:suite +npm run eval:suite -- --difficulty simple # subset by difficulty +npm run eval:suite -- --ids 5,11,12 # specific question IDs +npm run eval:suite -- --agent-harness codex --agent-model gpt-5.6-codex +``` + +### Grade a suite run (sources all metrics from LangSmith) + +```bash +npm run eval:grade:suite # most recent suite run +npm run eval:grade:suite -- evals/suite-runs/YYYY-MM-DD/ +``` + +Output: `evals/grades/YYYY-MM-DD//suite-grade.json`. + +### Grade a single run + +```bash +npm run eval:grade:bird -- evals/runs/YYYY-MM-DD/ +``` + +Output: `evals/grades/YYYY-MM-DD//bird-result.json`. + +The semantic judge runs headless through `GRADER_HARNESS`/`GRADER_MODEL` (temperature 0 +where the CLI supports it). Trace poll behavior is tunable with +`GRADE_TRACE_TIMEOUT_MS` (default 60000) and `GRADE_TRACE_POLL_MS` (default 5000). + +### Ad hoc runs + +```bash +npm run eval:run -- input "how many schools have SAT scores above 1200?" +npm run eval:grade -- evals/runs/YYYY-MM-DD/ # tool-coverage grading (trace-sourced) +``` + +--- + +## Longitudinal Report + +Roll up every graded case into cohorts by (harness × normalized model) and over time: + +```bash +npm run eval:report +npm run eval:report -- --since 2026-07-01 +npm run eval:report -- --harness codex --model gpt-5.6-codex +``` + +Output under `evals/reports//`: `longitudinal.json`, `longitudinal.csv`, +`summary.md`. Per-case metrics: **accuracy** (verdict-derived), **latency** +(`wall_s`, `ttft_s`), **cost** (USD; LangSmith-reported or estimated from +`pricing.ts`), **tool-call count**, **error count**, token totals, and the four +quality signals. + +--- + +## Grading Signals + +| Signal | Method | What it checks | +|---|---|---| +| `numeric_match` | Code | Expected value / row count appears in the agent's final message (±1% for floats, substring for strings) | +| `semantic_match` | LLM judge via `GRADER_HARNESS` | Final message correctly answers the question vs. the gold summary | +| `columns_match` | Trace tool-call inspection | Expected VizQL field captions present in a `query-datasource` call | +| `filters_match` | Trace tool-call inspection | Expected filter field captions present in a `query-datasource` call | + +`numeric_match` and `semantic_match` drive the verdict; `columns_match` / `filters_match` +are diagnostic (subset matching — extra columns/filters are fine). + +### Verdicts + +| Verdict | Meaning | +|---|---| +| `pass` | Both `numeric_match` and `semantic_match` passed | +| `partial` | One outcome signal passed (or one was unavailable) | +| `fail` | Both outcome signals failed | +| `error` | The agent exited non-zero, or the trace shows an error and no answer | +| `skip` | Both outcome signals were unavailable | +| `grading_error` | No matching LangSmith trace was found within the poll window | + +--- + +## Architecture + +| Module | Responsibility | +|---|---| +| `adapters/types.ts` | `AgentAdapter` interface + shared eval-metadata builder | +| `adapters/claude-code.ts`, `cursor.ts`, `codex.ts` | Per-harness CLI invocation, MCP config, plugin tracing env | +| `adapters/index.ts` | Harness registry + `resolveHarness` | +| `adapters/run-headless.ts` | One-shot headless prompt (no MCP) for the judge + JSON extraction | +| `langsmith-reader.ts` | Fetch a trace by `eval_run_id`; normalize tools/tokens/cost/timings | +| `model-normalize.ts` | Canonical model ids so the same model lines up across harnesses | +| `pricing.ts` | Per-model price map for cost fallback when LangSmith has no cost | +| `run-case.ts` | Run one case through the selected adapter; write `run.json` | +| `run-suite.ts` | Fan out cases; pass harness/model/suite-run-id through | +| `grade-bird.ts` | Trace-sourced BIRD grader + headless semantic judge | +| `grade-suite.ts` | Batch grade a suite run; aggregate quality/latency/cost | +| `grade.ts` | Ad hoc tool-coverage grader (trace-sourced) | +| `report.ts` | Longitudinal cohort report (JSON/CSV/Markdown) | + +--- + +## Local Artifacts + +Output directories are organized by date (`YYYY-MM-DD`). + +``` +evals/ + runs/YYYY-MM-DD// + run.json metadata, eval_run_id, harness/model, timing, exit code + agent-output.jsonl raw agent stream (debugging only; NOT a grading input) + mcp-config.json / codex-home/ / cursor-workspace/ per-run MCP config (ephemeral) + judge/ headless judge working dir (ephemeral) + logs/ MCP server file logs + suite-runs/YYYY-MM-DD// + suite-summary.json run-time metadata (wall/exit/timeout); metrics deferred to grading + cases/ ephemeral per-case JSON files + grades/YYYY-MM-DD/ + /bird-result.json per-case verdict + full metric set + /suite-grade.json aggregate grading output for a suite run + reports// longitudinal.json / longitudinal.csv / summary.md +``` + +`evals/runs/`, `evals/suite-runs/`, `evals/grades/`, and `evals/reports/` are gitignored. + +--- + +## Regenerating Expected Answers + +The suite ships with precomputed answers. To regenerate (rarely needed; requires the +BIRD SQLite snapshot): + +```bash +python3 evals/scripts/precompute-bird-answers.py +``` + +This executes the gold SQL from `bird_mini/data/test_cases/mini_dev_sqlite.json` against +`bird_mini/data/dev_databases/california_schools/california_schools.sqlite` and writes +`expected_value`, `expected_row_count`, `expected_columns`, `expected_filter_fields`, and +`ai_summarized_answer` into `evals/suites/bird-california-schools.json`. +``` diff --git a/evals/adapters/claude-code.ts b/evals/adapters/claude-code.ts new file mode 100644 index 000000000..d9d9c789b --- /dev/null +++ b/evals/adapters/claude-code.ts @@ -0,0 +1,161 @@ +/** + * Claude Code adapter. + * + * Invocation (agent-under-test): `claude -p [--model M] --mcp-config + * --strict-mcp-config --allowedTools "ToolSearch,mcp__tableau__*" + * --output-format stream-json --verbose` + * Tracing: the LangSmith Claude Code plugin, enabled via TRACE_TO_LANGSMITH + + * CC_LANGSMITH_* env; correlation carried in CC_LANGSMITH_METADATA. + * Determinism (judge): CLAUDE_CODE_TEMPERATURE (verified against installed CLI docs; + * startup-only env var). + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +import { + AgentAdapter, + AgentInvocation, + buildEvalMetadata, + HeadlessContext, + RunContext, +} from './types.js'; + +function mcpConfigPath(runDir: string): string { + return path.join(runDir, 'mcp-config.json'); +} + +function baseTraceEnv(langsmith: { + apiKey: string; + project: string; + endpoint: string; +}): Record { + return { + TRACE_TO_LANGSMITH: 'true', + CC_LANGSMITH_API_KEY: langsmith.apiKey, + CC_LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_API_KEY: langsmith.apiKey, + LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_ENDPOINT: langsmith.endpoint, + }; +} + +export const claudeCodeAdapter: AgentAdapter = { + harness: 'claude-code', + + resolveModel(requested) { + return requested?.trim() ?? ''; + }, + + writeConfig(ctx: RunContext) { + const logsDir = path.join(ctx.runDir, 'logs'); + fs.mkdirSync(logsDir, { recursive: true }); + const mcpConfig = { + mcpServers: { + tableau: { + command: 'node', + args: [ctx.mcpServerEntry], + env: { + ...ctx.mcpServerEnv, + TRANSPORT: 'stdio', + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }, + }; + fs.writeFileSync(mcpConfigPath(ctx.runDir), JSON.stringify(mcpConfig, null, 2)); + }, + + buildInvocation(ctx: RunContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + suiteRunId: ctx.suiteRunId, + harness: 'claude-code', + model: ctx.model, + questionId: ctx.questionId, + }); + const args = [ + '-p', + ctx.prompt, + '--mcp-config', + mcpConfigPath(ctx.runDir), + '--strict-mcp-config', + '--allowedTools', + 'ToolSearch,mcp__tableau__*', + '--output-format', + 'stream-json', + '--verbose', + ]; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'claude', + args, + env: { + ...baseTraceEnv(ctx.langsmith), + CC_LANGSMITH_METADATA: JSON.stringify(metadata), + }, + cwd: process.cwd(), + timeoutMs: ctx.budget.maxWallMs + 10_000, + }; + }, + + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + harness: 'claude-code', + model: ctx.model, + role: ctx.role, + }); + const args = ['-p', ctx.prompt, '--output-format', 'json']; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'claude', + args, + env: { + ...baseTraceEnv(ctx.langsmith), + CC_LANGSMITH_METADATA: JSON.stringify(metadata), + CLAUDE_CODE_TEMPERATURE: String(ctx.temperature), + }, + cwd: process.cwd(), + timeoutMs: ctx.timeoutMs, + }; + }, + + extractFinalText(stdout: string): string { + // Headless judge uses --output-format json → a single JSON object with `.result`. + const trimmed = stdout.trim(); + try { + const obj = JSON.parse(trimmed) as { result?: string; text?: string }; + if (typeof obj.result === 'string') return obj.result; + if (typeof obj.text === 'string') return obj.text; + } catch { + // Fall through to stream-json line scan. + } + // stream-json fallback: last assistant text / result event. + const lines = trimmed.split('\n').filter((l) => l.trim()); + for (let i = lines.length - 1; i >= 0; i--) { + try { + const ev = JSON.parse(lines[i]) as { + type?: string; + result?: string; + message?: { content?: Array<{ type?: string; text?: string }> | string }; + }; + if (ev.type === 'result' && typeof ev.result === 'string') return ev.result; + const content = ev.message?.content; + if (Array.isArray(content)) { + const text = content + .filter((b) => b.type === 'text' && b.text) + .map((b) => b.text) + .join('\n'); + if (text) return text; + } else if (typeof content === 'string' && content.trim()) { + return content; + } + } catch { + continue; + } + } + return trimmed; + }, +}; diff --git a/evals/adapters/codex.ts b/evals/adapters/codex.ts new file mode 100644 index 000000000..8ae9490a3 --- /dev/null +++ b/evals/adapters/codex.ts @@ -0,0 +1,195 @@ +/** + * OpenAI Codex CLI adapter. + * + * Invocation (agent-under-test): `codex exec --json -m -C + * --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox ` + * MCP: configured via `[mcp_servers.tableau]` in an isolated `$CODEX_HOME/config.toml`. + * Tracing: the LangSmith Codex plugin, enabled via TRACE_TO_LANGSMITH + + * LANGSMITH_CODEX_* env (falls back to LANGSMITH_*); plugin hooks live in config.toml. + * Determinism (judge): `-c temperature=0` (reasoning models may ignore it; JSON output + * is the primary stability mechanism). + * + * We set CODEX_HOME to a per-run directory so the tableau MCP config and plugin + * settings never touch the user's global ~/.codex. The exact plugin-metadata wiring + * must be verified against the installed Codex LangSmith plugin. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +import { + AgentAdapter, + AgentInvocation, + buildEvalMetadata, + HeadlessContext, + RunContext, +} from './types.js'; + +function codexHome(runDir: string): string { + return path.join(runDir, 'codex-home'); +} + +function tomlString(value: string): string { + return JSON.stringify(value); // TOML basic strings share JSON escaping for our inputs. +} + +function tomlStringArray(values: Array): string { + return `[${values.map(tomlString).join(', ')}]`; +} + +/** Render a minimal config.toml with an optional MCP server + env table. */ +function renderConfigToml(opts: { mcp?: { entry: string; env: Record } }): string { + const lines: Array = []; + if (opts.mcp) { + lines.push('[mcp_servers.tableau]'); + lines.push(`command = ${tomlString('node')}`); + lines.push(`args = ${tomlStringArray([opts.mcp.entry])}`); + lines.push(''); + lines.push('[mcp_servers.tableau.env]'); + for (const [k, v] of Object.entries(opts.mcp.env)) { + lines.push(`${k} = ${tomlString(v)}`); + } + lines.push(''); + } + return `${lines.join('\n')}\n`; +} + +function baseTraceEnv( + runDir: string, + langsmith: { apiKey: string; project: string; endpoint: string }, + metadata: Record, +): Record { + const home = codexHome(runDir); + fs.mkdirSync(home, { recursive: true }); + // Best-effort correlation channel (verify against installed Codex plugin). + fs.writeFileSync( + path.join(home, 'langsmith.json'), + JSON.stringify({ enabled: true, project: langsmith.project, metadata }, null, 2), + ); + return { + CODEX_HOME: home, + TRACE_TO_LANGSMITH: 'true', + LANGSMITH_CODEX_API_KEY: langsmith.apiKey, + LANGSMITH_CODEX_PROJECT: langsmith.project, + LANGSMITH_CODEX_ENDPOINT: langsmith.endpoint, + LANGSMITH_API_KEY: langsmith.apiKey, + LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_ENDPOINT: langsmith.endpoint, + LANGSMITH_CODEX_METADATA: JSON.stringify(metadata), + }; +} + +export const codexAdapter: AgentAdapter = { + harness: 'codex', + + resolveModel(requested) { + return requested?.trim() ?? ''; + }, + + writeConfig(ctx: RunContext) { + const home = codexHome(ctx.runDir); + const logsDir = path.join(ctx.runDir, 'logs'); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(logsDir, { recursive: true }); + const toml = renderConfigToml({ + mcp: { + entry: ctx.mcpServerEntry, + env: { + ...ctx.mcpServerEnv, + TRANSPORT: 'stdio', + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }); + fs.writeFileSync(path.join(home, 'config.toml'), toml); + }, + + buildInvocation(ctx: RunContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + suiteRunId: ctx.suiteRunId, + harness: 'codex', + model: ctx.model, + questionId: ctx.questionId, + }); + const args = [ + 'exec', + '--json', + '--skip-git-repo-check', + '--dangerously-bypass-approvals-and-sandbox', + '-C', + process.cwd(), + ]; + if (ctx.model) args.push('-m', ctx.model); + args.push(ctx.prompt); + return { + command: 'codex', + args, + env: baseTraceEnv(ctx.runDir, ctx.langsmith, metadata), + cwd: process.cwd(), + timeoutMs: ctx.budget.maxWallMs + 10_000, + }; + }, + + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation { + const home = codexHome(ctx.runDir); + fs.mkdirSync(home, { recursive: true }); + // Empty config (no MCP servers) for the judge. + fs.writeFileSync(path.join(home, 'config.toml'), renderConfigToml({})); + const metadata = buildEvalMetadata({ + runId: ctx.runId, + harness: 'codex', + model: ctx.model, + role: ctx.role, + }); + const args = [ + 'exec', + '--json', + '--skip-git-repo-check', + '--dangerously-bypass-approvals-and-sandbox', + '-c', + `temperature=${ctx.temperature}`, + '-C', + ctx.runDir, + ]; + if (ctx.model) args.push('-m', ctx.model); + args.push(ctx.prompt); + return { + command: 'codex', + args, + env: baseTraceEnv(ctx.runDir, ctx.langsmith, metadata), + cwd: ctx.runDir, + timeoutMs: ctx.timeoutMs, + }; + }, + + extractFinalText(stdout: string): string { + const lines = stdout + .trim() + .split('\n') + .filter((l) => l.trim()); + let lastText = ''; + for (const line of lines) { + let ev: Record; + try { + ev = JSON.parse(line) as Record; + } catch { + continue; + } + // Tolerate several codex event shapes across versions. + const item = ev.item as { text?: string; item_type?: string; type?: string } | undefined; + const msg = ev.msg as { type?: string; message?: string } | undefined; + if (item && typeof item.text === 'string' && item.text.trim()) { + lastText = item.text; + } else if (msg && typeof msg.message === 'string' && msg.message.trim()) { + lastText = msg.message; + } else if (typeof ev.message === 'string' && (ev.message as string).trim()) { + lastText = ev.message as string; + } else if (typeof ev.text === 'string' && (ev.text as string).trim()) { + lastText = ev.text as string; + } + } + return lastText || stdout.trim(); + }, +}; diff --git a/evals/adapters/cursor.ts b/evals/adapters/cursor.ts new file mode 100644 index 000000000..28f45d291 --- /dev/null +++ b/evals/adapters/cursor.ts @@ -0,0 +1,168 @@ +/** + * Cursor CLI (`cursor-agent`) adapter. + * + * Invocation (agent-under-test): `cursor-agent -p [--model M] + * --output-format stream-json --force --approve-mcps --trust --workspace ` + * MCP: cursor-agent discovers `.cursor/mcp.json` in the workspace dir; the adapter + * writes an isolated workspace per run. + * Tracing: the LangSmith Cursor plugin, enabled via TRACE_TO_LANGSMITH + + * LANGSMITH_CURSOR_* env (falls back to LANGSMITH_*). Requires Node >= 22.13. + * Determinism (judge): cursor-agent exposes NO temperature flag — the judge relies on + * JSON output only. + * + * NOTE: `cursor-agent` has no `--mcp-config` flag and no per-tool allowlist, so we + * isolate via a per-run workspace. The custom-metadata propagation mechanism for the + * Cursor plugin is set via env here and must be verified against the installed plugin. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +import { + AgentAdapter, + AgentInvocation, + buildEvalMetadata, + HeadlessContext, + RunContext, +} from './types.js'; + +function workspaceDir(runDir: string): string { + return path.join(runDir, 'cursor-workspace'); +} + +function baseTraceEnv( + langsmith: { apiKey: string; project: string; endpoint: string }, + metadata: Record, +): Record { + return { + TRACE_TO_LANGSMITH: 'true', + LANGSMITH_CURSOR_API_KEY: langsmith.apiKey, + LANGSMITH_CURSOR_PROJECT: langsmith.project, + LANGSMITH_CURSOR_ENDPOINT: langsmith.endpoint, + LANGSMITH_API_KEY: langsmith.apiKey, + LANGSMITH_PROJECT: langsmith.project, + LANGSMITH_ENDPOINT: langsmith.endpoint, + // Best-effort correlation channel (verify against installed Cursor plugin). + LANGSMITH_CURSOR_METADATA: JSON.stringify(metadata), + }; +} + +export const cursorAdapter: AgentAdapter = { + harness: 'cursor', + + resolveModel(requested) { + return requested?.trim() ?? ''; + }, + + writeConfig(ctx: RunContext) { + const ws = workspaceDir(ctx.runDir); + const cursorDir = path.join(ws, '.cursor'); + const logsDir = path.join(ctx.runDir, 'logs'); + fs.mkdirSync(cursorDir, { recursive: true }); + fs.mkdirSync(logsDir, { recursive: true }); + const mcpConfig = { + mcpServers: { + tableau: { + command: 'node', + args: [ctx.mcpServerEntry], + env: { + ...ctx.mcpServerEnv, + TRANSPORT: 'stdio', + FILE_LOGGER_DIRECTORY: logsDir, + ENABLED_LOGGERS: 'fileLogger', + }, + }, + }, + }; + fs.writeFileSync(path.join(cursorDir, 'mcp.json'), JSON.stringify(mcpConfig, null, 2)); + }, + + buildInvocation(ctx: RunContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + suiteRunId: ctx.suiteRunId, + harness: 'cursor', + model: ctx.model, + questionId: ctx.questionId, + }); + const ws = workspaceDir(ctx.runDir); + const args = [ + '-p', + ctx.prompt, + '--output-format', + 'stream-json', + '--force', + '--approve-mcps', + '--trust', + '--workspace', + ws, + ]; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'cursor-agent', + args, + env: baseTraceEnv(ctx.langsmith, metadata), + cwd: process.cwd(), + timeoutMs: ctx.budget.maxWallMs + 10_000, + }; + }, + + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation { + const metadata = buildEvalMetadata({ + runId: ctx.runId, + harness: 'cursor', + model: ctx.model, + role: ctx.role, + }); + // No MCP tools for the judge; ask/read-only mode keeps it from taking actions. + const args = ['-p', ctx.prompt, '--output-format', 'json', '--mode', 'ask']; + if (ctx.model) args.push('--model', ctx.model); + return { + command: 'cursor-agent', + args, + env: baseTraceEnv(ctx.langsmith, metadata), + cwd: process.cwd(), + timeoutMs: ctx.timeoutMs, + }; + }, + + extractFinalText(stdout: string): string { + const trimmed = stdout.trim(); + // `--output-format json` → single object; try direct parse first. + try { + const obj = JSON.parse(trimmed) as { + result?: string; + text?: string; + message?: { content?: Array<{ type?: string; text?: string }> | string }; + }; + if (typeof obj.result === 'string') return obj.result; + if (typeof obj.text === 'string') return obj.text; + } catch { + // Fall through to JSONL scan. + } + const lines = trimmed.split('\n').filter((l) => l.trim()); + for (let i = lines.length - 1; i >= 0; i--) { + try { + const ev = JSON.parse(lines[i]) as { + type?: string; + result?: string; + message?: { content?: Array<{ type?: string; text?: string }> | string }; + }; + if (ev.type === 'result' && typeof ev.result === 'string') return ev.result; + const content = ev.message?.content; + if (Array.isArray(content)) { + const text = content + .filter((b) => b.type === 'text' && b.text) + .map((b) => b.text) + .join('\n'); + if (text) return text; + } else if (typeof content === 'string' && content.trim()) { + return content; + } + } catch { + continue; + } + } + return trimmed; + }, +}; diff --git a/evals/adapters/index.ts b/evals/adapters/index.ts new file mode 100644 index 000000000..750ec6081 --- /dev/null +++ b/evals/adapters/index.ts @@ -0,0 +1,35 @@ +/** Adapter registry: resolve an AgentAdapter by harness id. */ + +import { claudeCodeAdapter } from './claude-code.js'; +import { codexAdapter } from './codex.js'; +import { cursorAdapter } from './cursor.js'; +import { AGENT_HARNESSES, AgentAdapter, AgentHarness, isAgentHarness } from './types.js'; + +const REGISTRY: Record = { + 'claude-code': claudeCodeAdapter, + cursor: cursorAdapter, + codex: codexAdapter, +}; + +export function getAdapter(harness: AgentHarness): AgentAdapter { + return REGISTRY[harness]; +} + +/** + * Resolve the harness from an explicit value or `process.env`, defaulting to + * claude-code. Throws on an unrecognized value. + */ +export function resolveHarness(value: string | undefined, envVar: string): AgentHarness { + const raw = value ?? process.env[envVar]; + if (raw == null || raw.trim() === '') return 'claude-code'; + const normalized = raw.trim(); + if (!isAgentHarness(normalized)) { + throw new Error( + `Invalid ${envVar}="${normalized}". Expected one of: ${AGENT_HARNESSES.join(', ')}.`, + ); + } + return normalized; +} + +export * from './types.js'; +export { claudeCodeAdapter, codexAdapter, cursorAdapter }; diff --git a/evals/adapters/run-headless.ts b/evals/adapters/run-headless.ts new file mode 100644 index 000000000..0ff54bc07 --- /dev/null +++ b/evals/adapters/run-headless.ts @@ -0,0 +1,70 @@ +/** + * Headless one-shot helper: send a single prompt to a coding-agent CLI (no MCP + * tools) and return the final assistant text. Used by the grader's semantic judge. + */ + +import { execFileSync } from 'child_process'; + +import { AgentAdapter, HeadlessContext } from './types.js'; + +export type HeadlessRunResult = { + text: string; + exitCode: number; + stdout: string; + stderr: string; +}; + +export function runHeadless(adapter: AgentAdapter, ctx: HeadlessContext): HeadlessRunResult { + const invocation = adapter.buildHeadlessInvocation(ctx); + let stdout = ''; + let stderr = ''; + let exitCode = 0; + try { + stdout = execFileSync(invocation.command, invocation.args, { + env: { ...process.env, ...invocation.env }, + cwd: invocation.cwd, + timeout: invocation.timeoutMs, + maxBuffer: 25 * 1024 * 1024, + }).toString(); + } catch (error: unknown) { + const e = error as { status?: number; stdout?: Buffer; stderr?: Buffer; message?: string }; + exitCode = e.status ?? 1; + stdout = e.stdout?.toString() ?? ''; + stderr = [e.message, e.stderr?.toString()].filter(Boolean).join('\n'); + } + return { text: adapter.extractFinalText(stdout), exitCode, stdout, stderr }; +} + +/** + * Extract the first balanced JSON object from a text blob. Coding-agent CLIs often + * wrap the judge's JSON in prose/markdown fences, so we scan for `{ ... }`. + */ +export function extractJsonObject(text: string): T | null { + // Prefer fenced ```json blocks. + const fenceMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/i); + const candidates: Array = []; + if (fenceMatch?.[1]) candidates.push(fenceMatch[1].trim()); + candidates.push(text.trim()); + + for (const candidate of candidates) { + const start = candidate.indexOf('{'); + if (start === -1) continue; + let depth = 0; + for (let i = start; i < candidate.length; i++) { + const ch = candidate[i]; + if (ch === '{') depth += 1; + else if (ch === '}') { + depth -= 1; + if (depth === 0) { + const slice = candidate.slice(start, i + 1); + try { + return JSON.parse(slice) as T; + } catch { + break; + } + } + } + } + } + return null; +} diff --git a/evals/adapters/types.ts b/evals/adapters/types.ts new file mode 100644 index 000000000..653ea0eb1 --- /dev/null +++ b/evals/adapters/types.ts @@ -0,0 +1,115 @@ +/** + * Agent harness abstraction for the Tableau MCP eval harness. + * + * An `AgentAdapter` encapsulates everything agent-specific about invoking a + * coding-agent CLI: the command + argv (including the model flag), the MCP + * server config in the agent's own format, and the environment that enables the + * official LangSmith coding-agent tracing plugin. Adapters do NOT post traces or + * compute grades — the plugin traces automatically and the grader reads the trace + * back from LangSmith (single source of truth). + */ + +export type AgentHarness = 'claude-code' | 'cursor' | 'codex'; + +export const AGENT_HARNESSES: ReadonlyArray = ['claude-code', 'cursor', 'codex']; + +export function isAgentHarness(value: string | undefined): value is AgentHarness { + return value != null && (AGENT_HARNESSES as ReadonlyArray).includes(value); +} + +/** A fully-formed CLI invocation the runner can hand to `execFileSync`. */ +export type AgentInvocation = { + command: string; + args: Array; + env: Record; + cwd: string; + timeoutMs: number; +}; + +/** Result of spawning an agent CLI (persisted verbatim for debugging only). */ +export type AgentResult = { + exitCode: number; + stdout: string; + stderr: string; +}; + +/** LangSmith connection + correlation shared by every invocation. */ +export type LangSmithConfig = { + apiKey: string; + project: string; + endpoint: string; +}; + +/** Context for a full agent-under-test run (with MCP tools attached). */ +export type RunContext = { + runId: string; + suiteRunId: string | null; + runDir: string; + prompt: string; + /** Resolved model id ('' means "let the CLI use its default"). */ + model: string; + mcpServerEntry: string; + mcpServerEnv: Record; + questionId: number | null; + langsmith: LangSmithConfig; + budget: { maxWallMs: number; maxToolCalls: number }; +}; + +/** Context for a headless one-shot prompt (no MCP tools) — e.g. the grader judge. */ +export type HeadlessContext = { + runId: string; + runDir: string; + prompt: string; + model: string; + /** 0 for deterministic judging; applied only where the CLI supports it. */ + temperature: number; + /** Attribution role for trace metadata, e.g. 'judge'. */ + role: string; + langsmith: LangSmithConfig; + timeoutMs: number; +}; + +export interface AgentAdapter { + readonly harness: AgentHarness; + + /** Resolve the model to use: the requested id, or the adapter's default (''=CLI default). */ + resolveModel(requested: string | undefined): string; + + /** + * Write any per-run config files the CLI needs (MCP server config in the + * agent's own format). Called before `buildInvocation`. + */ + writeConfig(ctx: RunContext): void; + + /** Build the CLI invocation for a full agent-under-test run (MCP tools attached). */ + buildInvocation(ctx: RunContext): AgentInvocation; + + /** + * Build the CLI invocation for a headless one-shot prompt with no MCP tools, + * requesting structured/JSON-friendly output. Used by the grader's judge. + */ + buildHeadlessInvocation(ctx: HeadlessContext): AgentInvocation; + + /** Extract the final assistant text from this agent's raw stdout. */ + extractFinalText(stdout: string): string; +} + +/** Build the `coding-agent` custom metadata blob shared across adapters. */ +export function buildEvalMetadata(ctx: { + runId: string; + suiteRunId?: string | null; + harness: AgentHarness; + model: string; + questionId?: number | null; + role?: string; +}): Record { + const meta: Record = { + eval_run_id: ctx.runId, + harness: ctx.harness, + }; + if (ctx.suiteRunId) meta.suite_run_id = ctx.suiteRunId; + if (ctx.model) meta.model = ctx.model; + if (ctx.questionId != null) meta.question_id = ctx.questionId; + if (ctx.role) meta.eval_role = ctx.role; + return meta; +} diff --git a/evals/bird_mini b/evals/bird_mini new file mode 160000 index 000000000..387f095bf --- /dev/null +++ b/evals/bird_mini @@ -0,0 +1 @@ +Subproject commit 387f095bf8e6fbf1692c6435fff57dcfd70c2dc0 diff --git a/evals/cases/get-datasource-metadata.json b/evals/cases/get-datasource-metadata.json new file mode 100644 index 000000000..81a757dbf --- /dev/null +++ b/evals/cases/get-datasource-metadata.json @@ -0,0 +1,12 @@ +{ + "id": "get-datasource-metadata", + "name": "Get Datasource Metadata", + "description": "Exercises metadata retrieval for a known datasource LUID provided by EVAL_DATASOURCE_LUID.", + "tags": ["datasource", "metadata", "real-server"], + "prompt": "Use the Tableau MCP server to get metadata for datasource LUID {{env.EVAL_DATASOURCE_LUID}}. Summarize the datasource name, fields, and any warnings returned by the tool. Do not query data rows.", + "expected_tools": ["get-datasource-metadata"], + "budget": { + "max_tool_calls": 5, + "max_wall_ms": 120000 + } +} diff --git a/evals/cases/list-datasources.json b/evals/cases/list-datasources.json new file mode 100644 index 000000000..26aadc48b --- /dev/null +++ b/evals/cases/list-datasources.json @@ -0,0 +1,12 @@ +{ + "id": "list-datasources", + "name": "List Datasources", + "description": "Smoke test for a real Tableau Cloud/Server connection. Claude Code should use the Tableau MCP server to list datasources on the configured site.", + "tags": ["smoke", "datasource", "real-server"], + "prompt": "Use the Tableau MCP server to list data sources available on the configured Tableau site. Return a concise summary with datasource names and IDs when available. Do not invent data if the server returns no results.", + "expected_tools": ["list-datasources"], + "budget": { + "max_tool_calls": 5, + "max_wall_ms": 120000 + } +} diff --git a/evals/cases/query-datasource.json b/evals/cases/query-datasource.json new file mode 100644 index 000000000..b06b51fd5 --- /dev/null +++ b/evals/cases/query-datasource.json @@ -0,0 +1,12 @@ +{ + "id": "query-datasource", + "name": "Query Datasource", + "description": "Runs a small, bounded query against a known datasource LUID provided by EVAL_DATASOURCE_LUID.", + "tags": ["datasource", "query", "real-server"], + "prompt": "Use the Tableau MCP server to query datasource LUID {{env.EVAL_DATASOURCE_LUID}}. First inspect or use known metadata if needed, then run this query exactly: {{env.EVAL_DATASOURCE_QUERY}}. Keep the row limit small and summarize the returned rows.", + "expected_tools": ["query-datasource"], + "budget": { + "max_tool_calls": 8, + "max_wall_ms": 180000 + } +} diff --git a/evals/grade-bird.ts b/evals/grade-bird.ts new file mode 100644 index 000000000..145359861 --- /dev/null +++ b/evals/grade-bird.ts @@ -0,0 +1,536 @@ +/** + * BIRD-specific grader for California Schools eval runs — trace-sourced. + * + * All per-case signals are derived from the coding-agent trace in LangSmith + * (fetched by `eval_run_id`), not from local artifacts. Signals: + * columns_match — required VizQL fields present in a query-datasource tool call + * filters_match — required filter fields present in a query-datasource tool call + * numeric_match — expected value / row count found in the agent's final message + * semantic_match — LLM judge (run via GRADER_HARNESS/GRADER_MODEL, headless) + * + * Usage: + * npx tsx evals/grade-bird.ts evals/runs// + * + * Required environment: + * LANGSMITH_API_KEY (or LANGCHAIN_API_KEY), LANGSMITH_PROJECT (fallback for older runs) + * Optional: + * GRADER_HARNESS=claude-code|cursor|codex (default claude-code), GRADER_MODEL= + * GRADE_TRACE_TIMEOUT_MS (default 60000), GRADE_TRACE_POLL_MS (default 5000) + */ + +/* eslint-disable no-console */ + +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +import { getAdapter, HeadlessContext, resolveHarness } from './adapters/index.js'; +import { extractJsonObject, runHeadless } from './adapters/run-headless.js'; +import { fetchTraceSummary, findVizqlQuery, makeClient, TraceSummary } from './langsmith-reader.js'; +import { normalizeModel } from './model-normalize.js'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const GRADES_DIR = path.join(EVALS_DIR, 'grades'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} + +const runDirArg = process.argv[2]; +if (!runDirArg) { + console.error('Usage: npx tsx evals/grade-bird.ts '); + process.exit(1); +} +const absRunDir = path.resolve(runDirArg); + +// ─── Types ─────────────────────────────────────────────────────────────────── + +type RunMeta = { + run_id: string; + case_id: string; + eval_run_id?: string; + harness?: string; + model?: string | null; + langsmith_project?: string; + wall_ms?: number; + agent_exit_code?: number; + timed_out?: boolean; + metadata?: { + question_id?: number; + difficulty?: string; + suite_file?: string; + }; +}; + +type VizqlField = { fieldCaption?: string }; +type VizqlFilter = { field?: { fieldCaption?: string } }; +type VizqlQuery = { fields?: Array; filters?: Array }; + +type BirdCase = { + question_id: number; + question: string; + difficulty: string; + answer_type: 'scalar' | 'list'; + expected_value: number | string | null; + expected_row_count: number | null; + ai_summarized_answer: string; + expected_columns: Array; + expected_filter_fields: Array; +}; + +type LlmJudgeResult = { correct: boolean; score: number; reason: string }; + +type BirdGradeResult = { + run_id: string; + eval_run_id: string; + question_id: number; + difficulty: string; + graded_at: string; + harness: string | null; + model: string | null; + model_normalized: string | null; + grader_harness: string; + grader_model: string | null; + // Latency / cost / volume metrics (from trace). + wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; + cost_source: TraceSummary['costSource'] | 'n/a'; + tokens: { + input_tokens: number | null; + output_tokens: number | null; + cache_read_tokens: number | null; + cache_creation_tokens: number | null; + total_tokens: number | null; + }; + tool_calls: number; + tools_used: Array; + llm_calls: number; + subagent_count: number; + error_count: number; + // Quality signals. + signals: { + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + }; + accuracy: number | null; + details: { + expected_columns: Array; + actual_columns: Array; + missing_columns: Array; + expected_filter_fields: Array; + actual_filter_fields: Array; + missing_filter_fields: Array; + expected_value: number | string | null; + expected_row_count: number | null; + extracted_number: number | null; + final_message_preview: string; + llm_judge: LlmJudgeResult | null; + llm_judge_error: string | null; + trace_error: string | null; + }; + verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip' | 'grading_error'; +}; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +function readOptionalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T; + } catch { + return null; + } +} + +function collectFieldCaptions(fields: Array | undefined): Array { + return (fields ?? []) + .map((f) => f.fieldCaption ?? '') + .filter(Boolean) + .map((c) => c.toLowerCase()); +} + +function collectFilterCaptions(filters: Array | undefined): Array { + return (filters ?? []) + .map((f) => f.field?.fieldCaption ?? '') + .filter(Boolean) + .map((c) => c.toLowerCase()); +} + +function extractMatchingNumber(text: string, expected: number | string | null): number | null { + if (expected === null) return null; + let expectedNum: number | null = null; + if (typeof expected === 'number') { + expectedNum = expected; + } else { + const parsed = parseFloat(String(expected).replace(/,/g, '')); + if (!isNaN(parsed)) expectedNum = parsed; + } + if (expectedNum === null) return null; + + const clean = text.replace(/,/g, ''); + const matches = clean.match(/-?\d+(?:\.\d+)?/g) ?? []; + const candidates = matches.map((m) => parseFloat(m)).filter((n) => !isNaN(n)); + + const isClose = (a: number, b: number): boolean => { + if (b === 0) return Math.abs(a) < 0.001; + return Math.abs(a - b) / Math.abs(b) <= 0.01 || Math.abs(a - b) <= 0.001; + }; + const percentageCandidates = candidates.map((n) => n / 100); + for (const candidate of [...candidates, ...percentageCandidates]) { + if (isClose(candidate, expectedNum)) return candidate; + } + return null; +} + +function checkNumericMatch( + finalMessage: string, + birdCase: BirdCase, +): { matched: boolean; extracted: number | null } { + const target = + birdCase.answer_type === 'scalar' ? birdCase.expected_value : birdCase.expected_row_count; + if (target === null) return { matched: false, extracted: null }; + if (typeof target === 'string') { + return { matched: finalMessage.toLowerCase().includes(target.toLowerCase()), extracted: null }; + } + const extracted = extractMatchingNumber(finalMessage, target); + return { matched: extracted !== null, extracted }; +} + +// ─── LLM Judge (via GRADER_HARNESS) ──────────────────────────────────────────── + +function runJudge( + birdCase: BirdCase, + finalMessage: string, +): { + result: LlmJudgeResult | null; + error: string | null; + harness: string; + model: string | null; +} { + const graderHarness = resolveHarness(undefined, 'GRADER_HARNESS'); + const adapter = getAdapter(graderHarness); + const graderModel = adapter.resolveModel(process.env.GRADER_MODEL); + + const targetValue = + birdCase.answer_type === 'scalar' + ? `Expected value: ${String(birdCase.expected_value)}` + : `Expected row count: ${String(birdCase.expected_row_count)}`; + + const prompt = [ + 'You are evaluating whether an AI agent correctly answered a data question about California schools.', + '', + `Question: ${birdCase.question}`, + '', + `Gold answer summary: ${birdCase.ai_summarized_answer}`, + `${targetValue}`, + '', + "Agent's response:", + finalMessage.slice(0, 3000), + '', + "Does the agent's response correctly answer the question?", + 'Consider:', + '- Numeric values must match (allow rounding/formatting differences, e.g. "90.5%" and "0.905" are the same)', + '- The agent must address the right entities (not a proxy or wrong dimension)', + '- Partial credit if the agent got the right approach but a slightly wrong number', + '', + 'Respond with a single JSON object and nothing else:', + '{"correct": true|false, "score": 0.0-1.0, "reason": "one sentence"}', + ].join('\n'); + + const judgeDir = path.join(absRunDir, 'judge'); + fs.mkdirSync(judgeDir, { recursive: true }); + + const ctx: HeadlessContext = { + runId: `${path.basename(absRunDir)}-judge`, + runDir: judgeDir, + prompt, + model: graderModel, + temperature: 0, + role: 'judge', + langsmith: { + apiKey: process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY ?? '', + project: process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals', + endpoint: process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com', + }, + timeoutMs: Number(process.env.GRADE_JUDGE_TIMEOUT_MS ?? 120_000), + }; + + const outcome = runHeadless(adapter, ctx); + if (outcome.exitCode !== 0 && !outcome.text) { + return { + result: null, + error: `judge exited ${outcome.exitCode}: ${outcome.stderr}`, + harness: graderHarness, + model: graderModel || null, + }; + } + const parsed = extractJsonObject>(outcome.text); + if (!parsed) { + return { + result: null, + error: `could not parse judge JSON from output: ${outcome.text.slice(0, 200)}`, + harness: graderHarness, + model: graderModel || null, + }; + } + return { + result: { + correct: parsed.correct ?? false, + score: typeof parsed.score === 'number' ? parsed.score : parsed.correct ? 1.0 : 0.0, + reason: parsed.reason ?? '', + }, + error: null, + harness: graderHarness, + model: graderModel || null, + }; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +function writeResult(result: BirdGradeResult): string { + const gradeDir = path.join(GRADES_DIR, dateSlug(), result.run_id); + fs.mkdirSync(gradeDir, { recursive: true }); + const resultPath = path.join(gradeDir, 'bird-result.json'); + fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); + return resultPath; +} + +async function main(): Promise { + const runMeta = readOptionalJson(path.join(absRunDir, 'run.json')); + if (!runMeta) { + console.error(`run.json not found in ${absRunDir}`); + process.exit(1); + } + + const questionId = runMeta.metadata?.question_id; + const suiteFilePath = runMeta.metadata?.suite_file; + if (!questionId || !suiteFilePath) { + console.error( + 'run.json is missing metadata.question_id or metadata.suite_file.\n' + + 'This grader only works on runs produced by run-suite.ts.', + ); + process.exit(1); + } + if (!fs.existsSync(suiteFilePath)) { + console.error(`Suite file not found: ${suiteFilePath}`); + process.exit(1); + } + + const suite = JSON.parse(fs.readFileSync(suiteFilePath, 'utf-8')) as Array; + const birdCase = suite.find((c) => c.question_id === questionId); + if (!birdCase) { + console.error(`Question ID ${questionId} not found in suite file.`); + process.exit(1); + } + + const runId = runMeta.run_id ?? path.basename(absRunDir); + const evalRunId = runMeta.eval_run_id ?? runId; + const projectName = + runMeta.langsmith_project ?? process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; + const difficulty = runMeta.metadata?.difficulty ?? birdCase.difficulty ?? 'unknown'; + + const baseResult: BirdGradeResult = { + run_id: runId, + eval_run_id: evalRunId, + question_id: questionId, + difficulty, + graded_at: new Date().toISOString(), + harness: runMeta.harness ?? null, + model: runMeta.model ?? null, + model_normalized: null, + grader_harness: resolveHarness(undefined, 'GRADER_HARNESS'), + grader_model: null, + wall_s: runMeta.wall_ms != null ? Math.round(runMeta.wall_ms / 1000) : null, + ttft_s: null, + cost_usd: null, + cost_source: 'n/a', + tokens: { + input_tokens: null, + output_tokens: null, + cache_read_tokens: null, + cache_creation_tokens: null, + total_tokens: null, + }, + tool_calls: 0, + tools_used: [], + llm_calls: 0, + subagent_count: 0, + error_count: 0, + signals: { + numeric_match: null, + semantic_match: null, + columns_match: null, + filters_match: null, + }, + accuracy: null, + details: { + expected_columns: birdCase.expected_columns, + actual_columns: [], + missing_columns: [], + expected_filter_fields: birdCase.expected_filter_fields, + actual_filter_fields: [], + missing_filter_fields: [], + expected_value: birdCase.answer_type === 'scalar' ? birdCase.expected_value : null, + expected_row_count: birdCase.expected_row_count, + extracted_number: null, + final_message_preview: '', + llm_judge: null, + llm_judge_error: null, + trace_error: null, + }, + verdict: 'grading_error', + }; + + // ── Fetch trace (single source of truth) ───────────────────────────────── + const client = makeClient(); + const summary = await fetchTraceSummary(client, { + projectName, + evalRunId, + pollIntervalMs: Number(process.env.GRADE_TRACE_POLL_MS ?? 5_000), + timeoutMs: Number(process.env.GRADE_TRACE_TIMEOUT_MS ?? 60_000), + }); + + if (!summary) { + baseResult.details.trace_error = `No LangSmith trace found for eval_run_id=${evalRunId} in project "${projectName}" within the timeout. Ensure the coding-agent LangSmith plugin is installed and configured for the ${runMeta.harness ?? 'agent'} harness.`; + baseResult.verdict = 'grading_error'; + const p = writeResult(baseResult); + console.error(`\nGrade: Q${questionId} — GRADING_ERROR (no trace)`); + console.error(baseResult.details.trace_error); + console.error(`Result: ${p}`); + return; + } + + // ── Metrics from trace ──────────────────────────────────────────────────── + baseResult.model = summary.model ?? runMeta.model ?? null; + baseResult.model_normalized = normalizeModel(baseResult.model); + baseResult.ttft_s = summary.ttftMs != null ? Math.round(summary.ttftMs / 100) / 10 : null; + if (summary.wallMs != null) baseResult.wall_s = Math.round(summary.wallMs / 1000); + baseResult.cost_usd = summary.costUsd; + baseResult.cost_source = summary.costSource; + baseResult.tokens = { + input_tokens: summary.tokens.input, + output_tokens: summary.tokens.output, + cache_read_tokens: summary.tokens.cacheRead, + cache_creation_tokens: summary.tokens.cacheWrite, + total_tokens: summary.tokens.total, + }; + baseResult.tool_calls = summary.toolCalls.length; + baseResult.tools_used = [...new Set(summary.toolCalls.map((t) => t.normalizedName))]; + baseResult.llm_calls = summary.llmRunCount; + baseResult.subagent_count = summary.subagentCount; + baseResult.error_count = summary.errorCount; + + const finalMessage = summary.finalText; + baseResult.details.final_message_preview = finalMessage.slice(0, 500); + + // ── Signals 1 & 2: columns_match / filters_match ───────────────────────── + const queryCalls = summary.toolCalls.filter((t) => t.normalizedName === 'query-datasource'); + if (queryCalls.length > 0) { + const allActualColumns = new Set(); + const allActualFilters = new Set(); + for (const call of queryCalls) { + const query = findVizqlQuery(call.inputs) as VizqlQuery | null; + for (const c of collectFieldCaptions(query?.fields)) allActualColumns.add(c); + for (const f of collectFilterCaptions(query?.filters)) allActualFilters.add(f); + } + baseResult.details.actual_columns = [...allActualColumns]; + baseResult.details.actual_filter_fields = [...allActualFilters]; + baseResult.details.missing_columns = birdCase.expected_columns.filter( + (c) => !allActualColumns.has(c.toLowerCase()), + ); + baseResult.details.missing_filter_fields = birdCase.expected_filter_fields.filter( + (f) => !allActualFilters.has(f.toLowerCase()), + ); + baseResult.signals.columns_match = + baseResult.details.missing_columns.length === 0 && birdCase.expected_columns.length > 0; + baseResult.signals.filters_match = + baseResult.details.missing_filter_fields.length === 0 && + birdCase.expected_filter_fields.length > 0; + } + + // ── Signal 3: numeric_match ─────────────────────────────────────────────── + if (finalMessage) { + const numeric = checkNumericMatch(finalMessage, birdCase); + baseResult.signals.numeric_match = numeric.matched; + baseResult.details.extracted_number = numeric.extracted; + } + + // ── Signal 4: semantic_match (judge via GRADER_HARNESS) ────────────────── + if (!finalMessage) { + baseResult.details.llm_judge_error = 'No final message found in trace outputs.'; + } else { + const judge = runJudge(birdCase, finalMessage); + baseResult.grader_harness = judge.harness; + baseResult.grader_model = judge.model; + baseResult.details.llm_judge = judge.result; + baseResult.details.llm_judge_error = judge.error; + baseResult.signals.semantic_match = judge.result?.score ?? null; + } + + // ── Verdict + accuracy ──────────────────────────────────────────────────── + const s = baseResult.signals; + baseResult.verdict = (() => { + if (runMeta.agent_exit_code != null && runMeta.agent_exit_code !== 0) return 'error'; + if (summary.hadError && s.semantic_match === null && s.numeric_match === null) return 'error'; + if (s.semantic_match === null && s.numeric_match === null) return 'skip'; + const semanticOk = s.semantic_match != null && s.semantic_match >= 0.8; + const numericOk = s.numeric_match === true; + if (semanticOk && numericOk) return 'pass'; + if (semanticOk || numericOk) return 'partial'; + return 'fail'; + })(); + baseResult.accuracy = + baseResult.verdict === 'pass' + ? 1 + : baseResult.verdict === 'partial' + ? 0.5 + : baseResult.verdict === 'fail' + ? 0 + : null; + + const resultPath = writeResult(baseResult); + + // ── Print summary ───────────────────────────────────────────────────────── + console.log(`\nGrade: Q${questionId} (${difficulty}) — ${baseResult.verdict.toUpperCase()}`); + console.log(`Harness/model: ${baseResult.harness ?? '?'} / ${baseResult.model ?? 'n/a'}`); + console.log( + `numeric_match: ${s.numeric_match === null ? 'n/a' : s.numeric_match ? 'YES' : 'NO'}`, + ); + console.log( + `semantic_match: ${s.semantic_match === null ? 'n/a' : s.semantic_match.toFixed(2)}` + + (baseResult.details.llm_judge + ? ` — ${baseResult.details.llm_judge.reason}` + : baseResult.details.llm_judge_error + ? ` (${baseResult.details.llm_judge_error})` + : ''), + ); + console.log( + `columns_match: ${s.columns_match === null ? 'n/a' : s.columns_match ? 'YES' : `NO (missing: ${baseResult.details.missing_columns.join(', ')})`}`, + ); + console.log( + `filters_match: ${s.filters_match === null ? 'n/a' : s.filters_match ? 'YES' : `NO (missing: ${baseResult.details.missing_filter_fields.join(', ')})`}`, + ); + console.log(`Wall / TTFT: ${baseResult.wall_s ?? '?'}s / ${baseResult.ttft_s ?? '?'}s`); + console.log( + `Cost: ${baseResult.cost_usd != null ? `$${baseResult.cost_usd.toFixed(4)} (${baseResult.cost_source})` : 'n/a'}`, + ); + console.log( + `Tokens (total): ${baseResult.tokens.total_tokens ?? 'n/a'} | LLM calls: ${baseResult.llm_calls} | errors: ${baseResult.error_count}`, + ); + console.log( + `Tool calls: ${baseResult.tool_calls} (${baseResult.tools_used.join(', ') || 'none'})`, + ); + console.log(`Result: ${resultPath}`); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/evals/grade-suite.ts b/evals/grade-suite.ts new file mode 100644 index 000000000..da959c592 --- /dev/null +++ b/evals/grade-suite.ts @@ -0,0 +1,328 @@ +/** + * Batch grader for a completed BIRD suite run. + * + * Grades every case in a suite run and writes a single suite-grade.json + * to evals/grades/YYYY-MM-DD//. + * + * Usage: + * npx tsx evals/grade-suite.ts evals/suite-runs/YYYY-MM-DD/ + * npx tsx evals/grade-suite.ts # auto-discovers most recent suite run + * + * Optional environment: + * OPENAI_API_KEY — enables semantic_match (LLM judge); verdict degrades to numeric-only without it + * BIRD_GRADE_MODEL — override LLM judge model (default: gpt-4o-mini) + */ + +/* eslint-disable no-console */ + +import { execFileSync } from 'child_process'; +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const SUITE_RUNS_DIR = path.join(EVALS_DIR, 'suite-runs'); +const GRADES_DIR = path.join(EVALS_DIR, 'grades'); +const GRADE_BIRD_SCRIPT = path.join(EVALS_DIR, 'grade-bird.ts'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} + +// ─── Types ──────────────────────────────────────────────────────────────────── + +type SuiteSummary = { + suite_run_id: string; + suite_file: string; + started_at: string; + finished_at: string; + total_wall_ms: number; + harness?: string | null; + model?: string | null; + cases: Array<{ + question_id: number; + difficulty: string; + run_id: string; + run_dir: string; + exit_code: number | null; + wall_ms: number | null; + timed_out: boolean; + error: string | null; + }>; +}; + +type BirdResult = { + run_id: string; + eval_run_id: string; + question_id: number; + difficulty: string; + graded_at: string; + harness: string | null; + model: string | null; + model_normalized: string | null; + wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; + tokens: { + input_tokens: number | null; + output_tokens: number | null; + cache_creation_tokens: number | null; + cache_read_tokens: number | null; + total_tokens: number | null; + }; + tool_calls: number; + tools_used: Array; + llm_calls: number; + error_count: number; + signals: { + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + }; + accuracy: number | null; + verdict: 'pass' | 'partial' | 'fail' | 'error' | 'skip' | 'grading_error'; +}; + +type CaseGrade = { + question_id: number; + difficulty: string; + run_id: string; + verdict: BirdResult['verdict']; + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + accuracy: number | null; + harness: string | null; + model: string | null; + model_normalized: string | null; + wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; + tool_calls: number; + tools_used: Array; + llm_calls: number | null; + error_count: number | null; + tokens: BirdResult['tokens'] | null; + grade_file: string | null; + grading_error: string | null; +}; + +// ─── Discovery ──────────────────────────────────────────────────────────────── + +function findMostRecentSuiteRunDir(): string { + const allSummaries: Array<{ mtime: number; dir: string }> = []; + + for (const entry of fs.readdirSync(SUITE_RUNS_DIR, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const sub = path.join(SUITE_RUNS_DIR, entry.name); + + // Support both flat (legacy) and date-nested layouts + const candidates = [ + sub, + ...fs + .readdirSync(sub, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => path.join(sub, e.name)), + ]; + + for (const dir of candidates) { + const summaryPath = path.join(dir, 'suite-summary.json'); + if (fs.existsSync(summaryPath)) { + allSummaries.push({ mtime: fs.statSync(summaryPath).mtimeMs, dir }); + } + } + } + + if (allSummaries.length === 0) { + console.error(`No suite runs found under ${SUITE_RUNS_DIR}`); + process.exit(1); + } + + allSummaries.sort((a, b) => b.mtime - a.mtime); + return allSummaries[0].dir; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main(): Promise { + const suiteRunDirArg = process.argv[2]; + const suiteRunDir = suiteRunDirArg ? path.resolve(suiteRunDirArg) : findMostRecentSuiteRunDir(); + + const summaryPath = path.join(suiteRunDir, 'suite-summary.json'); + if (!fs.existsSync(summaryPath)) { + console.error(`suite-summary.json not found in: ${suiteRunDir}`); + process.exit(1); + } + + const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf-8')) as SuiteSummary; + const { suite_run_id, cases } = summary; + const today = dateSlug(); + + console.log(`\nGrading suite: ${suite_run_id}`); + console.log(`Cases: ${cases.length}`); + console.log(`Run dir: ${suiteRunDir}\n`); + + const caseGrades: Array = []; + + for (let i = 0; i < cases.length; i++) { + const c = cases[i]; + const label = `Q${c.question_id} (${c.difficulty}) [${i + 1}/${cases.length}]`; + process.stdout.write(` ${label.padEnd(32)}`); + + // Derive where grade-bird.ts will write its output + const gradeFile = path.join(GRADES_DIR, today, c.run_id, 'bird-result.json'); + + let gradeResult: BirdResult | null = null; + let gradingError: string | null = null; + + try { + execFileSync('npx', ['tsx', GRADE_BIRD_SCRIPT, c.run_dir], { + env: process.env, + cwd: REPO_ROOT, + stdio: 'pipe', + }); + if (fs.existsSync(gradeFile)) { + gradeResult = JSON.parse(fs.readFileSync(gradeFile, 'utf-8')) as BirdResult; + } else { + gradingError = 'bird-result.json not found after grading'; + } + } catch (err: unknown) { + const e = err as { message?: string; stderr?: Buffer }; + gradingError = e.stderr?.toString().trim() || e.message || 'unknown grading error'; + } + + const verdict = gradeResult?.verdict ?? 'grading_error'; + const verdictLabel = + verdict === 'pass' + ? '✓ PASS' + : verdict === 'partial' + ? '~ PARTIAL' + : verdict === 'fail' + ? '✗ FAIL' + : verdict === 'error' + ? '! ERROR' + : verdict === 'skip' + ? '- SKIP' + : '? GRADING_ERROR'; + + process.stdout.write(`${verdictLabel}\n`); + + caseGrades.push({ + question_id: c.question_id, + difficulty: c.difficulty, + run_id: c.run_id, + verdict, + numeric_match: gradeResult?.signals.numeric_match ?? null, + semantic_match: gradeResult?.signals.semantic_match ?? null, + columns_match: gradeResult?.signals.columns_match ?? null, + filters_match: gradeResult?.signals.filters_match ?? null, + accuracy: gradeResult?.accuracy ?? null, + harness: gradeResult?.harness ?? null, + model: gradeResult?.model ?? null, + model_normalized: gradeResult?.model_normalized ?? null, + wall_s: gradeResult?.wall_s ?? null, + ttft_s: gradeResult?.ttft_s ?? null, + cost_usd: gradeResult?.cost_usd ?? null, + tool_calls: gradeResult?.tool_calls ?? 0, + tools_used: gradeResult?.tools_used ?? [], + llm_calls: gradeResult?.llm_calls ?? null, + error_count: gradeResult?.error_count ?? null, + tokens: gradeResult?.tokens ?? null, + grade_file: gradeResult ? gradeFile : null, + grading_error: gradingError, + }); + } + + // ── Aggregate stats ────────────────────────────────────────────────────────── + + const counts = { + pass: caseGrades.filter((g) => g.verdict === 'pass').length, + partial: caseGrades.filter((g) => g.verdict === 'partial').length, + fail: caseGrades.filter((g) => g.verdict === 'fail').length, + error: caseGrades.filter((g) => g.verdict === 'error').length, + skip: caseGrades.filter((g) => g.verdict === 'skip').length, + grading_error: caseGrades.filter((g) => g.verdict === 'grading_error').length, + }; + const total = caseGrades.length; + const passRate = total > 0 ? counts.pass / total : 0; + + const mean = (values: Array): number | null => + values.length > 0 ? values.reduce((a, b) => a + b, 0) / values.length : null; + const sum = (values: Array): number => values.reduce((a, b) => a + b, 0); + + const accuracyValues = caseGrades.map((g) => g.accuracy).filter((v): v is number => v != null); + const wallValues = caseGrades.map((g) => g.wall_s).filter((v): v is number => v != null); + const ttftValues = caseGrades.map((g) => g.ttft_s).filter((v): v is number => v != null); + const costValues = caseGrades.map((g) => g.cost_usd).filter((v): v is number => v != null); + const toolCallValues = caseGrades.map((g) => g.tool_calls); + const errorValues = caseGrades.map((g) => g.error_count).filter((v): v is number => v != null); + + const suiteGrade = { + suite_run_id, + suite_file: summary.suite_file, + suite_started_at: summary.started_at, + suite_finished_at: summary.finished_at, + graded_at: new Date().toISOString(), + harness: summary.harness ?? caseGrades.find((g) => g.harness)?.harness ?? null, + model: caseGrades.find((g) => g.model_normalized)?.model_normalized ?? summary.model ?? null, + summary: { + total, + pass: counts.pass, + partial: counts.partial, + fail: counts.fail, + error: counts.error, + skip: counts.skip, + grading_error: counts.grading_error, + pass_rate: Math.round(passRate * 1000) / 1000, + }, + metrics: { + mean_accuracy: accuracyValues.length + ? Math.round((mean(accuracyValues) ?? 0) * 1000) / 1000 + : null, + mean_wall_s: wallValues.length ? Math.round((mean(wallValues) ?? 0) * 10) / 10 : null, + mean_ttft_s: ttftValues.length ? Math.round((mean(ttftValues) ?? 0) * 10) / 10 : null, + total_cost_usd: costValues.length ? Math.round(sum(costValues) * 1e4) / 1e4 : null, + mean_cost_usd: costValues.length ? Math.round((mean(costValues) ?? 0) * 1e6) / 1e6 : null, + mean_tool_calls: toolCallValues.length + ? Math.round((mean(toolCallValues) ?? 0) * 10) / 10 + : null, + total_errors: errorValues.length ? sum(errorValues) : null, + }, + cases: caseGrades, + }; + + const gradeOutDir = path.join(GRADES_DIR, today, suite_run_id); + fs.mkdirSync(gradeOutDir, { recursive: true }); + const suiteGradePath = path.join(gradeOutDir, 'suite-grade.json'); + fs.writeFileSync(suiteGradePath, JSON.stringify(suiteGrade, null, 2)); + + // ── Print summary ───────────────────────────────────────────────────────── + + console.log('\n═══════════════════════════════════════'); + console.log(`Suite grade: ${suite_run_id}`); + console.log(` Harness/model: ${suiteGrade.harness ?? '?'} / ${suiteGrade.model ?? '?'}`); + console.log(` Pass rate: ${counts.pass}/${total} (${(passRate * 100).toFixed(1)}%)`); + console.log( + ` pass=${counts.pass} partial=${counts.partial} fail=${counts.fail}` + + ` error=${counts.error} skip=${counts.skip}` + + (counts.grading_error > 0 ? ` grading_error=${counts.grading_error}` : ''), + ); + const m = suiteGrade.metrics; + console.log( + ` accuracy=${m.mean_accuracy ?? '?'} mean_wall=${m.mean_wall_s ?? '?'}s ` + + `mean_ttft=${m.mean_ttft_s ?? '?'}s total_cost=${m.total_cost_usd != null ? `$${m.total_cost_usd}` : '?'} ` + + `mean_tools=${m.mean_tool_calls ?? '?'} errors=${m.total_errors ?? '?'}`, + ); + console.log(` Grade file: ${suiteGradePath}`); +} + +main().catch((err: unknown) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/evals/grade.ts b/evals/grade.ts new file mode 100644 index 000000000..6926118c9 --- /dev/null +++ b/evals/grade.ts @@ -0,0 +1,117 @@ +/** + * Ad hoc grader for coding-agent + LangSmith eval runs. + * + * Tool coverage is sourced from the LangSmith trace (by `eval_run_id`), not local + * artifacts. Grades tool coverage + budget/timeout/exit only; use grade-bird.ts for + * answer-quality grading. + * + * Usage: + * npx tsx evals/grade.ts evals/runs// + */ + +/* eslint-disable no-console */ + +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +import { fetchTraceSummary, makeClient, normalizeToolName } from './langsmith-reader.js'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const runDir = process.argv[2]; +if (!runDir) { + console.error('Usage: npx tsx evals/grade.ts '); + process.exit(1); +} + +type RunMeta = { + run_id: string; + case_id: string; + eval_run_id?: string; + harness?: string; + model?: string | null; + langsmith_project?: string; + expected_tools?: Array; + budget: { max_tool_calls: number; max_wall_ms: number }; + wall_ms?: number; + agent_exit_code?: number; + timed_out?: boolean; +}; + +type Outcome = 'pass' | 'fail' | 'timeout' | 'budget_exceeded' | 'error' | 'grading_error'; + +const absRunDir = path.resolve(runDir); +const runMeta = JSON.parse(fs.readFileSync(path.join(absRunDir, 'run.json'), 'utf-8')) as RunMeta; + +async function main(): Promise { + const evalRunId = runMeta.eval_run_id ?? runMeta.run_id; + const projectName = + runMeta.langsmith_project ?? process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; + + const client = makeClient(); + const summary = await fetchTraceSummary(client, { + projectName, + evalRunId, + pollIntervalMs: Number(process.env.GRADE_TRACE_POLL_MS ?? 5_000), + timeoutMs: Number(process.env.GRADE_TRACE_TIMEOUT_MS ?? 60_000), + }); + + const expectedTools = runMeta.expected_tools ?? []; + const observedTools = summary ? [...new Set(summary.toolCalls.map((t) => t.normalizedName))] : []; + const toolCalls = summary?.toolCalls.length ?? 0; + const missingTools = expectedTools.filter( + (tool) => !observedTools.includes(normalizeToolName(tool)), + ); + + const outcome: Outcome = (() => { + if (!summary) return 'grading_error'; + if (runMeta.timed_out) return 'timeout'; + if (runMeta.agent_exit_code != null && runMeta.agent_exit_code !== 0) return 'error'; + if (toolCalls > runMeta.budget.max_tool_calls) return 'budget_exceeded'; + if (missingTools.length > 0) return 'fail'; + return 'pass'; + })(); + + const result = { + run_id: runMeta.run_id, + case_id: runMeta.case_id, + eval_run_id: evalRunId, + graded_at: new Date().toISOString(), + outcome, + harness: runMeta.harness ?? null, + model: summary?.model ?? runMeta.model ?? null, + langsmith_project: projectName, + trace_id: summary?.traceId ?? null, + expected_tools: expectedTools, + observed_tools: observedTools, + missing_tools: missingTools, + tool_calls: toolCalls, + llm_calls: summary?.llmRunCount ?? null, + cost_usd: summary?.costUsd ?? null, + wall_ms: runMeta.wall_ms ?? summary?.wallMs ?? null, + budget: runMeta.budget, + trace_error: summary + ? null + : `No LangSmith trace found for eval_run_id=${evalRunId} in project "${projectName}".`, + }; + + const resultPath = path.join(absRunDir, 'result.json'); + fs.writeFileSync(resultPath, JSON.stringify(result, null, 2)); + + console.log(`\nGrade: ${runMeta.run_id}`); + console.log(`Outcome: ${result.outcome.toUpperCase()}`); + console.log(`Harness: ${result.harness ?? 'n/a'} / ${result.model ?? 'n/a'}`); + console.log(`Tool calls: ${result.tool_calls} (budget: ${runMeta.budget.max_tool_calls})`); + console.log(`Expected: ${expectedTools.length ? expectedTools.join(', ') : 'n/a'}`); + console.log(`Observed: ${observedTools.length ? observedTools.join(', ') : 'none'}`); + console.log(`Missing: ${missingTools.length ? missingTools.join(', ') : 'none'}`); + if (result.trace_error) console.log(`Trace: ${result.trace_error}`); + console.log(`Result: ${resultPath}`); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/evals/langsmith-reader.ts b/evals/langsmith-reader.ts new file mode 100644 index 000000000..ca40ef0b4 --- /dev/null +++ b/evals/langsmith-reader.ts @@ -0,0 +1,284 @@ +/** + * LangSmith trace reader. + * + * The graders source all per-case metrics from the coding-agent trace in LangSmith + * (single source of truth). This module fetches the root run for an eval by its + * `eval_run_id` metadata (written by the runner into the plugin's custom metadata), + * pulls the child runs of that trace, and normalizes them into a `TraceSummary`. + */ + +import { Client, Run } from 'langsmith'; + +import { estimateCostUsd } from './pricing.js'; + +export type TraceToolCall = { + name: string; + normalizedName: string; + inputs: unknown; + outputs: unknown; + error: string | null; + startMs: number | null; + endMs: number | null; +}; + +export type TraceTokens = { + input: number | null; + output: number | null; + cacheRead: number | null; + cacheWrite: number | null; + total: number | null; +}; + +export type TraceSummary = { + traceId: string; + rootRunId: string; + model: string | null; + finalText: string; + toolCalls: Array; + tokens: TraceTokens; + costUsd: number | null; + costSource: 'langsmith' | 'estimated' | 'unavailable'; + wallMs: number | null; + ttftMs: number | null; + llmRunCount: number; + subagentCount: number; + errorCount: number; + hadError: boolean; +}; + +type KVMap = Record; + +export function makeClient(): Client { + return new Client({ + apiKey: process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY, + apiUrl: process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com', + }); +} + +export function normalizeToolName(toolName: string): string { + const parts = toolName.split('__'); + const raw = parts[parts.length - 1] || toolName; + return raw.replace(/^tableau_/, '').replace(/^tableau-/, ''); +} + +function toMs(value: number | string | undefined): number | null { + if (value == null) return null; + if (typeof value === 'number') return value; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function getMetadata(run: Run): KVMap { + const extra = (run.extra ?? {}) as KVMap; + return (extra.metadata as KVMap) ?? {}; +} + +/** Deeply search an object for the first `{ fields?, filters? }` VizQL-shaped query. */ +export function findVizqlQuery(value: unknown): { fields?: unknown; filters?: unknown } | null { + if (value == null || typeof value !== 'object') return null; + const obj = value as KVMap; + if ('fields' in obj || 'filters' in obj) { + return obj as { fields?: unknown; filters?: unknown }; + } + for (const v of Object.values(obj)) { + if (typeof v === 'object' && v !== null) { + const found = findVizqlQuery(v); + if (found) return found; + } + } + return null; +} + +/** Extract the assistant final text from a run's outputs, tolerating several shapes. */ +export function extractTextFromOutputs(outputs: KVMap | undefined): string { + if (!outputs) return ''; + const visit = (value: unknown, depth: number): string => { + if (depth > 6 || value == null) return ''; + if (typeof value === 'string') return value; + if (Array.isArray(value)) { + // Message arrays: prefer the last element with text content. + for (let i = value.length - 1; i >= 0; i--) { + const text = visit(value[i], depth + 1); + if (text.trim()) return text; + } + return ''; + } + if (typeof value === 'object') { + const obj = value as KVMap; + for (const key of ['output', 'result', 'text', 'content', 'message', 'messages', 'choices']) { + if (key in obj) { + const text = visit(obj[key], depth + 1); + if (text.trim()) return text; + } + } + } + return ''; + }; + return visit(outputs, 0).trim(); +} + +function extractServerCost(run: Run): number | null { + const anyRun = run as unknown as KVMap; + for (const key of ['total_cost', 'totalCost']) { + const v = anyRun[key]; + if (typeof v === 'number') return v; + if (typeof v === 'string' && v.trim() && Number.isFinite(Number(v))) return Number(v); + } + return null; +} + +function accumulateTokens(runs: Array): TraceTokens { + let input = 0; + let output = 0; + let cacheRead = 0; + let cacheWrite = 0; + let found = false; + + for (const run of runs) { + if (run.run_type !== 'llm') continue; + const prompt = run.prompt_tokens ?? 0; + const completion = run.completion_tokens ?? 0; + if (run.prompt_tokens != null || run.completion_tokens != null) found = true; + input += prompt; + output += completion; + + // Cache detail lives on usage_metadata (coding-agent-v1 preserves it). + const meta = getMetadata(run); + const usage = (meta.usage_metadata ?? (run.extra as KVMap)?.usage_metadata) as + | KVMap + | undefined; + if (usage) { + const inputDetails = usage.input_token_details as KVMap | undefined; + if (inputDetails) { + const cr = Number(inputDetails.cache_read ?? 0); + const cc = Number(inputDetails.cache_creation ?? inputDetails.cache_write ?? 0); + if (Number.isFinite(cr)) cacheRead += cr; + if (Number.isFinite(cc)) cacheWrite += cc; + found = true; + } + } + } + + if (!found) { + return { input: null, output: null, cacheRead: null, cacheWrite: null, total: null }; + } + return { + input, + output, + cacheRead, + cacheWrite, + total: input + output + cacheRead + cacheWrite, + }; +} + +async function collect(iterable: AsyncIterable): Promise> { + const out: Array = []; + for await (const item of iterable) out.push(item); + return out; +} + +async function findRootRun( + client: Client, + projectName: string, + evalRunId: string, +): Promise { + const filter = `has(metadata, '{"eval_run_id": "${evalRunId}"}')`; + const runs = await collect(client.listRuns({ projectName, filter, isRoot: true, limit: 1 })); + return runs[0] ?? null; +} + +export type FetchOptions = { + projectName: string; + evalRunId: string; + pollIntervalMs?: number; + timeoutMs?: number; +}; + +/** + * Fetch and summarize the trace for an eval run, polling to absorb ingestion latency. + * Returns null if no matching trace appears within the timeout (grader → grading_error). + */ +export async function fetchTraceSummary( + client: Client, + opts: FetchOptions, +): Promise { + const pollIntervalMs = opts.pollIntervalMs ?? 5_000; + const timeoutMs = opts.timeoutMs ?? 60_000; + const deadline = Date.now() + timeoutMs; + + let root: Run | null = null; + for (;;) { + root = await findRootRun(client, opts.projectName, opts.evalRunId); + if (root || Date.now() >= deadline) break; + await new Promise((r) => setTimeout(r, pollIntervalMs)); + } + if (!root || !root.trace_id) return null; + + const allRuns = await collect(client.listRuns({ traceId: root.trace_id })); + + const toolRuns = allRuns.filter((r) => r.run_type === 'tool'); + const llmRuns = allRuns.filter((r) => r.run_type === 'llm'); + const subagentRuns = allRuns.filter( + (r) => r.run_type === 'chain' && String(getMetadata(r).ls_run_type ?? '') === 'subagent', + ); + + const toolCalls: Array = toolRuns.map((r) => ({ + name: r.name, + normalizedName: normalizeToolName(r.name), + inputs: r.inputs, + outputs: r.outputs, + error: r.error ?? null, + startMs: toMs(r.start_time), + endMs: toMs(r.end_time), + })); + + const tokens = accumulateTokens(allRuns); + const rootMeta = getMetadata(root); + const model = + (rootMeta.ls_model_name as string | undefined) ?? + llmRuns.map((r) => getMetadata(r).ls_model_name as string | undefined).find(Boolean) ?? + null; + + const rootStart = toMs(root.start_time); + const rootEnd = toMs(root.end_time); + const wallMs = rootStart != null && rootEnd != null ? rootEnd - rootStart : null; + const firstLlmStart = llmRuns + .map((r) => toMs(r.start_time)) + .filter((v): v is number => v != null) + .sort((a, b) => a - b)[0]; + const ttftMs = rootStart != null && firstLlmStart != null ? firstLlmStart - rootStart : null; + + const errorRuns = allRuns.filter((r) => r.error != null && r.error !== ''); + + let costUsd = extractServerCost(root); + let costSource: TraceSummary['costSource'] = costUsd != null ? 'langsmith' : 'unavailable'; + if (costUsd == null) { + const estimate = estimateCostUsd(model, { + input: tokens.input, + output: tokens.output, + cacheRead: tokens.cacheRead, + cacheWrite: tokens.cacheWrite, + }); + if (estimate != null) { + costUsd = estimate; + costSource = 'estimated'; + } + } + + return { + traceId: root.trace_id, + rootRunId: root.id, + model, + finalText: extractTextFromOutputs(root.outputs as KVMap | undefined), + toolCalls, + tokens, + costUsd, + costSource, + wallMs, + ttftMs, + llmRunCount: llmRuns.length, + subagentCount: subagentRuns.length, + errorCount: errorRuns.length, + hadError: (root.error != null && root.error !== '') || errorRuns.length > 0, + }; +} diff --git a/evals/model-normalize.ts b/evals/model-normalize.ts new file mode 100644 index 000000000..580c529ee --- /dev/null +++ b/evals/model-normalize.ts @@ -0,0 +1,45 @@ +/** + * Model id normalization. + * + * Different harnesses report the same underlying model with different ids + * (Cursor `sonnet-4.5`, Claude `claude-sonnet-4-5-20250101`, LangSmith + * `ls_model_name`, etc.). Reports group on a single canonical id so the same model + * run through different harnesses lines up. Both the raw id and the normalized id + * are recorded on each grade. + */ + +/** Ordered rules: the first regex that matches maps to the canonical id. */ +const RULES: Array<{ pattern: RegExp; canonical: string }> = [ + // Anthropic Claude family + { pattern: /opus-4[.\- ]?8/i, canonical: 'claude-opus-4.8' }, + { pattern: /opus-4[.\- ]?7/i, canonical: 'claude-opus-4.7' }, + { pattern: /opus-4[.\- ]?6/i, canonical: 'claude-opus-4.6' }, + { pattern: /opus-4/i, canonical: 'claude-opus-4' }, + { pattern: /sonnet-4[.\- ]?6|sonnet-4-6/i, canonical: 'claude-sonnet-4.6' }, + { pattern: /sonnet-4[.\- ]?5|sonnet-4-5/i, canonical: 'claude-sonnet-4.5' }, + { pattern: /sonnet-4/i, canonical: 'claude-sonnet-4' }, + { pattern: /haiku-4[.\- ]?5|haiku-4-5/i, canonical: 'claude-haiku-4.5' }, + { pattern: /claude.*3[.\- ]?7.*sonnet|3-7-sonnet/i, canonical: 'claude-3.7-sonnet' }, + // OpenAI / Codex family + { pattern: /gpt-5[.\- ]?6.*codex|gpt-5-6-codex/i, canonical: 'gpt-5.6-codex' }, + { pattern: /gpt-5.*codex/i, canonical: 'gpt-5-codex' }, + { pattern: /gpt-5[.\- ]?6/i, canonical: 'gpt-5.6' }, + { pattern: /gpt-5/i, canonical: 'gpt-5' }, + { pattern: /gpt-4o[.\- ]?mini/i, canonical: 'gpt-4o-mini' }, + { pattern: /gpt-4o/i, canonical: 'gpt-4o' }, + { pattern: /o4[.\- ]?mini/i, canonical: 'o4-mini' }, + { pattern: /o3/i, canonical: 'o3' }, + // Google Gemini + { pattern: /gemini-3[.\- ]?1.*pro/i, canonical: 'gemini-3.1-pro' }, + { pattern: /gemini.*pro/i, canonical: 'gemini-pro' }, +]; + +export function normalizeModel(raw: string | null | undefined): string { + if (!raw) return 'unknown'; + const trimmed = raw.trim(); + for (const rule of RULES) { + if (rule.pattern.test(trimmed)) return rule.canonical; + } + // Fall back to a lightly-cleaned raw id (strip trailing date stamps). + return trimmed.replace(/-\d{8}$/, '').toLowerCase(); +} diff --git a/evals/pricing.ts b/evals/pricing.ts new file mode 100644 index 000000000..a9dad39d2 --- /dev/null +++ b/evals/pricing.ts @@ -0,0 +1,68 @@ +/** + * Per-model pricing map (USD per 1M tokens), used only as a fallback when LangSmith + * does not report a computed cost for a trace. Keyed by canonical (normalized) model + * id from `model-normalize.ts`. Keep this current; prices drift. + * + * Rates are approximate list prices and intended for relative comparison in reports, + * not billing. Cache-read is typically a fraction of input; cache-write a small + * premium. Where a model's exact rates are unknown, omit it — cost will be null. + */ + +import { normalizeModel } from './model-normalize.js'; + +export type ModelPricePer1M = { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; +}; + +// USD per 1,000,000 tokens. +const PRICES: Record = { + 'claude-opus-4.8': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-opus-4.7': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-opus-4.6': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-opus-4': { input: 15, output: 75, cacheRead: 1.5, cacheWrite: 18.75 }, + 'claude-sonnet-4.6': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'claude-sonnet-4.5': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'claude-sonnet-4': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'claude-haiku-4.5': { input: 1, output: 5, cacheRead: 0.1, cacheWrite: 1.25 }, + 'claude-3.7-sonnet': { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + 'gpt-5.6-codex': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-5-codex': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-5.6': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-5': { input: 1.25, output: 10, cacheRead: 0.125, cacheWrite: 1.25 }, + 'gpt-4o': { input: 2.5, output: 10, cacheRead: 1.25, cacheWrite: 2.5 }, + 'gpt-4o-mini': { input: 0.15, output: 0.6, cacheRead: 0.075, cacheWrite: 0.15 }, +}; + +export type TokenCounts = { + input: number | null; + output: number | null; + cacheRead: number | null; + cacheWrite: number | null; +}; + +/** + * Estimate cost in USD from token counts. Returns null if the model has no pricing + * entry (so callers can distinguish "free" from "unknown"). + */ +export function estimateCostUsd( + model: string | null | undefined, + tokens: TokenCounts, +): number | null { + const canonical = normalizeModel(model); + const price = PRICES[canonical]; + if (!price) return null; + const input = tokens.input ?? 0; + const output = tokens.output ?? 0; + const cacheRead = tokens.cacheRead ?? 0; + const cacheWrite = tokens.cacheWrite ?? 0; + const cost = + (input * price.input + + output * price.output + + cacheRead * price.cacheRead + + cacheWrite * price.cacheWrite) / + 1_000_000; + return Math.round(cost * 1e6) / 1e6; +} diff --git a/evals/report.ts b/evals/report.ts new file mode 100644 index 000000000..dc8558d38 --- /dev/null +++ b/evals/report.ts @@ -0,0 +1,320 @@ +/** + * Longitudinal quality report for the Tableau MCP eval suite. + * + * Scans every graded case (bird-result.json under evals/grades/) and rolls the + * per-case metrics up into cohorts by (harness, normalized model) and over time, + * emitting JSON + CSV + a Markdown summary under `evals/reports//`. + * + * Per-case metrics reported: accuracy (verdict-derived), latency (wall_s, ttft_s), + * cost ($), tool-call count, error count, token totals, and the four quality signals + * (numeric/semantic/columns/filters match). + * + * Usage: + * npx tsx evals/report.ts # scan all grades + * npx tsx evals/report.ts --since 2026-07-01 + * npx tsx evals/report.ts --harness codex --model gpt-5.6-codex + */ + +/* eslint-disable no-console */ + +import * as fs from 'fs'; +import * as path from 'path'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const GRADES_DIR = path.join(EVALS_DIR, 'grades'); +const REPORTS_DIR = path.join(EVALS_DIR, 'reports'); + +const args = process.argv.slice(2); +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} +const sinceFilter = getArgValue('--since'); +const harnessFilter = getArgValue('--harness'); +const modelFilter = getArgValue('--model'); + +type BirdResult = { + run_id: string; + eval_run_id?: string; + question_id: number; + difficulty: string; + graded_at: string; + harness: string | null; + model: string | null; + model_normalized: string | null; + wall_s: number | null; + ttft_s: number | null; + cost_usd: number | null; + tokens: { + input_tokens: number | null; + output_tokens: number | null; + total_tokens: number | null; + }; + tool_calls: number; + llm_calls?: number; + error_count?: number; + signals: { + numeric_match: boolean | null; + semantic_match: number | null; + columns_match: boolean | null; + filters_match: boolean | null; + }; + accuracy: number | null; + verdict: string; +}; + +type CohortStats = { + cohort: string; + harness: string; + model: string; + n: number; + pass: number; + partial: number; + fail: number; + error: number; + skip: number; + grading_error: number; + pass_rate: number | null; + mean_accuracy: number | null; + mean_wall_s: number | null; + mean_ttft_s: number | null; + total_cost_usd: number | null; + mean_cost_usd: number | null; + mean_tool_calls: number | null; + total_tokens: number | null; + total_errors: number | null; + numeric_match_rate: number | null; + semantic_match_rate: number | null; + columns_match_rate: number | null; + filters_match_rate: number | null; + first_graded_at: string; + last_graded_at: string; +}; + +// ─── Collection ───────────────────────────────────────────────────────────── + +function walkBirdResults(dir: string): Array { + const out: Array = []; + if (!fs.existsSync(dir)) return out; + const stack = [dir]; + while (stack.length) { + const current = stack.pop() as string; + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) { + stack.push(full); + } else if (entry.name === 'bird-result.json') { + try { + out.push(JSON.parse(fs.readFileSync(full, 'utf-8')) as BirdResult); + } catch { + // Skip malformed files. + } + } + } + } + return out; +} + +function mean(values: Array): number | null { + return values.length ? values.reduce((a, b) => a + b, 0) / values.length : null; +} +function sum(values: Array): number { + return values.reduce((a, b) => a + b, 0); +} +function round(value: number | null, places: number): number | null { + if (value == null) return null; + const f = 10 ** places; + return Math.round(value * f) / f; +} +function rate(values: Array): number | null { + const defined = values.filter((v): v is boolean => v != null); + return defined.length ? defined.filter(Boolean).length / defined.length : null; +} + +function computeCohort(cohortKey: string, results: Array): CohortStats { + const [harness, model] = cohortKey.split('||'); + const verdicts = results.map((r) => r.verdict); + const countVerdict = (v: string): number => verdicts.filter((x) => x === v).length; + const graded = results.filter((r) => r.verdict !== 'grading_error').length; + + const gradedAt = results.map((r) => r.graded_at).sort(); + + return { + cohort: cohortKey, + harness, + model, + n: results.length, + pass: countVerdict('pass'), + partial: countVerdict('partial'), + fail: countVerdict('fail'), + error: countVerdict('error'), + skip: countVerdict('skip'), + grading_error: countVerdict('grading_error'), + pass_rate: graded > 0 ? round(countVerdict('pass') / graded, 3) : null, + mean_accuracy: round( + mean(results.map((r) => r.accuracy).filter((v): v is number => v != null)), + 3, + ), + mean_wall_s: round(mean(results.map((r) => r.wall_s).filter((v): v is number => v != null)), 1), + mean_ttft_s: round(mean(results.map((r) => r.ttft_s).filter((v): v is number => v != null)), 1), + total_cost_usd: round(sum(results.map((r) => r.cost_usd ?? 0)), 4), + mean_cost_usd: round( + mean(results.map((r) => r.cost_usd).filter((v): v is number => v != null)), + 6, + ), + mean_tool_calls: round(mean(results.map((r) => r.tool_calls)), 1), + total_tokens: round(sum(results.map((r) => r.tokens?.total_tokens ?? 0)), 0), + total_errors: round(sum(results.map((r) => r.error_count ?? 0)), 0), + numeric_match_rate: round(rate(results.map((r) => r.signals.numeric_match)), 3), + semantic_match_rate: round( + mean(results.map((r) => r.signals.semantic_match).filter((v): v is number => v != null)), + 3, + ), + columns_match_rate: round(rate(results.map((r) => r.signals.columns_match)), 3), + filters_match_rate: round(rate(results.map((r) => r.signals.filters_match)), 3), + first_graded_at: gradedAt[0] ?? '', + last_graded_at: gradedAt[gradedAt.length - 1] ?? '', + }; +} + +// ─── CSV / Markdown ─────────────────────────────────────────────────────────── + +const COHORT_COLUMNS: Array = [ + 'harness', + 'model', + 'n', + 'pass_rate', + 'mean_accuracy', + 'mean_wall_s', + 'mean_ttft_s', + 'total_cost_usd', + 'mean_cost_usd', + 'mean_tool_calls', + 'total_errors', + 'numeric_match_rate', + 'semantic_match_rate', + 'columns_match_rate', + 'filters_match_rate', + 'first_graded_at', + 'last_graded_at', +]; + +function toCsv(cohorts: Array): string { + const header = COHORT_COLUMNS.join(','); + const rows = cohorts.map((c) => + COHORT_COLUMNS.map((col) => { + const value = c[col]; + const str = value == null ? '' : String(value); + return /[",\n]/.test(str) ? `"${str.replace(/"/g, '""')}"` : str; + }).join(','), + ); + return [header, ...rows].join('\n'); +} + +function toMarkdown(cohorts: Array, totalCases: number): string { + const lines: Array = []; + lines.push('# Tableau MCP Eval — Longitudinal Report'); + lines.push(''); + lines.push(`Generated: ${new Date().toISOString()}`); + lines.push(`Graded cases scanned: ${totalCases}`); + if (sinceFilter) lines.push(`Since: ${sinceFilter}`); + if (harnessFilter) lines.push(`Harness filter: ${harnessFilter}`); + if (modelFilter) lines.push(`Model filter: ${modelFilter}`); + lines.push(''); + lines.push('## Cohorts (harness × normalized model)'); + lines.push(''); + lines.push( + '| Harness | Model | N | Pass rate | Accuracy | Wall (s) | TTFT (s) | Total $ | $/case | Tools/case | Errors | Semantic | Cols | Filters |', + ); + lines.push( + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ); + for (const c of cohorts) { + lines.push( + `| ${c.harness} | ${c.model} | ${c.n} | ${fmt(c.pass_rate)} | ${fmt(c.mean_accuracy)} | ` + + `${fmt(c.mean_wall_s)} | ${fmt(c.mean_ttft_s)} | ${fmtMoney(c.total_cost_usd)} | ${fmtMoney(c.mean_cost_usd)} | ` + + `${fmt(c.mean_tool_calls)} | ${c.total_errors ?? ''} | ${fmt(c.semantic_match_rate)} | ` + + `${fmt(c.columns_match_rate)} | ${fmt(c.filters_match_rate)} |`, + ); + } + lines.push(''); + return lines.join('\n'); +} + +function fmt(value: number | null): string { + return value == null ? '—' : String(value); +} +function fmtMoney(value: number | null): string { + return value == null ? '—' : `$${value}`; +} + +// ─── Main ─────────────────────────────────────────────────────────────────── + +function main(): void { + let results = walkBirdResults(GRADES_DIR); + + if (sinceFilter) results = results.filter((r) => r.graded_at >= sinceFilter); + if (harnessFilter) results = results.filter((r) => (r.harness ?? '') === harnessFilter); + if (modelFilter) { + results = results.filter((r) => (r.model_normalized ?? r.model ?? '') === modelFilter); + } + + if (results.length === 0) { + console.error( + `No graded cases found under ${GRADES_DIR}` + + (sinceFilter || harnessFilter || modelFilter ? ' matching the given filters.' : '.'), + ); + process.exit(1); + } + + const cohortMap = new Map>(); + for (const r of results) { + const key = `${r.harness ?? 'unknown'}||${r.model_normalized ?? r.model ?? 'unknown'}`; + const bucket = cohortMap.get(key) ?? []; + bucket.push(r); + cohortMap.set(key, bucket); + } + + const cohorts = [...cohortMap.entries()] + .map(([key, rs]) => computeCohort(key, rs)) + .sort((a, b) => (b.pass_rate ?? -1) - (a.pass_rate ?? -1)); + + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const outDir = path.join(REPORTS_DIR, stamp); + fs.mkdirSync(outDir, { recursive: true }); + + const jsonReport = { + generated_at: new Date().toISOString(), + filters: { + since: sinceFilter ?? null, + harness: harnessFilter ?? null, + model: modelFilter ?? null, + }, + total_cases: results.length, + cohorts, + }; + + const jsonPath = path.join(outDir, 'longitudinal.json'); + const csvPath = path.join(outDir, 'longitudinal.csv'); + const mdPath = path.join(outDir, 'summary.md'); + fs.writeFileSync(jsonPath, JSON.stringify(jsonReport, null, 2)); + fs.writeFileSync(csvPath, toCsv(cohorts)); + fs.writeFileSync(mdPath, toMarkdown(cohorts, results.length)); + + console.log( + `\nLongitudinal report over ${results.length} graded cases, ${cohorts.length} cohorts:\n`, + ); + for (const c of cohorts) { + console.log( + ` ${c.harness} / ${c.model}: n=${c.n} pass_rate=${fmt(c.pass_rate)} ` + + `acc=${fmt(c.mean_accuracy)} wall=${fmt(c.mean_wall_s)}s $=${fmtMoney(c.total_cost_usd)} ` + + `tools/case=${fmt(c.mean_tool_calls)} errors=${c.total_errors ?? 0}`, + ); + } + console.log(`\n JSON: ${jsonPath}`); + console.log(` CSV: ${csvPath}`); + console.log(` MD: ${mdPath}`); +} + +main(); diff --git a/evals/run-case.ts b/evals/run-case.ts new file mode 100644 index 000000000..f37c3fd47 --- /dev/null +++ b/evals/run-case.ts @@ -0,0 +1,283 @@ +/** + * Multi-agent eval runner for the Tableau MCP server. + * + * Drives Claude Code, Cursor, or Codex (selected by AGENT_HARNESS) against the + * Tableau MCP server for a single case. Tracing is handled by the official LangSmith + * coding-agent plugin for the selected agent (enabled via env in the adapter); this + * runner only spawns the CLI, stamps the `eval_run_id` correlation metadata, and + * records run metadata. Grading reads the trace back from LangSmith. + * + * Usage: + * npx tsx evals/run-case.ts evals/cases/list-datasources.json [--run-id ] + * npx tsx evals/run-case.ts input "sales by region" [--run-id ] + * + * Required environment: + * LANGSMITH_API_KEY (or LANGCHAIN_API_KEY), LANGSMITH_PROJECT + * SERVER plus a supported Tableau auth config, e.g. + * AUTH=pat SITE_NAME= PAT_NAME= PAT_VALUE= + * Optional: + * AGENT_HARNESS=claude-code|cursor|codex (default claude-code), AGENT_MODEL= + */ + +/* eslint-disable no-console */ + +import { execFileSync } from 'child_process'; +import { randomUUID } from 'crypto'; +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +import { AgentInvocation, getAdapter, resolveHarness, RunContext } from './adapters/index.js'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const RUNS_DIR = path.join(EVALS_DIR, 'runs'); +const MCP_SERVER_ENTRY = path.join(REPO_ROOT, 'build', 'index.js'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} + +type EvalCase = { + id: string; + name?: string; + description?: string; + prompt: string; + expected_tools?: Array; + tags?: Array; + metadata?: Record; + budget?: { + max_tool_calls?: number; + max_wall_ms?: number; + }; +}; + +const args = process.argv.slice(2); +const positionalArgs = args.filter((arg) => !arg.startsWith('--')); + +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + +if (positionalArgs.length === 0) { + console.error( + 'Usage: npx tsx evals/run-case.ts [--run-id ] [--agent-harness ] [--agent-model ]\n' + + ' or: npx tsx evals/run-case.ts input "" [--run-id ]', + ); + process.exit(1); +} + +const runIdArg = getArgValue('--run-id'); +const suiteRunId = getArgValue('--suite-run-id') ?? null; + +const harness = resolveHarness(getArgValue('--agent-harness'), 'AGENT_HARNESS'); +const adapter = getAdapter(harness); +const model = adapter.resolveModel(getArgValue('--agent-model') ?? process.env.AGENT_MODEL); + +const langSmithApiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY ?? ''; +const langSmithProject = process.env.LANGSMITH_PROJECT ?? 'tableau-mcp-evals'; +const langSmithEndpoint = process.env.LANGSMITH_ENDPOINT ?? 'https://api.smith.langchain.com'; + +const evalCase = loadEvalCase(positionalArgs); +const runId = runIdArg ?? `${evalCase.id}-${Date.now()}-${randomUUID().slice(0, 8)}`; +const runDir = path.join(RUNS_DIR, dateSlug(), runId); + +const budget = { + maxToolCalls: evalCase.budget?.max_tool_calls ?? 20, + maxWallMs: evalCase.budget?.max_wall_ms ?? 5 * 60 * 1000, +}; + +preflight(); +fs.mkdirSync(runDir, { recursive: true }); + +const prompt = expandTemplate(evalCase.prompt); +const startedAt = new Date().toISOString(); + +const questionId = + typeof evalCase.metadata?.question_id === 'number' + ? (evalCase.metadata.question_id as number) + : null; + +const runContext: RunContext = { + runId, + suiteRunId, + runDir, + prompt, + model, + mcpServerEntry: MCP_SERVER_ENTRY, + mcpServerEnv: tableauServerEnv(), + questionId, + langsmith: { apiKey: langSmithApiKey, project: langSmithProject, endpoint: langSmithEndpoint }, + budget, +}; + +const runMeta = { + run_id: runId, + suite_run_id: suiteRunId, + case_id: evalCase.id, + name: evalCase.name ?? evalCase.id, + description: evalCase.description ?? null, + started_at: startedAt, + prompt, + harness, + model: model || null, + eval_run_id: runId, + langsmith_project: langSmithProject, + expected_tools: evalCase.expected_tools ?? [], + tags: evalCase.tags ?? [], + metadata: evalCase.metadata ?? {}, + budget, + git_sha: tryExec('git', ['rev-parse', 'HEAD']), + git_branch: tryExec('git', ['rev-parse', '--abbrev-ref', 'HEAD']), +}; + +adapter.writeConfig(runContext); +fs.writeFileSync(path.join(runDir, 'run.json'), JSON.stringify(runMeta, null, 2)); + +console.log(`\nRun ID: ${runId}`); +console.log(`Harness / model: ${harness} / ${model || '(cli default)'}`); +console.log(`LangSmith project: ${langSmithProject}`); +console.log(`Run dir: ${runDir}`); + +const invocation: AgentInvocation = adapter.buildInvocation(runContext); + +const startMs = Date.now(); +let exitCode = 0; +let stdout = ''; +let errorMessage: string | undefined; + +try { + stdout = execFileSync(invocation.command, invocation.args, { + env: { ...process.env, ...invocation.env }, + cwd: invocation.cwd, + timeout: invocation.timeoutMs, + maxBuffer: 25 * 1024 * 1024, + }).toString(); +} catch (error: unknown) { + const e = error as { status?: number; stdout?: Buffer; stderr?: Buffer; message?: string }; + exitCode = e.status ?? 1; + stdout = e.stdout?.toString() ?? ''; + errorMessage = [e.message, e.stderr?.toString()].filter(Boolean).join('\n'); + console.error(`${invocation.command} exited with code ${exitCode}`); + if (errorMessage) console.error(errorMessage); +} + +const finishedAt = new Date().toISOString(); +const wallMs = Date.now() - startMs; + +// Raw agent stdout is persisted for human debugging only — never a grading input. +fs.writeFileSync(path.join(runDir, 'agent-output.jsonl'), stdout); + +const finishedMeta = { + ...runMeta, + finished_at: finishedAt, + wall_ms: wallMs, + agent_exit_code: exitCode, + timed_out: wallMs >= budget.maxWallMs, + error: errorMessage ?? null, +}; +fs.writeFileSync(path.join(runDir, 'run.json'), JSON.stringify(finishedMeta, null, 2)); + +console.log(`\nRun complete in ${wallMs}ms (exit ${exitCode})`); +console.log(`Run dir: ${runDir}`); +console.log(`To grade locally: npx tsx ${path.join(EVALS_DIR, 'grade.ts')} ${runDir}`); + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +function loadEvalCase(positional: Array): EvalCase { + const [modeOrPath, ...rest] = positional; + if (modeOrPath === 'input') { + const message = rest.join(' ').trim(); + if (!message) { + throw new Error('Ad hoc input mode requires a message, for example: input "sales by region"'); + } + return { + id: `adhoc-${slugify(message)}`, + name: `Ad Hoc: ${message.slice(0, 80)}`, + description: 'Ad hoc coding-agent session against the Tableau MCP server.', + prompt: message, + tags: ['adhoc', 'real-server'], + metadata: { source: 'cli-input' }, + budget: { max_tool_calls: 20, max_wall_ms: 5 * 60 * 1000 }, + }; + } + return JSON.parse(fs.readFileSync(path.resolve(modeOrPath), 'utf-8')) as EvalCase; +} + +function slugify(value: string): string { + const slug = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 48); + return slug || 'input'; +} + +/** Environment passed through to the Tableau MCP stdio server subprocess. */ +function tableauServerEnv(): Record { + const keys = [ + 'SERVER', + 'SITE_NAME', + 'AUTH', + 'PAT_NAME', + 'PAT_VALUE', + 'JWT', + 'USERNAME', + 'PASSWORD', + 'CONNECTED_APP_CLIENT_ID', + 'CONNECTED_APP_SECRET_ID', + 'CONNECTED_APP_SECRET_VALUE', + 'DATASOURCE_CREDENTIALS', + 'DEFAULT_LOG_LEVEL', + 'INCLUDE_TOOLS', + 'EXCLUDE_TOOLS', + 'MAX_RESULT_LIMIT', + 'DISABLE_QUERY_DATASOURCE_FILTER_VALIDATION', + 'DISABLE_METADATA_API_REQUESTS', + 'FLOW_TOOLS_ENABLED', + ]; + const env: Record = {}; + for (const key of keys) { + const value = process.env[key]; + if (value != null && value !== '') env[key] = value; + } + return env; +} + +function preflight(): void { + if (!langSmithApiKey) { + throw new Error('LANGSMITH_API_KEY or LANGCHAIN_API_KEY must be set.'); + } + if (!fs.existsSync(MCP_SERVER_ENTRY)) { + throw new Error(`MCP server build not found at ${MCP_SERVER_ENTRY}. Run npm run build first.`); + } + if (!process.env.SERVER && process.env.AUTH !== 'oauth') { + throw new Error('SERVER must be set for real Tableau Cloud/Server evals.'); + } + const auth = process.env.AUTH ?? 'pat'; + if (auth === 'pat' && (!process.env.PAT_NAME || !process.env.PAT_VALUE)) { + throw new Error('PAT_NAME and PAT_VALUE must be set when AUTH is "pat" or unset.'); + } +} + +function expandTemplate(value: string): string { + return value.replace(/\{\{env\.([A-Z0-9_]+)\}\}/g, (_match, name: string) => { + const envValue = process.env[name]; + if (!envValue) { + throw new Error(`Prompt references {{env.${name}}}, but ${name} is not set.`); + } + return envValue; + }); +} + +function tryExec(command: string, commandArgs: Array): string | null { + try { + return execFileSync(command, commandArgs, { cwd: REPO_ROOT }).toString().trim(); + } catch { + return null; + } +} diff --git a/evals/run-suite.ts b/evals/run-suite.ts new file mode 100644 index 000000000..bd67ef1eb --- /dev/null +++ b/evals/run-suite.ts @@ -0,0 +1,299 @@ +/** + * Suite runner for the BIRD California Schools eval. + * + * Runs all 30 cases (or a filtered subset) sequentially against the live + * Tableau MCP server. Each case is executed via run-case.ts, so agent selection + * (AGENT_HARNESS/AGENT_MODEL) and LangSmith plugin tracing are identical to ad-hoc + * runs. Per-case quality/latency/cost/token metrics are NOT collected here — they are + * sourced from the LangSmith trace at grading time (grade-suite.ts). + * + * Usage: + * npx tsx evals/run-suite.ts + * npx tsx evals/run-suite.ts --suite evals/suites/bird-california-schools.json + * npx tsx evals/run-suite.ts --difficulty simple + * npx tsx evals/run-suite.ts --ids 5,11,12 + * npx tsx evals/run-suite.ts --agent-harness codex --agent-model gpt-5.6-codex + * + * Required environment: same as run-case.ts (Tableau auth + LANGSMITH_API_KEY). + * Set EVAL_DATASOURCE_LUID to the LUID of the published California Schools datasource. + */ + +/* eslint-disable no-console */ + +import { execFileSync } from 'child_process'; +import { randomUUID } from 'crypto'; +import dotenv from 'dotenv'; +import * as fs from 'fs'; +import * as path from 'path'; + +import { getAdapter, resolveHarness } from './adapters/index.js'; + +const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +dotenv.config({ path: path.join(REPO_ROOT, '.env') }); + +const EVALS_DIR = path.join(REPO_ROOT, 'evals'); +const RUNS_DIR = path.join(EVALS_DIR, 'runs'); +const SUITE_RUNS_DIR = path.join(EVALS_DIR, 'suite-runs'); + +function dateSlug(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; +} +const DEFAULT_SUITE = path.join(EVALS_DIR, 'suites', 'bird-california-schools.json'); + +const args = process.argv.slice(2); + +function getArgValue(flag: string): string | undefined { + const index = args.indexOf(flag); + return index === -1 ? undefined : args[index + 1]; +} + +const suiteFile = getArgValue('--suite') ?? DEFAULT_SUITE; +const difficultyFilter = getArgValue('--difficulty'); +const idsFilter = getArgValue('--ids') + ?.split(',') + .map((s) => parseInt(s.trim(), 10)) + .filter(Number.isFinite); + +const harness = resolveHarness(getArgValue('--agent-harness'), 'AGENT_HARNESS'); +const model = getAdapter(harness).resolveModel( + getArgValue('--agent-model') ?? process.env.AGENT_MODEL, +); + +type BirdCase = { + question_id: number; + question: string; + evidence: string; + difficulty: string; + answer_type: 'scalar' | 'list'; + expected_value: number | string | null; + expected_row_count: number | null; + ai_summarized_answer: string; + expected_columns: Array; + expected_filter_fields: Array; + prompt: string; + expected_tools: Array; + budget: { max_wall_ms: number }; +}; + +type EvalCaseFile = { + id: string; + name: string; + description: string; + prompt: string; + expected_tools: Array; + tags: Array; + metadata: Record; + budget: { max_wall_ms: number }; +}; + +type CaseRunResult = { + question_id: number; + difficulty: string; + run_id: string; + run_dir: string; + eval_run_id: string; + exit_code: number | null; + wall_ms: number | null; + timed_out: boolean; + error: string | null; +}; + +function readOptionalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as T; + } catch { + return null; + } +} + +function buildEvalCaseFile(birdCase: BirdCase, absoluteSuitePath: string): EvalCaseFile { + return { + id: `bird-ca-schools-${birdCase.question_id}`, + name: `BIRD California Schools Q${birdCase.question_id}`, + description: birdCase.question, + prompt: birdCase.prompt, + expected_tools: birdCase.expected_tools, + tags: ['bird', 'california-schools', birdCase.difficulty], + metadata: { + question_id: birdCase.question_id, + difficulty: birdCase.difficulty, + suite: 'bird-california-schools', + suite_file: absoluteSuitePath, + }, + budget: birdCase.budget, + }; +} + +function collectRunMetrics(runDir: string): Omit { + const runMeta = readOptionalJson<{ + run_id: string; + eval_run_id?: string; + wall_ms?: number; + agent_exit_code?: number; + timed_out?: boolean; + error?: string | null; + }>(path.join(runDir, 'run.json')); + const runId = path.basename(runDir); + return { + run_id: runMeta?.run_id ?? runId, + run_dir: runDir, + eval_run_id: runMeta?.eval_run_id ?? runMeta?.run_id ?? runId, + exit_code: runMeta?.agent_exit_code ?? null, + wall_ms: runMeta?.wall_ms ?? null, + timed_out: runMeta?.timed_out ?? false, + error: runMeta?.error ?? null, + }; +} + +function main(): void { + if (!fs.existsSync(suiteFile)) { + console.error(`Suite file not found: ${suiteFile}`); + process.exit(1); + } + + if (!process.env.EVAL_DATASOURCE_LUID) { + console.error( + 'EVAL_DATASOURCE_LUID must be set to the LUID of the published datasource to evaluate against.', + ); + process.exit(1); + } + + const allCases = JSON.parse(fs.readFileSync(suiteFile, 'utf-8')) as Array; + + let cases = allCases; + if (difficultyFilter) { + cases = cases.filter((c) => c.difficulty === difficultyFilter); + console.log(`Filtered to difficulty="${difficultyFilter}": ${cases.length} cases`); + } + if (idsFilter?.length) { + cases = cases.filter((c) => idsFilter.includes(c.question_id)); + console.log(`Filtered to ids=${idsFilter.join(',')}: ${cases.length} cases`); + } + + if (cases.length === 0) { + console.error('No cases matched the filters.'); + process.exit(1); + } + + const today = dateSlug(); + const suiteRunId = `bird-ca-schools-${Date.now()}-${randomUUID().slice(0, 8)}`; + const suiteRunDir = path.join(SUITE_RUNS_DIR, today, suiteRunId); + const casesDir = path.join(suiteRunDir, 'cases'); + fs.mkdirSync(casesDir, { recursive: true }); + fs.mkdirSync(path.join(RUNS_DIR, today), { recursive: true }); + + const absoluteSuitePath = path.resolve(suiteFile); + const startedAt = new Date().toISOString(); + + console.log(`\nSuite run: ${suiteRunId}`); + console.log(`Suite: ${suiteFile}`); + console.log(`Cases: ${cases.length}`); + console.log(`Harness / model: ${harness} / ${model || '(cli default)'}`); + console.log(`Run dir: ${suiteRunDir}\n`); + + const runCaseScript = path.join(EVALS_DIR, 'run-case.ts'); + const results: Array = []; + + for (let i = 0; i < cases.length; i++) { + const birdCase = cases[i]; + const caseLabel = `Q${birdCase.question_id} (${birdCase.difficulty}) [${i + 1}/${cases.length}]`; + console.log(`\n─── ${caseLabel} ───`); + console.log( + ` ${birdCase.question.slice(0, 80)}${birdCase.question.length > 80 ? '...' : ''}`, + ); + + const evalCaseFile = buildEvalCaseFile(birdCase, absoluteSuitePath); + const caseFilePath = path.join(casesDir, `case-${birdCase.question_id}.json`); + fs.writeFileSync(caseFilePath, JSON.stringify(evalCaseFile, null, 2)); + + const runId = `${evalCaseFile.id}-${Date.now()}-${randomUUID().slice(0, 8)}`; + const runDir = path.join(RUNS_DIR, today, runId); + + const budgetSec = Math.round(birdCase.budget.max_wall_ms / 1000); + process.stdout.write(` Running ${harness} (up to ${budgetSec}s)...`); + + const runCaseArgs = [ + 'tsx', + runCaseScript, + caseFilePath, + '--run-id', + runId, + '--suite-run-id', + suiteRunId, + '--agent-harness', + harness, + ]; + if (model) runCaseArgs.push('--agent-model', model); + + let spawnError: string | null = null; + try { + execFileSync('npx', runCaseArgs, { + env: process.env, + cwd: REPO_ROOT, + timeout: birdCase.budget.max_wall_ms + 30_000, + stdio: 'pipe', + }); + } catch (error: unknown) { + const e = error as { message?: string }; + spawnError = e.message ?? 'unknown spawn error'; + } + process.stdout.write(' done\n'); + + const metrics = collectRunMetrics(runDir); + results.push({ + question_id: birdCase.question_id, + difficulty: birdCase.difficulty, + ...metrics, + error: metrics.error ?? spawnError, + }); + + const status = + metrics.exit_code === 0 ? 'OK' : metrics.timed_out ? 'TIMEOUT' : `EXIT ${metrics.exit_code}`; + console.log( + ` ${status} | ${metrics.wall_ms != null ? `${Math.round(metrics.wall_ms / 1000)}s` : '?s'}`, + ); + } + + const finishedAt = new Date().toISOString(); + const totalWallMs = results.reduce((sum, r) => sum + (r.wall_ms ?? 0), 0); + const completed = results.filter((r) => r.exit_code === 0).length; + const errored = results.filter((r) => r.exit_code !== 0 && !r.timed_out).length; + const timedOut = results.filter((r) => r.timed_out).length; + const avgWall = + results.filter((r) => r.wall_ms != null).length > 0 + ? Math.round(totalWallMs / results.filter((r) => r.wall_ms != null).length) + : null; + + const summary = { + suite_run_id: suiteRunId, + suite_file: absoluteSuitePath, + started_at: startedAt, + finished_at: finishedAt, + harness, + model: model || null, + total_wall_ms: totalWallMs, + cases: results, + aggregate: { + total_cases: results.length, + completed, + errored, + timed_out: timedOut, + avg_wall_ms: avgWall, + }, + }; + + const summaryPath = path.join(suiteRunDir, 'suite-summary.json'); + fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2)); + + console.log('\n═══════════════════════════════════════'); + console.log(`Suite complete: ${suiteRunId}`); + console.log(` ${completed}/${results.length} OK | ${errored} error | ${timedOut} timeout`); + console.log(` Total wall time: ${(totalWallMs / 1000).toFixed(1)}s`); + console.log(` Summary: ${summaryPath}`); + console.log('\nTo grade the whole suite (sources metrics from LangSmith traces):'); + console.log(` npx tsx ${path.join(EVALS_DIR, 'grade-suite.ts')} ${suiteRunDir}`); +} + +main(); diff --git a/evals/scripts/precompute-bird-answers.py b/evals/scripts/precompute-bird-answers.py new file mode 100644 index 000000000..357651e1d --- /dev/null +++ b/evals/scripts/precompute-bird-answers.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +One-time script to precompute expected answers for the BIRD California Schools eval suite. + +Reads the BIRD SQLite snapshot and VDS case file, executes gold SQL queries, and +writes evals/suites/bird-california-schools.json with all expected answers baked in. +The output file is committed to the repo so eval runners never need SQLite access. + +Requires: + bird_mini/data/dev_databases/california_schools/california_schools.sqlite + bird_mini/data/test_cases/mini_dev_sqlite.json + bird_mini/data/test_cases/mini_dev_postgresql_vds.json + +Usage: + python3 evals/scripts/precompute-bird-answers.py +""" + +import json +import sqlite3 +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent.parent +SQLITE_PATH = ( + REPO_ROOT + / "bird_mini" + / "data" + / "dev_databases" + / "california_schools" + / "california_schools.sqlite" +) +SQLITE_CASES_PATH = REPO_ROOT / "bird_mini" / "data" / "test_cases" / "mini_dev_sqlite.json" +VDS_CASES_PATH = ( + REPO_ROOT / "bird_mini" / "data" / "test_cases" / "mini_dev_postgresql_vds.json" +) +OUTPUT_PATH = REPO_ROOT / "evals" / "suites" / "bird-california-schools.json" + +DEFAULT_BUDGET = {"max_wall_ms": 180000} + + +DATASOURCE_LUID_PLACEHOLDER = "{{env.EVAL_DATASOURCE_LUID}}" + + +def build_prompt(question: str, evidence: str) -> str: + if not evidence: + return f"Using datasource {DATASOURCE_LUID_PLACEHOLDER}: {question}" + separator = " " if question.rstrip().endswith(("?", "!", ".")) else ". " + return f"Using datasource {DATASOURCE_LUID_PLACEHOLDER}: {question}{separator}{evidence}" + + +def run_sql(conn: sqlite3.Connection, sql: str): + """Execute SQL and return (rows, column_names, error).""" + try: + cursor = conn.execute(sql) + rows = cursor.fetchall() + columns = [d[0] for d in cursor.description] if cursor.description else [] + return rows, columns, None + except Exception as exc: + return None, None, str(exc) + + +def classify_result(rows, columns): + """ + Determine answer_type, expected_value, and expected_row_count. + + Scalar: result is exactly 1 row × 1 column — the cell value is the answer. + List: everything else — row count is the meaningful fact. + """ + if not rows and not columns: + return "list", None, 0 + + if len(rows) == 1 and len(columns) == 1: + raw = rows[0][0] + if isinstance(raw, float): + value = raw + elif isinstance(raw, int): + value = raw + else: + # String scalar (e.g. a year returned as text) + try: + value = int(raw) + except (TypeError, ValueError): + try: + value = float(raw) + except (TypeError, ValueError): + value = raw + return "scalar", value, 1 + + return "list", None, len(rows) + + +def extract_vds_columns(vds_query: dict) -> list[str]: + """Field captions from VDS_QUERY.fields (the SELECT clause).""" + captions = [] + for field in vds_query.get("fields", []): + caption = field.get("fieldCaption") + if caption: + captions.append(caption) + return captions + + +def extract_vds_filter_fields(vds_query: dict) -> list[str]: + """Field captions from VDS_QUERY.filters (the WHERE clause).""" + captions = [] + for f in vds_query.get("filters", []): + caption = f.get("field", {}).get("fieldCaption") + if caption: + captions.append(caption) + return captions + + +def main(): + for path in [SQLITE_PATH, SQLITE_CASES_PATH, VDS_CASES_PATH]: + if not path.exists(): + print(f"ERROR: required file not found: {path}", file=sys.stderr) + sys.exit(1) + + print(f"Loading SQLite cases from {SQLITE_CASES_PATH.relative_to(REPO_ROOT)}") + with open(SQLITE_CASES_PATH) as f: + sqlite_cases = { + c["question_id"]: c + for c in json.load(f) + if c["db_id"] == "california_schools" + } + + print(f"Loading VDS cases from {VDS_CASES_PATH.relative_to(REPO_ROOT)}") + with open(VDS_CASES_PATH) as f: + vds_cases = { + c["question_id"]: c + for c in json.load(f) + if c["db_id"] == "california_schools" + } + + print(f"Connecting to {SQLITE_PATH.relative_to(REPO_ROOT)}") + conn = sqlite3.connect(str(SQLITE_PATH)) + + suite = [] + errors = [] + + for qid, sqlite_case in sorted(sqlite_cases.items()): + vds_case = vds_cases.get(qid, {}) + question = sqlite_case["question"] + evidence = sqlite_case.get("evidence", "") + difficulty = sqlite_case.get("difficulty", "simple") + sql = sqlite_case["SQL"] + ai_summary = vds_case.get("ai_summarized_answer", "") + vds_query = vds_case.get("VDS_QUERY", {}) + + rows, columns, error = run_sql(conn, sql) + + if error: + print(f" Q{qid} ({difficulty}): SQL ERROR — {error}") + errors.append({"question_id": qid, "error": error}) + answer_type, expected_value, expected_row_count = "list", None, None + else: + answer_type, expected_value, expected_row_count = classify_result(rows, columns) + print( + f" Q{qid} ({difficulty}): {answer_type}, " + f"value={expected_value}, rows={expected_row_count}" + ) + + expected_columns = extract_vds_columns(vds_query) + expected_filter_fields = extract_vds_filter_fields(vds_query) + + suite.append( + { + "question_id": qid, + "question": question, + "evidence": evidence, + "difficulty": difficulty, + "answer_type": answer_type, + "expected_value": expected_value, + "expected_row_count": expected_row_count, + "ai_summarized_answer": ai_summary, + "expected_columns": expected_columns, + "expected_filter_fields": expected_filter_fields, + "prompt": build_prompt(question, evidence), + "expected_tools": ["query-datasource"], + "budget": DEFAULT_BUDGET, + } + ) + + conn.close() + + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(OUTPUT_PATH, "w") as f: + json.dump(suite, f, indent=2) + f.write("\n") + + print(f"\nWrote {len(suite)} cases to {OUTPUT_PATH.relative_to(REPO_ROOT)}") + if errors: + print(f"WARNING: {len(errors)} cases had SQL errors: {[e['question_id'] for e in errors]}") + + +if __name__ == "__main__": + main() diff --git a/evals/suites/bird-california-schools.json b/evals/suites/bird-california-schools.json new file mode 100644 index 000000000..f6167b4ed --- /dev/null +++ b/evals/suites/bird-california-schools.json @@ -0,0 +1,728 @@ +[ + { + "question_id": 5, + "question": "How many schools with an average score in Math greater than 400 in the SAT test are exclusively virtual?", + "evidence": "Exclusively virtual refers to Virtual = 'F'", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": 4, + "expected_row_count": 1, + "ai_summarized_answer": "There are 4 schools that are exclusively virtual and have an average Math SAT score greater than 400.", + "expected_columns": [ + "School" + ], + "expected_filter_fields": [ + "Virtual", + "Avgscrmath" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: How many schools with an average score in Math greater than 400 in the SAT test are exclusively virtual? Exclusively virtual refers to Virtual = 'F'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 11, + "question": "Please list the codes of the schools with a total enrollment of over 500.", + "evidence": "Total enrollment can be represented by `Enrollment (K-12)` + `Enrollment (Ages 5-17)`", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 7806, + "ai_summarized_answer": "The query returns 7,806 school codes with total enrollment over 500, including codes like 01100170109835, 01100170112607, 01100170124172, and 7,803 more.", + "expected_columns": [ + "Cdscode" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Please list the codes of the schools with a total enrollment of over 500. Total enrollment can be represented by `Enrollment (K-12)` + `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 12, + "question": "Among the schools with an SAT excellence rate of over 0.3, what is the highest eligible free rate for students aged 5-17?", + "evidence": "Excellence rate = NumGE1500 / NumTstTakr; Eligible free rates for students aged 5-17 = `Free Meal Count (Ages 5-17)` / `Enrollment (Ages 5-17)`", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": 0.9049079754601227, + "expected_row_count": 1, + "ai_summarized_answer": "The highest eligible free rate for students aged 5-17 among schools with an SAT excellence rate over 0.3 is approximately 0.905 or 90.5%.", + "expected_columns": [ + "Eligible Free Rate Max" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Among the schools with an SAT excellence rate of over 0.3, what is the highest eligible free rate for students aged 5-17? Excellence rate = NumGE1500 / NumTstTakr; Eligible free rates for students aged 5-17 = `Free Meal Count (Ages 5-17)` / `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 17, + "question": "Rank schools by their average score in Writing where the score is greater than 499, showing their charter numbers.", + "evidence": "Valid charter number means the number is not null", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 58, + "ai_summarized_answer": "The results show 58 schools ranked by writing scores above 499, with the top school (charter 0210) scoring 630 and rank 1, charter 0890 scoring 593 at rank 2, charter 0290 scoring 582 at rank 3, and 55 more schools ranked down to charter 1400 with a score of 501 at rank 58.", + "expected_columns": [ + "Cdscode", + "Charternum", + "Avgscrwrite" + ], + "expected_filter_fields": [ + "Avgscrwrite", + "Charternum" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Rank schools by their average score in Writing where the score is greater than 499, showing their charter numbers. Valid charter number means the number is not null", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 23, + "question": "List the names of schools with more than 30 difference in enrollements between K-12 and ages 5-17? Please also give the full street adress of the schools.", + "evidence": "Diffrence in enrollement = `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1239, + "ai_summarized_answer": "The data shows 1,239 schools with enrollment differences greater than 30, including Alameda County Community at 313 West Winton Avenue, California School for the Deaf-Fremont at 39350 Gallaudet Drive, Alameda High at 2201 Encinal Avenue, and 1,236 more schools.", + "expected_columns": [ + "Cdscode", + "School Name", + "Street" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: List the names of schools with more than 30 difference in enrollements between K-12 and ages 5-17? Please also give the full street adress of the schools. Diffrence in enrollement = `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 24, + "question": "Give the names of the schools with the percent eligible for free meals in K-12 is more than 0.1 and test takers whose test score is greater than or equal to 1500?", + "evidence": "Percent eligible for free meals = Free Meal Count (K-12) / Total (Enrollment (K-12)", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1165, + "ai_summarized_answer": "The results list 1,167 schools meeting both criteria, including FAME Public Charter, Envision Academy for Arts & Technology, Alameda Science and Technology Institute, Alameda High, Alternatives in Action, and 1,162 more.", + "expected_columns": [ + "School Name" + ], + "expected_filter_fields": [ + "Numge1500" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Give the names of the schools with the percent eligible for free meals in K-12 is more than 0.1 and test takers whose test score is greater than or equal to 1500? Percent eligible for free meals = Free Meal Count (K-12) / Total (Enrollment (K-12)", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 25, + "question": "Name schools in Riverside which the average of average math score for SAT is grater than 400, what is the funding type of these schools?", + "evidence": "Average of average math = sum(average math scores) / count(schools).", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 6, + "ai_summarized_answer": "Six schools in Riverside meet the criteria: Arlington High, John W. North High, Martin Luther King Jr. High, Polytechnic High, and Ramona High all have no charter funding (None), while River Springs Charter is directly funded.", + "expected_columns": [ + "Cds", + "School Name", + "Charter Funding Type" + ], + "expected_filter_fields": [ + "District Name", + "Avgscrmath" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Name schools in Riverside which the average of average math score for SAT is grater than 400, what is the funding type of these schools? Average of average math = sum(average math scores) / count(schools).", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 26, + "question": "State the names and full communication address of high schools in Monterey which has more than 800 free or reduced price meals for ages 15-17?", + "evidence": "Full communication address should include Street, City, State and zip code if any.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 5, + "ai_summarized_answer": "Five high schools meet the criteria: Alisal High at 777 Williams Road, Salinas, CA 93905-1907; Everett Alvarez High at 1900 Independence Boulevard, Salinas, CA 93906-5300; North Salinas High at 55 Kip Drive, Salinas, CA 93906-2908; Salinas High at 726 South Main Street, Salinas, CA 93901-3243; and Soledad High at 425 Gabilan Drive, Soledad, CA 93960-3207.", + "expected_columns": [ + "School Name", + "Street", + "City", + "State", + "Zip" + ], + "expected_filter_fields": [ + "County", + "Free Meal Count (Ages 5-17)", + "School Type" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: State the names and full communication address of high schools in Monterey which has more than 800 free or reduced price meals for ages 15-17? Full communication address should include Street, City, State and zip code if any.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 27, + "question": "What is the average score in writing for the schools that were opened after 1991 or closed before 2000? List the school names along with the score. Also, list the communication number of the schools if there is any.", + "evidence": "Communication number refers to phone number.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 8574, + "ai_summarized_answer": "The query returns 8,574 schools with their writing scores and phone numbers. Many schools have null writing scores. Examples include FAME Public Charter with a score of 505.0 and no phone number, Envision Academy for Arts & Technology with 395.0 and phone (510) 596-8901, and 8,572 more schools with varying scores and phone availability.", + "expected_columns": [ + "School", + "Avgscrwrite" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the average score in writing for the schools that were opened after 1991 or closed before 2000? List the school names along with the score. Also, list the communication number of the schools if there is any. Communication number refers to phone number.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 28, + "question": "Consider the average difference between K-12 enrollment and 15-17 enrollment of schools that are locally funded, list the names and DOC type of schools which has a difference above this average.", + "evidence": "Difference between K-12 enrollment and 15-17 enrollment can be computed by `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 57, + "ai_summarized_answer": "The results show 57 locally funded schools with enrollment differences above the average. Examples include Mountain Oaks (DOC 00), Castle Rock (DOC 00), Clovis Online Charter (DOC 54), Washington Elementary (DOC 52), West Park Charter Academy (DOC 52), and 52 more schools with various DOC types.", + "expected_columns": [ + "School", + "Doc" + ], + "expected_filter_fields": [ + "Fundingtype" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Consider the average difference between K-12 enrollment and 15-17 enrollment of schools that are locally funded, list the names and DOC type of schools which has a difference above this average. Difference between K-12 enrollment and 15-17 enrollment can be computed by `Enrollment (K-12)` - `Enrollment (Ages 5-17)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 31, + "question": "What is the eligible free rate of the 10th and 11th schools with the highest enrolment for students in grades 1 through 12?", + "evidence": "K-12 refers to students in grades 1 through 12; Eligible free rate for K-12 = `Free Meal Count (K-12)` / `Enrollment (K-12)`", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 2, + "ai_summarized_answer": "The eligible free rates for the 10th and 11th schools with the highest enrollment are approximately 0.134 (13.4%) and 0.291 (29.1%) respectively.", + "expected_columns": [ + "Cds", + "Percent (%) Eligible Free (K-12)" + ], + "expected_filter_fields": [ + "Cds" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the eligible free rate of the 10th and 11th schools with the highest enrolment for students in grades 1 through 12? K-12 refers to students in grades 1 through 12; Eligible free rate for K-12 = `Free Meal Count (K-12)` / `Enrollment (K-12)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 32, + "question": "What is the eligible free or reduced price meal rate for the top 5 schools in grades 1-12 with the highest free or reduced price meal count of the schools with the ownership code 66?", + "evidence": "grades 1-12 means K-12; Eligible free or reduced price meal rate for K-12 = `FRPM Count (K-12)` / `Enrollment (K-12)`", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 5, + "ai_summarized_answer": "The top 5 schools with ownership code 66 have the following FRPM rates: Paramount High at 91.8%, Calexico High at 99.9%, Bell Senior High at 89.6%, Anaheim High at 89.6%, and Bell Gardens High at 91.4%.", + "expected_columns": [ + "School Name", + "FRPM Rate" + ], + "expected_filter_fields": [ + "Soc", + "Cdscode" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the eligible free or reduced price meal rate for the top 5 schools in grades 1-12 with the highest free or reduced price meal count of the schools with the ownership code 66? grades 1-12 means K-12; Eligible free or reduced price meal rate for K-12 = `FRPM Count (K-12)` / `Enrollment (K-12)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 36, + "question": "Under whose administration is the school with the highest number of students scoring 1500 or more on the SAT? Indicate their full names.", + "evidence": "full name means first name, last name; There are at most 3 administrators for each school; SAT Scores are greater or equal to 1500 refers to NumGE1500", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The school with the highest number of students scoring 1500 or more on the SAT is administered by Michelle King. No additional administrators are listed for her school.", + "expected_columns": [ + "administrator's first name", + "administrator's last name", + "Admfname2", + "Admlname2", + "Admfname3", + "Admlname3", + "Numge1500" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Under whose administration is the school with the highest number of students scoring 1500 or more on the SAT? Indicate their full names. full name means first name, last name; There are at most 3 administrators for each school; SAT Scores are greater or equal to 1500 refers to NumGE1500", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 37, + "question": "What is the complete address of the school with the lowest excellence rate? Indicate the Street, City, Zip and State.", + "evidence": "Execellence Rate = NumGE1500 / NumTstTakr; complete address has Street, City, State, Zip code", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The school with the lowest excellence rate is located at 2125 Jefferson Avenue, Berkeley, CA 94703-1414.", + "expected_columns": [ + "School", + "Street", + "City", + "State", + "Zip" + ], + "expected_filter_fields": [ + "Street" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the complete address of the school with the lowest excellence rate? Indicate the Street, City, Zip and State. Execellence Rate = NumGE1500 / NumTstTakr; complete address has Street, City, State, Zip code", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 39, + "question": "What is the average number of test takers from Fresno schools that opened between 1/1/1980 and 12/31/1980?", + "evidence": "between 1/1/1980 and 12/31/1980 means the year = 1980", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": 137.88888888888889, + "expected_row_count": 1, + "ai_summarized_answer": "The average number of test takers from Fresno schools that opened in 1980 is approximately 137.89.", + "expected_columns": [ + "Numtsttakr" + ], + "expected_filter_fields": [ + "County", + "Opendate" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the average number of test takers from Fresno schools that opened between 1/1/1980 and 12/31/1980? between 1/1/1980 and 12/31/1980 means the year = 1980", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 40, + "question": "What is the telephone number for the school with the lowest average score in reading in Fresno Unified?", + "evidence": "Fresno Unified is a name of district;", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": "(559) 248-5100", + "expected_row_count": 1, + "ai_summarized_answer": "The phone number for the Fresno Unified school with the lowest average reading score is (559) 248-5100.", + "expected_columns": [ + "Phone", + "Avgscrread" + ], + "expected_filter_fields": [ + "District", + "Avgscrread" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the telephone number for the school with the lowest average score in reading in Fresno Unified? Fresno Unified is a name of district;", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 41, + "question": "List the names of virtual schools that are among the top 5 in their respective counties based on average reading scores.", + "evidence": "Exclusively virtual refers to Virtual = 'F'; respective counties means PARTITION BY County", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 34, + "ai_summarized_answer": "The results show 34 virtual schools that rank in the top 5 within their respective counties. Examples include Academy of Arts and Sciences: Fresno, Dunlap Leadership Academy, Insight School of California, California Virtual Academy @ Kings, National University Academy, Armona, and 29 more schools.", + "expected_columns": [], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: List the names of virtual schools that are among the top 5 in their respective counties based on average reading scores. Exclusively virtual refers to Virtual = 'F'; respective counties means PARTITION BY County", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 45, + "question": "What is the average writing score of each of the schools managed by Ricci Ulrich? List the schools and the corresponding average writing scores.", + "evidence": "Usually, administrators manage the school stuff.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "There is one school managed by Ricci Ulrich: Buchanan High with an average writing score of 507.", + "expected_columns": [ + "School", + "Avgscrwrite" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the average writing score of each of the schools managed by Ricci Ulrich? List the schools and the corresponding average writing scores. Usually, administrators manage the school stuff.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 46, + "question": "Which state special schools have the highest number of enrollees from grades 1 through 12?", + "evidence": "State Special Schools refers to DOC = 31; Grades 1 through 12 means K-12", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": "California School for the Deaf-Fremont", + "expected_row_count": 1, + "ai_summarized_answer": "The state special school with the highest K-12 enrollment is California School for the Deaf-Fremont.", + "expected_columns": [ + "School", + "Enrollment (K-12)" + ], + "expected_filter_fields": [ + "Doc" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Which state special schools have the highest number of enrollees from grades 1 through 12? State Special Schools refers to DOC = 31; Grades 1 through 12 means K-12", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 47, + "question": "What is the monthly average number of schools that opened in Alameda County under the jurisdiction of the Elementary School District in 1980?", + "evidence": "Elementary School District refers to DOC = 52; Monthly average number of schools that opened in 1980 = count(schools that opened in 1980) / 12", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": 1.4166666666666667, + "expected_row_count": 1, + "ai_summarized_answer": "The monthly average number of Elementary School District schools that opened in Alameda County in 1980 is approximately 1.42 schools per month.", + "expected_columns": [ + "Monthly Average Schools Opened" + ], + "expected_filter_fields": [ + "Doc", + "County", + "Opendate" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the monthly average number of schools that opened in Alameda County under the jurisdiction of the Elementary School District in 1980? Elementary School District refers to DOC = 52; Monthly average number of schools that opened in 1980 = count(schools that opened in 1980) / 12", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 48, + "question": "What is the ratio of merged Unified School District schools in Orange County to merged Elementary School District schools?", + "evidence": "Elementary School District refers to DOC = 52; Unified School District refers to DOC = 54.", + "difficulty": "moderate", + "answer_type": "scalar", + "expected_value": 0.5714285714285714, + "expected_row_count": 1, + "ai_summarized_answer": "The ratio of merged Unified School District schools to merged Elementary School District schools in Orange County is approximately 0.571, meaning there are about 0.57 unified schools for every elementary school.", + "expected_columns": [ + "Ratio Unified to Elementary" + ], + "expected_filter_fields": [ + "Statustype", + "County" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the ratio of merged Unified School District schools in Orange County to merged Elementary School District schools? Elementary School District refers to DOC = 52; Unified School District refers to DOC = 54.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 50, + "question": "What is the postal street address for the school with the 7th highest Math average? Indicate the school's name.", + "evidence": "Postal street and mailing street are synonyms.", + "difficulty": "simple", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The query returns the top 6 schools by math average. The 7th highest would be Saratoga High at 20300 Herriman Avenue (based on the ordering shown).", + "expected_columns": [ + "School", + "Mailstreet", + "Avgscrmath" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the postal street address for the school with the 7th highest Math average? Indicate the school's name. Postal street and mailing street are synonyms.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 62, + "question": "What is the total number of non-chartered schools in the county of Los Angeles with a percent (%) of eligible free meals for grades 1 through 12 that is less than 0.18%?", + "evidence": "non-chartered schools refer to schools whose Charter = 0; K-12 means grades 1 through 12; percent of eligible free rate for K-12 = `Free Meal Count (K-12)` * 100 / `Enrollment (K-12)`", + "difficulty": "challenging", + "answer_type": "scalar", + "expected_value": 1, + "expected_row_count": 1, + "ai_summarized_answer": "There is 1 non-chartered school in Los Angeles County with an eligible free meal rate less than 0.18%.", + "expected_columns": [ + "School" + ], + "expected_filter_fields": [ + "County", + "Charter" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the total number of non-chartered schools in the county of Los Angeles with a percent (%) of eligible free meals for grades 1 through 12 that is less than 0.18%? non-chartered schools refer to schools whose Charter = 0; K-12 means grades 1 through 12; percent of eligible free rate for K-12 = `Free Meal Count (K-12)` * 100 / `Enrollment (K-12)`", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 72, + "question": "How many students from the ages of 5 to 17 are enrolled at the State Special School school in Fremont for the 2014-2015 academic year?", + "evidence": "State Special School means EdOpsCode = 'SSS'", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 2, + "ai_summarized_answer": "There are two State Special Schools in Fremont for the 2014-2015 academic year with enrollments of 40.0 and 335.0 students aged 5-17.", + "expected_columns": [ + "Enrollment (Ages 5-17)" + ], + "expected_filter_fields": [ + "Edopscode", + "City", + "Academic Year" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: How many students from the ages of 5 to 17 are enrolled at the State Special School school in Fremont for the 2014-2015 academic year? State Special School means EdOpsCode = 'SSS'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 77, + "question": "Which schools served a grade span of Kindergarten to 9th grade in the county of Los Angeles and what is its Percent (%) Eligible FRPM (Ages 5-17)?", + "evidence": "Percent (%) Eligible FRPM (Ages 5-17) can be acquired by `FRPM Count (Ages 5-17)` / `Enrollment (Ages 5-17)` * 100", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 2, + "ai_summarized_answer": "Two schools in Los Angeles County serve grades K-9: White Oak Elementary with a 3.76% eligible FRPM rate and The Accelerated with a 97.64% eligible FRPM rate.", + "expected_columns": [ + "School", + "Percent Eligible FRPM (Ages 5-17)" + ], + "expected_filter_fields": [ + "County", + "Gsserved" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Which schools served a grade span of Kindergarten to 9th grade in the county of Los Angeles and what is its Percent (%) Eligible FRPM (Ages 5-17)? Percent (%) Eligible FRPM (Ages 5-17) can be acquired by `FRPM Count (Ages 5-17)` / `Enrollment (Ages 5-17)` * 100", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 79, + "question": "Between San Diego and Santa Barbara, which county offers the most number of schools that does not offer physical building? Indicate the amount.", + "evidence": "'Does not offer physical building' means Virtual = F in the database.", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "San Diego County offers the most virtual schools (those without physical buildings) with a count of 8 schools.", + "expected_columns": [ + "County", + "Virtual" + ], + "expected_filter_fields": [ + "County", + "Virtual" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Between San Diego and Santa Barbara, which county offers the most number of schools that does not offer physical building? Indicate the amount. 'Does not offer physical building' means Virtual = F in the database.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 82, + "question": "What is the grade span offered in the school with the highest longitude?", + "evidence": "the highest longitude refers to the school with the maximum absolute longitude value.", + "difficulty": "simple", + "answer_type": "scalar", + "expected_value": "K-8", + "expected_row_count": 1, + "ai_summarized_answer": "The school with the highest absolute longitude value has a null grade span (no grade span information available). The next highest is K-8 with an absolute longitude of 124.28481", + "expected_columns": [ + "Gsoffered", + "Max absolute longitude" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the grade span offered in the school with the highest longitude? the highest longitude refers to the school with the maximum absolute longitude value.", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 83, + "question": "Of the schools that offers a magnet program serving a grade span of Kindergarten to 8th grade, how many offers Multiple Provision Types? List the number of cities that offers a Kindergarten to 8th grade span and indicate how many schools are there serving such grade span for each city.", + "evidence": "Kindergarten to 8th grade refers to K-8; 'Offers a magnet program' means Magnet = 1; Multiple Provision Types refers to `NSLP Provision Status` = 'Multiple Provision Types'", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "There is 1 city (Adelanto) with a K-8 magnet school offering Multiple Provision Types, and it has 1 such school.", + "expected_columns": [ + "City", + "Cdscode" + ], + "expected_filter_fields": [ + "Magnet", + "Gsoffered", + "NSLP Provision Status" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: Of the schools that offers a magnet program serving a grade span of Kindergarten to 8th grade, how many offers Multiple Provision Types? List the number of cities that offers a Kindergarten to 8th grade span and indicate how many schools are there serving such grade span for each city. Kindergarten to 8th grade refers to K-8; 'Offers a magnet program' means Magnet = 1; Multiple Provision Types refers to `NSLP Provision Status` = 'Multiple Provision Types'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 85, + "question": "What is the Percent (%) Eligible Free (K-12) in the school administered by an administrator whose first name is Alusine. List the district code of the school.", + "evidence": "Percent (%) Eligible Free (K-12) = `Free Meal Count (K-12)` / `Enrollment (K-12)` * 100%", + "difficulty": "moderate", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The school administered by Alusine has a percent eligible free rate of approximately 70.15% for K-12 students, and the district code is 64857.", + "expected_columns": [ + "District Code", + "Percent Eligible Free (K-12)" + ], + "expected_filter_fields": [], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What is the Percent (%) Eligible Free (K-12) in the school administered by an administrator whose first name is Alusine. List the district code of the school. Percent (%) Eligible Free (K-12) = `Free Meal Count (K-12)` / `Enrollment (K-12)` * 100%", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + }, + { + "question_id": 87, + "question": "What are the valid e-mail addresses of the administrator of the school located in the San Bernardino county, City of San Bernardino City Unified that opened between 1/1/2009 to 12/31/2010 whose school types are public Intermediate/Middle Schools and Unified Schools?", + "evidence": "Intermediate/Middle Schools refers to SOC = 62; Unified School refers to DOC = 54; years between 2009 and 2010 can refer to 'between 1/1/2009 to 12/31/2010'", + "difficulty": "challenging", + "answer_type": "list", + "expected_value": null, + "expected_row_count": 1, + "ai_summarized_answer": "The administrator email addresses for the qualifying school are a.lucero@realjourney.org and j.hernandez@realjourney.org.", + "expected_columns": [ + "Admemail1", + "Admemail2", + "Admemail3" + ], + "expected_filter_fields": [ + "County", + "City", + "Doc", + "Soc", + "Opendate" + ], + "prompt": "Using datasource {{env.EVAL_DATASOURCE_LUID}}: What are the valid e-mail addresses of the administrator of the school located in the San Bernardino county, City of San Bernardino City Unified that opened between 1/1/2009 to 12/31/2010 whose school types are public Intermediate/Middle Schools and Unified Schools? Intermediate/Middle Schools refers to SOC = 62; Unified School refers to DOC = 54; years between 2009 and 2010 can refer to 'between 1/1/2009 to 12/31/2010'", + "expected_tools": [ + "query-datasource" + ], + "budget": { + "max_wall_ms": 180000 + } + } +] diff --git a/package-lock.json b/package-lock.json index 14e71054b..e0cf71a48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@tableau/mcp-server", - "version": "2.2.4", + "version": "2.2.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@tableau/mcp-server", - "version": "2.2.4", + "version": "2.2.5", "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/sdk": "^1.26.0", @@ -46,6 +46,7 @@ "eslint-config-prettier": "^10.1.2", "eslint-plugin-prettier": "^5.2.6", "eslint-plugin-simple-import-sort": "^12.1.1", + "langsmith": "^0.8.4", "npm-run-all": "^4.1.5", "openai": "^5.23.2", "prettier": "^3.5.3", @@ -1816,6 +1817,7 @@ "integrity": "sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -2009,6 +2011,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.33.1.tgz", "integrity": "sha512-qwxv6dq682yVvgKKp2qWwLgRbscDAYktPptK4JPojCwwi3R9cwrvIxS4lvBpzmcqzR4bdn54Z0IG1uHFskW4dA==", "dev": true, + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.33.1", "@typescript-eslint/types": "8.33.1", @@ -2389,6 +2392,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "dev": true, + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3359,6 +3363,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.28.0.tgz", "integrity": "sha512-ocgh41VhRlf9+fVpe7QKzwLj9c92fDiqOj8Y3Sd4/ZmVA4Btx4PlUYPq4pp9JDyupkf1upbEXecxL2mwNV7jPQ==", "dev": true, + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -3419,6 +3424,7 @@ "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.5.tgz", "integrity": "sha512-zc1UmCpNltmVY34vuLRV61r1K27sWuX39E+uyUnY8xS2Bex88VV9cugG+UZbRSRGtGyFboj+D8JODyme1plMpw==", "dev": true, + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -3596,6 +3602,13 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3629,6 +3642,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3915,7 +3929,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=4.0" }, @@ -4333,6 +4346,7 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.7.tgz", "integrity": "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -5020,6 +5034,40 @@ "json-buffer": "3.0.1" } }, + "node_modules/langsmith": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.4.tgz", + "integrity": "sha512-e2zdhPUV/mLwVv+Gde/0gLGnCOE/zvZ3gGNh/rvkzrxsDz7qgUT4CJVUmJE2GwQ1A3FPtWKzy08oo//VV1nBFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5640,6 +5688,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -5670,6 +5728,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -5911,6 +5999,7 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "dev": true, + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -5961,8 +6050,7 @@ "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "peer": true + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "node_modules/punycode": { "version": "2.3.1", @@ -7025,6 +7113,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -7717,6 +7806,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7824,6 +7914,7 @@ "integrity": "sha512-qTl3VF7BvOupTR85Zc561sPEgxyUSNSvTQ9fit7DEMP7yPgvvIGm5Zfa1dOM+kOwWGNviK9uFM9ra77+OjK7lQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -7939,6 +8030,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -7951,6 +8043,7 @@ "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.2.tgz", "integrity": "sha512-fyNn/Rp016Bt5qvY0OQvIUCwW2vnaEBLxP42PmKbNIoasSYjML+8xyeADOPvBe+Xfl/ubIw4og7Lt9jflRsCNw==", "dev": true, + "peer": true, "dependencies": { "@types/chai": "^5.2.2", "@vitest/expect": "3.2.2", @@ -8303,6 +8396,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 49816b403..b3d922627 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@tableau/mcp-server", "description": "Helping agents see and understand data.", - "version": "2.2.4", + "version": "2.2.5", "repository": { "type": "git", "url": "git+https://github.com/tableau/tableau-mcp.git" @@ -47,6 +47,13 @@ "test": "vitest --config ./vitest.config.ts", "test:e2e": "vitest --config ./vitest.config.e2e.ts", "test:eval": "vitest --config ./vitest.config.eval.ts", + "eval:run": "tsx evals/run-case.ts", + "eval:claude": "tsx evals/run-case.ts", + "eval:grade": "tsx evals/grade.ts", + "eval:suite": "tsx evals/run-suite.ts", + "eval:grade:bird": "tsx evals/grade-bird.ts", + "eval:grade:suite": "tsx evals/grade-suite.ts", + "eval:report": "tsx evals/report.ts", "test:oauth:embedded": "vitest --config ./vitest.config.oauth.embedded.ts", "test:oauth:tableau": "npx playwright test", "coverage": "vitest run --config ./vitest.config.ts --coverage", @@ -90,6 +97,7 @@ "eslint-config-prettier": "^10.1.2", "eslint-plugin-prettier": "^5.2.6", "eslint-plugin-simple-import-sort": "^12.1.1", + "langsmith": "^0.8.4", "npm-run-all": "^4.1.5", "openai": "^5.23.2", "prettier": "^3.5.3", diff --git a/team_context/eval-questions.md b/team_context/eval-questions.md new file mode 100644 index 000000000..ca23a9229 --- /dev/null +++ b/team_context/eval-questions.md @@ -0,0 +1,13 @@ +1. How many tableau data sources do I have access to? +2. What are the workbooks on my tableau site? +3. pull me some insights from the +4. pull me some insights from the +5. Show me the dashboard +6. what does the tell me +7. generate some insights for a metric based on the +8. let's analyze this workbook: +9. what's the most popular content on my site? +10. Do we have any data about +11. Who should I reach out to to get more information about +12. what does the dashboard tell me? +13. \ No newline at end of file