tableau · anyoung-tableau · Mar 5, 2026 · Mar 9, 2026 · Mar 10, 2026 · Mar 10, 2026
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -43,7 +43,7 @@
     "build:inspect:docker:http": "run-s build:docker inspect:docker:http",
     "test": "vitest --config ./vitest.config.ts",
     "test:e2e": "vitest --config ./vitest.config.e2e.ts",
-    "test:eval": "vitest --config ./vitest.config.eval.ts",
+    "test:eval": "tsx tests/eval/runEval.ts",
     "test:oauth:embedded": "vitest --config ./vitest.config.oauth.embedded.ts",
     "coverage": "vitest run --config ./vitest.config.ts --coverage",
     "version:major": "npm version major --no-git-tag-version",
@@ -68,7 +68,9 @@
     "@anthropic-ai/mcpb": "^1.0.0",
     "@eslint/eslintrc": "^3.3.1",
     "@eslint/js": "^9.25.1",
-    "@openai/agents": "^0.1.9",
+    "@langchain/core": "^1.1.31",
+    "@langchain/mcp-adapters": "^1.1.3",
+    "@langchain/openai": "^1.2.12",
     "@types/cors": "^2.8.19",
     "@types/eslint__js": "^8.42.3",
     "@types/express": "^5.0.3",
@@ -83,8 +85,9 @@
     "eslint-config-prettier": "^10.1.2",
     "eslint-plugin-prettier": "^5.2.6",
     "eslint-plugin-simple-import-sort": "^12.1.1",
+    "langchain": "^1.2.30",
+    "langsmith": "^0.5.7",
     "npm-run-all": "^4.1.5",
-    "openai": "^5.23.2",
     "prettier": "^3.5.3",
     "supertest": "^7.1.4",
     "tsx": "^4.21.0",

diff --git a/tests/eval/base.ts b/tests/eval/base.ts
@@ -1,27 +1,13 @@
-import { CallToolResultSchema } from '@modelcontextprotocol/sdk/types.js';
-import {
-  Agent,
-  getAllMcpTools,
-  MCPServerStdio,
-  OpenAIChatCompletionsModel,
-  StreamedRunResult,
-  withTrace,
-} from '@openai/agents';
-import { OpenAI } from 'openai/client.js';
-import { Err, Ok, Result } from 'ts-results-es';
-import z from 'zod';
-
-import invariant from '../../src/utils/invariant.js';
-
-type ToolExecution = {
-  name: string;
-  arguments: Record<string, unknown>;
-  output: string;
-};
+import { MultiServerMCPClient, StdioConnection } from '@langchain/mcp-adapters';
+import { ChatOpenAI } from '@langchain/openai';
+import { AIMessage, createAgent, HumanMessage, SystemMessage, ToolMessage } from 'langchain';
 
 const DEFAULT_MODEL = 'claude-sonnet-4-5-20250929';
 
-function getApiKey(): string {
+type Message = AIMessage | HumanMessage | SystemMessage | ToolMessage;
+type Agent = ReturnType<typeof createAgent>;
+
+export function getApiKey(): string {
   const { OPENAI_API_KEY } = process.env;
 
   if (!OPENAI_API_KEY) {
@@ -35,125 +21,64 @@ export function getModel(): string {
   return process.env.EVAL_TEST_MODEL || DEFAULT_MODEL;
 }
 
-export async function getMcpServer(env?: Record<string, string>): Promise<MCPServerStdio> {
-  const mcpServer = new MCPServerStdio({
+export function getMcpServer(env?: Record<string, string>): StdioConnection {
+  const mcpServer: StdioConnection = {
     command: 'node',
     args: ['build/index.js'],
     env,
-    cacheToolsList: true,
-  });
+  };
 
-  await mcpServer.connect();
   return mcpServer;
 }
 
 export async function getAgent({
-  systemPrompt,
   model,
   mcpServer,
 }: {
-  systemPrompt: string;
   model: string;
-  mcpServer?: MCPServerStdio;
-}): Promise<Agent> {
-  return await withTrace('get_agent', async () => {
-    const agentOptions = {
-      name: 'Assistant with Tableau MCP tools',
-      instructions: systemPrompt,
-      model: new OpenAIChatCompletionsModel(
-        new OpenAI({
-          baseURL: process.env.OPENAI_BASE_URL,
-          apiKey: getApiKey(),
-        }),
-        model,
-      ),
-    };
-
-    if (!mcpServer) {
-      return new Agent(agentOptions);
-    }
-
-    const tools = await getAllMcpTools([mcpServer]);
-    tools.forEach((tool) => {
-      tool.name = `tableau_${tool.name}`;
-    });
-
-    return new Agent({
-      ...agentOptions,
-      mcpServers: [mcpServer],
-      tools,
-      modelSettings: { toolChoice: 'required' },
-    });
+  mcpServer?: StdioConnection;
+}): Promise<{ agent: Agent; client?: MultiServerMCPClient }> {
+  const agentOptions = {
+    name: 'Assistant with Tableau MCP tools',
+    model: new ChatOpenAI({
+      model,
+      configuration: {
+        apiKey: getApiKey(),
+        baseURL: process.env.OPENAI_BASE_URL,
+      },
+    }),
+  };
+
+  if (!mcpServer) {
+    return { agent: createAgent({ ...agentOptions }) };
+  }
+
+  const client = new MultiServerMCPClient({
+    tableau: mcpServer,
   });
-}
 
-export async function getToolExecutions(
-  result: StreamedRunResult<undefined, any>,
-): Promise<Array<ToolExecution>> {
-  const toolExecutions: Map<string, ToolExecution> = new Map();
-
-  for (const item of result.history) {
-    if (item.type === 'function_call') {
-      toolExecutions.set(item.callId, {
-        name: item.name,
-        arguments: JSON.parse(item.arguments) as Record<string, unknown>,
-        output: '',
-      });
-    }
-  }
+  const tools = await client.getTools();
+  return {
+    agent: createAgent({ ...agentOptions, tools }),
+    client,
+  };
+}
 
-  for (const item of result.history) {
-    if (item.type === 'function_call_result') {
-      const call = toolExecutions.get(item.callId);
-      if (!call) {
-        throw new Error(`Could not find tool execution for callId ${item.callId}`);
-      }
-
-      call.output =
-        item.output.type === 'text'
-          ? item.output.text
-          : item.output.type === 'image'
-            ? item.output.data
-            : '';
-    }
-  }
+export async function prompt(agent: Agent, content: string): Promise<Array<Message>> {
+  console.log(`Invoking agent with input: ${content}...`);
 
-  log('🛠️ tool executions:');
-  const executions = [...toolExecutions.values()];
-  for (const execution of executions) {
-    log(`  🔨 ${execution.name}`);
-    log(`    👉 arguments: ${JSON.stringify(execution.arguments)}`);
-    log(`    👈 output: ${execution.output}`);
-    log('\n');
-  }
+  const start = performance.now();
 
-  return executions;
-}
+  const { messages } = (await agent.invoke({
+    messages: [{ role: 'user', content }],
+  })) as { messages: Array<Message> };
 
-export function getCallToolResult<Z extends z.ZodTypeAny = z.ZodNever>(
-  toolExecution: ToolExecution,
-  schema: Z,
-): z.infer<Z> {
-  const callToolResult = CallToolResultSchema.parse(JSON.parse(toolExecution.output));
-  invariant(callToolResult.type === 'text');
-  invariant(typeof callToolResult.text === 'string');
-  const result = schema.parse(JSON.parse(callToolResult.text));
-  return result;
-}
+  const end = performance.now();
 
-export function getCallToolResultSafe<Z extends z.ZodTypeAny = z.ZodNever>(
-  toolExecution: ToolExecution,
-  schema: Z,
-): Result<z.infer<Z>, Error> {
-  const callToolResult = CallToolResultSchema.safeParse(JSON.parse(toolExecution.output));
-  if (!callToolResult.success) {
-    return Err(callToolResult.error);
-  }
+  const duration = Math.round(end - start);
+  console.log(`Agent response received after ${duration}ms`);
 
-  invariant(callToolResult.data.type === 'text');
-  invariant(typeof callToolResult.data.text === 'string');
-  const result = schema.parse(JSON.parse(callToolResult.data.text));
-  return Ok(result);
+  return messages;
 }
 
 export function log(message?: any, force?: boolean): void {

diff --git a/tests/eval/ensureDatasetExists.ts b/tests/eval/ensureDatasetExists.ts
@@ -0,0 +1,119 @@
+import { Client } from 'langsmith';
+import { ExampleCreate } from 'langsmith/schemas';
+
+import { evalInputSchema, evalReferenceSchema } from './evaluators';
+import { EvalExample, examples } from './examples';
+
+export async function ensureDatasetExists({
+  client,
+  datasetName,
+  datasetDescription,
+}: {
+  client: Client;
+  datasetName: string;
+  datasetDescription: string;
+}): Promise<void> {
+  console.log(`Ensuring dataset "${datasetName}" exists`);
+  console.log('Getting current datasets...');
+  const datasets = client.listDatasets({ datasetName });
+
+  let dataset = await first(datasets);
+
+  if (dataset) {
+    console.log(`Dataset "${datasetName}" already exists`);
+  } else {
+    console.log(`Creating dataset "${datasetName}"...`);
+    dataset = await client.createDataset(datasetName, {
+      description: datasetDescription,
+    });
+    console.log(`Created dataset "${datasetName}"`);
+  }
+
+  console.log('Getting current examples...');
+  const datasetExamples = await toArray(client.listExamples({ datasetId: dataset.id }));
+
+  const datasetExamplesStr = datasetExamples.length === 1 ? 'example' : 'examples';
+  console.log(`Found ${datasetExamples.length} ${datasetExamplesStr} in dataset`);
+
+  const missingExamples: Array<EvalExample> = examples.filter((example) => {
+    return !datasetExamples.some((e) => e.inputs.question === example.inputs.question);
+  });
+
+  const mismatchedExamples: Array<EvalExample> = [];
+  for (const datasetExample of datasetExamples) {
+    const inputResult = evalInputSchema.safeParse(datasetExample.inputs);
+    if (!inputResult.success) {
+      console.log("Dataset example inputs don't match expected schema");
+      console.log(`Deleting example: ${datasetExample.id}...`);
+      await client.deleteExample(datasetExample.id);
+      continue;
+    }
+
+    const datasetInputs = inputResult.data;
+    const example = examples.find((e) => e.inputs.question === datasetInputs.question);
+    if (!example) {
+      console.log(`Dataset example not found: ${datasetInputs.question}`);
+      console.log(`Deleting example: ${datasetExample.id}...`);
+      await client.deleteExample(datasetExample.id);
+      continue;
+    }
+
+    const outputResult = evalReferenceSchema.safeParse(datasetExample.outputs);
+    if (!outputResult.success) {
+      console.log(`Example outputs don't match expected schema: ${datasetExample.inputs.question}`);
+      console.log(`Deleting example: ${datasetExample.id}...`);
+      await client.deleteExample(datasetExample.id);
+      mismatchedExamples.push(example);
+      continue;
+    }
+
+    const datasetOutputs = outputResult.data;
+    if (
+      datasetOutputs.rubric !== example.outputs.rubric ||
+      datasetOutputs.mustContain.length !== example.outputs.mustContain.length ||
+      !datasetOutputs.mustContain.every((mustContain) =>
+        example.outputs.mustContain.includes(mustContain),
+      ) ||
+      datasetOutputs.expectedTools.length !== example.outputs.expectedTools.length ||
+      !datasetOutputs.expectedTools.every((tool) => example.outputs.expectedTools.includes(tool))
+    ) {
+      console.log(`Example mismatch: ${datasetExample.inputs.question}`);
+      console.log('Deleting example...');
+      await client.deleteExample(datasetExample.id);
+      mismatchedExamples.push(example);
+    }
+  }
+
+  const uploads: Array<ExampleCreate> = [...missingExamples, ...mismatchedExamples].map(
+    (example) => ({
+      inputs: example.inputs,
+      outputs: example.outputs,
+      dataset_id: dataset.id,
+    }),
+  );
+
+  if (uploads.length === 0) {
+    console.log('Dataset examples are up to date');
+    return;
+  }
+
+  const examplesStr = uploads.length === 1 ? 'example' : 'examples';
+  console.log(`Creating ${uploads.length} ${examplesStr}...`);
+  await client.createExamples(uploads);
+  console.log(`Created ${uploads.length} ${examplesStr}`);
+}
+
+async function first<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
+  for await (const item of iter) {
+    return item;
+  }
+}
+
+async function toArray<T>(iter: AsyncIterable<T>): Promise<T[]> {
+  const result: Array<T> = [];
+  for await (const item of iter) {
+    result.push(item);
+  }
+
+  return result;
+}
diff --git a/tests/eval/evaluationResult.ts b/tests/eval/evaluationResult.ts