feat(search-attributes): add attribute list as default behaviour

packages/mcp-core/src/tools/catalog/search-attributes.test.ts

@@ -1,8 +1,47 @@
-import { describe, expect, it } from "vitest";
+import { mswServer } from "@sentry/mcp-server-mocks";
+import { APICallError, generateText } from "ai";
+import { http, HttpResponse } from "msw";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { z } from "zod";
+import { ApiNotFoundError } from "../../api-client";
 import { getServerContext } from "../../test-setup.js";
 import searchAttributes from "./search-attributes.js";
 
+vi.mock("@ai-sdk/openai", () => {
+  const mockModel = vi.fn(() => "mocked-model");
+  return {
+    openai: mockModel,
+    createOpenAI: vi.fn(() => mockModel),
+  };
+});
+
+vi.mock("ai", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("ai")>();
+  return {
+    ...actual,
+    generateText: vi.fn(),
+    Output: { object: vi.fn(() => ({})) },
+  };
+});
+
+const ATTRIBUTES_URL =
+  "https://sentry.io/api/0/organizations/sentry-mcp-evals/trace-items/attributes/";
+
+const mockGenerateText = vi.mocked(generateText);
+
+function mockAgentResponse(output: {
+  best: string | null;
+  candidates: string[];
+}) {
+  return {
+    text: JSON.stringify(output),
+    experimental_output: output,
+    finishReason: "stop" as const,
+    usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
+    warnings: [] as const,
+  } as any;
+}
+
 function search(params: Record<string, unknown>) {
   return searchAttributes.handler(
     z.object(searchAttributes.inputSchema).parse({
@@ -14,58 +53,233 @@ function search(params: Record<string, unknown>) {
 }
 
 describe("search_attributes", () => {
-  it("applies defaults and leaves dataset unset when only the organization is given", async () => {
-    expect(await search({})).toMatchInlineSnapshot(`
-      "# Attribute Search in **sentry-mcp-evals**
-
-      Not implemented yet. Parsed inputs:
-
-      - Dataset: undefined (API item type: undefined)
-      - Project: all projects
-      - Query: none (list all attributes)
-      - Period: not set
-      - Limit: 10
-      "
-    `);
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // Pin the provider so the local .env can't change which path runs.
+    vi.stubEnv("EMBEDDED_AGENT_PROVIDER", "openai");
+    vi.stubEnv("OPENAI_API_KEY", "test-key");
+    vi.stubEnv("ANTHROPIC_API_KEY", "");
+    vi.stubEnv("OPENROUTER_API_KEY", "");
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    mswServer.resetHandlers();
+  });
+
+  it("asks for a dataset when none is given", async () => {
+    await expect(search({})).rejects.toThrowErrorMatchingInlineSnapshot(
+      `[UserInputError: Choose a dataset: \`spans\` (requests, latency, database and AI calls), \`logs\` (log lines), or \`metrics\` (counters, gauges, distributions).]`,
+    );
   });
 
-  it("resolves the project and passes the remaining inputs through", async () => {
+  it("lists the dataset's attributes when there's a query because searching is not supported yet", async () => {
     expect(
-      await search({
-        projectSlug: "cloudflare-mcp",
-        query: "  payment provider  ",
-        dataset: "metrics",
-        period: "7d",
-        limit: 50,
-      }),
+      await search({ dataset: "spans", query: "endpoint status response" }),
     ).toMatchInlineSnapshot(`
-      "# Attribute Search in **sentry-mcp-evals**
+      "Matching by \`query\` is not available yet. Here are all of the dataset's attributes instead.
+      # Span Attributes in **sentry-mcp-evals** (last 14d)
+
+      **Suggested presentation:** A table works well here, with each attribute's type, description, deprecation status, replacement attributes, and whether it's added by Sentry.
+
+      Please tell the user each attribute's type, description, examples, whether they were added by sentry, and which attributes are deprecated and what replaces them.
+
+      | Attribute | Type | Description | Added by Sentry | Deprecated | Replacement Attribute |
+      | --- | --- | --- | --- | --- | --- |
+      | \`environment\` | string | The sentry environment. EXAMPLE VALUES: production | yes | yes | sentry.environment |
+      | \`http.method\` | string | The HTTP method used. EXAMPLE VALUES: GET | yes | yes | http.request.method |
+      | \`http.status_code\` | number | The status code of the HTTP response. EXAMPLE VALUES: 404 | yes | yes | http.response.status_code |
+      | \`http.url\` | string | The URL of the resource that was fetched. EXAMPLE VALUES: https://example.com/test?foo=bar#buzz | yes | yes | url.full |
+      | \`platform\` | string | The sdk platform that generated the event. EXAMPLE VALUES: php | yes |  |  |
+      | \`project\` | string | The name of the project. In some pages of sentry.io, you can also filter on project using a dropdown.  | yes |  |  |
+      | \`release\` | string | The sentry release. EXAMPLE VALUES: production | yes | yes | sentry.release |
+      | \`sdk.name\` | string | The sentry sdk name. EXAMPLE VALUES: @sentry/react | yes |  |  |
+      | \`sdk.version\` | string | The sentry sdk version. EXAMPLE VALUES: 7.0.0 | yes |  |  |
+      | \`span.description\` | string | Description of the span's operation.  | yes |  |  |
+      | \`span.duration\` | number | The total time taken by the span.  | yes |  |  |
+      | \`span.op\` | string | The operation of a span. EXAMPLE VALUES: http.client | yes |  |  |
+      | \`span.status\` | string | The span's status (either "ok" or "error"). Older SDKs may set this to a more specific error, but this behaviour is deprecated. EXAMPLE VALUES: ok | yes |  |  |
+      | \`transaction\` | string | The sentry transaction (segment name). EXAMPLE VALUES: GET / | yes | yes | sentry.segment.name |
+      | \`transaction.duration\` | number |  | yes |  |  |
+      | \`transaction.op\` | string | Operation of the containing transaction.  | yes |  |  |
+      | \`transaction.status\` | string |  | yes |  |  |
+      | \`user.email\` | string |  | yes |  |  |
+      | \`user.id\` | string |  | yes |  |  |
+      | \`browser.name\` | string |  | yes |  |  |
+      | \`custom.count\` | number |  |  |  |  |
+      | \`custom.db.pool_size\` | number |  |  |  |  |
+      | \`custom.feature_flag\` | string |  |  |  |  |
+      | \`custom.latency_ms\` | number |  |  |  |  |
+      | \`custom.score\` | number |  |  |  |  |
+      | \`custom.tier\` | string |  |  |  |  |
+      | \`device\` | string |  | yes |  |  |
+      | \`device.memory\` | number | The estimated total memory capacity of the device.  | yes |  |  |
+      | \`deviceMemory\` | number |  | yes |  |  |
+      | \`geo.city\` | string | The city of the user's location. EXAMPLE VALUES: Toronto | yes |  |  |
+      | \`geo.country_code\` | string | The country code of the user's location. EXAMPLE VALUES: US | yes |  |  |
+      | \`geo.region\` | string | The region of the user's location. EXAMPLE VALUES: Canada | yes | yes | user.geo.region |
+      | \`http.request.method\` | string | The HTTP method used. EXAMPLE VALUES: GET | yes |  |  |
+      | \`kiosk.firmware\` | string | The firmware revision currently installed and running on the kiosk. EXAMPLE VALUES: 4.1.3 | yes |  |  |
+      | \`kiosk.firmware_revision\` | string | The firmware revision downloaded and staged for the kiosk's next update. It is not installed until the kiosk restarts. EXAMPLE VALUES: 4.2.0 | yes |  |  |
+      | \`os.name\` | string | Human readable operating system name. EXAMPLE VALUES: Ubuntu | yes |  |  |
+      | \`user.username\` | string |  | yes |  |  |
+
+      ## Response Notes
+
+      - Use a key with \`search_traces\` to query its values."
+    `);
+
+    expect(mockGenerateText).not.toHaveBeenCalled();
+  });
+
+  it("lists the dataset's attributes when there's no query", async () => {
+    expect(await search({ dataset: "metrics" })).toMatchInlineSnapshot(`
+      "# Metric Attributes in **sentry-mcp-evals** (last 14d)
+
+      **Suggested presentation:** A table works well here, with each attribute's type, description, deprecation status, replacement attributes, and whether it's added by Sentry.
+
+      Please tell the user each attribute's type, description, examples, whether they were added by sentry, and which attributes are deprecated and what replaces them.
 
-      Not implemented yet. Parsed inputs:
+      | Attribute | Type | Description | Added by Sentry | Deprecated | Replacement Attribute |
+      | --- | --- | --- | --- | --- | --- |
+      | \`environment\` | string | The sentry environment. EXAMPLE VALUES: production | yes | yes | sentry.environment |
+      | \`metric.name\` | string | The name of the metric. EXAMPLE VALUES: http.request.duration | yes |  |  |
+      | \`metric.type\` | string | The type of the metric (counter, gauge, distribution). EXAMPLE VALUES: distribution | yes |  |  |
+      | \`metric.unit\` | string | The unit of the metric value. EXAMPLE VALUES: millisecond | yes |  |  |
+      | \`observed_timestamp\` | number | When Sentry received the metric sample.  | yes |  |  |
+      | \`project\` | string | The name of the project. In some pages of sentry.io, you can also filter on project using a dropdown.  | yes |  |  |
+      | \`timestamp_precise\` | number | When the metric sample was recorded, with nanosecond precision.  | yes |  |  |
+      | \`value\` | number | The recorded value of the metric sample.  | yes |  |  |
+      | \`fleet.deployment_region\` | string | The region the server is scheduled to move to in the next fleet deployment. It is not where the server runs now. EXAMPLE VALUES: eu-west-1 | yes |  |  |
+      | \`fleet.region\` | string | The region the server is currently running in. EXAMPLE VALUES: us-east-1 | yes |  |  |
+      | \`http.method\` | string | The HTTP method used. EXAMPLE VALUES: GET | yes | yes | http.request.method |
+      | \`payload_size\` | number |  | yes |  |  |
+      | \`transaction\` | string | The sentry transaction (segment name). EXAMPLE VALUES: GET / | yes | yes | sentry.segment.name |
 
-      - Dataset: metrics (API item type: tracemetrics)
-      - Project: cloudflare-mcp (ID 4509109104082945)
-      - Query: payment provider
-      - Period: 7d
-      - Limit: 50
-      "
+      ## Response Notes
+
+      - Use a key with \`search_metrics\` to query its values."
     `);
+
+    expect(mockGenerateText).not.toHaveBeenCalled();
+  });
+
+  it("says how many are shown when the list is cut off", async () => {
+    const output = await search({ dataset: "spans", limit: 5 });
+
+    expect(output).toContain("- Showing 5 of 37.");
+  });
+
+  it("lists recommended keys first", async () => {
+    const attribute = (key: string, source: "sentry" | "user" = "sentry") => ({
+      key,
+      name: key,
+      attributeType: "string",
+      attributeSource: { source_type: source },
+    });
+    mswServer.use(
+      http.get(ATTRIBUTES_URL, () =>
+        HttpResponse.json([
+          attribute("app.tier", "user"),
+          attribute("environment"),
+          attribute("http.method"),
+          attribute("span.op"),
+          attribute("browser.name"),
+          attribute("project"),
+        ]),
+      ),
+    );
+
+    const output = await search({ dataset: "spans" });
+    const keys = [...output.matchAll(/^\| `([^`]+)` \|/gm)].map(
+      ([, key]) => key,
+    );
+
+    expect(keys).toEqual([
+      // Search events' fields: environment is a common field, http.method a
+      // span field, and project and span.op are recommended.
+      "environment",
+      "http.method",
+      "project",
+      "span.op",
+      // Everything else
+      "app.tier",
+      "browser.name",
+    ]);
   });
 
   it("treats a blank query as no query", async () => {
-    expect(await search({ query: "   ", dataset: "logs" }))
-      .toMatchInlineSnapshot(`
-        "# Attribute Search in **sentry-mcp-evals**
-
-        Not implemented yet. Parsed inputs:
-
-        - Dataset: logs (API item type: logs)
-        - Project: all projects
-        - Query: none (list all attributes)
-        - Period: not set
-        - Limit: 10
-        "
-      `);
+    const output = await search({ query: "   ", dataset: "logs" });
+
+    expect(output).toMatch(/^# Log Attributes in \*\*sentry-mcp-evals\*\*/);
+    expect(mockGenerateText).not.toHaveBeenCalled();
+  });
+
+  it("lists attributes with a note when no AI provider is set up", async () => {
+    vi.stubEnv("OPENAI_API_KEY", "");
+
+    const output = await search({
+      query: "HTTP status code",
+      dataset: "spans",
+    });
+
+    expect(output).toMatch(
+      /^Matching by `query` needs an AI provider, so these are the dataset's attributes instead./,
+    );
+    expect(output).toContain("# Span Attributes in **sentry-mcp-evals**");
+    expect(mockGenerateText).not.toHaveBeenCalled();
+  });
+
+  // This isn't quite a case yet but i'm leaving it here so we don't forget about this case
+  it("lists attributes with a note when the AI provider fails", async () => {
+    const output = await search({
+      query: "HTTP status code",
+      dataset: "spans",
+    });
+
+    expect(output).toMatch(
+      /^Matching by `query` is not available yet. Here are all of the dataset's attributes instead./,
+    );
+    expect(output).toContain("# Span Attributes in **sentry-mcp-evals**");
+  });
+
+  it("passes the project ID, period, and context flag to the attributes API", async () => {
+    let requestUrl: URL | undefined;
+    mswServer.use(
+      http.get(ATTRIBUTES_URL, ({ request }) => {
+        requestUrl = new URL(request.url);
+        return HttpResponse.json([]);
+      }),
+    );
+
+    const output = await search({
+      projectSlug: "cloudflare-mcp",
+      query: "payment provider",
+      dataset: "metrics",
+      period: "7d",
+    });
+
+    expect(Object.fromEntries(requestUrl?.searchParams ?? [])).toEqual({
+      itemType: "tracemetrics",
+      project: "4509109104082945",
+      statsPeriod: "7d",
+      expand: "context",
+    });
+    expect(output).toContain("No metric attributes were seen in the last 7d.");
+    // Nothing to match, so the agent isn't called.
+    expect(mockGenerateText).not.toHaveBeenCalled();
+  });
+
+  it("surfaces a 404 from the attributes API", async () => {
+    mswServer.use(
+      http.get(ATTRIBUTES_URL, () =>
+        HttpResponse.json({ detail: "Not found" }, { status: 404 }),
+      ),
+    );
+
+    await expect(search({ dataset: "spans" })).rejects.toThrow(
+      ApiNotFoundError,
+    );
   });
 
   it("rejects datasets other than spans, logs, and metrics", () => {

packages/mcp-core/src/tools/catalog/search-attributes.ts

@@ -19,33 +19,26 @@ export default defineTool({
   includeInSkillDefinitions: false,
   name: "search_attributes",
   title: "Search Attributes",
-  // TODO: Edit this description, figure out what we want it to do
+  // TODO: Edit this description to incoporate searching
   description: [
-    "Find span, log, or metric attribute keys in Sentry: which attributes exist, their types, short descriptions, and whether they're deprecated or follow semantic conventions.",
+    "List span, log, or metric attribute keys in Sentry: which attributes exist, their types, short descriptions, examples, and whether they're deprecated.",
     "",
-    // TODO: put this in after "Use before search_traces, search_logs, or search_metrics"
-    "Use when you don't know the exact attribute key, or to check whether an attribute is being sent at all.",
+    "Use before search_traces, search_logs, or search_metrics when you don't know the exact attribute key, or to check whether an attribute is still being sent.",
     "",
-    "`query` is natural language (preferred) or part of an attribute name; a configured agent turns it into an attribute-name search.",
+    "Lists the dataset's attributes seen in `period`, with Sentry's common search fields first. Searching by `query` is not available yet, so a `query` returns the same list.",
     "",
     "NOT for attribute values or event data (use search_traces, search_logs, or search_metrics), or issue tag values (use get_issue_tag_values).",
     "",
     "<examples>",
-    "search_attributes(organizationSlug='my-org', query='which attribute holds the HTTP status code, so I can group failed requests by it', dataset='spans')",
-    "search_attributes(organizationSlug='my-org', query='database query text', dataset='spans')",
-    "search_attributes(organizationSlug='my-org', query='gen_ai.usage', dataset='spans', limit=50)",
-    "search_attributes(organizationSlug='my-org', query='are we still sending the deprecated HTTP method attribute, and what replaced it', dataset='spans')",
-    "search_attributes(organizationSlug='my-org', query='user.id', dataset='spans', period='24h')",
-    "search_attributes(organizationSlug='my-org', projectSlug='backend', query='which attribute carries the request ID that ties log lines back to a trace', dataset='logs')",
-    "search_attributes(organizationSlug='my-org', query='log level', dataset='logs')",
-    "search_attributes(organizationSlug='my-org', projectSlug='frontend', query='which attributes identify payment provider and region, to break down revenue by them', dataset='metrics', period='7d')",
+    "search_attributes(organizationSlug='my-org', dataset='spans')",
+    "search_attributes(organizationSlug='my-org', projectSlug='backend', dataset='logs')",
+    "search_attributes(organizationSlug='my-org', dataset='metrics', period='7d')",
     "</examples>",
     "",
     "<hints>",
     "- name/otherName notation means <organizationSlug>/<projectSlug>; parse it directly, don't call find_organizations/find_projects.",
     "- Infer dataset from the question: spans for requests, latency, DB, or AI calls; logs for log lines; metrics for counters, gauges, and distributions.",
-    "- Omit `query` to list all attributes in the dataset.",
-    "- Pass a returned key to the matching search tool to query its values.",
+    "- Show the user each returned attribute's type, description, examples, deprecation status and if they were added by Sentry. Don't reduce the result to a list of names.",
     "</hints>",
   ].join("\n"),
   inputSchema: {
@@ -56,21 +49,23 @@ export default defineTool({
       .trim()
       .optional()
       .describe(
-        "What attribute to find, in natural language or as part of an attribute name. Omit to list attributes in the dataset.",
+        "Not used yet. Searching by query is not available, so the dataset's attributes are listed either way. Omit it.",
       ),
     period: ParamPeriod.optional(),
     regionUrl: ParamRegionUrl.nullable().default(null),
     limit: z
       .number()
       .min(1)
       .max(100)
-      .default(10)
-      .describe("Maximum number of attributes to return (1-100)"),
+      .optional()
+      .describe(
+        "Maximum number of attributes to list (1-100). Defaults to 50.",
+      ),
     dataset: z
       .enum(SEARCH_ATTRIBUTES_DATASETS)
       .optional()
       .describe(
-        "Which dataset's attributes to search: spans, logs, or metrics. Infer it from the question; ask the user to clarify if it cannot be inferred.",
+        "Which dataset's attributes to list: spans, logs, or metrics. Infer it from the question; ask the user to clarify if it cannot be inferred.",
       ),
   },
   async handler(params, context: ServerContext) {