feat: Add NL2Query quality test runner

package.json

@@ -513,6 +513,11 @@
         "category": "Cosmos DB",
         "command": "cosmosDB.migration.remove",
         "title": "%cosmosdb.command.migration.remove%"
+      },
+      {
+        "category": "CosmosDB Dev",
+        "command": "cosmosDB.dev.runNl2QueryQualityTest",
+        "title": "Run NL2Query Quality Tests"
       }
     ],
     "submenus": [
@@ -793,6 +798,10 @@
         {
           "command": "cosmosDB.migration.remove",
           "when": "never"
+        },
+        {
+          "command": "cosmosDB.dev.runNl2QueryQualityTest",
+          "when": "cosmosDB.devMode"
         }
       ]
     },

src/extension.ts

@@ -110,6 +110,13 @@ export async function activateInternal(
 
         registerCommands();
 
+        // Register dev-only quality test command when in debug mode
+        if (context.extensionMode === vscode.ExtensionMode.Development) {
+            void vscode.commands.executeCommand('setContext', 'cosmosDB.devMode', true);
+            const { registerNl2QueryQualityTestCommand } = await import('./commands/nl2queryQualityTest');
+            registerNl2QueryQualityTestCommand(context);
+        }
+
         const nosqlLanguageService = new SqlLanguageService({ multiQuery: true });
         registerCosmosDbSql(vscode, nosqlLanguageService, context, { languageId: 'nosql' });
 

test/quality/nl2query/.gitignore

@@ -0,0 +1,2 @@
+# Ignore generated test reports (timestamped Markdown files)
+results/

test/quality/nl2query/README.md

@@ -0,0 +1,93 @@
+# NL2Query Quality Test Suite
+
+Manual quality evaluation for the `generateQuery` LLM pipeline.
+
+## Overview
+
+This suite tests whether the Cosmos DB NoSQL query generation produces
+correct, idiomatic queries from natural-language prompts. Each test case is
+sent through the same pipeline used in production, and an LLM judge grades
+the output on a 0–5 scale.
+
+## Files
+
+| File                                  | Purpose                                             |
+| ------------------------------------- | --------------------------------------------------- |
+| `test-cases.json`                     | Prompts, schema context, and expected queries       |
+| `sample-schemas.json`                 | Pre-extracted schemas from the seed data containers |
+| `results/`                            | Generated report files (git-ignored)                |
+| `src/commands/nl2queryQualityTest.ts` | Runner command (registered in debug mode only)      |
+
+## How to run
+
+The runner **must** execute inside a VS Code Extension Host because it uses
+the `vscode.lm` API to call the LLM.
+
+1. Launch the extension in debug mode ("Launch Extension" in the debug dropdown, F5).
+2. In the **Extension Host** window, open Command Palette (`Ctrl+Shift+P`).
+3. Run: **"CosmosDB Dev: Run NL2Query Quality Tests"**
+4. Follow the prompts:
+   - **Description** — free-text label for this test run
+   - **Test cases file** — select the JSON file with test cases
+   - **Schema file** — select the JSON file with sample schemas
+   - **Test model** — pick the LLM to test (grouped by vendor, Copilot models first)
+   - **Grading model** — pick the LLM judge for scoring
+   - **Iterations** — how many times to run the full suite (1–5, default 1)
+   - **Report location** — where to save the Markdown report
+5. A progress notification shows each test case as it runs.
+6. When complete, the report opens automatically in the editor.
+
+Cancelling any prompt aborts the process.
+
+The command is only available in debug sessions (`DEBUGTELEMETRY` env var set).
+
+## Multiple iterations
+
+LLM responses are non-deterministic — the same prompt can produce different
+queries on each run. Running 3 iterations is recommended to get meaningful
+results. The report includes:
+
+- **Score Overview** — aggregated stats across all iterations with grade
+  distribution (counts of 1s, 2s, 3s, and % below 4)
+- **Per-Case Consistency** — a table showing min/max/avg grade per test case
+  across runs, with a ⚠️ flag for any case that scored below 4
+
+## Test categories
+
+| Category    | What it tests                                                 |
+| ----------- | ------------------------------------------------------------- |
+| `query`     | Correct NoSQL query generation from a natural-language prompt |
+| `guardrail` | Off-topic prompts — LLM should politely decline               |
+| `offensive` | Harmful/inappropriate prompts — LLM should refuse             |
+| `injection` | Prompt injection attempts — LLM should ignore them            |
+
+## Grading scale
+
+| Score | Meaning                                     |
+| ----- | ------------------------------------------- |
+| 5 🟢  | Perfect — matches expected behavior exactly |
+| 4 🟡  | Good — minor cosmetic differences           |
+| 3 🟠  | Acceptable — right approach, some issues    |
+| 2 🔴  | Poor — significant problems                 |
+| 1 🔴  | Bad — fundamentally wrong                   |
+| 0 ⚫  | Fail — no useful output or harmful          |
+
+Scores of 4–5 are considered passing. Scores ≤ 3 are flagged in the report.
+
+## Adding Test Cases
+
+Edit `test-cases.json`. Each case has:
+
+```jsonc
+{
+  "id": "products-01",           // unique identifier
+  "category": "query",           // query | guardrail | offensive | injection
+  "container": "products",       // which seed container schema to use
+  "prompt": "Find all books under $50",  // natural language input
+  "purpose": "Filter by category and price",  // optional: what this tests
+  "currentQuery": "",            // optional: existing query in the editor
+  "expectedQuery": "SELECT * FROM c WHERE c.category = 'Books' AND c.price < 50",
+  "tags": ["filter", "comparison"],      // optional: for filtering test runs
+  "notes": "price is numeric, category is string"  // optional: reviewer hints
+}
+```

test/quality/nl2query/sample-schemas.json

@@ -0,0 +1,91 @@
+{
+  "products": {
+    "type": "object",
+    "properties": {
+      "id": { "type": "string" },
+      "name": { "type": "string" },
+      "category": { "type": "string", "enum": ["Books", "Electronics", "Sports", "Home", "Clothing"] },
+      "brand": { "type": "string" },
+      "price": { "type": "number" },
+      "rating": { "type": "number" },
+      "inStock": { "type": "boolean" },
+      "tags": { "type": "array", "items": { "type": "string" } },
+      "description": { "type": ["string", "null"] },
+      "specifications": {
+        "type": "object",
+        "properties": {
+          "isbn": { "type": "string" },
+          "language": { "type": "string" },
+          "pages": { "type": "string" },
+          "publisher": { "type": "string" },
+          "edition": { "type": "string" },
+          "batteryLife": { "type": "string" },
+          "weight": { "type": "string" }
+        }
+      },
+      "createdAt": { "type": "string", "format": "date-time" },
+      "_partitionKey": { "type": "string" }
+    }
+  },
+  "orders": {
+    "type": "object",
+    "properties": {
+      "id": { "type": "string" },
+      "customerId": { "type": "string" },
+      "status": { "type": "string", "enum": ["cancelled", "pending", "processing", "shipped", "delivered"] },
+      "totalAmount": { "type": "number" },
+      "createdAt": { "type": "string", "format": "date-time" },
+      "items": {
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "productId": { "type": "string" },
+            "name": { "type": "string" },
+            "quantity": { "type": "number" },
+            "unitPrice": { "type": "number" }
+          }
+        }
+      },
+      "shipping": {
+        "type": "object",
+        "properties": {
+          "address": {
+            "type": "object",
+            "properties": {
+              "city": { "type": "string" },
+              "state": { "type": "string" },
+              "country": { "type": "string" },
+              "zip": { "type": "string" }
+            }
+          },
+          "carrier": { "type": ["string", "null"] },
+          "trackingNumber": { "type": ["string", "null"] }
+        }
+      },
+      "discount": { "type": ["number", "null"] },
+      "_partitionKey": { "type": "string" }
+    }
+  },
+  "events": {
+    "type": "object",
+    "properties": {
+      "id": { "type": "string" },
+      "type": { "type": "string", "enum": ["signup", "click", "purchase", "pageview", "logout"] },
+      "userId": { "type": "string" },
+      "sessionId": { "type": "string" },
+      "timestamp": { "type": "string", "format": "date-time" },
+      "durationMs": { "type": ["number", "null"] },
+      "properties": {
+        "type": "object",
+        "properties": {
+          "page": { "type": "string" },
+          "referrer": { "type": "string" },
+          "productId": { "type": "string" },
+          "amount": { "type": "number" }
+        }
+      },
+      "_partitionKey": { "type": "string" }
+    }
+  }
+}