From 6edb6f7203ac67264261ff37fb8e8d5cd62caa6b Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 14:25:16 -0400
Subject: [PATCH 01/10] Record how each benchmark point was measured

A chart line implies every point was measured the same way, but this series
spans single-iteration laptop runs, several Node versions and two runner
images -- shifts that move the numbers further than most real regressions do.
The result files knew all of this; the history step threw it away.

history.json now carries a schema version, the runner behind each point, and a
quality summary flagging the points that are not comparable with their
neighbors: too few iterations, a foreign platform, or an isolated spike. On
the current data that marks 37 laptop runs and the five ~9x spikes, and leaves
genuine step changes alone.

Rebuilding the history also stopped spawning a git process per result file,
which was most of its cost at 453 files: 67s and 198MB, now 3s and 31MB.
---
 packages/benchmark/src/generate-history.ts    | 198 ++++++++++++++----
 packages/benchmark/src/utils.ts               |  65 +++++-
 .../benchmark/test/generate-history.test.ts   | 161 ++++++++++++++
 3 files changed, 385 insertions(+), 39 deletions(-)
 create mode 100644 packages/benchmark/test/generate-history.test.ts

diff --git a/packages/benchmark/src/generate-history.ts b/packages/benchmark/src/generate-history.ts
index 8d3867cfae..bf530bdcce 100644
--- a/packages/benchmark/src/generate-history.ts
+++ b/packages/benchmark/src/generate-history.ts
@@ -1,9 +1,40 @@
 /* eslint-disable no-console */
-import { execSync } from "node:child_process";
 import { readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
-import type { BenchmarkResult, RuntimeStats, SpecBenchmarkResult } from "./types.js";
+import { median } from "./statistics.js";
+import type { BenchmarkResult, RunnerInfo, RuntimeStats, SpecBenchmarkResult } from "./types.js";
+import { DEFAULT_BRANCH, listResultBlobs, readBlobs } from "./utils.js";
+
+/**
+ * Schema version of `history.json`.
+ *
+ * Result files gained fields over time with no way to tell which ones a given
+ * file predates, so consumers had to guess. This lets them check instead.
+ *
+ * 1. Metrics only.
+ * 2. Adds `runner` and `quality` to every entry.
+ */
+export const HISTORY_VERSION = 2;
+
+/** Why a point may not be comparable with the ones around it. */
+export type EntryFlag =
+  /** Far enough from its neighbors to be a contended runner, not a change. */
+  | "outlier"
+  /** Averaged over too few iterations to separate signal from noise. */
+  | "low-iterations"
+  /** Measured on a different platform than the rest of the series. */
+  | "foreign-runner";
+
+/** How much weight a single point deserves. */
+export interface EntryQuality {
+  /** Measured iterations, lowest across the specs in this run. */
+  iterations: number;
+  /** Spread of `total`, highest across the specs. Null if the run predates it. */
+  cv: number | null;
+  /** Empty when the point is directly comparable with its neighbors. */
+  flags: EntryFlag[];
+}
 
 /** A single entry in the aggregated history. */
 export interface HistoryEntry {
@@ -13,10 +44,20 @@ export interface HistoryEntry {
   metrics: Record<string, number>;
   /** Per-spec metrics (spec name → flat metrics) */
   specMetrics: Record<string, Record<string, number>>;
+  /**
+   * Machine the run was measured on.
+   *
+   * Node and runner image changes move these numbers by more than most real
+   * regressions do, so a point is only meaningful alongside its environment.
+   */
+  runner?: RunnerInfo;
+  quality: EntryQuality;
 }
 
 /** The full history.json structure. */
 export interface HistoryData {
+  /** See {@link HISTORY_VERSION}. */
+  version: number;
   generated: string;
   labels: string[];
   /** All spec names found across all entries */
@@ -74,15 +115,98 @@ function averageAcrossSpecs(specs: Record<string, SpecBenchmarkResult>): Record<
   return avg;
 }
 
-function gitShow(path: string): string | null {
-  try {
-    return execSync(`git show benchmark-data:${path}`, {
-      encoding: "utf-8",
-      maxBuffer: 50_000_000,
-    });
-  } catch {
-    return null;
+/** Runs averaged over fewer iterations than this are too coarse to compare. */
+const MIN_TRUSTWORTHY_ITERATIONS = 5;
+
+/** Neighbors weighed when deciding whether a point is an isolated spike. */
+const OUTLIER_WINDOW = 11;
+
+/**
+ * How far from its neighbors a point has to sit to be called a spike.
+ *
+ * Deliberately far beyond any plausible regression: the point of this flag is
+ * to catch a contended runner, not to second-guess real changes. A shift that
+ * persists moves the neighboring median with it and is never flagged.
+ */
+const OUTLIER_RATIO = 2;
+
+/** Summarize how much weight a run's numbers deserve. */
+function measureQuality(specs: Record<string, SpecBenchmarkResult>): EntryQuality {
+  const results = Object.values(specs);
+  const spreads = results
+    .map((spec) => spec.variability?.total.cv)
+    .filter((cv): cv is number => cv !== undefined);
+
+  return {
+    // The weakest spec sets the confidence for the run as a whole.
+    iterations: results.length > 0 ? Math.min(...results.map((spec) => spec.iterations ?? 0)) : 0,
+    cv: spreads.length > 0 ? Math.max(...spreads) : null,
+    flags: [],
+  };
+}
+
+/** Platform identity, ignoring the kernel build that changes constantly. */
+function platformOf(runner: RunnerInfo | undefined): string | null {
+  if (!runner) return null;
+  return `${runner.os.split("-")[0]}-${runner.arch}`;
+}
+
+/** The platform most of the series was measured on. */
+function dominantPlatform(entries: HistoryEntry[]): string | null {
+  const counts = new Map<string, number>();
+  for (const entry of entries) {
+    const platform = platformOf(entry.runner);
+    if (platform) counts.set(platform, (counts.get(platform) ?? 0) + 1);
+  }
+  let best: string | null = null;
+  for (const [platform, count] of counts) {
+    if (best === null || count > counts.get(best)!) best = platform;
   }
+  return best;
+}
+
+/**
+ * Mark points that cannot be read as part of the same series.
+ *
+ * A chart line implies every point was measured the same way. This history
+ * spans laptop runs, single-iteration runs and several runner images, so the
+ * entries that break that assumption are called out rather than silently
+ * plotted alongside the rest.
+ */
+function flagEntries(entries: HistoryEntry[]): void {
+  const expectedPlatform = dominantPlatform(entries);
+  const totals = entries.map((entry) => entry.metrics["total"] ?? null);
+  const reach = (OUTLIER_WINDOW - 1) / 2;
+
+  entries.forEach((entry, index) => {
+    const flags = entry.quality.flags;
+
+    if (entry.quality.iterations > 0 && entry.quality.iterations < MIN_TRUSTWORTHY_ITERATIONS) {
+      flags.push("low-iterations");
+    }
+
+    const platform = platformOf(entry.runner);
+    if (platform && expectedPlatform && platform !== expectedPlatform) {
+      flags.push("foreign-runner");
+    }
+
+    const value = totals[index];
+    if (value === null || value <= 0) return;
+
+    const neighbors: number[] = [];
+    for (let i = index - reach; i <= index + reach; i++) {
+      if (i === index || i < 0 || i >= totals.length) continue;
+      const neighbor = totals[i];
+      if (neighbor !== null && neighbor > 0) neighbors.push(neighbor);
+    }
+    // Too few neighbors to tell a spike from the start of a trend.
+    if (neighbors.length < 4) return;
+
+    const expected = median(neighbors);
+    if (expected > 0 && (value > expected * OUTLIER_RATIO || value * OUTLIER_RATIO < expected)) {
+      flags.push("outlier");
+    }
+  });
 }
 
 interface ResultFile {
@@ -90,43 +214,36 @@ interface ResultFile {
   content: string;
 }
 
-function readFromDirectory(dir: string): ResultFile[] {
+/**
+ * Yield result files one at a time.
+ *
+ * Every result file carries its raw per-iteration stats, which is ~95% of its
+ * size and of no use here, so holding all of them in memory at once costs
+ * hundreds of megabytes for a history this long.
+ */
+function* readFromDirectory(dir: string): Generator<ResultFile> {
   const files = readdirSync(dir).filter(
     (f) => f.endsWith(".json") && f !== "latest.json" && f !== "history.json",
   );
   console.error(`Found ${files.length} result files in ${dir}`);
-  const results: ResultFile[] = [];
   for (const file of files) {
     try {
-      const content = readFileSync(join(dir, file), "utf-8");
-      results.push({ name: file, content });
+      yield { name: file, content: readFileSync(join(dir, file), "utf-8") };
     } catch {
       // skip unreadable files
     }
   }
-  return results;
-}
-
-function readFromGitBranch(): ResultFile[] {
-  const fileList = execSync("git ls-tree --name-only benchmark-data -- results/", {
-    encoding: "utf-8",
-  })
-    .trim()
-    .split("\n")
-    .filter(
-      (f) => f.endsWith(".json") && !f.includes("latest.json") && !f.includes("history.json"),
-    );
-  console.error(`Found ${fileList.length} result files on benchmark-data branch`);
-  const results: ResultFile[] = [];
-  for (const file of fileList) {
-    const content = gitShow(file);
-    if (content) results.push({ name: file, content });
-  }
-  return results;
+}
+
+/** Yield result files from the data branch without a subprocess per file. */
+function* readFromGitBranch(branch: string): Generator<ResultFile> {
+  const blobs = listResultBlobs(branch);
+  console.error(`Found ${blobs.length} result files on ${branch} branch`);
+  yield* readBlobs(blobs);
 }
 
 /** Generate a HistoryData object from a list of result files. */
-export function buildHistory(resultFiles: ResultFile[]): HistoryData {
+export function buildHistory(resultFiles: Iterable<ResultFile>): HistoryData {
   const entries: HistoryEntry[] = [];
   const allSpecNames = new Set<string>();
 
@@ -146,6 +263,8 @@ export function buildHistory(resultFiles: ResultFile[]): HistoryData {
         timestamp: result.timestamp,
         metrics,
         specMetrics,
+        runner: result.runner,
+        quality: measureQuality(result.specs),
       });
     } catch (e: any) {
       console.error(`Failed to parse ${name}: ${e.message}`);
@@ -153,6 +272,7 @@ export function buildHistory(resultFiles: ResultFile[]): HistoryData {
   }
 
   entries.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+  flagEntries(entries);
 
   const allLabels = new Set<string>();
   for (const entry of entries) {
@@ -162,6 +282,7 @@ export function buildHistory(resultFiles: ResultFile[]): HistoryData {
   }
 
   return {
+    version: HISTORY_VERSION,
     generated: new Date().toISOString(),
     labels: [...allLabels].sort(),
     specNames: [...allSpecNames].sort(),
@@ -172,12 +293,17 @@ export function buildHistory(resultFiles: ResultFile[]): HistoryData {
 export interface GenerateHistoryOptions {
   /** Read results from a directory instead of the benchmark-data git branch. */
   dir?: string;
+  /** Data branch to read results from when `dir` is not given. */
+  branch?: string;
 }
 
 /** Generate history data from result files. */
 export function generateHistory(options: GenerateHistoryOptions = {}): HistoryData {
-  const resultFiles = options.dir ? readFromDirectory(options.dir) : readFromGitBranch();
-  return buildHistory(resultFiles);
+  return buildHistory(
+    options.dir
+      ? readFromDirectory(options.dir)
+      : readFromGitBranch(options.branch ?? DEFAULT_BRANCH),
+  );
 }
 
 /** CLI entry point for generate-history. */
diff --git a/packages/benchmark/src/utils.ts b/packages/benchmark/src/utils.ts
index 43fc21743a..24393a35d6 100644
--- a/packages/benchmark/src/utils.ts
+++ b/packages/benchmark/src/utils.ts
@@ -36,10 +36,13 @@ export function execOk(cmd: string, options?: { cwd?: string }): boolean {
 }
 
 /** List existing result SHAs on the benchmark-data branch. */
-export function listExistingResults(branch: string = DEFAULT_BRANCH): Set<string> {
+export function listExistingResults(
+  branch: string = DEFAULT_BRANCH,
+  dir: string = "results",
+): Set<string> {
   const existing = new Set<string>();
   try {
-    const fileList = git(`ls-tree --name-only origin/${branch} -- results/`);
+    const fileList = git(`ls-tree --name-only origin/${branch} -- ${dir}/`);
     for (const line of fileList.split("\n")) {
       const trimmed = line.trim();
       if (
@@ -47,7 +50,7 @@ export function listExistingResults(branch: string = DEFAULT_BRANCH): Set<string
         !trimmed.includes("latest.json") &&
         !trimmed.includes("history.json")
       ) {
-        const sha = trimmed.replace("results/", "").replace(".json", "");
+        const sha = trimmed.replace(`${dir}/`, "").replace(".json", "");
         existing.add(sha);
       }
     }
@@ -56,3 +59,59 @@ export function listExistingResults(branch: string = DEFAULT_BRANCH): Set<string
   }
   return existing;
 }
+
+/** List result blob paths on a data branch, newest-first ordering not guaranteed. */
+export function listResultBlobs(
+  branch: string = DEFAULT_BRANCH,
+  dir: string = "results",
+): string[] {
+  // `:(top)` anchors the pathspec at the repo root so this works from any cwd.
+  return git(`ls-tree --name-only --full-tree ${branch} -- ":(top)${dir}/"`)
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(
+      (f) => f.endsWith(".json") && !f.includes("latest.json") && !f.includes("history.json"),
+    );
+}
+
+/** Blobs read per `git cat-file` call, balancing subprocess count against memory. */
+const BLOB_BATCH_SIZE = 50;
+
+/**
+ * Read blobs from a git branch in batches.
+ *
+ * One `git show` per file costs a process spawn per result, which dominates
+ * the history rebuild once the series is a few hundred runs long. `cat-file
+ * --batch` reads a whole group in one go, and batching keeps the decoded
+ * output bounded instead of materializing the entire branch at once.
+ */
+export function* readBlobs(
+  paths: string[],
+  branch: string = DEFAULT_BRANCH,
+): Generator<{ name: string; content: string }> {
+  for (let start = 0; start < paths.length; start += BLOB_BATCH_SIZE) {
+    const batch = paths.slice(start, start + BLOB_BATCH_SIZE);
+    // Kept as a buffer: `cat-file` reports blob sizes in bytes, which only
+    // line up with string offsets while every blob happens to be ASCII.
+    const stdout: Buffer = execSync(`git cat-file --batch`, {
+      input: batch.map((path) => `${branch}:${path}`).join("\n") + "\n",
+      maxBuffer: 500_000_000,
+    });
+
+    // Each blob arrives as `<sha> blob <size>\n<size bytes>\n`.
+    let offset = 0;
+    for (let i = 0; i < batch.length; i++) {
+      const headerEnd = stdout.indexOf(0x0a, offset);
+      if (headerEnd === -1) return;
+      const size = Number(stdout.toString("utf-8", offset, headerEnd).split(" ")[2]);
+      if (!Number.isFinite(size)) {
+        // Missing object: git emits `<name> missing` with no body.
+        offset = headerEnd + 1;
+        continue;
+      }
+      const bodyStart = headerEnd + 1;
+      yield { name: batch[i], content: stdout.toString("utf-8", bodyStart, bodyStart + size) };
+      offset = bodyStart + size + 1;
+    }
+  }
+}
diff --git a/packages/benchmark/test/generate-history.test.ts b/packages/benchmark/test/generate-history.test.ts
new file mode 100644
index 0000000000..2e9d17cfc0
--- /dev/null
+++ b/packages/benchmark/test/generate-history.test.ts
@@ -0,0 +1,161 @@
+import { expect, it } from "vitest";
+import { buildHistory, HISTORY_VERSION } from "../src/generate-history.js";
+import type { BenchmarkResult, RunnerInfo, SpecBenchmarkResult } from "../src/types.js";
+
+interface SpecOptions {
+  total: number;
+  iterations?: number;
+  cv?: number;
+}
+
+function spec({ total, iterations = 25, cv }: SpecOptions): SpecBenchmarkResult {
+  return {
+    name: "sample",
+    iterations,
+    rawIterations: [],
+    stats: {
+      complexity: { createdTypes: 0, finishedTypes: 0 },
+      runtime: {
+        total,
+        loader: 0,
+        resolver: 0,
+        checker: 0,
+        validation: { total: 0, validators: {} },
+        linter: { total: 0, rules: {} },
+        emit: { total: 0, emitters: {} },
+      },
+    },
+    ...(cv === undefined
+      ? {}
+      : {
+          variability: {
+            total: {
+              mean: total,
+              median: total,
+              stdDev: total * cv,
+              cv,
+              min: total,
+              max: total,
+              sampleCount: iterations,
+            },
+          },
+        }),
+  } as SpecBenchmarkResult;
+}
+
+const LINUX: RunnerInfo = { os: "linux-6.11.0", nodeVersion: "v24.15.0", arch: "x64" };
+
+interface RunOptions extends SpecOptions {
+  day: number;
+  runner?: RunnerInfo;
+}
+
+function run({ day, runner = LINUX, ...specOptions }: RunOptions): {
+  name: string;
+  content: string;
+} {
+  const result: BenchmarkResult = {
+    commit: `commit-${day}`,
+    timestamp: new Date(Date.UTC(2026, 0, day)).toISOString(),
+    runner,
+    specs: { sample: spec(specOptions) },
+  } as BenchmarkResult;
+  return { name: `commit-${day}.json`, content: JSON.stringify(result) };
+}
+
+/** A steady series long enough for the outlier window to have neighbors. */
+function steadySeries(count: number, total = 100) {
+  return Array.from({ length: count }, (_, i) => run({ day: i + 1, total }));
+}
+
+it("buildHistory stamps the schema version", () => {
+  const history = buildHistory(steadySeries(3));
+  expect(history.version).toBe(HISTORY_VERSION);
+});
+
+it("buildHistory carries the runner through to each entry", () => {
+  const history = buildHistory(steadySeries(3));
+  expect(history.entries[0].runner).toEqual(LINUX);
+});
+
+it("buildHistory records iteration count and spread per entry", () => {
+  const history = buildHistory([run({ day: 1, total: 100, iterations: 25, cv: 0.02 })]);
+  expect(history.entries[0].quality.iterations).toBe(25);
+  expect(history.entries[0].quality.cv).toBe(0.02);
+});
+
+it("buildHistory reports no spread for runs measured before it was recorded", () => {
+  const history = buildHistory([run({ day: 1, total: 100 })]);
+  expect(history.entries[0].quality.cv).toBeNull();
+});
+
+it("buildHistory leaves comparable entries unflagged", () => {
+  const history = buildHistory(steadySeries(11));
+  expect(history.entries.every((entry) => entry.quality.flags.length === 0)).toBe(true);
+});
+
+it("buildHistory flags runs averaged over too few iterations", () => {
+  const history = buildHistory([run({ day: 1, total: 100, iterations: 1 })]);
+  expect(history.entries[0].quality.flags).toContain("low-iterations");
+});
+
+it("buildHistory flags runs measured on a different platform than the rest", () => {
+  const files = steadySeries(10);
+  files.push(
+    run({
+      day: 11,
+      total: 100,
+      runner: { os: "darwin-25.4.0", nodeVersion: "v24.15.0", arch: "arm64" },
+    }),
+  );
+  const history = buildHistory(files);
+  expect(history.entries.at(-1)!.quality.flags).toContain("foreign-runner");
+  expect(history.entries[0].quality.flags).not.toContain("foreign-runner");
+});
+
+it("buildHistory ignores kernel version differences on the same platform", () => {
+  const files = steadySeries(10);
+  files.push(
+    run({
+      day: 11,
+      total: 100,
+      runner: { os: "linux-6.14.0", nodeVersion: "v24.17.0", arch: "x64" },
+    }),
+  );
+  const history = buildHistory(files);
+  expect(history.entries.at(-1)!.quality.flags).not.toContain("foreign-runner");
+});
+
+it("buildHistory flags an isolated spike", () => {
+  const files = steadySeries(11);
+  files[5] = run({ day: 6, total: 900 });
+  const history = buildHistory(files);
+  expect(history.entries[5].quality.flags).toContain("outlier");
+});
+
+it("buildHistory flags an isolated dip", () => {
+  const files = steadySeries(11);
+  files[5] = run({ day: 6, total: 10 });
+  const history = buildHistory(files);
+  expect(history.entries[5].quality.flags).toContain("outlier");
+});
+
+it("buildHistory does not flag a change that persists", () => {
+  const files = [
+    ...steadySeries(12),
+    ...Array.from({ length: 12 }, (_, i) => run({ day: i + 13, total: 300 })),
+  ];
+  const history = buildHistory(files);
+  const flagged = history.entries.filter((entry) => entry.quality.flags.includes("outlier"));
+  expect(flagged).toEqual([]);
+});
+
+it("buildHistory does not flag when there are too few neighbors to judge", () => {
+  const history = buildHistory([run({ day: 1, total: 100 }), run({ day: 2, total: 900 })]);
+  expect(history.entries[1].quality.flags).not.toContain("outlier");
+});
+
+it("buildHistory orders entries oldest first", () => {
+  const history = buildHistory([run({ day: 3, total: 100 }), run({ day: 1, total: 100 })]);
+  expect(history.entries.map((entry) => entry.commit)).toEqual(["commit-1", "commit-3"]);
+});

From 4473fa85dc3943ec34ba66fbc34925cd5824850d Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 14:25:16 -0400
Subject: [PATCH 02/10] Allow the whole benchmark series to be re-measured

Points measured with different iteration counts and noise gates cannot be
compared, so making the series uniform means measuring all of it again. That
was impossible: backfill skipped commits that already had results, wrote to a
hardcoded results/ directory regardless of --results-dir, and did not forward
the noise-gate flags, so a backfilled point was produced differently from a
live one.

    gh workflow run benchmark.yml \
      -f backfill_from=<oldest-sha> -f backfill_to=<sha> -f backfill_reset=true

--reset re-measures every commit in range and drops the stored results, and
backfill_to splits a long range into chunks that each fit in a job, so the
whole series can be rebuilt from Actions without a laptop in the loop.
---
 .github/workflows/benchmark-run.yml | 22 +++++++++++++
 .github/workflows/benchmark.yml     | 17 ++++++++++
 packages/benchmark/src/backfill.ts  | 48 ++++++++++++++++++++++++-----
 packages/benchmark/src/cli.ts       | 14 +++++++++
 4 files changed, 94 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/benchmark-run.yml b/.github/workflows/benchmark-run.yml
index 404de10518..485685e4d4 100644
--- a/.github/workflows/benchmark-run.yml
+++ b/.github/workflows/benchmark-run.yml
@@ -38,6 +38,21 @@ on:
         required: false
         type: string
         default: ""
+      backfill_to:
+        description: "Backfill up to this commit SHA, inclusive. Lets a long range be re-measured one chunk per run."
+        required: false
+        type: string
+        default: ""
+      backfill_reset:
+        description: "Re-measure commits that already have results, and drop the stored ones. Use on the first chunk only."
+        required: false
+        type: boolean
+        default: false
+      timeout-minutes:
+        description: "Job timeout. Re-measuring a long range needs far more than the 6h default."
+        required: false
+        type: number
+        default: 360
       branch:
         description: "Data branch to store results"
         required: false
@@ -51,6 +66,7 @@ jobs:
   benchmark:
     name: Run Benchmarks
     runs-on: ${{ inputs.runner }}
+    timeout-minutes: ${{ inputs.timeout-minutes }}
     env:
       TYPESPEC_VS_CI_BUILD: true
       TYPESPEC_SKIP_WEBSITE_BUILD: true
@@ -75,10 +91,16 @@ jobs:
         run: |
           node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js backfill \
             --from ${{ inputs.backfill_from }} \
+            ${{ inputs.backfill_to != '' && format('--to {0}', inputs.backfill_to) || '' }} \
             --specs-dir ${{ inputs.specs-dir }} \
+            --results-dir ${{ inputs.results-dir }} \
             --iterations ${{ inputs.iterations }} \
             --warmup ${{ inputs.warmup }} \
+            --noise-cv-threshold ${{ inputs.noise-cv }} \
+            --max-reruns 1 \
+            --rerun-iterations ${{ inputs.rerun-iterations }} \
             --branch ${{ inputs.branch }} \
+            ${{ inputs.backfill_reset && '--reset' || '' }} \
             --push
 
       - name: Run benchmarks
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 15d55ffba8..1d8bb27b61 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -10,6 +10,20 @@ on:
         description: "Backfill from: commit SHA or number of recent commits. Leave empty to run a normal benchmark."
         required: false
         type: string
+      backfill_to:
+        description: "Backfill up to this commit SHA, inclusive. Lets a long range be re-measured one chunk per run."
+        required: false
+        type: string
+      backfill_reset:
+        description: "Re-measure commits that already have results, and drop the stored ones. Use on the first chunk only."
+        required: false
+        type: boolean
+        default: false
+      timeout_minutes:
+        description: "Job timeout in minutes. Re-measuring a long range needs far more than the 6h default."
+        required: false
+        type: string
+        default: "360"
       branch:
         description: "Data branch to store results (default: benchmark-data)"
         required: false
@@ -40,6 +54,9 @@ jobs:
       rerun-iterations: 10
       runner: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }}
       backfill_from: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_from || '' }}
+      backfill_to: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_to || '' }}
+      backfill_reset: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_reset || false }}
+      timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && fromJSON(inputs.timeout_minutes) || 360 }}
       branch: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || 'benchmark-data' }}
     permissions:
       contents: write
diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index acbd946c4b..584b18c3c6 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -34,6 +34,22 @@ export interface BackfillOptions {
   specs?: string;
   /** Directory containing benchmark specs. Forwarded to the run command. */
   specsDir?: string;
+  /** Coefficient of variation above which a spec is re-run. Forwarded to the run command. */
+  noiseCvThreshold?: number;
+  /** Maximum re-runs of a noisy spec. Forwarded to the run command. */
+  maxReruns?: number;
+  /** Iterations per re-run of a noisy spec. Forwarded to the run command. */
+  rerunIterations?: number;
+  /** Directory on the data branch holding results. Defaults to `results`. */
+  resultsDir?: string;
+  /**
+   * Re-measure every commit in range, discarding the results already stored.
+   *
+   * Measurement settings have changed over the life of the series, so the only
+   * way to get points that can be compared with each other is to throw the old
+   * ones away and measure the whole range the same way.
+   */
+  reset?: boolean;
 }
 
 const DEFAULT_FROM = "100";
@@ -128,6 +144,8 @@ export function backfill(options: BackfillOptions = {}): void {
   const sourceBranch = options.sourceBranch ?? "main";
   const dataBranch = options.dataBranch ?? DEFAULT_BRANCH;
   const shouldPush = options.push ?? false;
+  const dataDirName = options.resultsDir ?? "results";
+  const reset = options.reset ?? false;
 
   // Build flags to forward to `cli.js run`
   const runFlags: string[] = [];
@@ -135,6 +153,15 @@ export function backfill(options: BackfillOptions = {}): void {
   if (options.warmup !== undefined) runFlags.push(`--warmup ${options.warmup}`);
   if (options.specs) runFlags.push(`--specs ${options.specs}`);
   if (options.specsDir) runFlags.push(`--specs-dir "${options.specsDir}"`);
+  // The noise gate is part of how a number is produced, so a backfilled point
+  // is only comparable with a live one if it was gated the same way.
+  if (options.noiseCvThreshold !== undefined) {
+    runFlags.push(`--noise-cv-threshold ${options.noiseCvThreshold}`);
+  }
+  if (options.maxReruns !== undefined) runFlags.push(`--max-reruns ${options.maxReruns}`);
+  if (options.rerunIterations !== undefined) {
+    runFlags.push(`--rerun-iterations ${options.rerunIterations}`);
+  }
   const runFlagsStr = runFlags.join(" ");
 
   const repoRoot = git("rev-parse --show-toplevel");
@@ -174,7 +201,7 @@ export function backfill(options: BackfillOptions = {}): void {
   if (gitSilent("fetch origin " + dataBranch)) {
     // fetched successfully
   }
-  const existingResults = listExistingResults(dataBranch);
+  const existingResults = reset ? new Set<string>() : listExistingResults(dataBranch, dataDirName);
 
   // Stash uncommitted changes
   let stashed = false;
@@ -306,26 +333,33 @@ export function backfill(options: BackfillOptions = {}): void {
     gitSilent("rm -rf . --quiet");
   }
 
-  mkdirSync("results", { recursive: true });
+  if (reset) {
+    console.log(`Clearing ${dataDirName}/ — every commit in range was re-measured.`);
+    rmSync(dataDirName, { recursive: true, force: true });
+  }
+
+  mkdirSync(dataDirName, { recursive: true });
   for (const file of newResults) {
-    copyFileSync(join(resultsDir, file), join("results", file));
+    copyFileSync(join(resultsDir, file), join(dataDirName, file));
   }
 
   // Update latest.json to the most recent result (by commit order, not lexicographic SHA)
   const resultShas = new Set(newResults.map((f) => f.replace(".json", "")));
   const latestSha = [...commits].reverse().find((sha) => resultShas.has(sha));
   if (latestSha) {
-    copyFileSync(join(resultsDir, `${latestSha}.json`), "results/latest.json");
+    copyFileSync(join(resultsDir, `${latestSha}.json`), join(dataDirName, "latest.json"));
   }
 
   // Generate aggregated history.json
-  const resultsPath = join(process.cwd(), "results");
+  const resultsPath = join(process.cwd(), dataDirName);
   const history = generateHistory({ dir: resultsPath });
   writeFileSync(join(resultsPath, "history.json"), JSON.stringify(history, null, 2));
   console.log("Generated history.json");
 
-  git("add results/");
-  const commitMsg = `benchmark: backfill results for ${succeeded} commits`;
+  git(`add ${dataDirName}/`);
+  const commitMsg = reset
+    ? `benchmark: re-measure ${succeeded} commits (${dataDirName})`
+    : `benchmark: backfill results for ${succeeded} commits`;
   gitSilent(`commit -m "${commitMsg}" --quiet`);
   console.log(`Results committed to ${dataBranch} branch.`);
 
diff --git a/packages/benchmark/src/cli.ts b/packages/benchmark/src/cli.ts
index 743c73184e..2ad2b33d6f 100644
--- a/packages/benchmark/src/cli.ts
+++ b/packages/benchmark/src/cli.ts
@@ -74,6 +74,13 @@ Backfill options:
   --iterations <n>      Number of measured iterations per spec (default: 5)
   --warmup <n>          Number of warmup iterations (default: 1)
   --specs <name,...>    Comma-separated list of specific specs to run
+  --results-dir <dir>   Directory on the data branch to store results/history (default: results)
+  --noise-cv-threshold <n>
+                        Rerun when total-runtime coefficient of variation is above this value
+  --max-reruns <n>      Max rerun cycles when noise gate triggers
+  --rerun-iterations <n>
+                        Extra measured iterations per rerun
+  --reset               Re-measure every commit in range, discarding stored results
 `);
 }
 
@@ -211,6 +218,13 @@ function backfillCommand(args: Record<string, string>): void {
     warmup: args["warmup"] ? parseInt(args["warmup"], 10) : undefined,
     specs: args["specs"],
     specsDir: args["specs-dir"],
+    noiseCvThreshold: args["noise-cv-threshold"]
+      ? parseFloat(args["noise-cv-threshold"])
+      : undefined,
+    maxReruns: args["max-reruns"] ? parseInt(args["max-reruns"], 10) : undefined,
+    rerunIterations: args["rerun-iterations"] ? parseInt(args["rerun-iterations"], 10) : undefined,
+    resultsDir: args["results-dir"],
+    reset: args["reset"] === "true",
   });
 }
 

From c54f804f75b45c1d4b55f5e06caa9c381142eaab Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 14:47:16 -0400
Subject: [PATCH 03/10] Let backfill run from a branch other than main

A CI checkout only creates the branch it was asked for, so `git log main`
fails on every branch except main -- including whichever branch a change to
the backfill is being tested on. Fall back to origin's copy, fetching it when
the checkout left it out.
---
 packages/benchmark/src/backfill.ts | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index 584b18c3c6..5f1b6dfa51 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -120,7 +120,31 @@ function restoreBenchmark(repoRoot: string, savedBenchmark: string): void {
 }
 
 /** Resolve a commit range from the from/to options. Returns commits oldest-first. */
-function resolveCommitRange(from: string, to: string | undefined, sourceBranch: string): string[] {
+/**
+ * Resolve the branch to read commits from.
+ *
+ * A CI checkout only creates the branch it was asked for, so `main` is not a
+ * local ref on any other branch -- including whichever branch a change to the
+ * backfill itself is being tested on. Fall back to the remote copy, fetching
+ * it if the checkout was shallow enough to have left it out.
+ */
+function resolveSourceBranch(sourceBranch: string): string {
+  if (execOk(`git rev-parse --verify --quiet ${sourceBranch}^{commit}`)) return sourceBranch;
+
+  const remote = `origin/${sourceBranch}`;
+  if (!execOk(`git rev-parse --verify --quiet ${remote}^{commit}`)) {
+    gitSilent(`fetch origin ${sourceBranch}:refs/remotes/${remote}`);
+  }
+  if (!execOk(`git rev-parse --verify --quiet ${remote}^{commit}`)) {
+    throw new Error(`Cannot resolve source branch '${sourceBranch}' locally or on origin.`);
+  }
+
+  console.log(`Source branch '${sourceBranch}' is not checked out; using ${remote}.`);
+  return remote;
+}
+
+function resolveCommitRange(from: string, to: string | undefined, branch: string): string[] {
+  const sourceBranch = resolveSourceBranch(branch);
   const isNumber = /^\d+$/.test(from);
 
   if (isNumber) {

From 4d44ed2d43c5aefda3feff13452fab7149e79ea7 Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 15:00:22 -0400
Subject: [PATCH 04/10] Report why a backfilled commit failed

Backfill pointed at a log file it never wrote -- the benchmark's output went
to /dev/null -- and a run where every commit failed still exited 0, so CI
reported green having measured nothing. Keep the output, print the tail of it
next to the failure, and fail the run when nothing was measured.
---
 packages/benchmark/src/backfill.ts | 35 ++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index 5f1b6dfa51..ec01dd11e8 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -1,11 +1,14 @@
 /* eslint-disable no-console */
 import { execSync } from "node:child_process";
 import {
+  closeSync,
   copyFileSync,
   cpSync,
   existsSync,
   mkdirSync,
+  openSync,
   readdirSync,
+  readFileSync,
   rmSync,
   symlinkSync,
   writeFileSync,
@@ -88,6 +91,22 @@ const BUILD_FILTER = [
   .map((p) => `--filter "${p}"`)
   .join(" ");
 
+/** Last few lines of a log, for explaining a failure without dumping the file. */
+function tail(file: string, lines = 20): string {
+  try {
+    return readFileSync(file, "utf-8").trimEnd().split("\n").slice(-lines).join("\n");
+  } catch {
+    return "(no output captured)";
+  }
+}
+
+function indent(text: string): string {
+  return text
+    .split("\n")
+    .map((line) => `    ${line}`)
+    .join("\n");
+}
+
 /** Restore the saved benchmark package into the repo with symlinks to workspace packages. */
 function restoreBenchmark(repoRoot: string, savedBenchmark: string): void {
   const benchDir = join(repoRoot, "packages/benchmark");
@@ -315,17 +334,23 @@ export function backfill(options: BackfillOptions = {}): void {
       // Use --specs-dir from runFlags if provided, otherwise use the saved specs
       const specsFlag = options.specsDir ? "" : `--specs-dir "${defaultSpecsDir}"`;
 
+      // Keep the output: it is the only account of why a commit failed, and
+      // it lives on a runner that is thrown away when the job ends.
+      const logFd = openSync(benchLog, "w");
       try {
         execSync(
           `node "${benchmarkCli}" run ${specsFlag} --output "${resultFile}" ${runFlagsStr}`.trim(),
-          { cwd: repoRoot, stdio: ["ignore", "ignore", "ignore"] },
+          { cwd: repoRoot, stdio: ["ignore", logFd, logFd] },
         );
         console.log("done ✓");
         succeeded++;
       } catch {
-        console.log(`benchmark failed (see ${benchLog})`);
+        console.log("benchmark failed");
+        console.log(indent(tail(benchLog)));
         failed++;
         rmSync(resultFile, { force: true });
+      } finally {
+        closeSync(logFd);
       }
     }
   } finally {
@@ -339,6 +364,12 @@ export function backfill(options: BackfillOptions = {}): void {
   console.log(`Failed: ${failed}`);
   console.log(`Skipped: ${skipped}`);
 
+  // A run where nothing was measured has produced no data, so let it report as
+  // a failure rather than a green job with an empty result.
+  if (succeeded === 0 && failed > 0) {
+    throw new Error(`Backfill measured nothing: all ${failed} commits failed.`);
+  }
+
   // Step 4: Push results to benchmark-data branch
   const newResults = readdirSync(resultsDir).filter((f) => f.endsWith(".json"));
   if (newResults.length === 0) {

From 675030fcb2d01c34edab00b79a7e43b36bcc65ed Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 15:11:39 -0400
Subject: [PATCH 05/10] Build the emitters the benchmark specs actually use

The specs emit with the Python, JS, TS and Java client emitters, but backfill
only ever built the core and Azure libraries, so every compile failed with
"Emitter not found" and no commit could be measured. Build those emitters and
link them where the specs resolve from.
---
 packages/benchmark/src/backfill.ts | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index ec01dd11e8..d3bae006e9 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -69,6 +69,7 @@ const TYPESPEC_PACKAGES = [
   "events",
   "streams",
   "sse",
+  "http-client-js",
 ];
 
 const AZURE_PACKAGES = [
@@ -77,8 +78,15 @@ const AZURE_PACKAGES = [
   "typespec-autorest",
   "typespec-client-generator-core",
   "typespec-azure-rulesets",
+  "typespec-python",
+  "typespec-ts",
+  "typespec-java",
 ];
 
+// Every emitter the specs `emit:`, plus what they depend on. A spec that names
+// an emitter this list forgets fails to compile outright, so it has to track
+// packages/benchmark/specs/*/tspconfig.yaml. Filters that match nothing at an
+// older commit are ignored by pnpm.
 const BUILD_FILTER = [
   "@typespec/compiler",
   "@azure-tools/typespec-azure-core",
@@ -87,6 +95,10 @@ const BUILD_FILTER = [
   "@typespec/openapi3",
   "@azure-tools/typespec-client-generator-core",
   "@azure-tools/typespec-azure-rulesets",
+  "@azure-tools/typespec-python",
+  "@typespec/http-client-js",
+  "@azure-tools/typespec-ts",
+  "@azure-tools/typespec-java",
 ]
   .map((p) => `--filter "${p}"`)
   .join(" ");

From f0d74a7b0ee7438139452dfba8731b2e1d86232f Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 16:08:05 -0400
Subject: [PATCH 06/10] Give backfill an identity to commit results under

A CI checkout has no git identity, so backfill's commit failed, and because
that commit was silent the branch was simply never created -- surfacing much
later as "src refspec does not match any" from the push. Configure the same
identity store-results uses, and let a failed commit say so.
---
 packages/benchmark/src/backfill.ts      | 16 ++++++++++++++--
 packages/benchmark/src/store-results.ts |  6 ++----
 packages/benchmark/src/utils.ts         |  6 ++++++
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index d3bae006e9..4c4d935a02 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -16,7 +16,15 @@ import {
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { generateHistory } from "./generate-history.js";
-import { DEFAULT_BRANCH, exec, execOk, git, gitSilent, listExistingResults } from "./utils.js";
+import {
+  configureGitIdentity,
+  DEFAULT_BRANCH,
+  exec,
+  execOk,
+  git,
+  gitSilent,
+  listExistingResults,
+} from "./utils.js";
 
 export interface BackfillOptions {
   /** Starting point: a commit SHA, or a number of recent commits to include. Defaults to 100. */
@@ -391,6 +399,8 @@ export function backfill(options: BackfillOptions = {}): void {
 
   console.log(`\nCommitting ${newResults.length} result(s) to ${dataBranch} branch...`);
 
+  configureGitIdentity();
+
   // Switch to benchmark-data branch
   if (gitSilent(`rev-parse --verify origin/${dataBranch}`)) {
     gitSilent(`checkout origin/${dataBranch} --force --quiet`);
@@ -427,7 +437,9 @@ export function backfill(options: BackfillOptions = {}): void {
   const commitMsg = reset
     ? `benchmark: re-measure ${succeeded} commits (${dataDirName})`
     : `benchmark: backfill results for ${succeeded} commits`;
-  gitSilent(`commit -m "${commitMsg}" --quiet`);
+  // Not silent: a failure here leaves the branch unborn, and the push that
+  // follows then fails with an unrelated-looking "src refspec" error.
+  git(`commit -m "${commitMsg}" --quiet`);
   console.log(`Results committed to ${dataBranch} branch.`);
 
   if (shouldPush) {
diff --git a/packages/benchmark/src/store-results.ts b/packages/benchmark/src/store-results.ts
index d451109e37..0e41c5979d 100644
--- a/packages/benchmark/src/store-results.ts
+++ b/packages/benchmark/src/store-results.ts
@@ -2,7 +2,7 @@
 import { copyFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { generateHistory } from "./generate-history.js";
-import { DEFAULT_BRANCH, git, gitSilent } from "./utils.js";
+import { DEFAULT_BRANCH, configureGitIdentity, git, gitSilent } from "./utils.js";
 
 export interface StoreResultsOptions {
   /** Path to the benchmark results JSON file. */
@@ -30,9 +30,7 @@ export function storeResults(options: StoreResultsOptions): void {
     throw new Error(`Results file not found: ${resultsFile}`);
   }
 
-  // Configure git
-  git('config user.name "github-actions[bot]"');
-  git('config user.email "github-actions[bot]@users.noreply.github.com"');
+  configureGitIdentity();
 
   try {
     // Set up worktree for the benchmark-data branch
diff --git a/packages/benchmark/src/utils.ts b/packages/benchmark/src/utils.ts
index 24393a35d6..be21fcddb1 100644
--- a/packages/benchmark/src/utils.ts
+++ b/packages/benchmark/src/utils.ts
@@ -35,6 +35,12 @@ export function execOk(cmd: string, options?: { cwd?: string }): boolean {
   }
 }
 
+/** Give git an identity to commit under, which a CI checkout does not have. */
+export function configureGitIdentity(cwd?: string): void {
+  git('config user.name "github-actions[bot]"', cwd);
+  git('config user.email "github-actions[bot]@users.noreply.github.com"', cwd);
+}
+
 /** List existing result SHAs on the benchmark-data branch. */
 export function listExistingResults(
   branch: string = DEFAULT_BRANCH,

From 9793af3e338e32552fce814b736cd10062245201 Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Fri, 4 Sep 2026 19:59:08 -0400
Subject: [PATCH 07/10] Build the workspace dependencies backfill needs

Filtering the build to the packages the specs use left their own workspace
dependencies unbuilt, so the compiler failed to resolve tmlanguage-generator
and every commit in the range died before it could be measured.

Install and build output was also discarded, which reported this as a bare
"build failed" that could not be diagnosed after the runner was gone.
---
 packages/benchmark/src/backfill.ts | 33 ++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index 4c4d935a02..4399451449 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -95,6 +95,10 @@ const AZURE_PACKAGES = [
 // an emitter this list forgets fails to compile outright, so it has to track
 // packages/benchmark/specs/*/tspconfig.yaml. Filters that match nothing at an
 // older commit are ignored by pnpm.
+//
+// The "..." suffix is load-bearing: it pulls in each package's workspace
+// dependencies. Without it the compiler builds against a tmlanguage-generator
+// that was never built and every commit dies in tsc.
 const BUILD_FILTER = [
   "@typespec/compiler",
   "@azure-tools/typespec-azure-core",
@@ -108,7 +112,7 @@ const BUILD_FILTER = [
   "@azure-tools/typespec-ts",
   "@azure-tools/typespec-java",
 ]
-  .map((p) => `--filter "${p}"`)
+  .map((p) => `--filter "${p}..."`)
   .join(" ");
 
 /** Last few lines of a log, for explaining a failure without dumping the file. */
@@ -127,6 +131,23 @@ function indent(text: string): string {
     .join("\n");
 }
 
+/**
+ * Run a preparation step, keeping its output. `execOk` throws it away, which
+ * turns any install or build failure into a bare "build failed" that cannot be
+ * diagnosed once the runner is gone.
+ */
+function execLogged(cmd: string, cwd: string, log: string): boolean {
+  const fd = openSync(log, "w");
+  try {
+    execSync(cmd, { cwd, stdio: ["ignore", fd, fd] });
+    return true;
+  } catch {
+    return false;
+  } finally {
+    closeSync(fd);
+  }
+}
+
 /** Restore the saved benchmark package into the repo with symlinks to workspace packages. */
 function restoreBenchmark(repoRoot: string, savedBenchmark: string): void {
   const benchDir = join(repoRoot, "packages/benchmark");
@@ -330,16 +351,20 @@ export function backfill(options: BackfillOptions = {}): void {
         continue;
       }
 
-      if (!execOk("pnpm install --frozen-lockfile --quiet", { cwd: repoRoot })) {
-        if (!execOk("pnpm install --quiet", { cwd: repoRoot })) {
+      const setupLog = join(resultsDir, `${sha}.setup.log`);
+
+      if (!execLogged("pnpm install --frozen-lockfile", repoRoot, setupLog)) {
+        if (!execLogged("pnpm install", repoRoot, setupLog)) {
           console.log("install failed, skipping");
+          console.log(indent(tail(setupLog)));
           failed++;
           continue;
         }
       }
 
-      if (!execOk(`pnpm -r ${BUILD_FILTER} build`, { cwd: repoRoot })) {
+      if (!execLogged(`pnpm -r ${BUILD_FILTER} build`, repoRoot, setupLog)) {
         console.log("build failed, skipping");
+        console.log(indent(tail(setupLog)));
         failed++;
         continue;
       }

From 063fc09c9faaa09af41d9b3fc366d9a6ef4a0770 Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Tue, 8 Sep 2026 13:08:53 -0400
Subject: [PATCH 08/10] Measure the machine so results from different runners
 compare

Every commit gets its own CI job, and CI hands out whichever runner is free.
Across 100 commits of main the same work varied by 63% depending on the machine:
spread between machines was 13.7% against 0.9% within one, so hardware outweighed
code changes 16 to 1. That noise lands directly between neighboring points, which
is why the chart jumps at commits that changed nothing relevant.

Machine speed scales TypeSpec workloads more or less uniformly, so it can be
divided out. Each run now also compiles a frozen reference workload on the same
machine, in the same job, and records how long it took. Dividing by it drops
between-machine spread below 1%, taking the smallest detectable regression from
roughly 41% to 3.5%.

The reference deliberately does not use the compiler being benchmarked. If it
moved with the repo, a real compiler regression would slow the reference by the
same amount and cancel itself out. It is a fixed spec built with a pinned
release from npm, materialized from constants, so it is identical for every
commit ever measured, including commits that predate it.

Calibration costs ~15s and never fails a run: if the pinned compiler cannot be
installed the run continues and the point is flagged uncalibrated. Raw numbers
are never rewritten; history stores the calibration next to them and exposes a
factor to multiply by, so the correction stays visible and reversible.

Also records the CPU model, which is what actually differs between runners and
which nothing captured before.
---
 packages/benchmark/README.md                  |  38 ++-
 packages/benchmark/src/calibration.ts         | 252 ++++++++++++++++++
 packages/benchmark/src/generate-history.ts    | 103 ++++++-
 packages/benchmark/src/run.ts                 |  14 +
 packages/benchmark/src/types.ts               |  24 ++
 .../benchmark/test/generate-history.test.ts   | 119 ++++++++-
 packages/benchmark/vitest.config.ts           |  11 +-
 7 files changed, 551 insertions(+), 10 deletions(-)
 create mode 100644 packages/benchmark/src/calibration.ts

diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md
index db520ed454..39c906607d 100644
--- a/packages/benchmark/README.md
+++ b/packages/benchmark/README.md
@@ -9,9 +9,10 @@ Performance benchmarking tool for TypeSpec Azure compilation. Tracks compilation
 3. Runtime metrics are aggregated with an outlier-resistant estimator (trimmed mean for 5+ samples, median for smaller sample sizes)
 4. Per-spec variability (standard deviation and coefficient of variation) is captured from raw iterations
 5. Optional noise-gating can auto-run extra iterations when variance is high
-6. PR baseline can be built from a rolling window of recent `main` results instead of only `latest.json`
-7. Results are stored as JSON — on CI, they're saved to the `benchmark-data` branch
-8. PR comments show a comparison table highlighting performance changes
+6. A frozen reference workload is measured in the same job so results from different CI runners can be compared (see [Machine calibration](#machine-calibration))
+7. PR baseline can be built from a rolling window of recent `main` results instead of only `latest.json`
+8. Results are stored as JSON — on CI, they're saved to the `benchmark-data` branch
+9. PR comments show a comparison table highlighting performance changes
 
 ## Local usage
 
@@ -115,6 +116,37 @@ The backfill command:
 3. Skips commits that already have results on the `benchmark-data` branch
 4. Commits all new results to the `benchmark-data` branch
 
+## Machine calibration
+
+CI hands out whichever runner is free, and those machines are not equally fast.
+Measured across 100 commits of `main`, the same work varied by 63% depending on
+the machine: spread between machines was 13.7% against 0.9% on a single machine,
+so hardware outweighed code changes roughly 16 to 1. Every commit gets its own
+job, so that noise lands directly between neighboring points and shows up as
+jumps no code change explains.
+
+Machine speed scales TypeSpec workloads more or less uniformly, so it can be
+divided out. Each run therefore also compiles a **frozen reference workload** —
+a fixed spec built with a pinned `@typespec/compiler` release from npm — on the
+same machine, in the same job. Dividing by it drops between-machine spread to
+under 1%, taking the smallest reliably detectable regression from ~41% to ~3.5%.
+
+The reference is deliberately _not_ the compiler being benchmarked. If it moved
+with the repo, a genuine compiler regression would slow the reference by the
+same amount and cancel itself out. It is materialized from constants in
+`src/calibration.ts`, so it is identical for every commit ever measured,
+including commits that predate the file. Changing `REFERENCE_COMPILER_VERSION`
+or the reference spec breaks comparability with existing points, so both are
+versioned by `WORKLOAD_ID` and only entries sharing the dominant workload are
+corrected.
+
+Calibration costs ~15s per job and never fails a run: if the pinned compiler
+cannot be installed, the run proceeds and the point is flagged `uncalibrated`.
+
+Raw measurements are never rewritten. `history.json` stores the calibration
+alongside them and exposes a per-entry `normalization` factor to multiply by, so
+the correction stays visible and reversible.
+
 ## What gets measured
 
 The TypeSpec compiler provides built-in `Stats` covering:
diff --git a/packages/benchmark/src/calibration.ts b/packages/benchmark/src/calibration.ts
new file mode 100644
index 0000000000..24aeea7074
--- /dev/null
+++ b/packages/benchmark/src/calibration.ts
@@ -0,0 +1,252 @@
+/* eslint-disable no-console */
+/**
+ * Machine calibration.
+ *
+ * Every benchmark point is measured on whatever runner CI happens to hand out,
+ * and those runners are not equally fast. Measured across 100 commits, the same
+ * code varied by 63% depending on the machine: between-machine spread was 13.7%
+ * against 0.9% within a machine, so hardware outweighed code changes 16 to 1.
+ *
+ * Machine speed acts as a multiplicative constant, which is why dividing one
+ * TypeSpec workload by another collapsed that 13.7% to 0.7%. This module
+ * measures the denominator: a reference workload that never changes, compiled
+ * on the same machine, in the same job, as the real benchmark.
+ *
+ * The reference must stay frozen. It deliberately does not use the compiler
+ * being benchmarked -- it installs a pinned release from npm -- because a
+ * reference that moved with the repo would cancel out the very regressions this
+ * is meant to expose. Everything it needs is materialized from the constants
+ * below, so it is identical for every commit ever measured, including commits
+ * predating this file.
+ */
+import { execFile, execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { median, summarize } from "./statistics.js";
+import type { CalibrationInfo } from "./types.js";
+
+/**
+ * Pinned reference compiler. Changing this invalidates comparability with every
+ * point measured before the change, so it should move rarely and deliberately,
+ * together with WORKLOAD_ID.
+ */
+const REFERENCE_COMPILER_VERSION = "1.14.0";
+
+/**
+ * Identifies the frozen workload. Bump whenever REFERENCE_SPEC or the pinned
+ * compiler changes, so old points are never silently compared against a
+ * different yardstick.
+ */
+const WORKLOAD_ID = "v1";
+
+/**
+ * Sized so the machine-speed estimate is far more precise than the regressions
+ * it needs to expose: measured across-run spread is ~0.5%, against a ~3.5%
+ * detection target, for ~15s of job time.
+ */
+const CALIBRATION_WARMUP = 5;
+const CALIBRATION_ITERATIONS = 25;
+
+/** Size of the frozen workload. Tuned so the reference takes long enough to average out scheduler noise. */
+const ENTITY_COUNT = 400;
+const FIELDS_PER_ENTITY = 20;
+const OPERATION_COUNT = 200;
+
+/**
+ * The frozen workload. Uses only compiler built-ins so calibration needs a
+ * single pinned package, and is sized to run long enough to average out
+ * scheduler noise without materially adding to job time.
+ */
+const REFERENCE_SPEC = `
+import "@typespec/compiler";
+
+@service(#{ title: "Calibration" })
+namespace Calibration;
+
+model Base {
+  id: string;
+  createdAt: utcDateTime;
+  updatedAt: utcDateTime;
+  tags: string[];
+}
+
+model Item<T> is Base {
+  value: T;
+  nested: Record<T>;
+  maybe?: T | null;
+}
+
+union Status {
+  active: "active",
+  inactive: "inactive",
+  pending: "pending",
+}
+
+${Array.from({ length: ENTITY_COUNT }, (_, i) => {
+  const props = Array.from(
+    { length: FIELDS_PER_ENTITY },
+    (_, p) => `  field${p}: string | int32 | boolean;`,
+  ).join("\n");
+  return `model Entity${i} is Item<string> {
+${props}
+  status: Status;
+  peer?: Entity${(i + 1) % ENTITY_COUNT};
+}`;
+}).join("\n\n")}
+
+${Array.from(
+  { length: OPERATION_COUNT },
+  (_, i) =>
+    `op operation${i}(input: Entity${i % ENTITY_COUNT}, status: Status): Entity${(i + 7) % ENTITY_COUNT};`,
+).join("\n")}
+`;
+
+/**
+ * Loader executed inside the calibration directory so that a bare
+ * "@typespec/compiler" import resolves to the pinned copy through normal Node
+ * resolution rather than to the workspace being benchmarked.
+ */
+const REFERENCE_RUNNER = `
+import { compile, NodeHost, resolveCompilerOptions } from "@typespec/compiler";
+import { join } from "node:path";
+
+const specDir = process.argv[2];
+const mainFile = join(specDir, "main.tsp");
+const [options] = await resolveCompilerOptions(NodeHost, { entrypoint: mainFile, cwd: specDir });
+const program = await compile(NodeHost, mainFile, {
+  ...options,
+  outputDir: join(specDir, "tsp-output"),
+  noEmit: true,
+});
+if (program.hasError()) {
+  const errors = program.diagnostics
+    .filter((d) => d.severity === "error")
+    .map((d) => "  " + d.message)
+    .join("\\n");
+  throw new Error("Calibration spec failed to compile:\\n" + errors);
+}
+const runtime = program.stats.runtime;
+const total =
+  (runtime.loader ?? 0) +
+  (runtime.resolver ?? 0) +
+  (runtime.checker ?? 0) +
+  (runtime.validation?.total ?? 0) +
+  (runtime.linter?.total ?? 0);
+process.stdout.write(JSON.stringify({ total }));
+`;
+
+/** Where the frozen environment is materialized. Reused across commits in a backfill job. */
+function calibrationDir(): string {
+  return join(
+    tmpdir(),
+    `typespec-benchmark-calibration-${REFERENCE_COMPILER_VERSION}-${WORKLOAD_ID}`,
+  );
+}
+
+/**
+ * Materialize the pinned compiler and frozen spec. Installs only when missing,
+ * so a backfill measuring many commits pays for it once.
+ */
+function prepare(dir: string): void {
+  const specDir = join(dir, "spec");
+  const marker = join(dir, "node_modules", "@typespec", "compiler", "package.json");
+
+  mkdirSync(specDir, { recursive: true });
+  writeFileSync(join(specDir, "main.tsp"), REFERENCE_SPEC);
+  writeFileSync(join(specDir, "tspconfig.yaml"), "emit: []\n");
+  writeFileSync(join(dir, "run.mjs"), REFERENCE_RUNNER);
+  writeFileSync(
+    join(dir, "package.json"),
+    JSON.stringify(
+      { name: "typespec-benchmark-calibration", private: true, type: "module" },
+      null,
+      2,
+    ),
+  );
+
+  if (existsSync(marker)) return;
+
+  try {
+    execFileSync(
+      "npm",
+      [
+        "install",
+        `@typespec/compiler@${REFERENCE_COMPILER_VERSION}`,
+        "--no-package-lock",
+        "--no-audit",
+        "--no-fund",
+        "--prefer-offline",
+        "--loglevel",
+        "error",
+      ],
+      { cwd: dir, stdio: ["ignore", "ignore", "pipe"], encoding: "utf-8" },
+    );
+  } catch (error: any) {
+    const details = String(error?.stderr ?? "").trim();
+    throw new Error(
+      `failed to install the reference compiler${details ? `: ${details.split("\n")[0]}` : ""}`,
+      { cause: error },
+    );
+  }
+}
+
+async function compileReference(dir: string): Promise<number> {
+  return await new Promise<number>((resolvePromise, reject) => {
+    execFile(
+      process.execPath,
+      [join(dir, "run.mjs"), join(dir, "spec")],
+      { cwd: dir },
+      (error, stdout, stderr) => {
+        if (error) {
+          reject(new Error(stderr.trim() || error.message));
+          return;
+        }
+        try {
+          resolvePromise((JSON.parse(stdout) as { total: number }).total);
+        } catch {
+          reject(new Error(`unexpected calibration output: ${stdout.slice(0, 200)}`));
+        }
+      },
+    );
+  });
+}
+
+/**
+ * Measure this machine against the frozen reference.
+ *
+ * Returns undefined rather than throwing: a machine-speed estimate is valuable
+ * but not worth losing a 20 minute benchmark run over, and consumers already
+ * treat calibration as optional so that points measured before it existed stay
+ * readable.
+ */
+export async function measureCalibration(): Promise<CalibrationInfo | undefined> {
+  const dir = calibrationDir();
+  try {
+    prepare(dir);
+
+    for (let i = 0; i < CALIBRATION_WARMUP; i++) {
+      await compileReference(dir);
+    }
+
+    const samples: number[] = [];
+    for (let i = 0; i < CALIBRATION_ITERATIONS; i++) {
+      samples.push(await compileReference(dir));
+    }
+
+    const stats = summarize(samples);
+    return {
+      compilerVersion: REFERENCE_COMPILER_VERSION,
+      workload: WORKLOAD_ID,
+      total: median(samples),
+      iterations: samples.length,
+      cv: stats.cv,
+    };
+  } catch (error) {
+    console.log(
+      `  Calibration unavailable (${error instanceof Error ? error.message.split("\n")[0] : error}); ` +
+        `results will not be machine-normalized.`,
+    );
+    return undefined;
+  }
+}
diff --git a/packages/benchmark/src/generate-history.ts b/packages/benchmark/src/generate-history.ts
index bf530bdcce..688b2aab05 100644
--- a/packages/benchmark/src/generate-history.ts
+++ b/packages/benchmark/src/generate-history.ts
@@ -3,7 +3,13 @@ import { readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
 import { median } from "./statistics.js";
-import type { BenchmarkResult, RunnerInfo, RuntimeStats, SpecBenchmarkResult } from "./types.js";
+import type {
+  BenchmarkResult,
+  CalibrationInfo,
+  RunnerInfo,
+  RuntimeStats,
+  SpecBenchmarkResult,
+} from "./types.js";
 import { DEFAULT_BRANCH, listResultBlobs, readBlobs } from "./utils.js";
 
 /**
@@ -14,8 +20,9 @@ import { DEFAULT_BRANCH, listResultBlobs, readBlobs } from "./utils.js";
  *
  * 1. Metrics only.
  * 2. Adds `runner` and `quality` to every entry.
+ * 3. Adds `calibration` and a `normalization` factor per entry.
  */
-export const HISTORY_VERSION = 2;
+export const HISTORY_VERSION = 3;
 
 /** Why a point may not be comparable with the ones around it. */
 export type EntryFlag =
@@ -24,7 +31,9 @@ export type EntryFlag =
   /** Averaged over too few iterations to separate signal from noise. */
   | "low-iterations"
   /** Measured on a different platform than the rest of the series. */
-  | "foreign-runner";
+  | "foreign-runner"
+  /** No machine-speed measurement, so its value still carries the runner's speed. */
+  | "uncalibrated";
 
 /** How much weight a single point deserves. */
 export interface EntryQuality {
@@ -51,9 +60,33 @@ export interface HistoryEntry {
    * regressions do, so a point is only meaningful alongside its environment.
    */
   runner?: RunnerInfo;
+  /** Speed of the machine this point was measured on, against a frozen reference. */
+  calibration?: CalibrationInfo;
+  /**
+   * Multiply any raw metric by this to compare it across machines.
+   *
+   * CI hands out runners that differ by more than 60% in speed, which swamps
+   * real changes. Machine speed scales TypeSpec workloads more or less
+   * uniformly, so dividing by a frozen reference measured in the same job
+   * removes it. Absent when the point has no usable calibration; raw values are
+   * never rewritten, so the correction stays auditable.
+   */
+  normalization?: number;
   quality: EntryQuality;
 }
 
+/** How raw values were made comparable across machines. */
+export interface NormalizationInfo {
+  /** Frozen workload the factors are anchored to. */
+  workload: string;
+  /** Pinned compiler the reference was measured with. */
+  compilerVersion: string;
+  /** Reference time factors are relative to; the median across calibrated entries. */
+  baseline: number;
+  /** How many entries carry a usable calibration. */
+  calibratedEntries: number;
+}
+
 /** The full history.json structure. */
 export interface HistoryData {
   /** See {@link HISTORY_VERSION}. */
@@ -62,6 +95,8 @@ export interface HistoryData {
   labels: string[];
   /** All spec names found across all entries */
   specNames: string[];
+  /** Absent when no entry carries a calibration. */
+  normalization?: NormalizationInfo;
   entries: HistoryEntry[];
 }
 
@@ -165,6 +200,50 @@ function dominantPlatform(entries: HistoryEntry[]): string | null {
   return best;
 }
 
+/**
+ * Work out how much of each measurement was the machine rather than the code.
+ *
+ * Calibration is only meaningful against the same frozen workload, so a change
+ * to the reference splits the series: the yardstick most of the history was
+ * measured against wins, and points measured against any other are left
+ * uncorrected rather than silently rescaled against a different reference.
+ *
+ * Factors are expressed relative to the median calibrated machine, so
+ * normalized values stay in familiar milliseconds instead of a unitless ratio.
+ */
+function applyNormalization(entries: HistoryEntry[]): NormalizationInfo | undefined {
+  const byWorkload = new Map<string, HistoryEntry[]>();
+  for (const entry of entries) {
+    const calibration = entry.calibration;
+    if (!calibration || !(calibration.total > 0)) continue;
+    const key = `${calibration.compilerVersion}@${calibration.workload}`;
+    const group = byWorkload.get(key);
+    if (group) group.push(entry);
+    else byWorkload.set(key, [entry]);
+  }
+
+  let dominant: HistoryEntry[] | undefined;
+  for (const group of byWorkload.values()) {
+    if (!dominant || group.length > dominant.length) dominant = group;
+  }
+  if (!dominant || dominant.length === 0) return undefined;
+
+  const baseline = median(dominant.map((entry) => entry.calibration!.total));
+  if (!(baseline > 0)) return undefined;
+
+  for (const entry of dominant) {
+    entry.normalization = baseline / entry.calibration!.total;
+  }
+
+  const reference = dominant[0].calibration!;
+  return {
+    workload: reference.workload,
+    compilerVersion: reference.compilerVersion,
+    baseline,
+    calibratedEntries: dominant.length,
+  };
+}
+
 /**
  * Mark points that cannot be read as part of the same series.
  *
@@ -175,7 +254,14 @@ function dominantPlatform(entries: HistoryEntry[]): string | null {
  */
 function flagEntries(entries: HistoryEntry[]): void {
   const expectedPlatform = dominantPlatform(entries);
-  const totals = entries.map((entry) => entry.metrics["total"] ?? null);
+  const anyCalibrated = entries.some((entry) => entry.normalization !== undefined);
+  // Spike detection reads the same values a chart would, so that a fast or slow
+  // runner is not mistaken for a spike once it has been corrected for.
+  const totals = entries.map((entry) => {
+    const total = entry.metrics["total"];
+    if (total === undefined) return null;
+    return total * (entry.normalization ?? 1);
+  });
   const reach = (OUTLIER_WINDOW - 1) / 2;
 
   entries.forEach((entry, index) => {
@@ -190,6 +276,12 @@ function flagEntries(entries: HistoryEntry[]): void {
       flags.push("foreign-runner");
     }
 
+    // Only worth pointing out once some of the series is corrected; a history
+    // with no calibration at all is uniformly uncorrected, not inconsistent.
+    if (anyCalibrated && entry.normalization === undefined) {
+      flags.push("uncalibrated");
+    }
+
     const value = totals[index];
     if (value === null || value <= 0) return;
 
@@ -264,6 +356,7 @@ export function buildHistory(resultFiles: Iterable<ResultFile>): HistoryData {
         metrics,
         specMetrics,
         runner: result.runner,
+        calibration: result.calibration,
         quality: measureQuality(result.specs),
       });
     } catch (e: any) {
@@ -272,6 +365,7 @@ export function buildHistory(resultFiles: Iterable<ResultFile>): HistoryData {
   }
 
   entries.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+  const normalization = applyNormalization(entries);
   flagEntries(entries);
 
   const allLabels = new Set<string>();
@@ -286,6 +380,7 @@ export function buildHistory(resultFiles: Iterable<ResultFile>): HistoryData {
     generated: new Date().toISOString(),
     labels: [...allLabels].sort(),
     specNames: [...allSpecNames].sort(),
+    normalization,
     entries,
   };
 }
diff --git a/packages/benchmark/src/run.ts b/packages/benchmark/src/run.ts
index e68aec1332..278b083181 100644
--- a/packages/benchmark/src/run.ts
+++ b/packages/benchmark/src/run.ts
@@ -6,6 +6,7 @@ import os from "os";
 import { join, resolve } from "path";
 import { fileURLToPath } from "url";
 import { aggregateDurations } from "./aggregate.js";
+import { measureCalibration } from "./calibration.js";
 import {
   EXTERNAL_SPEC_CONFIG,
   loadExternalSpecConfig,
@@ -211,10 +212,13 @@ function averageRuntimeStats(runtimes: RuntimeStats[]): RuntimeStats {
 }
 
 function getRunnerInfo(): RunnerInfo {
+  const cpus = os.cpus();
   return {
     os: `${os.platform()}-${os.release()}`,
     nodeVersion: process.version,
     arch: os.arch(),
+    cpu: cpus[0]?.model,
+    cores: cpus.length,
   };
 }
 
@@ -242,6 +246,15 @@ export async function runBenchmarks(options: RunOptions): Promise<BenchmarkResul
     `Running benchmarks: ${specSources.length} spec(s), ${warmup} warmup + ${iterations} iterations each`,
   );
 
+  console.log(`\n  Calibrating machine speed against the frozen reference workload...`);
+  const calibration = await measureCalibration();
+  if (calibration) {
+    console.log(
+      `    Reference (@typespec/compiler@${calibration.compilerVersion}, workload ${calibration.workload}): ` +
+        `${calibration.total.toFixed(1)}ms, CV ${(calibration.cv * 100).toFixed(1)}%`,
+    );
+  }
+
   const specs: Record<string, SpecBenchmarkResult> = {};
   const noiseCvThreshold = options.noiseCvThreshold;
   const maxReruns = options.maxReruns ?? 0;
@@ -320,6 +333,7 @@ export async function runBenchmarks(options: RunOptions): Promise<BenchmarkResul
     commit,
     timestamp: new Date().toISOString(),
     runner: getRunnerInfo(),
+    calibration,
     specs,
   };
 }
diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts
index 8bf91f1120..bbb8df87ad 100644
--- a/packages/benchmark/src/types.ts
+++ b/packages/benchmark/src/types.ts
@@ -80,6 +80,12 @@ export interface BenchmarkResult {
   timestamp: string;
   /** Runner environment info. */
   runner: RunnerInfo;
+  /**
+   * Speed of this machine against a frozen reference workload, used to make
+   * points measured on different runners comparable. Absent when calibration
+   * could not run, and on points measured before calibration existed.
+   */
+  calibration?: CalibrationInfo;
   /** Per-spec benchmark results, keyed by spec name. */
   specs: Record<string, SpecBenchmarkResult>;
 }
@@ -88,6 +94,24 @@ export interface RunnerInfo {
   os: string;
   nodeVersion: string;
   arch: string;
+  /** CPU model, which varies between CI runners and drives most of the spread. */
+  cpu?: string;
+  /** Logical core count. */
+  cores?: number;
+}
+
+/** Measurement of a frozen reference workload, used to normalize away machine speed. */
+export interface CalibrationInfo {
+  /** Pinned compiler release the reference was compiled with. */
+  compilerVersion: string;
+  /** Identifier of the frozen workload; changes invalidate cross-version comparison. */
+  workload: string;
+  /** Median reference compile time in ms on this machine. */
+  total: number;
+  /** Number of measured reference compiles. */
+  iterations: number;
+  /** Coefficient of variation across those compiles. */
+  cv: number;
 }
 
 /** A single metric comparison between baseline and current. */
diff --git a/packages/benchmark/test/generate-history.test.ts b/packages/benchmark/test/generate-history.test.ts
index 2e9d17cfc0..4a881651f0 100644
--- a/packages/benchmark/test/generate-history.test.ts
+++ b/packages/benchmark/test/generate-history.test.ts
@@ -1,6 +1,11 @@
 import { expect, it } from "vitest";
 import { buildHistory, HISTORY_VERSION } from "../src/generate-history.js";
-import type { BenchmarkResult, RunnerInfo, SpecBenchmarkResult } from "../src/types.js";
+import type {
+  BenchmarkResult,
+  CalibrationInfo,
+  RunnerInfo,
+  SpecBenchmarkResult,
+} from "../src/types.js";
 
 interface SpecOptions {
   total: number;
@@ -45,12 +50,17 @@ function spec({ total, iterations = 25, cv }: SpecOptions): SpecBenchmarkResult
 
 const LINUX: RunnerInfo = { os: "linux-6.11.0", nodeVersion: "v24.15.0", arch: "x64" };
 
+function calibration(total: number, workload = "v1"): CalibrationInfo {
+  return { compilerVersion: "1.14.0", workload, total, iterations: 25, cv: 0.005 };
+}
+
 interface RunOptions extends SpecOptions {
   day: number;
   runner?: RunnerInfo;
+  calibration?: CalibrationInfo;
 }
 
-function run({ day, runner = LINUX, ...specOptions }: RunOptions): {
+function run({ day, runner = LINUX, calibration, ...specOptions }: RunOptions): {
   name: string;
   content: string;
 } {
@@ -58,6 +68,7 @@ function run({ day, runner = LINUX, ...specOptions }: RunOptions): {
     commit: `commit-${day}`,
     timestamp: new Date(Date.UTC(2026, 0, day)).toISOString(),
     runner,
+    ...(calibration ? { calibration } : {}),
     specs: { sample: spec(specOptions) },
   } as BenchmarkResult;
   return { name: `commit-${day}.json`, content: JSON.stringify(result) };
@@ -159,3 +170,107 @@ it("buildHistory orders entries oldest first", () => {
   const history = buildHistory([run({ day: 3, total: 100 }), run({ day: 1, total: 100 })]);
   expect(history.entries.map((entry) => entry.commit)).toEqual(["commit-1", "commit-3"]);
 });
+
+it("buildHistory carries the calibration through to each entry", () => {
+  const history = buildHistory([run({ day: 1, total: 100, calibration: calibration(300) })]);
+  expect(history.entries[0].calibration).toEqual(calibration(300));
+});
+
+it("buildHistory scales a fast machine's numbers up and a slow one's down", () => {
+  const history = buildHistory([
+    run({ day: 1, total: 100, calibration: calibration(200) }),
+    run({ day: 2, total: 100, calibration: calibration(400) }),
+    run({ day: 3, total: 100, calibration: calibration(800) }),
+  ]);
+
+  // The median calibrated machine anchors the series, so it is left untouched.
+  expect(history.entries[1].normalization).toBe(1);
+  // Half the reference time means a machine twice as fast, so its numbers are doubled.
+  expect(history.entries[0].normalization).toBe(2);
+  expect(history.entries[2].normalization).toBe(0.5);
+});
+
+it("buildHistory reports which reference the factors are anchored to", () => {
+  const history = buildHistory([
+    run({ day: 1, total: 100, calibration: calibration(200) }),
+    run({ day: 2, total: 100, calibration: calibration(400) }),
+  ]);
+
+  expect(history.normalization).toEqual({
+    workload: "v1",
+    compilerVersion: "1.14.0",
+    baseline: 300,
+    calibratedEntries: 2,
+  });
+});
+
+it("buildHistory leaves an uncalibrated history uncorrected", () => {
+  const history = buildHistory(steadySeries(3));
+  expect(history.normalization).toBeUndefined();
+  expect(history.entries[0].normalization).toBeUndefined();
+  expect(history.entries[0].quality.flags).toEqual([]);
+});
+
+it("buildHistory never rewrites the raw measurement", () => {
+  const history = buildHistory([
+    run({ day: 1, total: 100, calibration: calibration(200) }),
+    run({ day: 2, total: 100, calibration: calibration(400) }),
+  ]);
+  expect(history.entries[0].metrics["total"]).toBe(100);
+});
+
+it("buildHistory flags points missing a calibration the rest of the series has", () => {
+  const history = buildHistory([
+    run({ day: 1, total: 100, calibration: calibration(300) }),
+    run({ day: 2, total: 100 }),
+  ]);
+
+  expect(history.entries[0].quality.flags).not.toContain("uncalibrated");
+  expect(history.entries[1].quality.flags).toContain("uncalibrated");
+  expect(history.entries[1].normalization).toBeUndefined();
+});
+
+it("buildHistory anchors to the reference most of the series was measured against", () => {
+  const history = buildHistory([
+    ...Array.from({ length: 3 }, (_, i) =>
+      run({ day: i + 1, total: 100, calibration: calibration(300) }),
+    ),
+    run({ day: 4, total: 100, calibration: calibration(600, "v2") }),
+  ]);
+
+  expect(history.normalization?.workload).toBe("v1");
+  // Rescaling against a different yardstick would invent a change that never happened.
+  expect(history.entries[3].normalization).toBeUndefined();
+  expect(history.entries[3].quality.flags).toContain("uncalibrated");
+});
+
+it("buildHistory does not call a slow runner a spike once it is corrected for", () => {
+  const entries = Array.from({ length: 11 }, (_, i) =>
+    run({ day: i + 1, total: 100, calibration: calibration(300) }),
+  );
+  // Same code on a machine three times slower: raw value looks like a spike.
+  entries[5] = run({ day: 6, total: 300, calibration: calibration(900) });
+
+  const history = buildHistory(entries);
+  expect(history.entries[5].quality.flags).not.toContain("outlier");
+});
+
+it("buildHistory still reports a spike the machine cannot explain", () => {
+  const entries = Array.from({ length: 11 }, (_, i) =>
+    run({ day: i + 1, total: 100, calibration: calibration(300) }),
+  );
+  entries[5] = run({ day: 6, total: 300, calibration: calibration(300) });
+
+  const history = buildHistory(entries);
+  expect(history.entries[5].quality.flags).toContain("outlier");
+});
+
+it("buildHistory ignores a calibration that measured nothing", () => {
+  const history = buildHistory([
+    run({ day: 1, total: 100, calibration: calibration(0) }),
+    run({ day: 2, total: 100, calibration: calibration(300) }),
+  ]);
+
+  expect(history.entries[0].normalization).toBeUndefined();
+  expect(history.normalization?.calibratedEntries).toBe(1);
+});
diff --git a/packages/benchmark/vitest.config.ts b/packages/benchmark/vitest.config.ts
index e236e91da2..dfe77909d6 100644
--- a/packages/benchmark/vitest.config.ts
+++ b/packages/benchmark/vitest.config.ts
@@ -1,4 +1,13 @@
 import { defineConfig, mergeConfig } from "vitest/config";
 import { defaultTypeSpecVitestConfig } from "../../core/vitest.config.js";
 
-export default mergeConfig(defaultTypeSpecVitestConfig, defineConfig({}));
+export default mergeConfig(
+  defaultTypeSpecVitestConfig,
+  defineConfig({
+    test: {
+      // Running the benchmarks locally emits generated clients next to the
+      // specs, and those ship their own sample tests.
+      exclude: ["**/node_modules/**", "**/dist/**", "specs/*/tsp-output/**"],
+    },
+  }),
+);

From 224a306a5e51e7073211585a19be71cee0e7b902 Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Tue, 8 Sep 2026 14:01:20 -0400
Subject: [PATCH 09/10] Place each point where its commit landed

Points carried only the time they were measured, which is fine while one commit
is measured per push but wrong the moment history is rebuilt: a backfill covers
months of commits in an afternoon, so all 100 points collapsed onto the two days
the backfill ran and the chart's x-axis lost any meaning.

Worse, entries were also ordered by measurement time. Re-measuring a single old
commit would have moved it to the end of the series, silently reordering history
around it.

Record the commit's committer date and use it for both placement and ordering,
falling back to measurement time for points recorded before it was captured.
---
 packages/benchmark/src/generate-history.ts    | 20 ++++++++-
 packages/benchmark/src/run.ts                 | 20 +++++++++
 packages/benchmark/src/types.ts               |  8 ++++
 .../benchmark/test/generate-history.test.ts   | 41 ++++++++++++++++++-
 4 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/packages/benchmark/src/generate-history.ts b/packages/benchmark/src/generate-history.ts
index 688b2aab05..4e8317fea6 100644
--- a/packages/benchmark/src/generate-history.ts
+++ b/packages/benchmark/src/generate-history.ts
@@ -49,6 +49,14 @@ export interface EntryQuality {
 export interface HistoryEntry {
   commit: string;
   timestamp: string;
+  /**
+   * When the commit landed, when known.
+   *
+   * Points are placed and ordered by this rather than by `timestamp`, so that a
+   * backfill measuring a year of history in an afternoon still lands each point
+   * where it belongs.
+   */
+  commitDate?: string;
   /** Averaged metrics across all specs */
   metrics: Record<string, number>;
   /** Per-spec metrics (spec name → flat metrics) */
@@ -244,6 +252,11 @@ function applyNormalization(entries: HistoryEntry[]): NormalizationInfo | undefi
   };
 }
 
+/** Where a point sits in history, preferring when the commit landed. */
+function orderOf(entry: HistoryEntry): number {
+  return new Date(entry.commitDate ?? entry.timestamp).getTime();
+}
+
 /**
  * Mark points that cannot be read as part of the same series.
  *
@@ -353,6 +366,7 @@ export function buildHistory(resultFiles: Iterable<ResultFile>): HistoryData {
       entries.push({
         commit: result.commit,
         timestamp: result.timestamp,
+        commitDate: result.commitDate,
         metrics,
         specMetrics,
         runner: result.runner,
@@ -364,7 +378,11 @@ export function buildHistory(resultFiles: Iterable<ResultFile>): HistoryData {
     }
   }
 
-  entries.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+  // Ordered by when each commit landed, falling back to measurement time for
+  // points recorded before the commit date was captured. Sorting by
+  // measurement time would put a re-measured commit at the end of the series
+  // rather than back in its place in history.
+  entries.sort((a, b) => orderOf(a) - orderOf(b));
   const normalization = applyNormalization(entries);
   flagEntries(entries);
 
diff --git a/packages/benchmark/src/run.ts b/packages/benchmark/src/run.ts
index 278b083181..e2ac9aabba 100644
--- a/packages/benchmark/src/run.ts
+++ b/packages/benchmark/src/run.ts
@@ -231,6 +231,25 @@ function getGitCommit(providedCommit?: string): string {
   }
 }
 
+/**
+ * When the commit landed, as opposed to when it was measured.
+ *
+ * Uses the committer date rather than the author date, because that is the
+ * order the commits reached the branch; an author date can predate its own
+ * parent after a rebase.
+ */
+function getCommitDate(commit: string): string | undefined {
+  if (commit === "unknown") return undefined;
+  try {
+    return execSync(`git show -s --format=%cI ${commit}`, {
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "ignore"],
+    }).trim();
+  } catch {
+    return undefined;
+  }
+}
+
 /** Run benchmarks for all discovered specs. */
 export async function runBenchmarks(options: RunOptions): Promise<BenchmarkResult> {
   const specsDir = resolve(options.specsDir);
@@ -332,6 +351,7 @@ export async function runBenchmarks(options: RunOptions): Promise<BenchmarkResul
   return {
     commit,
     timestamp: new Date().toISOString(),
+    commitDate: getCommitDate(commit),
     runner: getRunnerInfo(),
     calibration,
     specs,
diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts
index bbb8df87ad..8c75fcc0f6 100644
--- a/packages/benchmark/src/types.ts
+++ b/packages/benchmark/src/types.ts
@@ -78,6 +78,14 @@ export interface BenchmarkResult {
   commit: string;
   /** ISO 8601 timestamp of when the benchmark was run. */
   timestamp: string;
+  /**
+   * ISO 8601 date the benchmarked commit landed.
+   *
+   * Distinct from `timestamp`: a backfill measures months of history in an
+   * afternoon, so measurement time says nothing about where a point belongs on
+   * a timeline. Absent when the commit could not be resolved.
+   */
+  commitDate?: string;
   /** Runner environment info. */
   runner: RunnerInfo;
   /**
diff --git a/packages/benchmark/test/generate-history.test.ts b/packages/benchmark/test/generate-history.test.ts
index 4a881651f0..d7de4cb228 100644
--- a/packages/benchmark/test/generate-history.test.ts
+++ b/packages/benchmark/test/generate-history.test.ts
@@ -58,15 +58,25 @@ interface RunOptions extends SpecOptions {
   day: number;
   runner?: RunnerInfo;
   calibration?: CalibrationInfo;
+  commitDate?: string;
+  measuredAt?: string;
 }
 
-function run({ day, runner = LINUX, calibration, ...specOptions }: RunOptions): {
+function run({
+  day,
+  runner = LINUX,
+  calibration,
+  commitDate,
+  measuredAt,
+  ...specOptions
+}: RunOptions): {
   name: string;
   content: string;
 } {
   const result: BenchmarkResult = {
     commit: `commit-${day}`,
-    timestamp: new Date(Date.UTC(2026, 0, day)).toISOString(),
+    timestamp: measuredAt ?? new Date(Date.UTC(2026, 0, day)).toISOString(),
+    ...(commitDate ? { commitDate } : {}),
     runner,
     ...(calibration ? { calibration } : {}),
     specs: { sample: spec(specOptions) },
@@ -274,3 +284,30 @@ it("buildHistory ignores a calibration that measured nothing", () => {
   expect(history.entries[0].normalization).toBeUndefined();
   expect(history.normalization?.calibratedEntries).toBe(1);
 });
+
+it("buildHistory places a point where its commit landed, not where it was measured", () => {
+  // A backfill measures old commits today, so measurement time would order the
+  // series backwards.
+  const history = buildHistory([
+    run({
+      day: 1,
+      total: 100,
+      commitDate: "2025-03-01T00:00:00.000Z",
+      measuredAt: "2026-09-07T00:00:00.000Z",
+    }),
+    run({
+      day: 2,
+      total: 100,
+      commitDate: "2025-01-01T00:00:00.000Z",
+      measuredAt: "2026-09-08T00:00:00.000Z",
+    }),
+  ]);
+
+  expect(history.entries.map((entry) => entry.commit)).toEqual(["commit-2", "commit-1"]);
+  expect(history.entries[0].commitDate).toBe("2025-01-01T00:00:00.000Z");
+});
+
+it("buildHistory falls back to measurement time for points with no commit date", () => {
+  const history = buildHistory([run({ day: 3, total: 100 }), run({ day: 1, total: 100 })]);
+  expect(history.entries.map((entry) => entry.commit)).toEqual(["commit-1", "commit-3"]);
+});

From 68d54ec2ce2ac51d601353a7afa2afb4a7429385 Mon Sep 17 00:00:00 2001
From: Timothee Guerin <tiguerin@microsoft.com>
Date: Wed, 9 Sep 2026 09:40:24 -0400
Subject: [PATCH 10/10] Calibrate against a workload that resembles the real
 one

The first reference imported only @typespec/compiler and compiled synthetic
models. Measured across two CI machines it slowed 16% while the real specs
slowed 34%, so dividing by it removed only half the machine effect and left a
15% residual at the boundary -- still far more than the regressions this is
meant to expose.

Hardware sensitivity depends on the kind of work. loader is a third of the real
measurement and turned out to be the most sensitive phase of all (+41% between
those two machines), and a spec with no libraries to load barely exercises it.
A reference is only useful if it does the same mix of work.

Use a frozen copy of the azure-full spec against a pinned library stack and the
same linter ruleset. Measured cold, the way both are actually measured, its
phase mix now tracks the real one closely: loader 34.9% against 33.6%, where
before there was essentially no library loading at all.

The reference stays frozen and independent of the repo, so it still cannot
cancel out a real regression. Backfill carries the calibration directory
alongside dist and specs, so every commit is measured against the same
yardstick.
---
 packages/benchmark/README.md                  |  41 ++-
 packages/benchmark/calibration/package.json   |  15 +
 packages/benchmark/calibration/spec/main.tsp  | 323 ++++++++++++++++++
 .../benchmark/calibration/spec/tspconfig.yaml |   4 +
 packages/benchmark/src/backfill.ts            |   6 +
 packages/benchmark/src/calibration.ts         | 167 +++------
 6 files changed, 429 insertions(+), 127 deletions(-)
 create mode 100644 packages/benchmark/calibration/package.json
 create mode 100644 packages/benchmark/calibration/spec/main.tsp
 create mode 100644 packages/benchmark/calibration/spec/tspconfig.yaml

diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md
index 39c906607d..b68c846db2 100644
--- a/packages/benchmark/README.md
+++ b/packages/benchmark/README.md
@@ -126,21 +126,32 @@ job, so that noise lands directly between neighboring points and shows up as
 jumps no code change explains.
 
 Machine speed scales TypeSpec workloads more or less uniformly, so it can be
-divided out. Each run therefore also compiles a **frozen reference workload** —
-a fixed spec built with a pinned `@typespec/compiler` release from npm — on the
-same machine, in the same job. Dividing by it drops between-machine spread to
-under 1%, taking the smallest reliably detectable regression from ~41% to ~3.5%.
-
-The reference is deliberately _not_ the compiler being benchmarked. If it moved
-with the repo, a genuine compiler regression would slow the reference by the
-same amount and cancel itself out. It is materialized from constants in
-`src/calibration.ts`, so it is identical for every commit ever measured,
-including commits that predate the file. Changing `REFERENCE_COMPILER_VERSION`
-or the reference spec breaks comparability with existing points, so both are
-versioned by `WORKLOAD_ID` and only entries sharing the dominant workload are
-corrected.
-
-Calibration costs ~15s per job and never fails a run: if the pinned compiler
+divided out. Each run therefore also compiles a **frozen reference workload** on
+the same machine, in the same job. Dividing by it removes the machine factor
+from the comparison.
+
+The reference has to satisfy two competing requirements.
+
+It must be **frozen**, and deliberately does not use the packages being
+benchmarked. If it moved with the repo, a genuine regression would slow the
+reference by the same amount and cancel itself out. It lives in `calibration/`
+and installs exactly pinned releases from npm.
+
+It must also be **representative**. A first attempt imported only
+`@typespec/compiler` and compiled synthetic models; between two CI machines it
+slowed 16% while the real specs slowed 34%, removing only half the machine
+effect. Hardware sensitivity depends on the kind of work being done — `loader`
+is a third of the real measurement and proved the most sensitive phase of all,
+and a spec with no libraries to load barely exercises it. The reference is
+therefore a frozen copy of the `azure-full` spec compiled against the same
+pinned library stack and linter ruleset, which brings its phase mix in line with
+what is actually being measured.
+
+Changing the reference spec or any pinned version breaks comparability with
+existing points, so both are versioned by `WORKLOAD_ID` in `src/calibration.ts`
+and only entries sharing the dominant workload are corrected.
+
+Calibration costs ~20s per commit and never fails a run: if the pinned stack
 cannot be installed, the run proceeds and the point is flagged `uncalibrated`.
 
 Raw measurements are never rewritten. `history.json` stores the calibration
diff --git a/packages/benchmark/calibration/package.json b/packages/benchmark/calibration/package.json
new file mode 100644
index 0000000000..25e45eee41
--- /dev/null
+++ b/packages/benchmark/calibration/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "typespec-benchmark-calibration",
+  "private": true,
+  "type": "module",
+  "description": "Frozen reference workload used to measure how fast the machine is. Versions are pinned exactly and must not be changed without bumping WORKLOAD_ID in src/calibration.ts.",
+  "dependencies": {
+    "@typespec/compiler": "1.15.0",
+    "@typespec/http": "1.15.0",
+    "@typespec/rest": "0.85.0",
+    "@typespec/versioning": "0.85.0",
+    "@typespec/openapi": "1.15.0",
+    "@azure-tools/typespec-azure-core": "0.71.0",
+    "@azure-tools/typespec-azure-rulesets": "0.71.0"
+  }
+}
diff --git a/packages/benchmark/calibration/spec/main.tsp b/packages/benchmark/calibration/spec/main.tsp
new file mode 100644
index 0000000000..25baf7945a
--- /dev/null
+++ b/packages/benchmark/calibration/spec/main.tsp
@@ -0,0 +1,323 @@
+import "@typespec/http";
+import "@typespec/rest";
+import "@typespec/versioning";
+import "@azure-tools/typespec-azure-core";
+
+using Http;
+using Rest;
+using Versioning;
+using Azure.Core;
+using Azure.Core.Traits;
+
+// ============================================
+// A comprehensive data-plane service with many models, operations, and patterns
+// ============================================
+
+@service(#{ title: "Contoso Benchmark Full Service" })
+@versioned(Contoso.BenchmarkFull.Versions)
+@useAuth(ApiKeyAuth<ApiKeyLocation.header, "X-Api-Key">)
+namespace Contoso.BenchmarkFull;
+
+enum Versions {
+  `2024-01-01`,
+}
+
+// --- Catalog domain ---
+
+union ItemCategory {
+  string,
+  Electronics: "Electronics",
+  Furniture: "Furniture",
+  Clothing: "Clothing",
+  Food: "Food",
+  Books: "Books",
+  Sports: "Sports",
+}
+
+union ItemCondition {
+  string,
+  New: "New",
+  Refurbished: "Refurbished",
+  Used: "Used",
+}
+
+@resource("items")
+model Item {
+  @key("itemId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  name: string;
+  description?: string;
+  category: ItemCategory;
+  condition: ItemCondition;
+  price: float64;
+  quantity: int32;
+  sku: string;
+  tags?: Record<string>;
+  metadata?: Record<string>;
+  createdAt: utcDateTime;
+  updatedAt?: utcDateTime;
+  ...EtagProperty;
+}
+
+@resource("reviews")
+@parentResource(Item)
+model ItemReview {
+  @key("reviewId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  author: string;
+  rating: int32;
+  title?: string;
+  comment?: string;
+  helpful: int32;
+  createdAt: utcDateTime;
+}
+
+@resource("images")
+@parentResource(Item)
+model ItemImage {
+  @key("imageId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  url: string;
+  altText?: string;
+  width: int32;
+  height: int32;
+  isPrimary: boolean;
+}
+
+// --- Order domain ---
+
+@resource("orders")
+model Order {
+  @key("orderId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  customerName: string;
+  customerEmail: string;
+  shippingAddress: Address;
+  billingAddress?: Address;
+  items: OrderLineItem[];
+  totalAmount: float64;
+  currency: string;
+  status: OrderStatus;
+  createdAt: utcDateTime;
+  updatedAt?: utcDateTime;
+  ...EtagProperty;
+}
+
+model Address {
+  street: string;
+  city: string;
+  state: string;
+  postalCode: string;
+  country: string;
+}
+
+model OrderLineItem {
+  itemId: string;
+  itemName: string;
+  quantity: int32;
+  unitPrice: float64;
+  discount?: float64;
+}
+
+union OrderStatus {
+  string,
+  Pending: "Pending",
+  Confirmed: "Confirmed",
+  Processing: "Processing",
+  Shipped: "Shipped",
+  Delivered: "Delivered",
+  Canceled: "Canceled",
+  Refunded: "Refunded",
+}
+
+@resource("shipments")
+@parentResource(Order)
+model Shipment {
+  @key("shipmentId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  carrier: string;
+  trackingNumber: string;
+  status: ShipmentStatus;
+  estimatedDelivery?: utcDateTime;
+  shippedAt?: utcDateTime;
+  deliveredAt?: utcDateTime;
+}
+
+union ShipmentStatus {
+  string,
+  Pending: "Pending",
+  InTransit: "InTransit",
+  OutForDelivery: "OutForDelivery",
+  Delivered: "Delivered",
+  Failed: "Failed",
+}
+
+// --- Customer domain ---
+
+@resource("customers")
+model Customer {
+  @key("customerId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  firstName: string;
+  lastName: string;
+  email: string;
+  phone?: string;
+  address?: Address;
+  loyaltyTier: LoyaltyTier;
+  createdAt: utcDateTime;
+  ...EtagProperty;
+}
+
+union LoyaltyTier {
+  string,
+  Bronze: "Bronze",
+  Silver: "Silver",
+  Gold: "Gold",
+  Platinum: "Platinum",
+}
+
+@resource("wishlists")
+@parentResource(Customer)
+model WishlistItem {
+  @key("wishlistItemId")
+  @visibility(Lifecycle.Read)
+  id: string;
+
+  itemId: string;
+  addedAt: utcDateTime;
+  note?: string;
+}
+
+// --- Analytics ---
+
+model SalesReport {
+  period: string;
+  totalRevenue: float64;
+  totalOrders: int32;
+  averageOrderValue: float64;
+  topCategories: CategorySales[];
+}
+
+model CategorySales {
+  category: ItemCategory;
+  revenue: float64;
+  unitsSold: int32;
+}
+
+model InventoryAlert {
+  itemId: string;
+  itemName: string;
+  currentQuantity: int32;
+  threshold: int32;
+  severity: AlertSeverity;
+}
+
+union AlertSeverity {
+  string,
+  Low: "Low",
+  Medium: "Medium",
+  High: "High",
+  Critical: "Critical",
+}
+
+model InventoryAlertList {
+  alerts: InventoryAlert[];
+}
+
+// --- Operations ---
+
+alias ServiceTraits = SupportsRepeatableRequests &
+  SupportsConditionalRequests &
+  SupportsClientRequestId;
+
+alias Ops = Azure.Core.ResourceOperations<ServiceTraits>;
+
+interface Items {
+  create is Ops.ResourceCreateWithServiceProvidedName<Item>;
+  get is Ops.ResourceRead<Item>;
+  delete is Ops.ResourceDelete<Item>;
+  list is Ops.ResourceList<
+    Item,
+    ListQueryParametersTrait<StandardListQueryParameters & SelectQueryParameter>
+  >;
+}
+
+interface ItemReviews {
+  create is Ops.ResourceCreateWithServiceProvidedName<ItemReview>;
+  get is Ops.ResourceRead<ItemReview>;
+  list is Ops.ResourceList<ItemReview>;
+}
+
+interface ItemImages {
+  create is Ops.ResourceCreateWithServiceProvidedName<ItemImage>;
+  get is Ops.ResourceRead<ItemImage>;
+  delete is Ops.ResourceDelete<ItemImage>;
+  list is Ops.ResourceList<ItemImage>;
+}
+
+interface Orders {
+  create is Ops.ResourceCreateWithServiceProvidedName<Order>;
+  get is Ops.ResourceRead<Order>;
+  list is Ops.ResourceList<Order, ListQueryParametersTrait<StandardListQueryParameters>>;
+}
+
+interface Shipments {
+  create is Ops.ResourceCreateWithServiceProvidedName<Shipment>;
+  get is Ops.ResourceRead<Shipment>;
+  list is Ops.ResourceList<Shipment>;
+}
+
+interface Customers {
+  create is Ops.ResourceCreateWithServiceProvidedName<Customer>;
+  get is Ops.ResourceRead<Customer>;
+  delete is Ops.ResourceDelete<Customer>;
+  list is Ops.ResourceList<Customer, ListQueryParametersTrait<StandardListQueryParameters>>;
+}
+
+interface WishlistItems {
+  create is Ops.ResourceCreateWithServiceProvidedName<WishlistItem>;
+  get is Ops.ResourceRead<WishlistItem>;
+  delete is Ops.ResourceDelete<WishlistItem>;
+  list is Ops.ResourceList<WishlistItem>;
+}
+
+@route("analytics/sales")
+op getSalesReport is RpcOperation<
+  {
+    @query period: string;
+  },
+  SalesReport,
+  ServiceTraits
+>;
+
+@route("analytics/inventory-alerts")
+op getInventoryAlerts is RpcOperation<
+  {
+    @query severity?: string;
+  },
+  InventoryAlertList,
+  ServiceTraits
+>;
+
+@route("service-status")
+op getServiceStatus is RpcOperation<
+  {},
+  {
+    status: string;
+    version: string;
+    uptime: int64;
+  },
+  ServiceTraits
+>;
diff --git a/packages/benchmark/calibration/spec/tspconfig.yaml b/packages/benchmark/calibration/spec/tspconfig.yaml
new file mode 100644
index 0000000000..06278d8068
--- /dev/null
+++ b/packages/benchmark/calibration/spec/tspconfig.yaml
@@ -0,0 +1,4 @@
+emit: []
+linter:
+  extends:
+    - "@azure-tools/typespec-azure-rulesets/data-plane"
diff --git a/packages/benchmark/src/backfill.ts b/packages/benchmark/src/backfill.ts
index 4399451449..2ddcb70587 100644
--- a/packages/benchmark/src/backfill.ts
+++ b/packages/benchmark/src/backfill.ts
@@ -156,6 +156,9 @@ function restoreBenchmark(repoRoot: string, savedBenchmark: string): void {
 
   cpSync(join(savedBenchmark, "dist"), join(benchDir, "dist"), { recursive: true });
   cpSync(join(savedBenchmark, "specs"), join(benchDir, "specs"), { recursive: true });
+  // The frozen calibration workload has to travel with it, or old commits would
+  // be measured against no reference at all.
+  cpSync(join(savedBenchmark, "calibration"), join(benchDir, "calibration"), { recursive: true });
   copyFileSync(join(savedBenchmark, "package.json"), join(benchDir, "package.json"));
 
   // Create node_modules with symlinks to workspace packages
@@ -271,6 +274,9 @@ export function backfill(options: BackfillOptions = {}): void {
   cpSync(join(repoRoot, "packages/benchmark/specs"), join(savedBenchmark, "specs"), {
     recursive: true,
   });
+  cpSync(join(repoRoot, "packages/benchmark/calibration"), join(savedBenchmark, "calibration"), {
+    recursive: true,
+  });
   copyFileSync(
     join(repoRoot, "packages/benchmark/package.json"),
     join(savedBenchmark, "package.json"),
diff --git a/packages/benchmark/src/calibration.ts b/packages/benchmark/src/calibration.ts
index 24aeea7074..0aa873e9bd 100644
--- a/packages/benchmark/src/calibration.ts
+++ b/packages/benchmark/src/calibration.ts
@@ -7,105 +7,64 @@
  * code varied by 63% depending on the machine: between-machine spread was 13.7%
  * against 0.9% within a machine, so hardware outweighed code changes 16 to 1.
  *
- * Machine speed acts as a multiplicative constant, which is why dividing one
- * TypeSpec workload by another collapsed that 13.7% to 0.7%. This module
- * measures the denominator: a reference workload that never changes, compiled
- * on the same machine, in the same job, as the real benchmark.
+ * This module measures the denominator needed to divide that out: a reference
+ * workload that never changes, compiled on the same machine, in the same job,
+ * as the real benchmark.
  *
- * The reference must stay frozen. It deliberately does not use the compiler
- * being benchmarked -- it installs a pinned release from npm -- because a
- * reference that moved with the repo would cancel out the very regressions this
- * is meant to expose. Everything it needs is materialized from the constants
- * below, so it is identical for every commit ever measured, including commits
- * predating this file.
+ * Two properties matter, and they pull in opposite directions.
+ *
+ * The reference must be **frozen**. It deliberately does not use the packages
+ * being benchmarked -- it installs pinned releases from npm -- because a
+ * reference that moved with the repo would slow down alongside a real
+ * regression and cancel it out.
+ *
+ * The reference must also be **representative**. A first attempt used a
+ * synthetic spec importing only the compiler; between two CI machines it slowed
+ * 16% while the real specs slowed 34%, so it removed only half the machine
+ * effect. Hardware sensitivity varies with the kind of work: `loader` is a
+ * third of the real measurement and was the most sensitive phase of all, and a
+ * spec with no libraries to load barely exercises it. The reference is
+ * therefore a frozen copy of the azure-full benchmark spec, compiled against
+ * the same pinned library stack and the same linter ruleset, so that it does
+ * the same mix of work.
  */
 import { execFile, execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { median, summarize } from "./statistics.js";
 import type { CalibrationInfo } from "./types.js";
 
 /**
- * Pinned reference compiler. Changing this invalidates comparability with every
- * point measured before the change, so it should move rarely and deliberately,
- * together with WORKLOAD_ID.
+ * Identifies the frozen workload. Bump whenever the reference spec or any
+ * pinned version in calibration/package.json changes, so that points measured
+ * against different yardsticks are never compared with each other.
+ *
+ * v1: synthetic compiler-only spec. Withdrawn, it under-corrected by half.
+ * v2: frozen copy of the azure-full spec against the pinned library stack.
  */
-const REFERENCE_COMPILER_VERSION = "1.14.0";
+const WORKLOAD_ID = "v2";
 
-/**
- * Identifies the frozen workload. Bump whenever REFERENCE_SPEC or the pinned
- * compiler changes, so old points are never silently compared against a
- * different yardstick.
- */
-const WORKLOAD_ID = "v1";
+/** Reported alongside results so the reference in use is always identifiable. */
+const REFERENCE_COMPILER_VERSION = "1.15.0";
 
 /**
  * Sized so the machine-speed estimate is far more precise than the regressions
- * it needs to expose: measured across-run spread is ~0.5%, against a ~3.5%
- * detection target, for ~15s of job time.
+ * it needs to expose, without adding meaningfully to a ~25 minute job.
  */
-const CALIBRATION_WARMUP = 5;
+const CALIBRATION_WARMUP = 3;
 const CALIBRATION_ITERATIONS = 25;
 
-/** Size of the frozen workload. Tuned so the reference takes long enough to average out scheduler noise. */
-const ENTITY_COUNT = 400;
-const FIELDS_PER_ENTITY = 20;
-const OPERATION_COUNT = 200;
-
-/**
- * The frozen workload. Uses only compiler built-ins so calibration needs a
- * single pinned package, and is sized to run long enough to average out
- * scheduler noise without materially adding to job time.
- */
-const REFERENCE_SPEC = `
-import "@typespec/compiler";
-
-@service(#{ title: "Calibration" })
-namespace Calibration;
-
-model Base {
-  id: string;
-  createdAt: utcDateTime;
-  updatedAt: utcDateTime;
-  tags: string[];
-}
-
-model Item<T> is Base {
-  value: T;
-  nested: Record<T>;
-  maybe?: T | null;
-}
-
-union Status {
-  active: "active",
-  inactive: "inactive",
-  pending: "pending",
-}
-
-${Array.from({ length: ENTITY_COUNT }, (_, i) => {
-  const props = Array.from(
-    { length: FIELDS_PER_ENTITY },
-    (_, p) => `  field${p}: string | int32 | boolean;`,
-  ).join("\n");
-  return `model Entity${i} is Item<string> {
-${props}
-  status: Status;
-  peer?: Entity${(i + 1) % ENTITY_COUNT};
-}`;
-}).join("\n\n")}
-
-${Array.from(
-  { length: OPERATION_COUNT },
-  (_, i) =>
-    `op operation${i}(input: Entity${i % ENTITY_COUNT}, status: Status): Entity${(i + 7) % ENTITY_COUNT};`,
-).join("\n")}
-`;
+/** The frozen workload, shipped in the package rather than generated. */
+const sourceDir = join(dirname(fileURLToPath(import.meta.url)), "../../calibration");
 
 /**
- * Loader executed inside the calibration directory so that a bare
- * "@typespec/compiler" import resolves to the pinned copy through normal Node
- * resolution rather than to the workspace being benchmarked.
+ * Loader executed from inside the calibration directory so that bare imports
+ * resolve to the pinned copies through normal Node resolution rather than to
+ * the workspace being benchmarked.
+ *
+ * Measures what `total` measures for real specs: every phase except emit.
  */
 const REFERENCE_RUNNER = `
 import { compile, NodeHost, resolveCompilerOptions } from "@typespec/compiler";
@@ -138,54 +97,38 @@ process.stdout.write(JSON.stringify({ total }));
 
 /** Where the frozen environment is materialized. Reused across commits in a backfill job. */
 function calibrationDir(): string {
-  return join(
-    tmpdir(),
-    `typespec-benchmark-calibration-${REFERENCE_COMPILER_VERSION}-${WORKLOAD_ID}`,
-  );
+  return join(tmpdir(), `typespec-benchmark-calibration-${WORKLOAD_ID}`);
 }
 
 /**
- * Materialize the pinned compiler and frozen spec. Installs only when missing,
- * so a backfill measuring many commits pays for it once.
+ * Materialize the pinned stack and frozen spec. Installs only when missing, so
+ * a backfill measuring many commits pays for it once.
  */
 function prepare(dir: string): void {
-  const specDir = join(dir, "spec");
-  const marker = join(dir, "node_modules", "@typespec", "compiler", "package.json");
+  const marker = join(
+    dir,
+    "node_modules",
+    "@azure-tools",
+    "typespec-azure-rulesets",
+    "package.json",
+  );
 
-  mkdirSync(specDir, { recursive: true });
-  writeFileSync(join(specDir, "main.tsp"), REFERENCE_SPEC);
-  writeFileSync(join(specDir, "tspconfig.yaml"), "emit: []\n");
+  mkdirSync(dir, { recursive: true });
+  cpSync(sourceDir, dir, { recursive: true });
   writeFileSync(join(dir, "run.mjs"), REFERENCE_RUNNER);
-  writeFileSync(
-    join(dir, "package.json"),
-    JSON.stringify(
-      { name: "typespec-benchmark-calibration", private: true, type: "module" },
-      null,
-      2,
-    ),
-  );
 
   if (existsSync(marker)) return;
 
   try {
     execFileSync(
       "npm",
-      [
-        "install",
-        `@typespec/compiler@${REFERENCE_COMPILER_VERSION}`,
-        "--no-package-lock",
-        "--no-audit",
-        "--no-fund",
-        "--prefer-offline",
-        "--loglevel",
-        "error",
-      ],
+      ["install", "--no-package-lock", "--no-audit", "--no-fund", "--loglevel", "error"],
       { cwd: dir, stdio: ["ignore", "ignore", "pipe"], encoding: "utf-8" },
     );
   } catch (error: any) {
     const details = String(error?.stderr ?? "").trim();
     throw new Error(
-      `failed to install the reference compiler${details ? `: ${details.split("\n")[0]}` : ""}`,
+      `failed to install the reference stack${details ? `: ${details.split("\n")[0]}` : ""}`,
       { cause: error },
     );
   }
@@ -216,7 +159,7 @@ async function compileReference(dir: string): Promise<number> {
  * Measure this machine against the frozen reference.
  *
  * Returns undefined rather than throwing: a machine-speed estimate is valuable
- * but not worth losing a 20 minute benchmark run over, and consumers already
+ * but not worth losing a 25 minute benchmark run over, and consumers already
  * treat calibration as optional so that points measured before it existed stay
  * readable.
  */