From f5b381e8fc08cf4f957176390a652be47e88ff5d Mon Sep 17 00:00:00 2001
From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com>
Date: Fri, 25 Sep 2026 20:40:46 +0800
Subject: [PATCH] feat(presets): add Terminal-Bench 3 and 4 releases

Co-Authored-By: GPT-6 Sol <noreply@pi.dev>
Generated-By: pi 0.87.1 (https://pi.dev)
---
 .../test/terminal-bench-3-4-presets.test.ts   | 53 +++++++++++++++++++
 .../terminal-bench-3-0-all-tasks-1-trial.json | 23 ++++++++
 ...terminal-bench-3-0-all-tasks-5-trials.json | 23 ++++++++
 .../terminal-bench-3-0-one-task-1-trial.json  | 24 +++++++++
 .../terminal-bench-4-0-all-tasks-1-trial.json | 23 ++++++++
 ...terminal-bench-4-0-all-tasks-5-trials.json | 23 ++++++++
 .../terminal-bench-4-0-one-task-1-trial.json  | 24 +++++++++
 7 files changed, 193 insertions(+)
 create mode 100644 packages/control-core/test/terminal-bench-3-4-presets.test.ts
 create mode 100644 presets/benchmarks/terminal-bench-3-0-all-tasks-1-trial.json
 create mode 100644 presets/benchmarks/terminal-bench-3-0-all-tasks-5-trials.json
 create mode 100644 presets/benchmarks/terminal-bench-3-0-one-task-1-trial.json
 create mode 100644 presets/benchmarks/terminal-bench-4-0-all-tasks-1-trial.json
 create mode 100644 presets/benchmarks/terminal-bench-4-0-all-tasks-5-trials.json
 create mode 100644 presets/benchmarks/terminal-bench-4-0-one-task-1-trial.json

diff --git a/packages/control-core/test/terminal-bench-3-4-presets.test.ts b/packages/control-core/test/terminal-bench-3-4-presets.test.ts
new file mode 100644
index 00000000..d96a2c3f
--- /dev/null
+++ b/packages/control-core/test/terminal-bench-3-4-presets.test.ts
@@ -0,0 +1,53 @@
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+import { PresetCatalog } from "../src/presets.js";
+
+const catalog = await PresetCatalog.load(
+  fileURLToPath(new URL("../../../presets", import.meta.url)),
+);
+
+const versions = [
+  {
+    benchmark: "terminal-bench-3-0",
+    revision: "2b0442c3c583b710ca8da14c8e601b99f2f1f244",
+  },
+  {
+    benchmark: "terminal-bench-4-0",
+    revision: "452bf305c6daa62fc59061d22133a7cbc7c1572e",
+  },
+] as const;
+
+const variants = [
+  { preset: "one-task-1-trial", attempts: 1, concurrency: 1, final: false },
+  { preset: "all-tasks-1-trial", attempts: 1, concurrency: 8, final: false },
+  { preset: "all-tasks-5-trials", attempts: 5, concurrency: 8, final: true },
+] as const;
+
+describe.each(versions)("$benchmark built-in presets", ({ benchmark, revision }) => {
+  it.each(variants)(
+    "$preset uses its exact release",
+    ({ preset, attempts, concurrency, final }) => {
+      const actual = catalog.benchmark(benchmark, preset);
+      expect(actual.leaderboard_eligible).toBe(final);
+      expect(actual.job).toEqual({
+        datasets: [
+          {
+            repo: `https://github.com/harbor-framework/terminal-bench.git@${revision}`,
+            path: "tasks",
+            ...(preset === "one-task-1-trial"
+              ? { task_names: ["interleaved-vigenere"] }
+              : {}),
+          },
+        ],
+        n_attempts: attempts,
+        n_concurrent_trials: concurrency,
+        environment: {
+          type: "hf-sandbox",
+          kwargs: { flavor: "cpu-upgrade", job_timeout: "none" },
+        },
+      });
+      expect(actual.job.agent_timeout_multiplier).toBeUndefined();
+      expect(actual.job.retry).toBeUndefined();
+    },
+  );
+});
diff --git a/presets/benchmarks/terminal-bench-3-0-all-tasks-1-trial.json b/presets/benchmarks/terminal-bench-3-0-all-tasks-1-trial.json
new file mode 100644
index 00000000..acfd6e1a
--- /dev/null
+++ b/presets/benchmarks/terminal-bench-3-0-all-tasks-1-trial.json
@@ -0,0 +1,23 @@
+{
+  "schema_version": "v1",
+  "benchmark": "terminal-bench-3-0",
+  "preset": "all-tasks-1-trial",
+  "leaderboard_eligible": false,
+  "job": {
+    "datasets": [
+      {
+        "repo": "https://github.com/harbor-framework/terminal-bench.git@2b0442c3c583b710ca8da14c8e601b99f2f1f244",
+        "path": "tasks"
+      }
+    ],
+    "n_attempts": 1,
+    "n_concurrent_trials": 8,
+    "environment": {
+      "type": "hf-sandbox",
+      "kwargs": {
+        "flavor": "cpu-upgrade",
+        "job_timeout": "none"
+      }
+    }
+  }
+}
diff --git a/presets/benchmarks/terminal-bench-3-0-all-tasks-5-trials.json b/presets/benchmarks/terminal-bench-3-0-all-tasks-5-trials.json
new file mode 100644
index 00000000..f0aee752
--- /dev/null
+++ b/presets/benchmarks/terminal-bench-3-0-all-tasks-5-trials.json
@@ -0,0 +1,23 @@
+{
+  "schema_version": "v1",
+  "benchmark": "terminal-bench-3-0",
+  "preset": "all-tasks-5-trials",
+  "leaderboard_eligible": true,
+  "job": {
+    "datasets": [
+      {
+        "repo": "https://github.com/harbor-framework/terminal-bench.git@2b0442c3c583b710ca8da14c8e601b99f2f1f244",
+        "path": "tasks"
+      }
+    ],
+    "n_attempts": 5,
+    "n_concurrent_trials": 8,
+    "environment": {
+      "type": "hf-sandbox",
+      "kwargs": {
+        "flavor": "cpu-upgrade",
+        "job_timeout": "none"
+      }
+    }
+  }
+}
diff --git a/presets/benchmarks/terminal-bench-3-0-one-task-1-trial.json b/presets/benchmarks/terminal-bench-3-0-one-task-1-trial.json
new file mode 100644
index 00000000..fe4a653c
--- /dev/null
+++ b/presets/benchmarks/terminal-bench-3-0-one-task-1-trial.json
@@ -0,0 +1,24 @@
+{
+  "schema_version": "v1",
+  "benchmark": "terminal-bench-3-0",
+  "preset": "one-task-1-trial",
+  "leaderboard_eligible": false,
+  "job": {
+    "datasets": [
+      {
+        "repo": "https://github.com/harbor-framework/terminal-bench.git@2b0442c3c583b710ca8da14c8e601b99f2f1f244",
+        "path": "tasks",
+        "task_names": ["interleaved-vigenere"]
+      }
+    ],
+    "n_attempts": 1,
+    "n_concurrent_trials": 1,
+    "environment": {
+      "type": "hf-sandbox",
+      "kwargs": {
+        "flavor": "cpu-upgrade",
+        "job_timeout": "none"
+      }
+    }
+  }
+}
diff --git a/presets/benchmarks/terminal-bench-4-0-all-tasks-1-trial.json b/presets/benchmarks/terminal-bench-4-0-all-tasks-1-trial.json
new file mode 100644
index 00000000..3e44f638
--- /dev/null
+++ b/presets/benchmarks/terminal-bench-4-0-all-tasks-1-trial.json
@@ -0,0 +1,23 @@
+{
+  "schema_version": "v1",
+  "benchmark": "terminal-bench-4-0",
+  "preset": "all-tasks-1-trial",
+  "leaderboard_eligible": false,
+  "job": {
+    "datasets": [
+      {
+        "repo": "https://github.com/harbor-framework/terminal-bench.git@452bf305c6daa62fc59061d22133a7cbc7c1572e",
+        "path": "tasks"
+      }
+    ],
+    "n_attempts": 1,
+    "n_concurrent_trials": 8,
+    "environment": {
+      "type": "hf-sandbox",
+      "kwargs": {
+        "flavor": "cpu-upgrade",
+        "job_timeout": "none"
+      }
+    }
+  }
+}
diff --git a/presets/benchmarks/terminal-bench-4-0-all-tasks-5-trials.json b/presets/benchmarks/terminal-bench-4-0-all-tasks-5-trials.json
new file mode 100644
index 00000000..7a477f07
--- /dev/null
+++ b/presets/benchmarks/terminal-bench-4-0-all-tasks-5-trials.json
@@ -0,0 +1,23 @@
+{
+  "schema_version": "v1",
+  "benchmark": "terminal-bench-4-0",
+  "preset": "all-tasks-5-trials",
+  "leaderboard_eligible": true,
+  "job": {
+    "datasets": [
+      {
+        "repo": "https://github.com/harbor-framework/terminal-bench.git@452bf305c6daa62fc59061d22133a7cbc7c1572e",
+        "path": "tasks"
+      }
+    ],
+    "n_attempts": 5,
+    "n_concurrent_trials": 8,
+    "environment": {
+      "type": "hf-sandbox",
+      "kwargs": {
+        "flavor": "cpu-upgrade",
+        "job_timeout": "none"
+      }
+    }
+  }
+}
diff --git a/presets/benchmarks/terminal-bench-4-0-one-task-1-trial.json b/presets/benchmarks/terminal-bench-4-0-one-task-1-trial.json
new file mode 100644
index 00000000..cdf16448
--- /dev/null
+++ b/presets/benchmarks/terminal-bench-4-0-one-task-1-trial.json
@@ -0,0 +1,24 @@
+{
+  "schema_version": "v1",
+  "benchmark": "terminal-bench-4-0",
+  "preset": "one-task-1-trial",
+  "leaderboard_eligible": false,
+  "job": {
+    "datasets": [
+      {
+        "repo": "https://github.com/harbor-framework/terminal-bench.git@452bf305c6daa62fc59061d22133a7cbc7c1572e",
+        "path": "tasks",
+        "task_names": ["interleaved-vigenere"]
+      }
+    ],
+    "n_attempts": 1,
+    "n_concurrent_trials": 1,
+    "environment": {
+      "type": "hf-sandbox",
+      "kwargs": {
+        "flavor": "cpu-upgrade",
+        "job_timeout": "none"
+      }
+    }
+  }
+}