Close the accuracy-gate gaps that let a broken model be promoted

docs/reference/session-breakdown.md

@@ -326,6 +326,29 @@ preserves the available `donor_canonical_id`, `donor_model`,
 `donor_breakdown_link`. Fields absent from the source recipe remain absent
 rather than being inferred.
 
+`kb_provenance.warm_replay` additionally records what the replay was judged
+on, whether it passed or failed. A replayed recipe is evidence from another
+session on another machine, so reproducing its throughput says nothing about
+whether it still computes correctly here.
+
+| Field | Type | Description |
+|---|---|---|
+| `eval_ran` | bool | Whether an eval produced output for this replay. Separates a model that answered nothing (`eval_ran` true, `replay_accuracy` `0.0`) from a replay nothing checked (`eval_ran` false, `replay_accuracy` `null`). |
+| `replay_accuracy` | float \| null | Score measured on the replayed config. `null` when no score could be read — not a score of zero. |
+| `baseline_accuracy` | float \| null | Reference the replay was compared against. `null` when the session recorded none, in which case the replay is judged against an absolute floor instead of a relative drop. |
+| `eval_error` | string \| null | Why no score could be read. Distinguishes a contract with the eval switched off, an eval that produced an unreadable file, and a results file carrying no metric this parser knows. |
+
+A replay whose accuracy could not be measured is still promoted — a failed
+measurement is not evidence the config broke the model — so `eval_ran` is what
+tells an unjudged promotion apart from a judged one.
+
+The `optimization_stack` entry a warm replay pushes carries the same score as
+`accuracy`, so the promotion and the evidence behind it are readable from one
+place. `null` there means the lane recorded no verdict.
+
+Sessions started with `--no-eval` run no eval at all, warm replay included, so
+these fields record the absence rather than a score.
+
 ## `session` — `SessionMeta`
 
 The `session` section contains the following metadata fields.

src/hyperloom/inference_optimizer/breakdown/schema.py

@@ -1446,12 +1446,28 @@ class KBFlusherStatus(TypedDict, total=False):
 
 
 class WarmReplayOutcome(TypedDict, total=False):
-    """GAP 1 — warm-recipe replay result. Empty {} when it never fired; else ``status`` + per-status fields."""
+    """GAP 1 — warm-recipe replay result. Empty {} when it never fired; else ``status`` + per-status fields.
+
+    ``eval_ran`` / ``replay_accuracy`` / ``baseline_accuracy`` are recorded on
+    every replay that reached a throughput measurement, not only on rejection:
+    a config that was checked and passed is a different record from one that
+    was never checked. ``eval_ran`` is what separates "the model scored 0.0"
+    from "no score exists", which are otherwise both a null accuracy.
+
+    A measurement that fails never stops the run. The replay is admitted and
+    ``eval_error`` carries why no score could be read, so an unjudged promotion
+    is visible after the fact rather than silently indistinguishable from a
+    judged one.
+    """
 
     status: str
     expected_gain_pct: float
     actual_gain_pct: float
     throughput_after: float
+    eval_ran: bool
+    eval_error: str | None
+    replay_accuracy: float | None
+    baseline_accuracy: float | None
     warm_recipe_tier: str
     warm_recipe_conf: float
     config_source: str

src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py

@@ -1249,6 +1249,86 @@ def test_baseline_double_run_by_default(tmp_path, monkeypatch):
     assert captured[1]["benchmark"]["server_lifecycle"]["cleanup"] is True
 
 
+def test_replay_warm_recipe_double_run_forces_warmup_eval(tmp_path):
+    """Warm replay evaluates in the warmup round and measures in the second."""
+    base = tmp_path / "base.yaml"
+    _write_yaml(base, framework="vllm")
+    output_dir = tmp_path / "ws"
+    captured: list = []
+    fake_run, state = _cold_then_hot_fake_run(captured)
+    executor = BaselineExecutor(
+        magpie_python=sys.executable,
+        default_config_path=base,
+        session_dir=tmp_path,
+        shared_state=SimpleNamespace(baseline_double_run=True),
+    )
+    task = SimpleNamespace(
+        task_id="t-replay-warm",
+        kind="replay_warm_recipe",
+        params={
+            "output_dir": str(output_dir),
+            "timeout_sec": 10,
+            "gpu_type": "mi300x",
+            "model_path": "/wekafs/models/Qwen-Qwen3-8B",
+        },
+    )
+    ctx = SimpleNamespace(task=task, extra={})
+    with patch(
+        "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill",
+        side_effect=fake_run,
+    ):
+        result = _run(executor(ctx))
+
+    assert result["status"] == "succeeded"
+    assert state["calls"] == 2
+    assert captured[0]["benchmark"]["envs"]["RUN_EVAL"] == "true"
+    assert captured[1]["benchmark"]["envs"]["RUN_EVAL"] == "false"
+
+
+def test_replay_warm_recipe_honours_no_eval(tmp_path):
+    """``--no-eval`` outranks the replay's forced warmup eval.
+
+    The flag is the operator saying no eval runs this session. Forcing one on
+    the warmup round would spend the time the flag was passed to save, and do
+    it silently -- the baseline path on the same executor already honours the
+    flag, so a replay that did not would be the odd one out.
+    """
+    base = tmp_path / "base.yaml"
+    _write_yaml(base, framework="vllm")
+    captured: list = []
+    fake_run, state = _cold_then_hot_fake_run(captured)
+    shared = SimpleNamespace(baseline_double_run=True, eval_disabled=True)
+    executor = BaselineExecutor(
+        magpie_python=sys.executable,
+        default_config_path=base,
+        session_dir=tmp_path,
+        shared_state=shared,
+    )
+    task = SimpleNamespace(
+        task_id="t-replay-no-eval",
+        kind="replay_warm_recipe",
+        params={
+            "output_dir": str(tmp_path / "ws"),
+            "timeout_sec": 10,
+            "gpu_type": "mi300x",
+            "model_path": "/wekafs/models/Qwen-Qwen3-8B",
+        },
+    )
+    ctx = SimpleNamespace(task=task, extra={"shared_state": shared})
+    with patch(
+        "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill",
+        side_effect=fake_run,
+    ):
+        result = _run(executor(ctx))
+
+    assert result["status"] == "succeeded"
+    assert state["calls"] == 2
+    assert [cfg["benchmark"]["envs"]["RUN_EVAL"] for cfg in captured] == [
+        "false",
+        "false",
+    ]
+
+
 def test_baseline_double_run_can_be_disabled_by_task_param(tmp_path, monkeypatch):
     """Focused callers may explicitly opt out of the default cold+hot baseline."""
     base = tmp_path / "base.yaml"

src/hyperloom/inference_optimizer/tests/test_eval_context_feasibility.py

@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+"""The accuracy gate must not spend a KEEP attempt on an eval that cannot run.
+
+Observed on 195 sessions: the parameter search picks ``--max-model-len 2048``
+for throughput, the gsm8k harness asks for a 2048-token completion on top of a
+~1k-token five-shot prompt, and every request comes back HTTP 400. No verdict is
+ever produced, so ``accuracy_pass`` stays ``None``. Because a positive baseline
+accuracy (measured earlier, under the larger context the run started with) is
+read as proof that eval works here, the missing verdict blocks the KEEP and the
+round is recorded as a fair attempt. Three of those and the kernel is discarded
+for a reason that has nothing to do with the kernel.
+"""
+
+import pytest
+
+from hyperloom.orchestrator.actions.executors import _accuracy_gate as ag
+from hyperloom.orchestrator.state.shared_state import SharedState
+
+# The generation budget the harness requests for gsm8k.
+GSM8K_GEN_TOKENS = 2048
+
+
+class TestServedContextHostsEval:
+    """Can the serving configuration physically answer an eval request?"""
+
+    def test_context_equal_to_the_generation_budget_cannot_host_a_prompt(self):
+        """2048 of context and 2048 requested output leaves nothing for the prompt."""
+        fits, reason = ag.served_context_hosts_eval(
+            served_max_model_len=2048,
+            eval_max_tokens=GSM8K_GEN_TOKENS,
+        )
+        assert fits is False
+        assert "2048" in reason
+
+    def test_the_real_session_configuration_is_rejected(self):
+        """The exact shape seen in session e268b0be: env asks 6144, the server
+        args override it to 2048, and the override is what the server honours."""
+        served = ag.resolve_served_context(
+            server_args=("--kv-cache-dtype fp8 --max-num-batched-tokens 32768 --max-model-len 2048 --async-scheduling"),
+            env_max_model_len=6144,
+        )
+        assert served == 2048
+        fits, _ = ag.served_context_hosts_eval(
+            served_max_model_len=served,
+            eval_max_tokens=GSM8K_GEN_TOKENS,
+        )
+        assert fits is False
+
+    def test_the_context_the_env_asked_for_does_host_the_eval(self):
+        fits, reason = ag.served_context_hosts_eval(
+            served_max_model_len=6144,
+            eval_max_tokens=GSM8K_GEN_TOKENS,
+        )
+        assert fits is True
+        assert reason == ""
+
+    def test_equals_form_of_the_flag_is_understood(self):
+        assert (
+            ag.resolve_served_context(
+                server_args="--max-model-len=2048",
+                env_max_model_len=6144,
+            )
+            == 2048
+        )
+
+    def test_env_is_used_when_the_server_args_are_silent(self):
+        assert (
+            ag.resolve_served_context(
+                server_args="--kv-cache-dtype fp8",
+                env_max_model_len=6144,
+            )
+            == 6144
+        )
+
+    def test_an_unknown_context_is_not_treated_as_infeasible(self):
+        """Nothing is known, so nothing is claimed: never block on a guess."""
+        fits, _ = ag.served_context_hosts_eval(
+            served_max_model_len=0,
+            eval_max_tokens=GSM8K_GEN_TOKENS,
+        )
+        assert fits is True
+
+    @pytest.mark.parametrize("budget", [0, -1])
+    def test_an_unbounded_generation_budget_is_not_treated_as_infeasible(self, budget):
+        fits, _ = ag.served_context_hosts_eval(
+            served_max_model_len=2048,
+            eval_max_tokens=budget,
+        )
+        assert fits is True
+
+
+class TestInfeasibleEvalIsAFault:
+    """An eval that could not run is an environment fault, not a gate verdict."""
+
+    def test_the_error_class_routes_to_the_fault_budget(self):
+        """Faults get their own retry budget and never burn the REVERT quota."""
+        assert SharedState._is_integrate_fault({"status": "ok", "error_class": ag.EVAL_KIND_CONTEXT_TOO_SMALL}) is True
+
+    def test_a_genuine_regression_is_still_a_verdict_not_a_fault(self):
+        assert SharedState._is_integrate_fault({"status": "ok", "error_class": "accuracy_regression"}) is False
+
+
+class TestGradeMarksTheRoundInfeasible:
+    """``_grade_integrate_accuracy`` must separate "eval broke" from "eval
+    cannot run here"."""
+
+    @staticmethod
+    def _grade(monkeypatch, tmp_path, server_args):
+        from hyperloom.orchestrator.kernel import request_handlers as rh
+
+        # No score anywhere: the state this bug is about.
+        monkeypatch.setattr(rh, "_maybe_revert_kernel_patch", lambda *_a, **_k: {})
+        monkeypatch.setenv("MAX_MODEL_LEN", "6144")
+        monkeypatch.delenv("HYPERLOOM_EVAL_MAX_TOKENS", raising=False)
+        return rh._grade_integrate_accuracy(
+            {"accuracy": None},
+            session_dir=tmp_path,
+            workspace=tmp_path,
+            server_args=server_args,
+        )
+
+    def test_a_context_that_cannot_host_the_eval_is_flagged(self, monkeypatch, tmp_path):
+        out = self._grade(monkeypatch, tmp_path, "--max-model-len 2048")
+        assert out["infeasible"] is True
+        assert out["accuracy_pass"] is None
+        assert "2048" in out["reason"]
+
+    def test_a_sufficient_context_is_not_flagged(self, monkeypatch, tmp_path):
+        out = self._grade(monkeypatch, tmp_path, "--max-model-len 16384")
+        assert out["infeasible"] is False
+
+    def test_the_env_context_is_used_when_no_flag_is_present(self, monkeypatch, tmp_path):
+        """MAX_MODEL_LEN 6144 against a 4096 budget leaves 2048 for the prompt."""
+        out = self._grade(monkeypatch, tmp_path, "--kv-cache-dtype fp8")
+        assert out["infeasible"] is False