diff --git a/docs/reference/session-breakdown.md b/docs/reference/session-breakdown.md index 241cf24da6..772035641a 100644 --- a/docs/reference/session-breakdown.md +++ b/docs/reference/session-breakdown.md @@ -326,6 +326,29 @@ preserves the available `donor_canonical_id`, `donor_model`, `donor_breakdown_link`. Fields absent from the source recipe remain absent rather than being inferred. +`kb_provenance.warm_replay` additionally records what the replay was judged +on, whether it passed or failed. A replayed recipe is evidence from another +session on another machine, so reproducing its throughput says nothing about +whether it still computes correctly here. + +| Field | Type | Description | +|---|---|---| +| `eval_ran` | bool | Whether an eval produced output for this replay. Separates a model that answered nothing (`eval_ran` true, `replay_accuracy` `0.0`) from a replay nothing checked (`eval_ran` false, `replay_accuracy` `null`). | +| `replay_accuracy` | float \| null | Score measured on the replayed config. `null` when no score could be read — not a score of zero. | +| `baseline_accuracy` | float \| null | Reference the replay was compared against. `null` when the session recorded none, in which case the replay is judged against an absolute floor instead of a relative drop. | +| `eval_error` | string \| null | Why no score could be read. Distinguishes a contract with the eval switched off, an eval that produced an unreadable file, and a results file carrying no metric this parser knows. | + +A replay whose accuracy could not be measured is still promoted — a failed +measurement is not evidence the config broke the model — so `eval_ran` is what +tells an unjudged promotion apart from a judged one. + +The `optimization_stack` entry a warm replay pushes carries the same score as +`accuracy`, so the promotion and the evidence behind it are readable from one +place. `null` there means the lane recorded no verdict. + +Sessions started with `--no-eval` run no eval at all, warm replay included, so +these fields record the absence rather than a score. + ## `session` — `SessionMeta` The `session` section contains the following metadata fields. diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index 5fafb94363..e62dcb910d 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -1446,12 +1446,28 @@ class KBFlusherStatus(TypedDict, total=False): class WarmReplayOutcome(TypedDict, total=False): - """GAP 1 — warm-recipe replay result. Empty {} when it never fired; else ``status`` + per-status fields.""" + """GAP 1 — warm-recipe replay result. Empty {} when it never fired; else ``status`` + per-status fields. + + ``eval_ran`` / ``replay_accuracy`` / ``baseline_accuracy`` are recorded on + every replay that reached a throughput measurement, not only on rejection: + a config that was checked and passed is a different record from one that + was never checked. ``eval_ran`` is what separates "the model scored 0.0" + from "no score exists", which are otherwise both a null accuracy. + + A measurement that fails never stops the run. The replay is admitted and + ``eval_error`` carries why no score could be read, so an unjudged promotion + is visible after the fact rather than silently indistinguishable from a + judged one. + """ status: str expected_gain_pct: float actual_gain_pct: float throughput_after: float + eval_ran: bool + eval_error: str | None + replay_accuracy: float | None + baseline_accuracy: float | None warm_recipe_tier: str warm_recipe_conf: float config_source: str diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 428d668c8e..97ebf72c5f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -1249,6 +1249,86 @@ def test_baseline_double_run_by_default(tmp_path, monkeypatch): assert captured[1]["benchmark"]["server_lifecycle"]["cleanup"] is True +def test_replay_warm_recipe_double_run_forces_warmup_eval(tmp_path): + """Warm replay evaluates in the warmup round and measures in the second.""" + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + output_dir = tmp_path / "ws" + captured: list = [] + fake_run, state = _cold_then_hot_fake_run(captured) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=SimpleNamespace(baseline_double_run=True), + ) + task = SimpleNamespace( + task_id="t-replay-warm", + kind="replay_warm_recipe", + params={ + "output_dir": str(output_dir), + "timeout_sec": 10, + "gpu_type": "mi300x", + "model_path": "/wekafs/models/Qwen-Qwen3-8B", + }, + ) + ctx = SimpleNamespace(task=task, extra={}) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert result["status"] == "succeeded" + assert state["calls"] == 2 + assert captured[0]["benchmark"]["envs"]["RUN_EVAL"] == "true" + assert captured[1]["benchmark"]["envs"]["RUN_EVAL"] == "false" + + +def test_replay_warm_recipe_honours_no_eval(tmp_path): + """``--no-eval`` outranks the replay's forced warmup eval. + + The flag is the operator saying no eval runs this session. Forcing one on + the warmup round would spend the time the flag was passed to save, and do + it silently -- the baseline path on the same executor already honours the + flag, so a replay that did not would be the odd one out. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + captured: list = [] + fake_run, state = _cold_then_hot_fake_run(captured) + shared = SimpleNamespace(baseline_double_run=True, eval_disabled=True) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=shared, + ) + task = SimpleNamespace( + task_id="t-replay-no-eval", + kind="replay_warm_recipe", + params={ + "output_dir": str(tmp_path / "ws"), + "timeout_sec": 10, + "gpu_type": "mi300x", + "model_path": "/wekafs/models/Qwen-Qwen3-8B", + }, + ) + ctx = SimpleNamespace(task=task, extra={"shared_state": shared}) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert result["status"] == "succeeded" + assert state["calls"] == 2 + assert [cfg["benchmark"]["envs"]["RUN_EVAL"] for cfg in captured] == [ + "false", + "false", + ] + + def test_baseline_double_run_can_be_disabled_by_task_param(tmp_path, monkeypatch): """Focused callers may explicitly opt out of the default cold+hot baseline.""" base = tmp_path / "base.yaml" diff --git a/src/hyperloom/inference_optimizer/tests/test_eval_context_feasibility.py b/src/hyperloom/inference_optimizer/tests/test_eval_context_feasibility.py new file mode 100644 index 0000000000..686c17b312 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_eval_context_feasibility.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +"""The accuracy gate must not spend a KEEP attempt on an eval that cannot run. + +Observed on 195 sessions: the parameter search picks ``--max-model-len 2048`` +for throughput, the gsm8k harness asks for a 2048-token completion on top of a +~1k-token five-shot prompt, and every request comes back HTTP 400. No verdict is +ever produced, so ``accuracy_pass`` stays ``None``. Because a positive baseline +accuracy (measured earlier, under the larger context the run started with) is +read as proof that eval works here, the missing verdict blocks the KEEP and the +round is recorded as a fair attempt. Three of those and the kernel is discarded +for a reason that has nothing to do with the kernel. +""" + +import pytest + +from hyperloom.orchestrator.actions.executors import _accuracy_gate as ag +from hyperloom.orchestrator.state.shared_state import SharedState + +# The generation budget the harness requests for gsm8k. +GSM8K_GEN_TOKENS = 2048 + + +class TestServedContextHostsEval: + """Can the serving configuration physically answer an eval request?""" + + def test_context_equal_to_the_generation_budget_cannot_host_a_prompt(self): + """2048 of context and 2048 requested output leaves nothing for the prompt.""" + fits, reason = ag.served_context_hosts_eval( + served_max_model_len=2048, + eval_max_tokens=GSM8K_GEN_TOKENS, + ) + assert fits is False + assert "2048" in reason + + def test_the_real_session_configuration_is_rejected(self): + """The exact shape seen in session e268b0be: env asks 6144, the server + args override it to 2048, and the override is what the server honours.""" + served = ag.resolve_served_context( + server_args=("--kv-cache-dtype fp8 --max-num-batched-tokens 32768 --max-model-len 2048 --async-scheduling"), + env_max_model_len=6144, + ) + assert served == 2048 + fits, _ = ag.served_context_hosts_eval( + served_max_model_len=served, + eval_max_tokens=GSM8K_GEN_TOKENS, + ) + assert fits is False + + def test_the_context_the_env_asked_for_does_host_the_eval(self): + fits, reason = ag.served_context_hosts_eval( + served_max_model_len=6144, + eval_max_tokens=GSM8K_GEN_TOKENS, + ) + assert fits is True + assert reason == "" + + def test_equals_form_of_the_flag_is_understood(self): + assert ( + ag.resolve_served_context( + server_args="--max-model-len=2048", + env_max_model_len=6144, + ) + == 2048 + ) + + def test_env_is_used_when_the_server_args_are_silent(self): + assert ( + ag.resolve_served_context( + server_args="--kv-cache-dtype fp8", + env_max_model_len=6144, + ) + == 6144 + ) + + def test_an_unknown_context_is_not_treated_as_infeasible(self): + """Nothing is known, so nothing is claimed: never block on a guess.""" + fits, _ = ag.served_context_hosts_eval( + served_max_model_len=0, + eval_max_tokens=GSM8K_GEN_TOKENS, + ) + assert fits is True + + @pytest.mark.parametrize("budget", [0, -1]) + def test_an_unbounded_generation_budget_is_not_treated_as_infeasible(self, budget): + fits, _ = ag.served_context_hosts_eval( + served_max_model_len=2048, + eval_max_tokens=budget, + ) + assert fits is True + + +class TestInfeasibleEvalIsAFault: + """An eval that could not run is an environment fault, not a gate verdict.""" + + def test_the_error_class_routes_to_the_fault_budget(self): + """Faults get their own retry budget and never burn the REVERT quota.""" + assert SharedState._is_integrate_fault({"status": "ok", "error_class": ag.EVAL_KIND_CONTEXT_TOO_SMALL}) is True + + def test_a_genuine_regression_is_still_a_verdict_not_a_fault(self): + assert SharedState._is_integrate_fault({"status": "ok", "error_class": "accuracy_regression"}) is False + + +class TestGradeMarksTheRoundInfeasible: + """``_grade_integrate_accuracy`` must separate "eval broke" from "eval + cannot run here".""" + + @staticmethod + def _grade(monkeypatch, tmp_path, server_args): + from hyperloom.orchestrator.kernel import request_handlers as rh + + # No score anywhere: the state this bug is about. + monkeypatch.setattr(rh, "_maybe_revert_kernel_patch", lambda *_a, **_k: {}) + monkeypatch.setenv("MAX_MODEL_LEN", "6144") + monkeypatch.delenv("HYPERLOOM_EVAL_MAX_TOKENS", raising=False) + return rh._grade_integrate_accuracy( + {"accuracy": None}, + session_dir=tmp_path, + workspace=tmp_path, + server_args=server_args, + ) + + def test_a_context_that_cannot_host_the_eval_is_flagged(self, monkeypatch, tmp_path): + out = self._grade(monkeypatch, tmp_path, "--max-model-len 2048") + assert out["infeasible"] is True + assert out["accuracy_pass"] is None + assert "2048" in out["reason"] + + def test_a_sufficient_context_is_not_flagged(self, monkeypatch, tmp_path): + out = self._grade(monkeypatch, tmp_path, "--max-model-len 16384") + assert out["infeasible"] is False + + def test_the_env_context_is_used_when_no_flag_is_present(self, monkeypatch, tmp_path): + """MAX_MODEL_LEN 6144 against a 4096 budget leaves 2048 for the prompt.""" + out = self._grade(monkeypatch, tmp_path, "--kv-cache-dtype fp8") + assert out["infeasible"] is False diff --git a/src/hyperloom/inference_optimizer/tests/test_warm_replay_accuracy_gate.py b/src/hyperloom/inference_optimizer/tests/test_warm_replay_accuracy_gate.py new file mode 100644 index 0000000000..1e692c6dc1 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_warm_replay_accuracy_gate.py @@ -0,0 +1,328 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +"""Warm replay must not promote a config that broke the model. + +Warm replay promoted on throughput alone. Across the retained session pool, 45 +of 241 promoted replays (19%) were promoted while the replayed config was +producing garbage — in the worst case a +23.95% "gain" on a config scoring +0.0000 on gsm8k against a 0.9014 baseline. The promoted config then becomes the +base every later measurement in that session is taken against. + +Two properties this file pins down, both learned from the recorded sessions: + +* The score lives in the *warmup* round. The cold-start guard evaluates once, + in the warmup round, and decides on the measure round, so a gate that reads + the deciding round's own workspace finds nothing: across 852 recorded replays + the score sat in ``warmup_round`` 320 times and in ``measure_round`` never. +* A replay with no score is admitted, not rejected. Rejecting on absent + evidence would have blocked every double-run replay, since the deciding + round never carries a score of its own. +""" + +import json +from pathlib import Path + +import pytest + +from hyperloom.orchestrator.loop.coordinator import Coordinator + +from .test_warm_replay import _make_coord, _StubTask, _warm_recipe_t1 + +RISKY_ENVS = {"VLLM_ROCM_USE_AITER": "1"} +SAFE_ARGS = "--max-num-seqs 2048" +BASELINE_ACC = 0.90 + + +def _coord_with_baseline(tmp_path: Path, accuracy: float) -> Coordinator: + coord = _make_coord(tmp_path, warm_start_recipe=_warm_recipe_t1()) + coord.shared_state.baseline_accuracy = accuracy + coord.shared_state.warm_replay_outcome = { + "status": "in_flight", + "warm_recipe_tier": "exact", + "warm_recipe_conf": 0.85, + "expected_gain_pct": 25.0, + "replay_task_id": "task-warm-replay-prelude", + } + return coord + + +def _risky_task() -> _StubTask: + return _StubTask( + params={ + "extra_server_args": "--attention-backend AITER", + "extra_envs": dict(RISKY_ENVS), + } + ) + + +def _safe_task() -> _StubTask: + return _StubTask(params={"extra_server_args": SAFE_ARGS, "extra_envs": {}}) + + +def _promoted(coord: Coordinator) -> bool: + return bool(coord.shared_state.optimization_stack) + + +def _double_run_dirs(tmp_path: Path, warmup_score: float | None) -> dict: + """Build a replay task directory shaped like a real cold-start double run. + + Returns the result envelope the executor hands back: it decides on the + measure round, so ``output_dir`` and ``workspace`` both point there while + any score sits under the sibling warmup round. + """ + root = tmp_path / "runs" / "replay_warm_recipe" / "task-warm-replay-prelude" + warm_bench = root / "warmup_round" / "benchmark_vllm_1" + measure_bench = root / "measure_round" / "benchmark_vllm_2" + warm_bench.mkdir(parents=True, exist_ok=True) + measure_bench.mkdir(parents=True, exist_ok=True) + if warmup_score is not None: + (warm_bench / "results_2026-08-19T00-00-00.json").write_text( + json.dumps( + {"results": {"gsm8k": {"exact_match,strict-match": warmup_score}}} + ), + encoding="utf-8", + ) + return { + "status": "succeeded", + "output_throughput": 738.0, + "output_dir": str(root / "measure_round"), + "workspace": str(measure_bench), + } + + +class TestScoreIsFoundWhereTheDoubleRunWritesIt: + """The deciding round carries no score of its own; the warmup round does.""" + + def test_a_warmup_round_score_is_read(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + result = _double_run_dirs(tmp_path, warmup_score=0.89) + coord._promote_warm_replay(result, task=_risky_task()) + + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is True + assert outcome["replay_accuracy"] == pytest.approx(0.89) + assert _promoted(coord) is True + + def test_a_collapsed_warmup_round_score_blocks_promotion(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + result = _double_run_dirs(tmp_path, warmup_score=0.0076) + coord._promote_warm_replay(result, task=_risky_task()) + + assert _promoted(coord) is False + outcome = coord.shared_state.warm_replay_outcome + assert outcome["status"] == "accuracy_failed" + assert outcome["replay_accuracy"] == pytest.approx(0.0076) + + def test_no_results_file_anywhere_records_that_no_eval_ran(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + result = _double_run_dirs(tmp_path, warmup_score=None) + coord._promote_warm_replay(result, task=_risky_task()) + + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is False + assert outcome["replay_accuracy"] is None + assert "no results" in outcome["eval_error"] + + +class TestWarmReplayRejectsBrokenConfigs: + def test_a_collapsed_score_blocks_promotion(self, tmp_path): + """The case observed 45 times: big throughput win, ruined accuracy.""" + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.20}, + task=_risky_task(), + ) + assert _promoted(coord) is False + outcome = coord.shared_state.warm_replay_outcome + assert outcome["status"] != "reproduced" + assert "accuracy" in str(outcome.get("reason") or "").lower() + + def test_an_intact_score_still_promotes(self, tmp_path): + """The gate must not cost a genuine win.""" + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.89}, + task=_risky_task(), + ) + assert _promoted(coord) is True + assert coord.shared_state.warm_replay_outcome["status"] == "reproduced" + + @pytest.mark.parametrize("drop", [0.0, 0.04]) + def test_a_drop_within_tolerance_is_not_a_regression(self, tmp_path, drop): + """Healthy run-to-run spread reaches 0.037 in the observed pool, so the + 0.05 tolerance must survive it.""" + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + { + "status": "succeeded", + "output_throughput": 738.0, + "accuracy": BASELINE_ACC - drop, + }, + task=_risky_task(), + ) + assert _promoted(coord) is True + + +class TestEveryReplayIsJudged: + """A KB recipe is another machine's evidence, so reproducing its throughput + says nothing about whether it still computes correctly here. The high-risk + trigger the other lanes use is deliberately not applied.""" + + def test_a_config_with_no_high_risk_knob_is_still_judged(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.20}, + task=_safe_task(), + ) + assert _promoted(coord) is False + assert coord.shared_state.warm_replay_outcome["status"] == "accuracy_failed" + + def test_a_sound_config_with_no_high_risk_knob_promotes(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.89}, + task=_safe_task(), + ) + assert _promoted(coord) is True + + +class TestAbsentEvidenceDoesNotBlock: + """A failed measurement never stops the run. It is not evidence the config + broke the model, and rejecting on it would block every double-run replay, + since the deciding round never carries a score of its own.""" + + def test_a_missing_verdict_still_promotes_and_is_marked(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0}, + task=_risky_task(), + ) + assert _promoted(coord) is True + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is False + assert outcome["replay_accuracy"] is None + assert outcome["eval_error"] + + def test_an_unscorable_results_file_promotes_and_records_why(self, tmp_path): + """The eval ran and produced a file with no metric this parser knows — + a different state from an eval that never ran, and still not a reason + to stop.""" + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + result = _double_run_dirs(tmp_path, warmup_score=None) + bench = Path(result["output_dir"]).parent / "warmup_round" / "benchmark_vllm_1" + (bench / "results_2026-08-19T00-00-00.json").write_text( + json.dumps({"results": {"gsm8k": {"unknown_metric": 1.0}}}), + encoding="utf-8", + ) + coord._promote_warm_replay(result, task=_risky_task()) + + assert _promoted(coord) is True + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is True + assert outcome["replay_accuracy"] is None + assert "no recognized metric" in outcome["eval_error"] + + def test_an_undecodable_results_file_is_an_eval_that_ran(self, tmp_path): + """A file the parser could not decode still proves the eval ran.""" + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + result = _double_run_dirs(tmp_path, warmup_score=None) + bench = Path(result["output_dir"]).parent / "warmup_round" / "benchmark_vllm_1" + (bench / "results_2026-08-19T00-00-00.json").write_text( + "{ this is not json", + encoding="utf-8", + ) + coord._promote_warm_replay(result, task=_risky_task()) + + assert _promoted(coord) is True + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is True + assert outcome["replay_accuracy"] is None + assert "parse error" in outcome["eval_error"] + + def test_a_parser_crash_is_not_an_eval_that_ran(self, tmp_path, monkeypatch): + """A parser that raised read no file, so nothing says the eval ran. + + Recording this as "ran" reads as a model that answered nothing, which + is the one state an operator must be able to tell it apart from: the + first is a broken config, the second is broken infrastructure. + """ + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + result = _double_run_dirs(tmp_path, warmup_score=None) + + from hyperloom.orchestrator.actions.executors import _accuracy_gate + + def _raise(*_args, **_kwargs): + raise OSError("results directory vanished mid-read") + + monkeypatch.setattr(_accuracy_gate, "parse_eval_results", _raise) + coord._promote_warm_replay(result, task=_risky_task()) + + assert _promoted(coord) is True + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is False + assert outcome["replay_accuracy"] is None + assert "eval parse raised" in outcome["eval_error"] + + def test_a_passing_replay_records_no_eval_error(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.89}, + task=_risky_task(), + ) + assert coord.shared_state.warm_replay_outcome["eval_error"] is None + + def test_no_baseline_missing_verdict_still_promotes(self, tmp_path): + coord = _coord_with_baseline(tmp_path, 0.0) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0}, + task=_risky_task(), + ) + assert _promoted(coord) is True + assert coord.shared_state.warm_replay_outcome["baseline_accuracy"] is None + + def test_no_baseline_collapsed_score_rejected_by_absolute_floor(self, tmp_path): + """``--no-eval`` sessions carry no baseline reference; a collapsed replay + must still be caught by the enablement absolute floor.""" + coord = _coord_with_baseline(tmp_path, 0.0) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.20}, + task=_risky_task(), + ) + assert _promoted(coord) is False + outcome = coord.shared_state.warm_replay_outcome + assert outcome["status"] == "accuracy_failed" + assert "absolute floor" in str(outcome.get("reason") or "").lower() + + def test_no_baseline_sound_score_passes_absolute_floor(self, tmp_path): + coord = _coord_with_baseline(tmp_path, 0.0) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.89}, + task=_risky_task(), + ) + assert _promoted(coord) is True + + +class TestAccuracyIsRecordedOnSuccess: + """A promotion that was checked and passed is not the same record as one + that was never checked.""" + + def test_a_passing_replay_records_both_scores(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.89}, + task=_risky_task(), + ) + outcome = coord.shared_state.warm_replay_outcome + assert outcome["eval_ran"] is True + assert outcome["replay_accuracy"] == pytest.approx(0.89) + assert outcome["baseline_accuracy"] == pytest.approx(BASELINE_ACC) + + def test_the_promoted_stack_entry_carries_the_score(self, tmp_path): + coord = _coord_with_baseline(tmp_path, BASELINE_ACC) + coord._promote_warm_replay( + {"status": "succeeded", "output_throughput": 738.0, "accuracy": 0.89}, + task=_risky_task(), + ) + entry = coord.shared_state.optimization_stack[-1] + assert entry["action"] == "replay_warm_recipe" + assert entry["accuracy"] == pytest.approx(0.89) diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index 4c1364f3fd..68e67ac059 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -17,6 +17,7 @@ import logging import math import os +import shlex from pathlib import Path from typing import Any @@ -70,6 +71,18 @@ # Distinct eval-failure kinds. EVAL_KIND_RUNTIME_FAILURE = "eval_runtime_failure" + +# The serving configuration cannot answer an eval request at all, so no verdict +# can ever be produced under it. Distinct from every other eval failure because +# it is a property of the configuration rather than of the run: retrying the +# same round reproduces it exactly. +EVAL_KIND_CONTEXT_TOO_SMALL = "eval_context_too_small" + +# Smallest prompt an eval task is assumed to send. Deliberately conservative: a +# five-shot gsm8k prompt runs to roughly a thousand tokens, so a context that +# cannot hold even 256 on top of the generation budget cannot hold any real +# task. Being conservative keeps this a proof of infeasibility, never a guess. +_MIN_EVAL_PROMPT_TOKENS = 256 EVAL_KIND_ACCURACY_UNAVAILABLE = "accuracy_unavailable" EVAL_KIND_ACCURACY_BELOW_FLOOR = "accuracy_below_floor" # The model never emitted EOS, so the eval was cut short and scored ~0. @@ -340,6 +353,96 @@ def eval_contract_fingerprint( return hashlib.sha256(payload.encode("utf-8", "replace")).hexdigest()[:16] +def resolve_served_context( + *, + server_args: str | None, + env_max_model_len: Any = 0, +) -> int: + """Resolve the context length the server was actually started with. + + ``--max-model-len`` inside the server-args string wins over the + ``MAX_MODEL_LEN`` env: the env is a request, the CLI flag is what the + process honours, and the parameter search rewrites the flag while leaving + the env untouched. + + Args: + server_args: The server-args string (e.g. ``EXTRA_VLLM_ARGS``). + env_max_model_len: The ``MAX_MODEL_LEN`` env value, used only when the + server args do not carry the flag. + + Returns: + The served context in tokens, or ``0`` when neither source resolves. + """ + raw = str(server_args or "") + if raw: + try: + toks = shlex.split(raw) + except ValueError: + toks = raw.split() + flag = "--max-model-len" + prefix = flag + "=" + for i, tok in enumerate(toks): + value = None + if tok == flag and i + 1 < len(toks): + value = toks[i + 1] + elif tok.startswith(prefix): + value = tok[len(prefix):] + if value is not None: + try: + parsed = int(value) + except (TypeError, ValueError): + break + if parsed > 0: + return parsed + break + try: + return max(0, int(env_max_model_len or 0)) + except (TypeError, ValueError): + return 0 + + +def served_context_hosts_eval( + *, + served_max_model_len: Any, + eval_max_tokens: Any, +) -> tuple[bool, str]: + """Whether the served context can hold an eval prompt plus its completion. + + Answers only the question it can answer from configuration alone: is the + context provably too small for ANY prompt once the generation budget is + reserved. An unknown context or an unbounded generation budget yields + ``True`` — the point is to identify configurations that cannot work, never + to guess at ones that might not. + + Args: + served_max_model_len: Context the server was started with; ``0`` when + unknown. + eval_max_tokens: Completion tokens the harness requests per sample; + ``0`` or negative means unbounded. + + Returns: + ``(fits, reason)``. ``reason`` is empty when it fits. + """ + try: + ctx = int(served_max_model_len or 0) + except (TypeError, ValueError): + ctx = 0 + try: + gen = int(eval_max_tokens or 0) + except (TypeError, ValueError): + gen = 0 + if ctx <= 0 or gen <= 0: + return True, "" + room = ctx - gen + if room >= _MIN_EVAL_PROMPT_TOKENS: + return True, "" + return False, ( + f"served --max-model-len {ctx} cannot host the eval: {gen} completion " + f"tokens leave {room} for the prompt, below the {_MIN_EVAL_PROMPT_TOKENS} " + "token minimum, so every request is rejected before it reaches the model" + ) + + def accuracy_keep_block( accuracy_pass: bool | None, *, @@ -730,6 +833,7 @@ def accuracy_passed( "EVAL_KIND_ACCURACY_BELOW_FLOOR", "EVAL_KIND_ACCURACY_UNAVAILABLE", "EVAL_KIND_GENERATION_PATHOLOGY", + "EVAL_KIND_CONTEXT_TOO_SMALL", "EVAL_KIND_RUNTIME_FAILURE", "EVAL_PROBE_FILENAME", "_extract_eval_contract_fields", @@ -746,6 +850,8 @@ def accuracy_passed( "read_eval_probe", "request_baseline_accuracy_stop", "resolve_enablement_mode", + "resolve_served_context", "require_framework_accuracy_default", "require_kernel_accuracy_default", + "served_context_hosts_eval", ] diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index f4f1845012..2a297dac8b 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -3440,12 +3440,33 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: # Round 1 (warmup): boot + run, leave running so round 2 can # re-attach. Throughput discarded (cold-contaminated). warmup_dir = output_dir / "warmup_round" + # A replayed KB config is promoted onto ``current_best`` and becomes + # the reference every later measurement in the session is taken + # against, so it may not be adopted on throughput alone. The warmup + # round is the only round that evaluates, so forcing it here is what + # gives the promotion gate a score to judge; inheriting a contract + # with RUN_EVAL off would leave the gate with no evidence. The + # staged-accuracy lane owns its own eval schedule and is left alone. + # ``force_disable_eval`` marks the salvage retry taken when the eval + # is itself what aborted the run; forcing it back on there would + # reproduce the failure and lose the throughput baseline too. + # ``--no-eval`` is the operator saying no eval runs this session, and + # forcing one here would spend the time it was passed to save while + # silently overriding that. A replay is then promoted unjudged, which + # is the trade the flag already makes everywhere else. + force_warmup_eval = ( + str(getattr(ctx.task, "kind", "") or "") == "replay_warm_recipe" + and not defer_accuracy_until_after_measure + and not force_disable_eval + and not eval_disabled + ) warmup_cfg = self._write_lifecycle_config( materialized_config_path, warmup_dir, cleanup=False, pid_dir=pid_dir, port=port, + run_eval=True if force_warmup_eval else None, ) log.info( "baseline_executor: cold-start guard — warmup round (discarded, boots persistent server) in %s", diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 4aeb68f212..80e090611c 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -1641,6 +1641,12 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la getattr(gv, "accepted_kernels", []) or [] ), "gain_pct": gain, + # The verdict this KEEP rests on. ``None`` means the + # variant was not gated (not high-risk, or no + # baseline to compare against) rather than that it + # scored nothing — without it the ledger cannot say + # afterwards whether a kept config was ever checked. + "accuracy": accuracy_value, "tput": decision_tput, "decision_tput": decision_tput, "single_workspace": r.workspace, diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 00b961283f..cb37b806b9 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -93,6 +93,10 @@ async def handler(payload: dict, *, session_dir: Path) -> dict: _FRAMEWORK_APPLYBACK_ARTIFACT_KIND = "framework_applyback" _INTEGRATE_ACCURACY_VALIDATION_TIER = "integrate_e2e_accuracy" +# Mirrors the completion ceiling the inferencex eval shim installs; kept in sync +# so the feasibility check reasons about the budget the eval will really ask for. +_EVAL_DEFAULT_MAX_TOKENS = 4096 + def _vram_guarded_server_args(extra_args: str) -> str: """Optionally cap ``--gpu-memory-utilization`` for the integrate re-baseline. @@ -7159,12 +7163,30 @@ async def _run_integrate_rebaseline_with_lock_retry( return retry_result +def _eval_generation_budget() -> int: + """Completion tokens the eval harness reserves per sample. + + Mirrors the clamp installed by the inferencex shim: ``HYPERLOOM_EVAL_MAX_TOKENS`` + when it parses as a positive integer, else the shim's own default. ``0`` + means the operator disabled the clamp, so no budget can be assumed. + """ + raw = (os.environ.get("HYPERLOOM_EVAL_MAX_TOKENS") or "").strip() + if not raw: + return _EVAL_DEFAULT_MAX_TOKENS + try: + value = int(raw) + except (TypeError, ValueError): + return _EVAL_DEFAULT_MAX_TOKENS + return value if value >= 0 else _EVAL_DEFAULT_MAX_TOKENS + + def _grade_integrate_accuracy( bench_result: dict[str, Any], *, session_dir: Path, workspace: Path, strict: bool = False, + server_args: str = "", ) -> dict[str, Any]: """Grade a kernel re-baseline's accuracy against the session baseline. @@ -7198,6 +7220,8 @@ def _grade_integrate_accuracy( accuracy_passed, parse_eval_results, require_kernel_accuracy_default, + resolve_served_context, + served_context_hosts_eval, ) baseline_accuracy = 0.0 @@ -7240,6 +7264,28 @@ def _grade_integrate_accuracy( "accuracy gate produced no eval result and this artifact has no " "other end-to-end correctness evidence" ) + # A verdict can be missing because the eval broke, or because the serving + # configuration cannot answer an eval request at all. Only the first says + # anything about the patch. The second reproduces on every retry, so + # charging it to the patch discards a kernel over a configuration choice. + infeasible = False + if accuracy_pass is None: + fits, why = served_context_hosts_eval( + served_max_model_len=resolve_served_context( + server_args=server_args, + env_max_model_len=os.environ.get("MAX_MODEL_LEN", 0), + ), + eval_max_tokens=_eval_generation_budget(), + ) + if not fits: + infeasible = True + reason = why + log.warning( + "integrate_handler: the accuracy gate cannot run under this " + "serving configuration, so no kernel can clear it until the " + "configuration changes: %s", + why, + ) log.info( "integrate_handler: accuracy gate pass=%s blocked=%s degraded=%s new=%s baseline=%.4f source=%s", accuracy_pass, @@ -7254,6 +7300,7 @@ def _grade_integrate_accuracy( "accuracy_pass": accuracy_pass, "reason": reason, "degraded": degraded, + "infeasible": infeasible, "accuracy": new_accuracy, "baseline_accuracy": baseline_accuracy, "task": task, @@ -7745,8 +7792,28 @@ def _restore_aiter_rebuild_env() -> None: session_dir=session_dir, workspace=workspace, strict=applyback_pending, + server_args=extra_args, ) if accuracy_gate["blocked"]: + if accuracy_gate.get("infeasible"): + # The gate cannot run under this configuration, so this round + # measured nothing about the patch. Report it as an integration + # fault: faults carry their own budget and never consume one of + # the three attempts a patch gets to prove itself. + from ..actions.executors._accuracy_gate import ( + EVAL_KIND_CONTEXT_TOO_SMALL, + ) + + revert_result = _maybe_revert_kernel_patch(apply_result) + return { + "status": "failed", + "error_class": EVAL_KIND_CONTEXT_TOO_SMALL, + "error": accuracy_gate["reason"], + "decision": "NEEDS_REVIEW", + "gain_pct": gain_pct, + "accuracy_gate": accuracy_gate, + "revert_result": revert_result, + } # A measured regression is hard negative evidence -> REVERT. A # missing verdict is only an evidence gap -> NEEDS_REVIEW. decision = "REVERT" if accuracy_gate["accuracy_pass"] is False else "NEEDS_REVIEW" diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index a49768ca67..82c58ddc5a 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -2668,6 +2668,10 @@ def _lift_to_current_best( "candidate_extra_server_args": candidate_args, "extra_server_args": full_args, "extra_envs": (dict(bv.get("extra_envs") or {}) if isinstance(bv, dict) else {}), + # Carry the promoting lane's accuracy verdict onto the stack + # so CLOSE reads one place instead of reconstructing which + # lane promoted the champion. ``None`` means "not gated". + "accuracy": (bv.get("accuracy") if isinstance(bv, dict) else None), "tput": float(best_tput), "workspace": (bv.get("workspace") if isinstance(bv, dict) else None), "ts": datetime.now(timezone.utc).isoformat(), diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index 8a2ca25d5b..371eff91e0 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -31,6 +31,13 @@ log = _logging.getLogger(__name__) +# The ``parse_eval_results`` reasons that prove an eval produced output: a +# results file it could not decode, and one carrying no metric it recognises. +# Every other reason -- no file at all, or the parser itself raising -- leaves +# no evidence the eval ran, and calling those "ran" makes an infrastructure +# fault read as a model that answered nothing. +_EVAL_RAN_BUT_UNSCORABLE = ("parse error:", "no recognized metric in") + def _merge_current_recipe_configs( explore: Mapping[str, Any], @@ -1978,6 +1985,151 @@ def _require_combined_warm_rollback( self.shared_state.save(self.session_dir) return False + @staticmethod + def _replay_eval_search_root(result: dict) -> Path | None: + """Directory holding every round of this replay task. + + The cold-start guard evaluates only in the warmup round but decides on + the measure round, so the score lands in a sibling directory rather + than under the deciding round's own workspace. Searching the workspace + alone finds nothing on every double-run replay: across 852 recorded + replays the score sat in ``warmup_round`` 320 times and in + ``measure_round`` never. + + Args: + result: The ``replay_warm_recipe`` result envelope. + + Returns: + The task-level directory containing both round directories, or + ``None`` when the result names no usable path. + """ + from ..actions.executors.baseline import ( + _MEASURE_ROUND_DIR, + _WARMUP_ROUND_DIR, + ) + + round_dirs = {_WARMUP_ROUND_DIR, _MEASURE_ROUND_DIR} + for key in ("output_dir", "workspace"): + raw = str(result.get(key) or "").strip() + if not raw: + continue + path = Path(raw) + for candidate in (path, *path.parents): + if candidate.name in round_dirs: + return candidate.parent + return path + return None + + def _warm_replay_accuracy_ok( + self, + result: dict, + task: "Task | None", + outcome: dict, + ) -> bool: + """Whether a replayed config may be promoted on accuracy grounds. + + Every replay is judged, not just the ones touching a knob known to be + risky: a KB recipe is evidence from another session and another + machine, so reproducing its throughput says nothing about whether it + still computes correctly here. The measured score is recorded either + way — a promotion that was checked and passed is not the same record as + one that was never checked. + + ``eval_ran`` separates the two ways ``replay_accuracy`` can be absent. + A score of 0.0 means the model answered nothing; no score at all means + no evidence either way, and those must not collapse into one state. + + Args: + result: The ``replay_warm_recipe`` result envelope. + task: The originating task, carrying the replayed args/envs. + outcome: The warm-replay outcome dict, stamped either way. + + Returns: + ``True`` when promotion may proceed; ``False`` when the caller must + stop (the rollback and outcome have already been recorded). + """ + from ..actions.executors._accuracy_gate import ( + DEFAULT_ENABLEMENT_ACCURACY_FLOOR, + accuracy_meets_floor, + accuracy_passed, + parse_eval_results, + ) + + state = self.shared_state + try: + baseline_accuracy = float(getattr(state, "baseline_accuracy", 0.0) or 0.0) + except (TypeError, ValueError): + baseline_accuracy = 0.0 + + measured = result.get("accuracy") + eval_ran = isinstance(measured, (int, float)) + eval_error = "" + if not eval_ran: + measured = None + root = self._replay_eval_search_root(result) + if root is None: + eval_error = "replay result names no round directory" + else: + try: + eval_out = parse_eval_results(root) + except Exception as exc: # noqa: BLE001 — an unreadable eval is "no verdict" + eval_out = {"error": f"eval parse raised: {type(exc).__name__}"} + parsed = eval_out.get("accuracy") + if isinstance(parsed, (int, float)): + measured = float(parsed) + eval_ran = True + else: + eval_error = str( + eval_out.get("error") or "no accuracy in eval output" + ) + # Only a results file the parser reached can say the eval + # ran; a parser that never got one says nothing either way. + eval_ran = eval_error.startswith(_EVAL_RAN_BUT_UNSCORABLE) + + outcome["eval_ran"] = bool(eval_ran) + outcome["eval_error"] = eval_error or None + outcome["replay_accuracy"] = float(measured) if measured is not None else None + outcome["baseline_accuracy"] = ( + baseline_accuracy if baseline_accuracy > 0 else None + ) + + if measured is None: + # A measurement that failed is not evidence the config broke the + # model, so it must not stop the run: the replay is admitted and the + # reason it could not be judged is recorded instead. ``eval_ran`` + # says whether an eval produced nothing or never ran at all. + log.warning( + "warm-replay admitted without an accuracy verdict " + "(eval_ran=%s, baseline %.4f): %s", + eval_ran, + baseline_accuracy, + eval_error or "no reason recorded", + ) + return True + if baseline_accuracy > 0: + if accuracy_passed(baseline_accuracy, float(measured)): + return True + reason = ( + "accuracy regression on the replayed config " + f"(baseline {baseline_accuracy:.4f}, replay {measured:.4f})" + ) + elif accuracy_meets_floor(measured, DEFAULT_ENABLEMENT_ACCURACY_FLOOR): + return True + else: + reason = ( + "accuracy below absolute floor on the replayed config " + f"(replay {measured:.4f}, " + f"floor {DEFAULT_ENABLEMENT_ACCURACY_FLOOR:.2f})" + ) + if not self._require_combined_warm_rollback(result, task, outcome): + return False + outcome["status"] = "accuracy_failed" + outcome["reason"] = reason + state.warm_replay_outcome = outcome + state.save(self.session_dir) + log.info("warm-replay REJECTED on accuracy: %s", reason) + return False + def _promote_warm_replay( self, result: dict, @@ -2071,6 +2223,16 @@ def _promote_warm_replay( state.save(self.session_dir) log.info("warm-replay REJECTED by quality gate: %s", qg) return + # A replayed config lands on ``current_best``, so every later + # measurement in the session is taken against it. Promoting one on + # throughput alone is how a config that makes the model emit garbage + # becomes the session's reference: breaking the numerics is itself a + # large throughput win, so the objective actively selects for it. + # Every replay is judged here, not only high-risk knobs: a KB recipe is + # evidence from another session, so reproducing its throughput says + # nothing about whether it still computes correctly here. + if not self._warm_replay_accuracy_ok(result, task, outcome): + return measured_gain = (single_round_tput / baseline_tput - 1.0) * 100.0 result["combined_gain_pct"] = round(measured_gain, 3) decision_params = (task.params if task is not None else {}) or {} @@ -2198,6 +2360,11 @@ def _promote_warm_replay( "gain_pct": round(measured_gain, 3), "hot_tput": float(hot_tput), "cold_tput": float(cold_round_tput) if cold_round_tput > 0 else None, + # The score this promotion was judged on, recorded alongside the + # throughput it was judged with. ``None`` means no score could be + # read, not that the model scored nothing — ``eval_ran`` on the + # outcome separates those. + "accuracy": outcome.get("replay_accuracy"), # source_tier records the warm-recipe tier for breakdown attribution. "source_tier": outcome.get("warm_recipe_tier", ""), "source_confidence": outcome.get("warm_recipe_conf", 0.0), diff --git a/src/hyperloom/orchestrator/state/_shared_state/explore_state.py b/src/hyperloom/orchestrator/state/_shared_state/explore_state.py index 078d2396d2..720c4d25e4 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/explore_state.py +++ b/src/hyperloom/orchestrator/state/_shared_state/explore_state.py @@ -599,6 +599,9 @@ def _list_field(key: str) -> list[str]: "note": str(variant.get("note") or ""), "tput": variant.get("output_throughput") or variant.get("tput"), "gain_pct": variant.get("gain_pct"), + # Carried through so the ledger records what the KEEP was judged on; + # ``None`` means the variant was never gated, not that it scored 0. + "accuracy": variant.get("accuracy"), "stack_index": variant.get("stack_index"), "accepted_at_round": str(variant.get("accepted_at_round") or ""), "ts": str(variant.get("ts") or _shared_state_module()._now_iso()), diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 29b32c349f..aa147d28be 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -248,6 +248,10 @@ def render_model_arch_compact(arch: dict | None) -> str: "missing_integration_inputs", "patch_not_applied", "apply_failed", + # The served context cannot host an eval request, so the accuracy gate + # can never return a verdict under this configuration. The patch was + # never fairly measured, so it must not spend a KEEP attempt. + "eval_context_too_small", "mn_server_restart_failed_post_patch", "rebaseline_exception", "cpp_itfs_rebuild_not_verified",