diff --git a/selfcheck_gates.py b/selfcheck_gates.py new file mode 100644 index 0000000000..c5fcce8f98 --- /dev/null +++ b/selfcheck_gates.py @@ -0,0 +1,124 @@ +"""Throwaway self-review: sweep the real pricing functions for disagreements.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from hyperloom.orchestrator.phases import machine_state as ms + +BOOT, COLD_BENCH, HOT = 350.0, 550.0, 400.0 +COLD_ROUND = BOOT + COLD_BENCH + + +def state(usable, *, phase="PRELUDE", warm=HOT, post_ready=COLD_BENCH, marked=False, double=True): + return SimpleNamespace( + phase=phase, + max_minutes=180, + baseline_tput=1000.0, + baseline_runtime_sec=COLD_ROUND, + baseline_post_ready_runtime_sec=post_ready, + baseline_warm_runtime_sec=warm, + baseline_measure_round_dropped=marked, + baseline_double_run=double, + session_budget_usable_sec=lambda: usable, + ) + + +def gate1_need(s): + """What the pre-ignition gate demands, mirroring _round_affordable.""" + cold = ms.measured_seconds(s, "baseline_runtime_sec") + rnd = ms.baseline_round_cost_sec(s, double_run=bool(s.baseline_double_run)) + if cold is None or rnd is None: + return None + use = (ms.one_more_measurement_sec(s) or cold) if s.phase == "PRELUDE" else 0.0 + return rnd + use + + +def gate2_need(s, *, warmup_sec, warmup_post_ready): + """What the post-warmup gate demands, mirroring _measure_round_affordable.""" + bench = ms.measured_seconds(s, "baseline_warm_runtime_sec") + if bench is None: + bench = warmup_post_ready + if bench is None or warmup_sec is None: + return None + use = (ms.one_more_measurement_sec(s) or warmup_sec) if s.phase == "PRELUDE" else 0.0 + return bench + use + + +def main() -> int: + bad = 0 + + # 1. The band: is there a budget gate 1 admits and gate 2 then certainly refuses? + # Gate 2 is asked after the warmup has spent a cold pass. + band = [] + for usable in range(0, 6001, 10): + s = state(float(usable)) + need1 = gate1_need(s) + admitted = need1 is not None and usable >= need1 + if not admitted: + continue + after = state(float(usable) - COLD_ROUND) + need2 = gate2_need(after, warmup_sec=COLD_ROUND, warmup_post_ready=COLD_BENCH) + if need2 is not None and (usable - COLD_ROUND) < need2: + band.append(usable) + if band: + bad += 1 + print(f"BAND gate 1 admits and gate 2 refuses for usable in {band[0]}..{band[-1]}") + else: + print("ok no budget is admitted before ignition only to be refused after the cold pass") + + # 2. Livelock: with the mark set, does every budget either close or admit a retry? + stuck = [] + for usable in range(0, 8001, 10): + s = state(float(usable), marked=True) + closes = ms.exit_cold_anchor_prelude(s) is not None + need1 = gate1_need(s) + admits = need1 is not None and usable >= need1 + if not closes and not admits: + stuck.append(usable) + if stuck: + bad += 1 + print(f"LIVELOCK neither closes nor admits for usable in {stuck[0]}..{stuck[-1]}") + else: + print("ok a marked session always either closes or may retry") + + # 3. A session with no split measured (multi-node / scriptable shape). + s = state(3000.0, warm=0.0, post_ready=0.0) + need = gate1_need(s) + if need is None: + bad += 1 + print("UNGATED a round with no boot boundary is waved through") + else: + print(f"ok a round with no split is priced at {need:.0f}s (whole cold rounds)") + + # 4. A first baseline must never be judged. + first = SimpleNamespace( + phase="PRELUDE", + max_minutes=180, + baseline_tput=0.0, + baseline_runtime_sec=0.0, + baseline_post_ready_runtime_sec=0.0, + baseline_warm_runtime_sec=0.0, + baseline_measure_round_dropped=False, + baseline_double_run=True, + session_budget_usable_sec=lambda: 60.0, + ) + if gate1_need(first) is not None: + bad += 1 + print("PREDICTED a first baseline was priced from measurements it cannot have") + else: + print("ok a first baseline is not judged") + + # 5. Later phases ask only whether the round fits. + later = state(2000.0, phase="EXPLORE") + if gate1_need(later) != ms.baseline_round_cost_sec(later, double_run=True): + bad += 1 + print("SCOPED a re-baseline outside PRELUDE was charged for a successor") + else: + print("ok a re-baseline outside PRELUDE pays only for itself") + + return bad + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/hyperloom/agents/robustness/role/envelope.py b/src/hyperloom/agents/robustness/role/envelope.py index a9a8512163..4931f5b01d 100644 --- a/src/hyperloom/agents/robustness/role/envelope.py +++ b/src/hyperloom/agents/robustness/role/envelope.py @@ -114,6 +114,7 @@ class IntentType(str, Enum): "start_ts", "resumed_ts", "max_minutes", + "closing_grace_sec", "optimization_stack", "gain_per_stack_entry", "schema_version", diff --git a/src/hyperloom/inference_optimizer/actions/roofline.md b/src/hyperloom/inference_optimizer/actions/roofline.md index 9117f2215e..4ae92490fc 100644 --- a/src/hyperloom/inference_optimizer/actions/roofline.md +++ b/src/hyperloom/inference_optimizer/actions/roofline.md @@ -92,8 +92,11 @@ analysis task. ## Cost / runtime -* `cost_minutes_p50=8` / `cost_minutes_p75=15` — dominated by profile - (Magpie + torch profiler overhead) + trace_analyze (TraceLens - subprocess); `trace_split` inside TraceLens adds < 30s. +* `typical_runtime_min=10` — dominated by profile (Magpie + torch profiler + overhead) + trace_analyze (TraceLens subprocess); `trace_split` inside + TraceLens adds < 30s. The estimate is calibrated on small models: a field + session measured an 81-minute roofline, so the budget guards prefer this + session's own measured baseline round once one exists and fall back to this + number only before that. * `requires_lanes=[profile_lane]` — same lane as `profile` so we don't run two profile-class tasks concurrently against the server. diff --git a/src/hyperloom/inference_optimizer/tests/conftest.py b/src/hyperloom/inference_optimizer/tests/conftest.py index 3b0b9d4fcd..ccfb24d0a3 100644 --- a/src/hyperloom/inference_optimizer/tests/conftest.py +++ b/src/hyperloom/inference_optimizer/tests/conftest.py @@ -46,6 +46,43 @@ def _bootstrap_kernel_agent_env() -> None: _bootstrap_kernel_agent_env() +def enable_multi_node(monkeypatch, nodes: int = 2) -> None: + """Put the executors in multi-node mode with a no-op per-round server restart. + + Multi-node is what puts a discarded client-warmup pass in front of a measured + round, so it is the mode in which one round launches more than one benchmark + process -- and the restart between them is the part that needs a cluster. + + Args: + monkeypatch: The requesting test's monkeypatch fixture. + nodes: How many nodes to claim, which is what the executors read. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl + + async def _no_restart(*_args, **_kwargs) -> None: + return None + + monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) + monkeypatch.setattr(mnl, "restart_server_for_round", _no_restart) + + +def launches_by_round_slot(recorded: list[dict]) -> dict[str, dict]: + """Index recorded benchmark launches by the output slot each round ran in. + + A round is identified by the slot it writes into rather than by its position + in the launch order, so a test can assert on one pass of a round without + encoding how many passes precede it. + + Args: + recorded: Launch records, each carrying the ``round_slot`` name the + subprocess doubles stamp on every round they see. + + Returns: + dict[str, dict]: The last launch recorded per slot name. + """ + return {launch["round_slot"]: launch for launch in recorded} + + def seed_target_analysis_marker(session_dir: Path) -> Path: """Write a ``no_target_gpu_configured`` marker JSON at the session dir.""" from hyperloom.inference_optimizer.session.session_paths import target_baseline_json @@ -324,12 +361,97 @@ def suppression_window_s() -> float: return StallConfig().stall_timeout_s -def enable_multi_node(monkeypatch, nodes: int = 2) -> None: - """Put the executors in multi-node mode with a no-op per-round server restart.""" - from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl +class _RayDoubleActorClass: + """The ``@ray.remote`` class: ``.options(...)`` then ``.remote()`` for a handle.""" - async def _no_restart(*_args, **_kwargs) -> None: - return None + def __init__(self, cls: type) -> None: + self._cls = cls + self._options: dict = {} - monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) - monkeypatch.setattr(mnl, "restart_server_for_round", _no_restart) + def options(self, **opts): + self._options = dict(opts) + return self + + def remote(self, *args, **kwargs) -> "_RayDoubleActorHandle": + return _RayDoubleActorHandle(self._cls(*args, **kwargs), self._options) + + +class _RayDoubleActorHandle: + """An actor handle whose methods run in a pool sized like the real actor's. + + ``max_concurrency`` is read from the options the production code passed, not + assumed: an actor left at Ray's single method slot gets a single-worker pool + here too, so a method that has to reach a call already running blocks behind + it exactly as it would on a real cluster. + """ + + def __init__(self, obj, options: dict) -> None: + from concurrent.futures import ThreadPoolExecutor + + self._obj = obj + self._pool = ThreadPoolExecutor(max_workers=int(options.get("max_concurrency", 1) or 1)) + self.killed = False + + def __getattr__(self, name: str): + from types import SimpleNamespace + + method = getattr(self._obj, name) + return SimpleNamespace(remote=lambda *a, **kw: self._pool.submit(method, *a, **kw)) + + +class RayDouble: + """A ``ray`` module stand-in that runs actor methods in real threads. + + Ray is not a test dependency, and what these tests are about is what a lease + and its actor do to each other while work is in flight. So the transport is + the only thing faked: the actor is the real class, running real subprocesses, + and an ``ObjectRef`` is a :class:`~concurrent.futures.Future`. + """ + + class exceptions: # noqa: N801 — mirrors the ray.exceptions namespace + class RayActorError(Exception): + pass + + class RayTaskError(Exception): + pass + + class GetTimeoutError(Exception): + pass + + def __init__(self) -> None: + self.killed: list = [] + + def remote(self, cls: type) -> _RayDoubleActorClass: + return _RayDoubleActorClass(cls) + + def cluster_resources(self) -> dict: + return {"CPU": 8.0, "GPU": 8.0, "serving_slot": 1.0} + + def get(self, ref, timeout: float | None = None): + return ref.result(timeout) + + def wait(self, refs: list, *, num_returns: int = 1, timeout: float | None = None): + from concurrent.futures import FIRST_COMPLETED + from concurrent.futures import wait as futures_wait + + done, not_done = futures_wait(refs, timeout=timeout, return_when=FIRST_COMPLETED) + return list(done)[:num_returns], list(not_done) + + def kill(self, actor) -> None: + actor.killed = True + self.killed.append(actor) + + +@pytest.fixture +def serving_lease_on_a_ray_double(monkeypatch): + """A real :class:`ServingLease` over :class:`RayDouble`, closed on teardown.""" + import sys + from types import SimpleNamespace + + from hyperloom.orchestrator.actions.executors import _ray_backend as rb + from hyperloom.orchestrator.actions.executors import _ray_serving as rs + + monkeypatch.setitem(sys.modules, "ray", RayDouble()) + monkeypatch.setattr(rb, "get_ray_backend", lambda: SimpleNamespace(ensure=lambda **_kw: None)) + with rs.ServingLease(num_gpus=1) as lease: + yield lease diff --git a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py index b6474f8f5c..a5bf9a5a40 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py @@ -711,6 +711,16 @@ def test_gate_update_state_cannot_move_the_resume_boundary(gate): assert exc.value.rule == "state_field" +def test_the_model_cannot_rewrite_the_budget_the_closing_reserve_leaves_it(gate): + """The reserve decides how much of ``max_minutes`` is spendable, so it is budget too.""" + with pytest.raises(PolicyDenied) as exc: + gate.validate_intent( + "orchestration", + Intent(type=IntentType.UPDATE_STATE, payload={"changes": {"closing_grace_sec": 0.0}}), + ) + assert exc.value.rule == "state_field" + + def test_gate_update_state_cannot_move_a_session_end_time(gate): # stop_ts is the timestamp half of stop_reason, written by the same setter: # locking only the reason lets a model post-date the session's end. @@ -726,6 +736,21 @@ def test_gate_update_state_cannot_move_a_session_end_time(gate): assert exc.value.rule == "state_field" +def test_a_forged_closing_reserve_would_have_spent_the_session_outright(): + """Names what the lock prevents: one field, and the run has no usable time left.""" + state = SharedState(session_id="s", max_minutes=100) + # Freeze elapsed time: two live ``session_budget_usable_sec`` reads race + # the clock by tens of microseconds, which is enough for ``==`` to fail. + state.elapsed_minutes = lambda **_kw: 90.0 # type: ignore[method-assign] + honest = state.session_budget_usable_sec() + + applied = state.apply_changes({"closing_grace_sec": 1e9}, allow_core=False) + + assert applied == {} + assert honest > 0.0 + assert state.session_budget_usable_sec() == honest + + def test_core_state_fields_synced_with_robustness_envelope(): # gate.CORE_STATE_FIELDS and the robustness # envelope copy must stay byte-identical. This direct assertion never skips diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 6215789206..428d668c8e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -4,8 +4,10 @@ """Regression tests for the baseline cold-start "warmup artifact". Covers the cold+hot double-run and its server-lifecycle reuse, the pre-start / -teardown cleanup around the reused port, the local InferenceX mirror, and the -subprocess-failure classifier. +teardown cleanup around the reused port, the local InferenceX mirror, the +subprocess-failure classifier, and the session wall-clock budget's reach into +the round (the deadline the reaper is handed, the clamp on the hang backstop, +and how a round the run stopped is told apart from one that failed). """ from __future__ import annotations @@ -16,6 +18,7 @@ import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -24,16 +27,36 @@ import yaml from hyperloom.orchestrator.actions.executors.baseline import ( + BASELINE_DEFAULT_TIMEOUT_SEC, + MEASURE_ROUND_DROPPED_WARNING, BaselineExecutor, ) +from hyperloom.orchestrator.actions.executors.profile import ( + PROFILE_DEFAULT_TIMEOUT_SEC, + ProfileExecutor, +) from hyperloom.orchestrator.actions.executors._grid_runner import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, GridVariant, + _SESSION_KILL_GRACE_SEC, run_grid, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, + _stamp_server_ready, +) +from hyperloom.orchestrator.actions.stop_attribution import STOPPED_BY_THE_RUN from hyperloom.orchestrator.state.shared_state import SharedState from hyperloom.orchestrator.trace.task_progress import progress_scope -from .conftest import chatty_child, enable_multi_node, suppression_window_s +from .conftest import ( + chatty_child, + enable_multi_node, + launches_by_round_slot, + suppression_window_s, +) @pytest.fixture(autouse=True) @@ -103,9 +126,20 @@ def _run(coro): _HOT_TPUT = 4701.6 -def _cold_then_hot_fake_run(captured: list | None = None): +def _cold_then_hot_fake_run( + captured: list | None = None, + *, + clock: _AClockOnlyThePassesMove | None = None, + boot_sec: float = 0.0, + benchmark_sec: float = 0.0, +): """Return a ``run_with_session_kill`` stand-in that emits a cold throughput - on its first call and a hot throughput thereafter.""" + on its first call and a hot throughput thereafter. + + Given a ``clock``, the first call spends it in the two parts a cold pass + spends it in and announces the server ready between them; later calls + re-attach, so they spend only the benchmark and announce nothing. + """ state = {"calls": 0} def fake_run(cmd, *args, **kwargs): @@ -116,6 +150,14 @@ def fake_run(cmd, *args, **kwargs): cfg = yaml.safe_load(Path(cmd[cfg_idx + 1]).read_text()) captured.append(cfg) tput = _COLD_TPUT if state["calls"] == 0 else _HOT_TPUT + if clock is not None: + server_log_path = kwargs.get("server_log_path") + if state["calls"] == 0: + clock.advance(boot_sec) + if server_log_path: + Path(server_log_path).parent.mkdir(parents=True, exist_ok=True) + _stamp_server_ready(server_log_path, boot_sec) + clock.advance(benchmark_sec) state["calls"] += 1 _fake_workspace(slot, tput=tput) return subprocess.CompletedProcess(cmd, 0, "ok", "") @@ -329,6 +371,666 @@ def test_a_failing_warmup_round_still_reported_that_it_started(tmp_path): assert [(n["label"], n["status"]) for n in notes] == [("warmup", "started")] +def _prelude_shared_state(*, usable_sec: float, phase: str = "PRELUDE") -> SimpleNamespace: + """A session state with an explicit clock, as the budget policy reads it. + + Only the usable remainder, because that is all the policy reads. An earlier + version also carried a phase ledger and a phase start, from when preparation + answered to a share of its own; a double that still offers them invites a + reader to believe they decide something. + + The phase is offered because the gates ask it one thing: whether the round's + worth depends on a variant following it. + """ + return SimpleNamespace( + baseline_double_run=True, + phase=phase, + max_minutes=180, + session_budget_usable_sec=lambda: usable_sec, + ) + + +def _a_session_the_passes_spend( + *, + usable_sec: float, + clock: _AClockOnlyThePassesMove, + **measured: float, +) -> SimpleNamespace: + """A PRELUDE session whose remaining budget falls as the passes spend it. + + The fixed-remainder double cannot show the two gates disagreeing, because the + second one is asked after the warmup has spent its share and a budget that + never moves hides exactly that. This reads the same clock the passes move. + """ + started = clock() + return SimpleNamespace( + baseline_double_run=True, + phase="PRELUDE", + max_minutes=180, + session_budget_usable_sec=lambda: usable_sec - (clock() - started), + **measured, + ) + + +def _run_double_run_baseline( + tmp_path, + shared_state, + *, + clock: _AClockOnlyThePassesMove | None = None, + boot_sec: float = 0.0, + benchmark_sec: float = 0.0, +) -> dict: + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + output_dir = tmp_path / "ws" + fake_run, state = _cold_then_hot_fake_run( + clock=clock, + boot_sec=boot_sec, + benchmark_sec=benchmark_sec, + ) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=shared_state, + ) + ctx = _make_ctx({"output_dir": str(output_dir), "timeout_sec": 10, "gpu_type": "mi300x"}) + with ( + _passes_time_the_executor_believes(clock or _AClockOnlyThePassesMove()), + patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ), + ): + result = _run(executor(ctx)) + result["_rounds_run"] = state["calls"] + return result + + +def test_a_budget_that_cannot_pay_for_the_measured_round_keeps_the_cold_warmup(tmp_path): + """The session clock cannot pay for the hot pass and a use for it; the cold one ran. + + Nothing is predicted before the warmup on a first baseline, so the round + starts and the warmup's GPU time is spent before the shortfall is known. + Refusing to keep its figure would throw that away and leave the session with + no anchor at all, which is strictly worse than the cold anchor a single-round + baseline would have produced. So the warmup is promoted and marked: the + number is depressed, and the marker is what tells a reader of the session's + later gains that their denominator is. + + The warmup boots for 350s and benchmarks for 550s, so the hot pass that would + follow costs 550s and a variant to read against it costs 900s. 1200s covers + the pass alone with room to spare and the pair not at all. + """ + clock = _AClockOnlyThePassesMove() + + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=1200.0), + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["status"] == "succeeded" + assert result["_rounds_run"] == 1, "the measured round ran on a budget that cannot pay for it" + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] + dropped = result["measure_round_dropped"] + assert dropped["bound"] == "session_usable" + assert dropped["priced_by"] == "warmup_post_ready" + assert dropped["measure_round_sec"] == pytest.approx(550.0, abs=1.0) + assert dropped["one_more_measurement_sec"] == pytest.approx(900.0, abs=1.0) + assert dropped["measure_round_sec"] < 1200.0, ( + "the pass alone did not fit, so this case does not show what it claims to" + ) + + +def test_a_round_admitted_before_ignition_is_not_refused_after_its_cold_pass(tmp_path): + """The two gates price the same second pass, so they must reach the same answer. + + A gate before ignition that admits what the gate after the cold pass will + certainly refuse spends a whole cold pass to learn something it already knew. + The disagreement is in the ruler: this session has measured a 400s hot pass, + and pricing the pass to come at the warmup's 550s post-ready segment instead + -- a segment that also paid the first request's compile -- demands 300s more + than ignition was allowed to require. + + 2100s is inside that band. Ignition needs the 1300s round and a 750s variant; + after the warmup spends 900s, the hot pass and its variant need 1150s of the + 1200s left, while the post-ready ruler would have called for 1450s. + """ + clock = _AClockOnlyThePassesMove() + state = _a_session_the_passes_spend( + usable_sec=2100.0, + clock=clock, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + baseline_warm_runtime_sec=400.0, + ) + + result = _run_double_run_baseline( + tmp_path, + state, + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 2, ( + "a cold pass was spent on a round the gate after it was always going to refuse" + ) + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + + +def test_a_warmup_that_overran_its_prediction_still_drops_the_hot_pass(tmp_path): + """Agreeing with the earlier gate is not the same as admitting everything. + + Once the two price the same work, the only rounds left for this gate to + refuse are the ones that cost more than they were admitted on -- which is + precisely what a gate asked after the pass, against the clock rather than + against a prediction, exists for. This round was admitted at 2050s and its + warmup then took 1250s instead of 900s, leaving 850s where 1150s is needed. + """ + clock = _AClockOnlyThePassesMove() + state = _a_session_the_passes_spend( + usable_sec=2100.0, + clock=clock, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + baseline_warm_runtime_sec=400.0, + ) + + result = _run_double_run_baseline( + tmp_path, + state, + clock=clock, + boot_sec=700.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 1 + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert result["measure_round_dropped"]["priced_by"] == "session_hot_pass" + + +def test_a_rebaselines_hot_pass_needs_only_its_own_wall_clock(tmp_path): + """The same 1200s that drops the hot pass in PRELUDE runs it here. + + In PRELUDE the hot pass buys a denominator, so a session that cannot follow + it with a variant gains nothing by running it. A re-baseline's hot pass is + the measurement the session came for, and covering it is the whole question. + """ + clock = _AClockOnlyThePassesMove() + + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=1200.0, phase="EXPLORE"), + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 2, "the measurement the round exists for was dropped" + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + assert MEASURE_ROUND_DROPPED_WARNING not in (result.get("nonfatal_warnings") or []) + + +def test_a_rebaseline_that_cannot_cover_its_hot_pass_keeps_the_cold_warmup(tmp_path): + """A later phase asks a narrower question, not no question. + + 550s of benchmarking does not fit in 400s, so the pass would be reaped + mid-flight and the warmup's figure lost with it. + """ + clock = _AClockOnlyThePassesMove() + + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=400.0, phase="EXPLORE"), + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 1 + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert result["measure_round_dropped"]["one_more_measurement_sec"] == pytest.approx(0.0) + + +def test_a_rounds_boot_is_priced_even_though_no_cap_bounded_it(tmp_path): + """The gate prices the round on what it spent, not on what its cap allowed. + + A warmup can keep well inside its own timeout and still leave the round unable + to pay for what should follow: the server boot in front of the pass is + wall-clock the cap never bounded. Asking after the pass, against the clock + rather than against the cap, is what catches that -- and is why nothing is + predicted before the pass on a first baseline instead. + + The cap here is 10 seconds and the pass spends 2600 of them, 1400 of which is + the boot. A gate priced on the cap would have waved through a round costing + three hundred times what it was allowed. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + clock = _AClockOnlyThePassesMove() + state = _BudgetedState(remaining_sec=3600.0, double_run=True) + fake_run, calls = _capturing_fake_run( + state=state, + clock=clock, + boot_sec=1400.0, + benchmark_sec=1200.0, + ) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=state, + ) + ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) + with ( + _passes_time_the_executor_believes(clock), + patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ), + ): + result = _run(executor(ctx)) + + assert [c["round_slot"] for c in calls] == ["warmup_round"], ( + f"the measured round ran on a budget the round had already spent: {[c['round_slot'] for c in calls]}" + ) + assert float(calls[0]["timeout"]) <= 10.0, "the cap was not the small one this case rests on" + assert result["status"] == "succeeded" + assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] + dropped = result["measure_round_dropped"] + assert dropped["one_more_measurement_sec"] == pytest.approx(2600.0, abs=1.0) + assert dropped["expected_cost_sec"] == pytest.approx(3800.0, abs=1.0) + + +def _warmup_then_reaped_fake_run(tmp_path): + """A double run whose warmup lands and whose measured pass the clock takes. + + The regime a prediction cannot rule out: the gate before the measured pass + admitted it on what the warmup had just cost, and the pass overran that. + """ + state = {"calls": 0} + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + state["calls"] += 1 + if state["calls"] == 1: + _fake_workspace(slot, tput=_COLD_TPUT) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + return subprocess.CompletedProcess(cmd, SESSION_TIME_EXHAUSTED_RETURNCODE, "", "") + + return fake_run, state + + +def test_a_measured_round_the_clock_takes_mid_flight_keeps_the_cold_warmup(tmp_path): + """The GPU time behind the warmup's figure is spent either way. + + Reporting the round as failed would discard it and leave the session with + nothing to show for a pass that ran to completion, so the warmup is kept and + marked exactly as a refused measured round keeps it. A session that reaches + this state has already paid for a cold anchor; what it must not do is pay + again for nothing. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + fake_run, state = _warmup_then_reaped_fake_run(tmp_path) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=_prelude_shared_state(usable_sec=10_000.0), + ) + ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) + + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert state["calls"] == 2, "the measured round did not run, so it cannot have been reaped" + assert result["status"] == "succeeded" + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] + assert result["measure_round_dropped"]["reason"] == "measure_round_reaped_by_the_run" + + +def test_a_measured_round_that_fails_on_its_own_is_still_a_failure(tmp_path): + """Only the run's clock earns the fallback. + + A pass that broke for a reason of its own is a fact about the configuration, + and the warmup having succeeded does not make the round's figure comparable. + Promoting a cold anchor here would bury a real failure under a warning. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + calls = {"n": 0} + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + calls["n"] += 1 + if calls["n"] == 1: + _fake_workspace(slot, tput=_COLD_TPUT) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + return subprocess.CompletedProcess(cmd, 1, "", "CUDA error") + + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=_prelude_shared_state(usable_sec=10_000.0), + ) + ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) + + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert result["status"] == "failed" + assert result["error_class"] != SESSION_TIME_EXHAUSTED_CLASS + assert MEASURE_ROUND_DROPPED_WARNING not in (result.get("nonfatal_warnings") or []) + + +def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): + """The guard must not turn every double-run into a single one.""" + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=10_000.0), + clock=_AClockOnlyThePassesMove(), + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 2 + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + assert "budget_shortfall" not in result + + +def test_a_double_run_reports_the_boot_split_of_the_pass_that_paid_it(tmp_path): + """The round's total and the part of it that was the benchmark, from one pass. + + Round 1 boots for 350s and benchmarks for 550s; round 2 re-attaches and + benchmarks for 550s more. The split belongs to round 1, whose 900s total is + also what the round reports: their difference is published as what booting + this workload costs, so two rounds cannot each supply one of them. Round 2's + own split says nothing, having never booted. + """ + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=10_000.0), + clock=_AClockOnlyThePassesMove(), + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["subprocess_runtime_sec"] == pytest.approx(900.0, abs=1.0) + assert result["post_ready_runtime_sec"] == pytest.approx(550.0, abs=1.0) + assert result["measure_round_runtime_sec"] == pytest.approx(550.0, abs=1.0) + boot_sec = result["subprocess_runtime_sec"] - result["post_ready_runtime_sec"] + assert boot_sec == pytest.approx(350.0, abs=1.0) + + +# The workload every case in the gate class below is priced against, and the +# figures the pricing derives from it. A round that boots for 350s and then +# benchmarks is what a variant costs too, because a variant's config differs in +# the knobs that decide how a server comes up and it has to bring up its own. +_COLD_ROUND_SEC = 900.0 +_COLD_POST_READY_SEC = 550.0 +_HOT_ROUND_SEC = 400.0 +# 900 - 550: the part of the cold round that was not the benchmark. +_BOOT_SEC = 350.0 +# One further measured variant: its own boot, then its own benchmark. The +# benchmark is priced hot because the variant runs on a JIT cache this session +# has already populated. +_ONE_MORE_SEC = _BOOT_SEC + _HOT_ROUND_SEC +# A round's first pass is the cold one, measured whole rather than rebuilt from +# its halves -- rebuilding it as boot-plus-hot would drop the compile it paid and +# under-price the round by 150s. A double run adds a second benchmark, which +# re-attaches and so buys no second boot. +_SINGLE_ROUND_SEC = _COLD_ROUND_SEC +_DOUBLE_ROUND_SEC = _COLD_ROUND_SEC + _HOT_ROUND_SEC + + +class TestARoundThatCannotFinishIsNotIgnited: + """The gate in front of a round, and the two things it must be asked with. + + A round is refused before it boots only on what earlier rounds measured, so + the session's first one is never refused: it has nothing to be judged by, and + a gate that guessed would either refuse every first baseline or wave every + one through. From the second on -- and on a resumed session's first, which + carries the earlier leg's figures -- the answer is available before a second + of GPU time is spent. + + What must fit is the round *and one further measured variant*. A baseline is + not a result; it is the denominator results are read against and the anchor + their overtime kill uses. A round no variant can follow buys neither, so the + wall-clock it would spend produces nothing. + + Priced on this workload: boot 350s, benchmark 400s, so one variant costs 750s, + a single-pass round costs 750s and a double-run round 1150s. + """ + + def test_a_first_round_is_not_judged_at_all(self, tmp_path): + """Nothing measured yet, so nothing to refuse it with.""" + result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert result["status"] == "succeeded" + assert calls, "a first baseline was refused on a prediction the session cannot have" + + def test_a_round_larger_than_what_is_left_boots_nothing(self, tmp_path): + """750s for the round and 750s for a variant to use it: 1400s cannot.""" + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1400.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert calls == [], "GPU time was spent on a round the session cannot finish" + assert result["status"] == "failed" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert result["returncode"] is None, "a round that never launched reported a returncode" + assert result["error"] == STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS].never_started + shortfall = result["budget_shortfall"] + assert shortfall["expected_cost_sec"] == pytest.approx(_SINGLE_ROUND_SEC + _ONE_MORE_SEC) + assert shortfall["round_sec"] == pytest.approx(_SINGLE_ROUND_SEC) + assert shortfall["one_more_measurement_sec"] == pytest.approx(_ONE_MORE_SEC) + assert shortfall["affordable_sec"] == pytest.approx(1400.0, abs=1.0) + + def test_a_round_the_budget_covers_is_ignited(self, tmp_path): + """The gate must not turn a merely expensive round into a refused one. + + Given a margin over the 1500s the round and its use need, rather than + exactly that: the headroom is read from a live clock, so a case pinned to + the boundary would decide on how long the test itself took to get there. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=_SINGLE_ROUND_SEC + _ONE_MORE_SEC + 60.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert result["status"] == "succeeded" + assert calls + + def test_a_round_the_budget_covers_with_nothing_left_to_use_it_is_refused(self, tmp_path): + """The requirement that is not about finishing the round. + + 1000s covers the 750s round with room to spare, and the round would run + to completion. It is still refused, because what it produces is a + denominator, and 250s buys no variant to read against it. Wall-clock + spent on a number nothing is ever compared to is wall-clock wasted, and + a session that stops here keeps the anchor it already had. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1000.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert calls == [], "a round ran that nothing could be measured against" + assert result["budget_shortfall"]["round_sec"] == pytest.approx(_SINGLE_ROUND_SEC) + assert result["budget_shortfall"]["round_sec"] < 1000.0, ( + "the round itself did not fit, so this case is not the one it claims to be" + ) + + def test_a_rebaseline_in_a_later_phase_needs_no_successor(self, tmp_path): + """The same 1000s that refuses a PRELUDE round admits this one. + + A re-baseline re-measures the stack the session has assembled, and that + measurement is what the session is for. Requiring a variant after it + would refuse the round that validates the run's own answer, at the point + in the budget where it is most likely to be the last thing left. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1000.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + phase="EXPLORE", + ) + + assert calls, "the round that validates the stack was refused for lack of a successor" + assert result["status"] == "succeeded" + + def test_a_rebaseline_larger_than_what_is_left_is_still_refused(self, tmp_path): + """Dropping the successor does not drop the round's own cost. + + A later phase is a narrower question, not an absent one: a round that + cannot finish inside the budget burns a boot and a compile for a number + the reaper takes away before it lands. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=_SINGLE_ROUND_SEC - 100.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + phase="EXPLORE", + ) + + assert calls == [], "GPU time was spent on a round the session cannot finish" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + shortfall = result["budget_shortfall"] + assert shortfall["one_more_measurement_sec"] == pytest.approx(0.0) + assert shortfall["expected_cost_sec"] == pytest.approx(_SINGLE_ROUND_SEC) + + def test_a_double_run_pays_for_the_second_pass_but_not_a_second_boot(self, tmp_path): + """One budget, two answers: it covers a single-pass round, not a double one. + + The difference between them is one benchmark and no second boot, because + the second pass re-attaches to the server the first left running. Both + sides run against the same figure so the refusal can only come from that. + """ + budget_sec = _SINGLE_ROUND_SEC + _ONE_MORE_SEC + 60.0 + for side in ("single", "double"): + (tmp_path / side).mkdir() + + fits, fitting_calls = _run_baseline_under_budget( + tmp_path / "single", + remaining_sec=budget_sec, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + refused, refused_calls = _run_baseline_under_budget( + tmp_path / "double", + remaining_sec=budget_sec, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + double_run=True, + ) + + assert fits["status"] == "succeeded" and fitting_calls + assert refused_calls == [], "the round was priced on one pass while planning to run two" + shortfall = refused["budget_shortfall"] + assert shortfall["round_sec"] == pytest.approx(_DOUBLE_ROUND_SEC) + assert shortfall["expected_cost_sec"] == pytest.approx(_DOUBLE_ROUND_SEC + _ONE_MORE_SEC) + + def test_a_session_with_no_hot_figure_prices_the_variant_from_the_cold_pass(self, tmp_path): + """The state a previous cold-anchor drop leaves, and the one to catch. + + A round whose measured pass was dropped for budget promotes its cold + number and has no hot number to write. Going inert on such a session would + exempt exactly the one that already ran out of budget once, so the cold + round's post-ready segment stands in for the variant's benchmark. It + over-predicts, having also paid the first request's compile, which is why + the hot figure wins whenever one exists. + """ + one_more_sec = _BOOT_SEC + _COLD_POST_READY_SEC + + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=_SINGLE_ROUND_SEC + one_more_sec - 1.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + ) + + assert calls == [] + shortfall = result["budget_shortfall"] + assert shortfall["one_more_measurement_sec"] == pytest.approx(one_more_sec) + assert shortfall["one_more_measurement_sec"] > _ONE_MORE_SEC, ( + "the fallback did not over-predict, so it cannot be the post-ready segment" + ) + + def test_a_round_whose_boot_was_never_measured_is_still_judged(self, tmp_path): + """A round with no split is priced at whole cold rounds, not waved through. + + Multi-node and scriptable workloads never report a boot boundary -- one + brings its server up outside the round, the other runs no server at all -- + so a gate that goes inert without the split exempts them permanently. Both + terms fall back to what the session did measure: the cold round's own + wall-clock, which is a boot and a benchmark, and so is what a variant + costs too. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=100.0, + cold_round_sec=_COLD_ROUND_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert calls == [], "a workload with no boot boundary was exempted from the gate" + shortfall = result["budget_shortfall"] + assert shortfall["round_sec"] == pytest.approx(_COLD_ROUND_SEC) + assert shortfall["one_more_measurement_sec"] == pytest.approx(_COLD_ROUND_SEC) + + def test_a_refused_round_carries_nothing_that_could_replace_the_anchor(self, tmp_path): + """The property the whole gate rests on, asserted rather than assumed. + + A refusal is only cheap if what the session already measured survives it. + The anchor is held in session state, not in this result, so the refused + round must come back with no throughput of its own to promote over it. + """ + result, _calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1400.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert result.get("output_throughput") in (None, 0, 0.0) + assert not result.get("nonfatal_warnings"), "a refused round volunteered a warning to promote" + + def test_deferred_accuracy_skips_eval_when_hot_throughput_regresses( tmp_path, ): @@ -1757,6 +2459,523 @@ def test_teardown_lifecycle_server_removes_state_files(tmp_path): assert not (pid_dir / "vllm_8888.json").exists() +# Every output slot a multi-node baseline round launches a benchmark process +# into, in launch order. The discarded client warmup is a full pass and costs the +# same wall-clock as the measured round it precedes, so both need the deadline. +# ``mn_warmup`` is production's name for the warmup slot; the measured round runs +# in the task's own output dir, which these tests name. +_MEASURED_ROUND_SLOT = "measured_round" +_BASELINE_ROUND_SLOTS = ("mn_warmup", _MEASURED_ROUND_SLOT) + + +# ``--max-hours`` defaults to 2.0 (``cli/parser.py``), so this is the session +# shape almost every run has. It matters here because PRELUDE's share of it is +# 48 minutes while a baseline round's declared cap is 130 minutes warm and 150 +# cold: any rule that prices the pair at the declared cap refuses every default +# session, and any rule that prices it at nothing lets the warmup eat the round. +_DEFAULT_SESSION_MINUTES = 120.0 + + +class _BudgetedState: + """A session state whose budget accounting moves as the passes spend it. + + Production reads one session through two accessors -- the deadline the round + reaper is handed and the usable-seconds figure the phase policy reads -- and + a double that lets them drift cannot see the regime where they disagree. + Both are derived from one deadline here. + + A pass calls :meth:`charge` for the wall-clock it burned, which is the only + way a test can reach the case this whole mechanism exists for: a round whose + first pass leaves the second one nothing. + + It carries no phase clock on purpose. An earlier version set + ``phase_started_unix`` to zero, which the budget policy of the day read as a + preparation phase running since the epoch, and a test meant to show a + round's overheads exhausting the session passed on that arithmetic instead. + The policy answers to the session clock alone, so that is all this offers. + """ + + def __init__( + self, + *, + remaining_sec: float | None, + cold_round_sec: float = 0.0, + cold_post_ready_sec: float = 0.0, + hot_round_sec: float = 0.0, + phase: str = "PRELUDE", + double_run: bool = False, + ) -> None: + self.baseline_double_run = double_run + self.baseline_runtime_sec = cold_round_sec + self.baseline_post_ready_runtime_sec = cold_post_ready_sec + self.baseline_warm_runtime_sec = hot_round_sec + self.phase = phase + self.max_minutes = 0.0 if remaining_sec is None else remaining_sec / 60.0 + self._deadline = None if remaining_sec is None else time.monotonic() + remaining_sec + + def charge(self, seconds: float) -> None: + """Spend ``seconds`` of the session, as a pass that ran that long does.""" + if self._deadline is not None: + self._deadline -= seconds + + def grid_session_deadline_sec(self) -> float | None: + return self._deadline + + def session_budget_usable_sec(self) -> float | None: + return None if self._deadline is None else self._deadline - time.monotonic() + + +class _AClockOnlyThePassesMove: + """The wall clock the executor prices rounds by, moved by hand. + + A round's price is wall-clock, so a test that needs a pass to cost twenty + minutes can sleep for them, assert on nothing, or hand the executor a clock it + can move. Only the third is both quick and about the thing under test. + + Installed over ``time.time``, which is what the executor times rounds with; + ``time.monotonic``, which the budget deadline runs on, is left alone so the + two cannot be confused for each other. + """ + + def __init__(self) -> None: + self._now = time.time() + + def __call__(self) -> float: + """Answer as ``time.time`` does.""" + return self._now + + def advance(self, seconds: float) -> None: + """Spend ``seconds``, as a pass that ran that long does.""" + self._now += float(seconds) + + +@contextmanager +def _passes_time_the_executor_believes(clock: _AClockOnlyThePassesMove): + """Run the block with ``clock`` standing in for the wall clock.""" + with patch("time.time", clock): + yield + + +def _capturing_fake_run( + returncode: int = 0, + *, + produces_workspace: bool = True, + pass_duration_sec: float = 0.0, + state: _BudgetedState | None = None, + charge_sec: float | None = None, + clock: _AClockOnlyThePassesMove | None = None, + boot_sec: float = 0.0, + benchmark_sec: float = 0.0, +): + """A ``run_with_session_kill`` stand-in that records how each round was launched. + + Every record carries the ``round_slot`` the round wrote into, so a launch can + be looked up by which pass it was rather than by the order it happened in. + + ``pass_duration_sec`` makes the double honour the cap it was handed the way a + real benchmark pass does, and it charges the budget for what it ran: a pass + granted less than the workload takes is killed rather than reporting, and it + is killed by whichever of the two limits it meets first -- the session + watchdog, which comes back with the sentinel returncode that says the run ran + out of time, or its own hard cap, which raises ``TimeoutExpired``. + + ``charge_sec`` charges the session more than the pass itself ran, which is + what a round with a server restart and a teardown around the pass costs. + + ``boot_sec`` and ``benchmark_sec`` spend ``clock`` in the two parts a real + pass spends it in, announcing the server ready between them exactly as the + gate loop does. A pass that models one duration cannot reach the pricing at + all, which is built on telling the two apart. + """ + calls: list[dict] = [] + + def fake_run(cmd, *args, **kwargs): + # The memoized interpreter probe is not a benchmark round. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + calls.append({"round_slot": slot.name, **kwargs}) + granted = float(kwargs.get("timeout") or 0.0) + deadline = kwargs.get("session_deadline_sec") + ran_sec = min(granted, pass_duration_sec) if pass_duration_sec else 0.0 + reaped_by_the_session = False + if deadline is not None and pass_duration_sec: + until_deadline = max(0.0, deadline - time.monotonic()) + reaped_by_the_session = until_deadline < ran_sec + ran_sec = min(ran_sec, until_deadline) + if clock is not None: + clock.advance(boot_sec) + server_log_path = kwargs.get("server_log_path") + if server_log_path: + Path(server_log_path).parent.mkdir(parents=True, exist_ok=True) + _stamp_server_ready(server_log_path, boot_sec) + clock.advance(benchmark_sec) + if state is not None: + state.charge(charge_sec if charge_sec is not None else ran_sec) + if pass_duration_sec and ran_sec < pass_duration_sec: + if reaped_by_the_session: + return subprocess.CompletedProcess(cmd, SESSION_TIME_EXHAUSTED_RETURNCODE, "", "") + raise subprocess.TimeoutExpired(cmd, granted) + if produces_workspace: + _fake_workspace(slot, tput=_HOT_TPUT) + return subprocess.CompletedProcess(cmd, returncode, "ok", "") + + return fake_run, calls + + +class _CapturingLease: + """A serving-lease stand-in that records how a round reached the Ray actor. + + The Ray path is the same round through a different door, and the door matters + here: the lease is handed what is left of the budget as a duration, because + the absolute deadline is a ``time.monotonic()`` instant that means nothing in + the actor's process. So it is a launch site of its own, with its own way of + losing the reaper. + """ + + def __init__(self) -> None: + self._run, self.calls = _capturing_fake_run() + + def run_session_kill(self, cmd, **kwargs) -> tuple[int | None, str, str]: + """Record the launch and answer as the actor does, with a bare triple.""" + proc = self._run(cmd, **kwargs) + return proc.returncode, proc.stdout, proc.stderr + + def close(self) -> None: + """The executor closes the lease it was given; nothing is held here.""" + + +def _run_baseline_under_budget( + tmp_path, + *, + remaining_sec: float | None, + timeout_sec: int = 7200, + returncode: int = 0, + produces_workspace: bool = True, + cold_round_sec: float = 0.0, + cold_post_ready_sec: float = 0.0, + hot_round_sec: float = 0.0, + pass_duration_sec: float = 0.0, + phase: str = "PRELUDE", + double_run: bool = False, + executor_cls=BaselineExecutor, +) -> tuple[dict, list[dict]]: + """Run one baseline round against a session with ``remaining_sec`` left.""" + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + state = _BudgetedState( + remaining_sec=remaining_sec, + cold_round_sec=cold_round_sec, + cold_post_ready_sec=cold_post_ready_sec, + hot_round_sec=hot_round_sec, + phase=phase, + double_run=double_run, + ) + fake_run, calls = _capturing_fake_run( + returncode, + produces_workspace=produces_workspace, + pass_duration_sec=pass_duration_sec, + state=state, + ) + executor = executor_cls( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + ) + ctx = _make_ctx( + { + "output_dir": str(tmp_path / _MEASURED_ROUND_SLOT), + "timeout_sec": timeout_sec, + "gpu_type": "mi300x", + } + ) + # The live state arrives on the context, the way the coordinator passes it. + ctx.extra["shared_state"] = state + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + return result, calls + + +def _mn_warmup_cap_sec(calls: list[dict]) -> int | None: + """The cap the multi-node warmup pass was granted, or ``None`` when it never ran. + + Both are outcomes a round is allowed to have -- a skipped warmup is what a + single-node baseline does -- so they are told apart here rather than by a + ``KeyError`` at the assertion. + """ + launch = launches_by_round_slot(calls).get("mn_warmup") + return None if launch is None else int(launch["timeout"]) + + +def _launch_one_grid_variant_under_budget( + tmp_path, + *, + remaining_sec: float, + variant_timeout_sec: int, + variant_expected_sec: float, +) -> list[dict]: + """Run one grid variant against the same budget a baseline round would get. + + The other arm that benches on the GPU, driven through its own entry point so + the two can be compared on what they grant the passes they both run. + """ + tmp_path.mkdir(parents=True, exist_ok=True) + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + fake_run, calls = _capturing_fake_run() + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + _run( + run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant(name="candidate")], + output_root=tmp_path / "out", + magpie_python=sys.executable, + variant_timeout_sec=variant_timeout_sec, + session_deadline_sec=time.monotonic() + remaining_sec, + variant_expected_sec=variant_expected_sec, + ) + ) + return calls + + +class TestTheSessionBudgetReachesTheBaselineRound: + """The arm #1146 names as the largest hole, and the one that motivated it. + + A baseline is admitted on a catalogue cost of five minutes and given a + two-hour hang backstop (four, cold), so the round that runs first and + longest was the one round no wall-clock defence covered: no deadline + reached the reaper, and nothing clamped the cap to what was left. + """ + + @pytest.mark.parametrize("round_slot", _BASELINE_ROUND_SLOTS) + def test_the_deadline_reaches_the_reaper(self, tmp_path, monkeypatch, round_slot): + """Parameterized over the passes a round launches, not just the measured one. + + The reaper is the only thing that attributes a budget kill correctly, and + it only knows about the deadline it was handed. A pass launched without + one runs until its own hard cap and comes back looking like a variant + that timed out. + """ + enable_multi_node(monkeypatch) + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert launches_by_round_slot(calls)[round_slot]["session_deadline_sec"] is not None + + def test_no_pass_of_a_round_is_launched_without_the_deadline(self, tmp_path, monkeypatch): + """The net for a pass added later, which no per-slot test would know about.""" + enable_multi_node(monkeypatch) + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert set(launches_by_round_slot(calls)) >= set(_BASELINE_ROUND_SLOTS) + assert [c["round_slot"] for c in calls if c.get("session_deadline_sec") is None] == [] + + def test_the_ray_path_is_handed_the_budget_as_a_duration(self, tmp_path, monkeypatch): + """The fourth launch site, and the one a parameterization cannot reach. + + Production runs a single-node round through a Ray lease, which is handed a + remaining duration rather than the deadline, by a different call. The + reaper in the actor's process has nothing else to go on. + """ + from hyperloom.orchestrator.actions.executors import _ray_serving + + lease = _CapturingLease() + monkeypatch.setattr(_ray_serving, "maybe_serving_lease", lambda **_kwargs: lease) + result, _calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert result["status"] == "succeeded" + launch = launches_by_round_slot(lease.calls)[_MEASURED_ROUND_SLOT] + remaining = launch.get("session_remaining_sec") + assert remaining is not None, f"the budget did not cross the process boundary: {sorted(launch)}" + assert 0 < remaining <= 3600.0 + + def test_the_hang_backstop_is_clamped_to_what_is_left(self, tmp_path): + """A cap larger than the budget outlives the session it belongs to.""" + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=120.0, timeout_sec=7200) + + assert 1 <= calls[0]["timeout"] <= 120 + _SESSION_KILL_GRACE_SEC + + def test_an_unbounded_budget_leaves_the_cap_alone(self, tmp_path): + """No session context means no budget to respect, not a budget of zero.""" + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=None, timeout_sec=7200) + + assert calls[0]["timeout"] == 7200 + assert calls[0]["session_deadline_sec"] is None + + def test_a_budget_kill_is_not_recorded_as_a_broken_model(self, tmp_path): + """A reaped round leaves exactly what a broken server leaves behind. + + No workspace, no report, a non-zero returncode -- so without this branch + the run that ran out of time is filed as a fact about the model. + """ + result, _calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1.0, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + produces_workspace=False, + ) + + assert result["status"] == "failed" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + + def test_a_cancel_is_told_apart_from_a_spent_budget(self, tmp_path): + """A resume meets the spent budget again and does not meet the shutdown.""" + result, _calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + returncode=ORCHESTRATOR_CANCELLED_RETURNCODE, + produces_workspace=False, + ) + + assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + + def test_a_cancelled_multi_node_warmup_does_not_go_on_to_the_measured_round( + self, + tmp_path, + monkeypatch, + ): + """The discarded warmup is a full pass, so a cancel there ends the round. + + Running the measured round anyway spends a second pass of GPU time the + run has already been told to stop spending -- and grades the baseline on + a round started after the stop. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + enable_multi_node(monkeypatch) + launched: list[str] = [] + + def fake_run(cmd, *args, **kwargs): + slot = Path(cmd[cmd.index("--output-dir") + 1]) + launched.append(slot.name) + if slot.name == "mn_warmup": + return subprocess.CompletedProcess(cmd, ORCHESTRATOR_CANCELLED_RETURNCODE, "", "") + _fake_workspace(slot, tput=_HOT_TPUT) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + ) + ctx = _make_ctx( + { + "output_dir": str(tmp_path / "ws"), + "timeout_sec": 600, + "gpu_type": "mi300x", + } + ) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert launched == ["mn_warmup"], f"the measured round ran after the cancel: {launched}" + assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + + def test_a_first_baseline_on_a_default_session_runs_both_of_its_passes( + self, + tmp_path, + monkeypatch, + ): + """The regime several rounds of this mechanism have failed in, pinned directly. + + A session's first baseline has measured nothing, so every rule that + shortens a pass by predicting the next one is guessing here. Each guess + tried so far killed a round that fit: the cold pass pays weight load and + graph capture, which the same file's cold-start cap sizes at up to 9000s, + so any share-of-the-budget cap lands under it on a default session. + + No pass is shortened now. The warmup is granted the round's own cap, and + this ten-minute workload runs both passes and yields the warm anchor the + round exists to produce. + """ + enable_multi_node(monkeypatch) + pass_sec = 600.0 + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=_DEFAULT_SESSION_MINUTES * 60.0, + timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, + pass_duration_sec=pass_sec, + ) + launches = launches_by_round_slot(calls) + + assert result["status"] == "succeeded", f"a first baseline was refused a default session: {result}" + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + assert set(launches) >= set(_BASELINE_ROUND_SLOTS), f"the round did not run both passes: {list(launches)}" + remaining_sec = _DEFAULT_SESSION_MINUTES * 60.0 + warmup = _mn_warmup_cap_sec(calls) + assert warmup is not None and warmup >= remaining_sec, ( + f"the warmup's cap was moved in front of the session deadline, so the " + f"pass can now be killed by its own timeout before the watchdog reaches " + f"it: {warmup}s against {remaining_sec}s left" + ) + + def test_a_multi_node_round_that_cannot_pay_for_both_passes_launches_neither( + self, + tmp_path, + monkeypatch, + ): + """A multi-node round is two client passes, and the figure covers one. + + The server comes up outside the round, so both passes are the same shape + and the recorded wall-clock -- taken after the warmup -- is one of them. + Pricing the round at that one figure admits a pair that cannot fit: the + warmup spends its half and the measured pass meets the deadline, leaving + the round with no anchor and the GPU time gone. + + 1500s is the band only this gate catches: the generic gate before it + prices the round at one 600s pass and a variant at another, admits at + 1200s, and has no way to know a second pass is coming. The pair plus a + variant needs 1800s. + """ + enable_multi_node(monkeypatch) + + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1500.0, + cold_round_sec=600.0, + ) + + assert calls == [], "a multi-node pair ran on a budget that covers one pass" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert result["returncode"] is None + assert result["budget_shortfall"]["round_sec"] == pytest.approx(1200.0) + + def test_a_multi_node_round_the_budget_covers_runs_both_passes(self, tmp_path, monkeypatch): + """The gate must not refuse the pairs that fit, only the ones that cannot.""" + enable_multi_node(monkeypatch) + + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + cold_round_sec=600.0, + ) + + assert set(launches_by_round_slot(calls)) >= set(_BASELINE_ROUND_SLOTS) + assert result["status"] == "succeeded" + + def test_the_profile_arm_gets_all_of_it(self, tmp_path): + """Profile is the same executor with a four-hour default -- longer than + any session budget it could be given.""" + _result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=120.0, + timeout_sec=PROFILE_DEFAULT_TIMEOUT_SEC, + executor_cls=ProfileExecutor, + ) + + assert calls[0]["session_deadline_sec"] is not None + assert 1 <= calls[0]["timeout"] <= 120 + _SESSION_KILL_GRACE_SEC + + # _classify_subprocess_error unit tests from hyperloom.orchestrator.actions.executors.baseline import ( diff --git a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py index 4772c6b4a4..a53a2688fb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py +++ b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py @@ -5,6 +5,7 @@ from __future__ import annotations +import time from dataclasses import dataclass, field from pathlib import Path from types import SimpleNamespace @@ -12,6 +13,7 @@ import pytest +from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE from hyperloom.orchestrator.knowledge.config import KnowledgeConfig, KnowledgeStoreMode from hyperloom.orchestrator.roles.agent_role import default_role_registry from hyperloom.orchestrator.roles.mock_backend import ( @@ -20,7 +22,12 @@ ScriptedPlan, ) from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.phases.close import ( + _CLOSE_STEP_WAIT_CEILING_SEC, + _CLOSE_STEP_WAIT_FLOOR_SEC, +) from hyperloom.orchestrator.policy.gate import CORE_STATE_FIELDS +from hyperloom.orchestrator.state.shared_state import effective_closing_grace_sec @dataclass @@ -36,6 +43,8 @@ class _BareState: recipe_finalize_attempts: int = 0 recipe_finalize_outcome: dict[str, Any] = field(default_factory=dict) phase_history: list[dict[str, Any]] = field(default_factory=list) + max_minutes: int = 0 + closing_grace_sec: float | None = None save_count: int = 0 def save(self, _session_dir: Path | None) -> None: @@ -44,6 +53,9 @@ def save(self, _session_dir: Path | None) -> None: def set_stop_reason(self, reason: str) -> None: self.stop_reason = reason + def closing_reserve_sec(self) -> float: + return effective_closing_grace_sec(self.max_minutes, self.closing_grace_sec) + @dataclass class _StubTaskRow: @@ -83,7 +95,8 @@ async def create_or_return_existing( row = _StubTaskRow( task_id=tid, kind=kind, - state="succeeded", + # Matches the registry's INSERT: a new row is always ``queued``. + state="queued", params=dict(params), idempotency_key=idempotency_key, ) @@ -249,6 +262,248 @@ async def test_enqueue_internal_report_task_reuses_existing(coord): assert "internal-report-close_phase_entry" not in coord.tasks._by_key +@pytest.mark.asyncio +async def test_enqueue_internal_report_task_replaces_a_cancelled_one(coord): + """A report the deadline path enqueued and then cancelled cannot be run; the sequencer needs a live one. + + Regression for the ``cannot transition from 'cancelled' to 'running'`` + crash: the helper used to answer "was one enqueued?" when the caller needs + "is one runnable?". + """ + dead = _StubTaskRow( + task_id="wallclock-report", + kind="report", + state="cancelled", + params={}, + idempotency_key="closing-report-1234", + ) + coord.tasks._by_id["wallclock-report"] = dead + coord.shared_state.closing_report_task_id = "wallclock-report" + + task = await coord._enqueue_internal_report_task(reason="close_phase_entry") + + assert task is not dead + assert task.state == "queued" + assert coord.shared_state.closing_report_task_id == task.task_id + + +@pytest.mark.asyncio +async def test_enqueue_internal_report_task_retries_past_a_dead_idempotent_row(coord): + """The idempotency key itself can resolve to a corpse; the retry key mints a runnable row.""" + coord.tasks._by_key["internal-report-close_phase_entry"] = _StubTaskRow( + task_id="dead-idempotent", + kind="report", + state="cancelled", + params={}, + idempotency_key="internal-report-close_phase_entry", + ) + + task = await coord._enqueue_internal_report_task(reason="close_phase_entry") + + assert task.task_id != "dead-idempotent" + assert task.idempotency_key == "internal-report-close_phase_entry-retry" + assert task.state == "queued" + + +@pytest.mark.asyncio +async def test_close_sequencer_still_reports_when_the_first_report_task_was_cancelled(coord): + """End to end: a session that hits its deadline is the one whose report matters most.""" + coord.shared_state.phase_history = [_close_phase_history_row()] + coord.tasks._by_id["wallclock-report"] = _StubTaskRow( + task_id="wallclock-report", + kind="report", + state="cancelled", + params={}, + idempotency_key="closing-report-1234", + ) + coord.shared_state.closing_report_task_id = "wallclock-report" + + await coord._on_enter_close(from_phase="SWEEP") + + rows = coord.shared_state.phase_history[-1]["evidence"]["close_steps"] + by_step = {r["step"]: r for r in rows} + assert by_step["report"]["status"] == "done" + assert [t.task_id for t in coord.sub.run_calls] != ["wallclock-report"] + + +@pytest.mark.asyncio +async def test_a_terminal_task_is_reported_not_run(coord): + """Backstop: nothing hands a terminal row to ``run_task``, whose ``queued -> running`` would raise.""" + done = _StubTaskRow( + task_id="already-done", + kind="report", + state="succeeded", + params={}, + idempotency_key="internal-report-close_phase_entry", + ) + + state = await coord._run_close_task(done, step="1 (report)") + + assert state == "succeeded" + assert coord.sub.run_calls == [] + + +class _FinishesWhileWaiting(_StubTaskRegistry): + """Registry whose running row lands terminal after ``lands_on`` lookups.""" + + def __init__(self, terminal_state: str, *, lands_on: int = 2): + super().__init__() + self._terminal_state = terminal_state + self._lands_on = lands_on + self.gets = 0 + + async def get(self, task_id): + row = await super().get(task_id) + self.gets += 1 + if self.gets >= self._lands_on: + row.state = self._terminal_state + return row + + +def _running_report_row(coord, *, kind: str = "report") -> _StubTaskRow: + """Register a close-step task the wall-clock deadline path already enqueued AND dispatched.""" + row = _StubTaskRow( + task_id="wallclock-report", + kind=kind, + state="running", + params={}, + idempotency_key="closing-report-1234", + ) + coord.tasks._by_id[row.task_id] = row + return row + + +def _clock_advancing_by(monkeypatch: pytest.MonkeyPatch, step_sec: float) -> None: + """Give the CLOSE module a monotonic clock that jumps ``step_sec`` per read. + + The wait under test is measured in minutes, so a test that spent it would + be a test nobody runs. Only the CLOSE module's view of the clock is + replaced, which leaves the event loop's own timekeeping alone. + """ + from hyperloom.orchestrator.phases import close as close_mod + + now = 0.0 + + def _monotonic() -> float: + nonlocal now + now += step_sec + return now + + monkeypatch.setattr( + close_mod, + "time", + SimpleNamespace(monotonic=_monotonic, time=time.time), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal_state", ["succeeded", "failed"]) +async def test_a_running_task_is_waited_for_not_re_run(coord, terminal_state: str): + """``running -> running`` is not a transition the registry has; asking for it kills the step. + + The deadline path dispatches the report before CLOSE is entered, so the + sequencer routinely meets its own step already under way. It was documented + as "the sequencer will wait for it" and implemented as a second dispatch. + """ + coord.tasks = _FinishesWhileWaiting(terminal_state) + coord._dispatcher_poll_sec = 0.01 + coord.shared_state.max_minutes = 60 + + state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") + + assert state == terminal_state + assert coord.sub.run_calls == [] + + +@pytest.mark.asyncio +async def test_a_running_task_that_never_lands_is_reported_not_waited_on_forever( + coord, + monkeypatch: pytest.MonkeyPatch, +): + """The wait is patient, not unbounded: a step that never lands is recorded, not awaited forever.""" + coord._dispatcher_poll_sec = 0.0 + coord.shared_state.max_minutes = 60 + _clock_advancing_by(monkeypatch, step_sec=30.0) + + state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") + + assert state == "running" + assert coord.sub.run_calls == [] + + +@pytest.mark.asyncio +async def test_the_wait_for_a_running_report_outlives_a_short_session_reserve( + coord, + monkeypatch: pytest.MonkeyPatch, +): + """A ten-minute session reserves twelve seconds for CLOSE; no report is written in twelve seconds. + + Bounding the wait by the reserve made it a wait on paper only — the step + the deadline path dispatched was declared failed while it was still + running, and the session that ran out of time is the one whose report is + worth the most. The bound belongs to the work, so it is the report's own + expected runtime. + """ + coord.tasks = _FinishesWhileWaiting("succeeded", lands_on=5) + coord._dispatcher_poll_sec = 0.0 + coord.shared_state.max_minutes = 10 + assert coord.shared_state.closing_reserve_sec() == pytest.approx(12.0) + # Five looks at five simulated seconds apiece: past the reserve, inside the + # two minutes the catalogue prices a report at. + _clock_advancing_by(monkeypatch, step_sec=5.0) + + state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") + + assert state == "succeeded" + assert coord.sub.run_calls == [] + + +def test_the_wait_is_the_step_s_own_expected_runtime(coord): + bound = coord.phase_close._close_step_wait_sec(_running_report_row(coord)) + + assert bound == pytest.approx(ACTION_CATALOGUE["report"].typical_runtime_min * 60.0) + + +def test_a_step_the_catalogue_prices_at_almost_nothing_still_gets_the_floor(coord): + """``session_breakdown`` is priced at 12s; giving up on it after 12s is giving up on it.""" + row = _running_report_row(coord, kind="session_breakdown") + + assert coord.phase_close._close_step_wait_sec(row) == pytest.approx(_CLOSE_STEP_WAIT_FLOOR_SEC) + + +def test_an_uncatalogued_step_gets_the_floor_too(coord): + row = _running_report_row(coord, kind="not_an_action") + + assert coord.phase_close._close_step_wait_sec(row) == pytest.approx(_CLOSE_STEP_WAIT_FLOOR_SEC) + + +def test_an_extravagantly_priced_step_is_capped(coord): + """A wedged step must not hold the process open for as long as its action might legitimately run.""" + coord.action_registry = {"report": SimpleNamespace(typical_runtime_min=1000.0)} + + bound = coord.phase_close._close_step_wait_sec(_running_report_row(coord)) + + assert bound == pytest.approx(_CLOSE_STEP_WAIT_CEILING_SEC) + + +@pytest.mark.asyncio +async def test_the_sequencer_records_the_state_a_running_report_ended_in(coord): + """End to end: the waited-for report is reported like any other outcome.""" + coord.tasks = _FinishesWhileWaiting("succeeded") + coord._dispatcher_poll_sec = 0.01 + coord.shared_state.max_minutes = 60 + coord.shared_state.phase_history = [_close_phase_history_row()] + _running_report_row(coord) + coord.shared_state.closing_report_task_id = "wallclock-report" + + await coord._on_enter_close(from_phase="SWEEP") + + rows = coord.shared_state.phase_history[-1]["evidence"]["close_steps"] + report = next(r for r in rows if r["step"] == "report") + assert report["status"] == "done" + assert "wallclock-report" not in [t.task_id for t in coord.sub.run_calls] + + @pytest.mark.asyncio async def test_enqueue_internal_session_breakdown_task(coord): task = await coord._enqueue_internal_session_breakdown_task( diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 0c818c0cc1..407b497fcc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -189,6 +189,22 @@ def coord(session_dir) -> Coordinator: return Coordinator(session_dir, backends=_build_backends()) +def test_every_delegated_name_resolves_on_its_collaborator(coord: Coordinator) -> None: + """A map entry naming a method its collaborator never defined is a crash at first call, not at import. + + A field run lost every EXPLORE variant-failure record to exactly that: the + entry was there, the method was not, and ``__getattr__`` raised only once + the reap loop reached for it. + """ + unresolved = [] + for name in Coordinator._DELEGATED: + try: + getattr(coord, name) + except AttributeError as exc: + unresolved.append(f"{name}: {exc}") + assert unresolved == [] + + # -- _context_inbox_reader -------------------------------------------------- def test_context_inbox_reader_empty(coord: Coordinator) -> None: out = coord._context_inbox_reader() diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py index fff4705067..88514af235 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py @@ -87,6 +87,157 @@ async def test_promote_single_round_baseline_clears_stale_warm_runtime(coord: Co assert coord.shared_state.baseline_warm_runtime_sec == 0.0 +@pytest.mark.asyncio +async def test_promote_baseline_carries_the_boot_and_benchmark_split(coord: Coordinator) -> None: + """The two figures that let later work be priced on what it will spend. + + The whole round and the part of it that ran after the server was ready; the + difference between them is what booting this workload costs, and every + variant boots again. + """ + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1000.0, + "subprocess_runtime_sec": 900.0, + "post_ready_runtime_sec": 550.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_runtime_sec == 900.0 + assert coord.shared_state.baseline_post_ready_runtime_sec == 550.0 + + +@pytest.mark.asyncio +async def test_promote_baseline_clears_a_split_a_later_round_did_not_report( + coord: Coordinator, +) -> None: + """A stale split would be subtracted from a fresh total and called the boot.""" + coord.shared_state.baseline_post_ready_runtime_sec = 550.0 + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1000.0, + "subprocess_runtime_sec": 900.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_post_ready_runtime_sec == 0.0 + + +@pytest.mark.asyncio +async def test_promote_baseline_carries_a_dropped_hot_pass_to_the_session( + coord: Coordinator, +) -> None: + """The marker drives a session-level decision, so it has to reach the session. + + PRELUDE routes to CLOSE on it rather than optimizing against a denominator + that was never the baseline, and it is cleared by the next baseline that does + land a hot figure -- otherwise a session resumed with a fresh clock stays + condemned by the earlier leg's shortfall. + """ + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1000.0, + "subprocess_runtime_sec": 900.0, + "measure_round_dropped": {"reason": "measure_round_reaped_by_the_run"}, + "workspace": "/tmp/ws", + }, + ) + assert coord.shared_state.baseline_measure_round_dropped is True + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1200.0, + "subprocess_runtime_sec": 900.0, + "measure_round_runtime_sec": 400.0, + "workspace": "/tmp/ws", + }, + ) + assert coord.shared_state.baseline_measure_round_dropped is False + + +class TestAHotPassCorrectsAColdAnchor: + """The escape from the marker, without which PRELUDE cannot finish. + + A cold anchor holds the phase open until a hot pass replaces it. The rule + that keeps a later, lower re-baseline from displacing the anchor would reject + that replacement whenever the cold figure reads higher -- which it does + whenever the "cold" pass was not really cold, its weights already in page + cache and its kernels already compiled by an earlier run. The session would + then re-measure whole baseline rounds until the clock killed it, each one + landing the very measurement that was supposed to release it. + """ + + @pytest.mark.asyncio + async def test_a_lower_hot_figure_replaces_a_marked_cold_one(self, coord: Coordinator) -> None: + coord.shared_state.baseline_tput = 1000.0 + coord.shared_state.baseline_measure_round_dropped = True + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 980.0, + "subprocess_runtime_sec": 900.0, + "measure_round_runtime_sec": 400.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_tput == 980.0 + assert coord.shared_state.baseline_measure_round_dropped is False + assert coord.shared_state.baseline_warm_runtime_sec == 400.0 + + @pytest.mark.asyncio + async def test_a_lower_cold_figure_does_not_replace_a_marked_cold_one( + self, + coord: Coordinator, + ) -> None: + """Only a hot pass corrects the anchor; another cold one is just noisier. + + Two cold figures are comparable to each other, so the ordinary rule + applies and the better one stands. Nothing has been corrected, so the + marker stays and the phase stays open. + """ + coord.shared_state.baseline_tput = 1000.0 + coord.shared_state.baseline_measure_round_dropped = True + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 980.0, + "subprocess_runtime_sec": 900.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_tput == 1000.0 + assert coord.shared_state.baseline_measure_round_dropped is True + + @pytest.mark.asyncio + async def test_a_lower_hot_figure_still_loses_to_a_hot_anchor(self, coord: Coordinator) -> None: + """With no marker there is nothing to correct, so drift is refused again.""" + coord.shared_state.baseline_tput = 1000.0 + coord.shared_state.baseline_measure_round_dropped = False + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 980.0, + "subprocess_runtime_sec": 900.0, + "measure_round_runtime_sec": 400.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_tput == 1000.0 + + @pytest.mark.asyncio async def test_promote_baseline_non_dict_is_noop(coord: Coordinator) -> None: await coord._promote_to_shared_state("baseline", "not-a-dict") # type: ignore[arg-type] diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py index ad3eb327d7..992225cfbb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py @@ -46,21 +46,6 @@ def test_infer_model_class_ignores_bool_experts(tmp_path): assert ch._infer_model_class_from_config(str(tmp_path)) == "dense" -# ---- effective_closing_grace_sec ---- - - -def test_closing_grace_explicit(): - assert ch.effective_closing_grace_sec(100, 0) == 0.0 - assert ch.effective_closing_grace_sec(100, 5) == 5.0 - - -def test_closing_grace_default(): - # min(120, max_minutes*60*0.02). - assert ch.effective_closing_grace_sec(200, None) == 120.0 - assert ch.effective_closing_grace_sec(10, None) == 12.0 - assert ch.effective_closing_grace_sec(None, None) == 0.0 - - # ---- _parse_iso_unix ---- diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 3e21e6baad..4f88a60951 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1179,6 +1179,56 @@ async def test_handle_unpromotable_baseline_third_failure_sets_stop_reason( await c.stop() +@pytest.mark.asyncio +@pytest.mark.parametrize("error_class", ["session_time_exhausted", "orchestrator_cancelled"]) +async def test_baseline_rounds_the_run_stopped_do_not_charge_the_failure_streak(session_dir, error_class): + """Three rounds the run stopped are not three baselines that failed. + + The executor refuses to grade a reaped round because it would put a verdict + on a model the round never reached; the streak has to agree, or the session + stops as ``baseline_failed`` on the evidence of its own clock. + """ + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + for i in range(3): + await c._handle_unpromotable_result( + _mk_task("baseline", f"t-stopped-{error_class}-{i}"), + { + "status": "failed", + "error_class": error_class, + "error": "the run stopped this round before it measured anything", + }, + ) + assert c.shared_state.baseline_failure_streak == 0 + assert c.shared_state.baseline_total_failures == 0 + assert c.shared_state.stop_reason in ("", None) + # The rounds are still recorded: not charging them is not hiding them. + assert len(c.shared_state.last_action_failures) == 3 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_a_stopped_baseline_round_does_not_clear_a_real_failure_streak(session_dir): + """A stop the run chose neither charges the streak nor forgives what preceded it.""" + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + await c._handle_unpromotable_result( + _mk_task("baseline", "t-real"), + {"status": "failed", "error_class": "no_report", "error": "missing"}, + ) + await c._handle_unpromotable_result( + _mk_task("baseline", "t-stopped"), + {"status": "failed", "error_class": "session_time_exhausted", "error": "reaped"}, + ) + assert c.shared_state.baseline_failure_streak == 1 + assert c.shared_state.baseline_total_failures == 1 + finally: + await c.stop() + + def _eval_failed_result() -> dict: return { "status": "failed", @@ -1460,6 +1510,203 @@ async def test_revalidation_boot_failure_clears_pending_and_rearmes(session_dir, await c.stop() +@pytest.mark.asyncio +@pytest.mark.parametrize("error_class", ["session_time_exhausted", "orchestrator_cancelled"]) +async def test_a_revalidation_the_run_stopped_does_not_burn_the_stall_streak( + session_dir, + monkeypatch, + error_class, +): + """The same round cannot be exempt from one ledger and charged to the other. + + A reaped revalidation baseline is exempted from the baseline failure streak + because nothing about the baseline was measured. Charging it to the + enablement stall streak reaches the cap on the evidence of a clock, and the + session's terminal reason becomes ``enablement_stalled`` for rounds nobody + ever ran. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = "t-reval-stopped" + st.enablement.stall_streak = 4 + await c._handle_unpromotable_result( + _mk_task("baseline", "t-reval-stopped"), + {"status": "failed", "error_class": error_class, "error": "reaped"}, + ) + assert st.enablement.stall_streak == 4 + assert st.stop_reason in ("", None) + assert st.baseline_failure_streak == 0 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_a_reaped_revalidation_leaves_the_window_open_for_a_resume(session_dir, monkeypatch): + """Nothing else reopens the window, so the stop must not close it. + + ``validation_pending`` is set only by an eval-origin KEEP, and the + revalidation enqueue is gated on it, so clearing it strands a KEEP'd patch + that was never revalidated. The generation is bumped for the same reason + opening the window bumps it: the idempotency key must not resolve to the row + the run just stopped. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = "t-reval-open" + st.enablement.revalidation_generation = 2 + await c._handle_unpromotable_result( + _mk_task("baseline", "t-reval-open"), + {"status": "failed", "error_class": "session_time_exhausted", "error": "reaped"}, + ) + assert st.enablement.validation_pending is True + assert st.enablement.revalidation_task_id == "" + assert st.enablement.revalidation_generation == 3 + assert st.enablement.inflight_task_id == "" + finally: + await c.stop() + + +async def _cancelled_revalidation_row(c: Coordinator, *, gen: int) -> Task: + """A revalidation row for ``gen`` that the queue scan cancelled before dispatch.""" + task, _existing = await c.tasks.create_or_return_existing( + kind="baseline", + params={"reason": "enablement_eval_revalidation"}, + idempotency_key=f"enablement_revalidation:gen{gen}", + ) + await c.tasks.transition(task.task_id, "cancelled", evidence={"reason": "time_budget"}) + return task + + +@pytest.mark.asyncio +async def test_a_revalidation_the_budget_cannot_fit_is_not_enqueued(session_dir, monkeypatch): + """Opening a row the dispatcher would cancel on sight is what wedges the window. + + A revalidation is a full baseline, and the queue scan drops a queued one the + wall-clock budget can no longer fit. That leaves a cancelled row owning this + window's idempotency key -- and a row cancelled at dispatch never produces a + result to route, so nothing advances the generation past it and every later + tick resolves the window to a row that measured nothing. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_generation = 3 + st.max_minutes = 60 + st.elapsed_minutes = lambda **_kw: 60.0 # type: ignore[method-assign] + + assert await c._maybe_enqueue_enablement_baseline_revalidation() == "" + + # The window survives the stop: same generation, still pending, and no + # row for the key a resume with budget left will need. + assert st.enablement.validation_pending is True + assert st.enablement.revalidation_generation == 3 + assert st.enablement.revalidation_task_id == "" + assert await c.tasks.by_state("cancelled") == [] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_a_revalidation_key_spent_on_a_cancelled_row_opens_the_next_one(session_dir, monkeypatch): + """A terminal row is a spent generation, not an enqueue. + + ``create_or_return_existing`` hands back the cancelled row for as long as the + key names it, so without recognising that the window stays open resolving to + it for the rest of the session. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_generation = 3 + spent = await _cancelled_revalidation_row(c, gen=3) + + tid = await c._maybe_enqueue_enablement_baseline_revalidation() + + assert tid and tid != spent.task_id, "the window resolved to the cancelled row" + assert st.enablement.revalidation_generation == 4 + assert (await c.tasks.get(tid)).state == "queued" + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_resume_does_not_charge_a_revalidation_the_run_cancelled( + session_dir, + monkeypatch, +): + """The exemption the reap grants must not be charged back by the resume. + + The reap path leaves the window open without charging the stall streak, + because a round the run stopped measured nothing. The resume-time recovery saw + only "tracked row is terminal" and closed the window with the increment the + reap went out of its way to avoid -- reaching the ``enablement_stalled`` cap on + the evidence of a clock, one resume later. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + cancelled = await _cancelled_revalidation_row(c, gen=3) + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = cancelled.task_id + st.enablement.revalidation_generation = 3 + st.enablement.stall_streak = 4 + + report: dict[str, Any] = {"fixes": []} + await c.writeback._resume_recover_pending_revalidation(report) + + assert st.enablement.stall_streak == 4 + assert st.stop_reason in ("", None) + # And the window is left usable rather than merely uncharged. + assert st.enablement.validation_pending is True + assert st.enablement.revalidation_task_id == "" + assert st.enablement.revalidation_generation == 4 + assert [f["kind"] for f in report["fixes"]] == ["reopened_revalidation_the_run_cancelled"] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_resume_still_closes_a_revalidation_window_that_had_its_chance(session_dir, monkeypatch): + """A row that is terminal for any other reason is evidence, and still charged.""" + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + task, _existing = await c.tasks.create_or_return_existing( + kind="baseline", + params={"reason": "enablement_eval_revalidation"}, + idempotency_key="enablement_revalidation:gen0", + ) + await c.tasks.transition(task.task_id, "running") + await c.tasks.transition(task.task_id, "succeeded") + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = task.task_id + st.enablement.stall_streak = 1 + + report: dict[str, Any] = {"fixes": []} + await c.writeback._resume_recover_pending_revalidation(report) + + assert st.enablement.validation_pending is False + assert st.enablement.revalidation_task_id == "" + assert st.enablement.stall_streak == 2 + assert [f["kind"] for f in report["fixes"]] == ["cleared_orphaned_revalidation_pending"] + finally: + await c.stop() + + @pytest.mark.asyncio async def test_handle_unpromotable_records_for_non_baseline_kinds(session_dir): c = Coordinator(session_dir, backends=_silent_backends()) diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index 5e3d58c696..6946d13193 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -1911,7 +1911,7 @@ def reset_policy_denial_streak(self, _action_name: str) -> None: c.bus = _StubBus() c._record_observation = AsyncMock() # type: ignore[method-assign] c._record_policy_denied = AsyncMock() # type: ignore[method-assign] - c._sequence_denial_for_action = lambda *a, **k: None # type: ignore[method-assign] + c._admission_denial_for_action = lambda *a, **k: None # type: ignore[method-assign] c._registry_lanes_ttl = lambda _name: (set(), 0) # type: ignore[method-assign] c.policy = None return c diff --git a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py index ef8f9c7410..fbed1ab62d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py @@ -818,3 +818,61 @@ def test_enablement_round_in_flight_allows_after_cleared(tmp_path, monkeypatch): payload={"action_name": "baseline", "params": {}}, ) gate.validate_intent("orchestration", intent) + + +class TestAColdAnchorIsNotAnEstablishedOne: + """The rule refuses a reference the run already has, and this is not one. + + A session that could not afford its hot pass keeps the warmup's cold figure + and marks it. PRELUDE will not finish while the mark is set, so the only way + on is another baseline -- and that is the round this rule would refuse, + leaving the phase with no way forward and no way out. The state arises on + resume, which is the whole point of keeping the figure recoverable. + """ + + def _a_baseline_is_proposed(self, gate): + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.DELEGATE, + payload={"action_name": "baseline", "params": {}}, + ), + ) + + def test_a_marked_cold_anchor_does_not_refuse_the_round_that_would_fix_it( + self, + tmp_path, + monkeypatch, + ): + gate, _ = _gate(tmp_path, monkeypatch) + gate.shared_state.baseline_tput = 1000.0 + gate.shared_state.baseline_measure_round_dropped = True + + self._a_baseline_is_proposed(gate) + + def test_an_established_anchor_still_refuses_a_repeat(self, tmp_path, monkeypatch): + gate, _ = _gate(tmp_path, monkeypatch) + gate.shared_state.baseline_tput = 1000.0 + gate.shared_state.baseline_measure_round_dropped = False + + with pytest.raises(PolicyDenied) as exc_info: + self._a_baseline_is_proposed(gate) + + assert exc_info.value.rule == "baseline_phase_singleton" + + def test_an_authoring_round_in_flight_still_wins(self, tmp_path, monkeypatch): + """The exemption is about which reference exists, not about what may run. + + A specialist rewriting the framework underneath a baseline is a reason to + wait whatever the anchor says; letting the cold mark through here would + launch a round against a stack that is being changed as it runs. + """ + gate, _ = _gate(tmp_path, monkeypatch) + gate.shared_state.baseline_tput = 1000.0 + gate.shared_state.baseline_measure_round_dropped = True + gate.shared_state.enablement.inflight_task_id = "spec-abc" + + with pytest.raises(PolicyDenied) as exc_info: + self._a_baseline_is_proposed(gate) + + assert exc_info.value.rule == "enablement_round_in_flight" diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py b/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py index 7259d62cd4..00f300ffa5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py @@ -14,6 +14,7 @@ import pytest +from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE from hyperloom.orchestrator.framework.build_actions import TargetedBuildAction, BuildResult, FrameworkRuntime from hyperloom.orchestrator.loop.build_lifecycle import BuildLifecycleCollaborator from hyperloom.orchestrator.loop.coordinator import Coordinator @@ -34,9 +35,19 @@ def coord(build_coord): phase delegates to (launch-probe enqueue, rearm capture, build lifecycle). """ build_coord._rearm_calls = [] - build_coord._enqueue_build_launch_probe = _types.MethodType( - Coordinator._enqueue_build_launch_probe, build_coord - ) + for name in ( + "_enqueue_build_launch_probe", + "_route_succeeded_build", + "_build_routing_record", + "_note_build_routed", + "_build_probe_was_cancelled", + "_open_row_past_spent_generations", + "_time_budget_denial_for_action", + ): + setattr(build_coord, name, _types.MethodType(getattr(Coordinator, name), build_coord)) + # The real wall-clock gate, on the real catalogue: with no budget set it + # admits everything, so a test that wants a denial sets one. + build_coord.action_registry = ACTION_CATALOGUE def _maybe_rearm_enablement(res): build_coord._rearm_calls.append(dict(res) if isinstance(res, dict) else {}) @@ -362,26 +373,51 @@ async def _enqueue_and_transition(coord, action, state): return task_id -@pytest.mark.asyncio -async def test_route_succeeded_row_enqueues_launch_probe(coord, tmp_path): - """A succeeded build must enqueue an integrate_patch launch probe, not call rearm directly.""" - root = tmp_path / "attempt_s" +async def _verified_build(coord, root, *, gap_id, framework="vllm", ref="v1", runtime_env=None): + """A succeeded ``targeted_build`` row whose result.json carries a usable runtime.""" root.mkdir(parents=True, exist_ok=True) - - rt = FrameworkRuntime(pythonpath_prefixes=(str(root),), runtime_env={"X": "1"}) + rt = FrameworkRuntime(pythonpath_prefixes=(str(root),), runtime_env=runtime_env or {}) br = BuildResult(ok=True, attempt_root=str(root), runtime=rt) (root / "result.json").write_text(json.dumps(br.to_state()), encoding="utf-8") + action = TargetedBuildAction( + gap_id=gap_id, + framework=framework, + component="aiter", + capability="fp4_moe", + ref=ref, + attempt_root=str(root), + ) + return await _enqueue_and_transition(coord, action, "succeeded") - action = TargetedBuildAction(gap_id="g2", framework="vllm", component="aiter", - capability="fp4_moe", ref="v1", attempt_root=str(root)) - await _enqueue_and_transition(coord, action, "succeeded") + +async def _queued_probes(coord): + """The launch-probe rows currently waiting to be dispatched.""" + return [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] + + +def _spend_the_budget(coord, *, minutes=60): + """Leave the session with no wall-clock budget for a probe to run in.""" + coord.shared_state.max_minutes = minutes + coord.shared_state.elapsed_minutes = lambda **_kw: float(minutes) + + +def _restore_the_budget(coord): + """Give the session a budget a probe fits in again, as a resume would.""" + coord.shared_state.max_minutes = 600 + coord.shared_state.elapsed_minutes = lambda **_kw: 0.0 + + +@pytest.mark.asyncio +async def test_route_succeeded_row_enqueues_launch_probe(coord, tmp_path): + """A succeeded build must enqueue an integrate_patch launch probe, not call rearm directly.""" + await _verified_build(coord, tmp_path / "attempt_s", gap_id="g2", runtime_env={"X": "1"}) await Coordinator._maybe_route_build_outcomes(coord) # Must NOT directly rearm with "kept" — KEEP comes from the probe. assert not any(r.get("status") == "kept" for r in coord._rearm_calls) # Must have queued a launch-probe task. - probes = [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] + probes = await _queued_probes(coord) assert len(probes) == 1 probe_params = probes[0].params assert probe_params.get("enablement_launch_only") is True @@ -393,21 +429,18 @@ async def test_route_succeeded_row_enqueues_launch_probe(coord, tmp_path): @pytest.mark.asyncio async def test_route_succeeded_row_probe_carries_config_path(coord, tmp_path): """Launch probe inherits baseline_config_path from shared state.""" - root = tmp_path / "attempt_cfg" - root.mkdir(parents=True, exist_ok=True) - - rt = FrameworkRuntime(pythonpath_prefixes=(str(root),)) - br = BuildResult(ok=True, attempt_root=str(root), runtime=rt) - (root / "result.json").write_text(json.dumps(br.to_state()), encoding="utf-8") - coord.shared_state.baseline_config_path = "/cfg/bench.yaml" - action = TargetedBuildAction(gap_id="g3", framework="sglang", component="aiter", - capability="fp4_moe", ref="v2", attempt_root=str(root)) - await _enqueue_and_transition(coord, action, "succeeded") + await _verified_build( + coord, + tmp_path / "attempt_cfg", + gap_id="g3", + framework="sglang", + ref="v2", + ) await Coordinator._maybe_route_build_outcomes(coord) - probes = [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] + probes = await _queued_probes(coord) assert len(probes) == 1 assert probes[0].params.get("config_path") == "/cfg/bench.yaml" @@ -456,22 +489,73 @@ async def test_route_succeeded_empty_runtime_override_calls_reverted(coord, tmp_ @pytest.mark.asyncio async def test_route_succeeded_probe_idempotent(coord, tmp_path): """Calling _maybe_route_build_outcomes twice for the same row only enqueues one probe.""" - root = tmp_path / "attempt_idem" - root.mkdir(parents=True, exist_ok=True) + await _verified_build(coord, tmp_path / "attempt_idem", gap_id="g6") - rt = FrameworkRuntime(pythonpath_prefixes=(str(root),)) - br = BuildResult(ok=True, attempt_root=str(root), runtime=rt) - (root / "result.json").write_text(json.dumps(br.to_state()), encoding="utf-8") + await Coordinator._maybe_route_build_outcomes(coord) + await Coordinator._maybe_route_build_outcomes(coord) - action = TargetedBuildAction(gap_id="g6", framework="vllm", component="aiter", - capability="fp4_moe", ref="v1", attempt_root=str(root)) - await _enqueue_and_transition(coord, action, "succeeded") + probes = await _queued_probes(coord) + assert len(probes) == 1 # idempotent + + +@pytest.mark.asyncio +async def test_a_launch_probe_the_budget_cannot_fit_is_not_enqueued(coord, tmp_path): + """A probe opened into a spent budget is cancelled at dispatch and lost. + + The probe is what declares KEEP for a build, and the queue scan drops a + queued row the wall-clock budget can no longer fit. Opening one anyway spends + the build's one routing pass on a row that will never run, so the build stays + verified and unlaunched with nothing left to notice it. + """ + build_tid = await _verified_build(coord, tmp_path / "attempt_broke", gap_id="g_budget") + _spend_the_budget(coord) + + await Coordinator._maybe_route_build_outcomes(coord) + assert await _queued_probes(coord) == [] + # Nothing was routed, so the build is still owed a probe. + assert Coordinator._build_routing_record(coord, build_tid) is None + assert coord._rearm_calls == [] + + +@pytest.mark.asyncio +async def test_a_build_whose_probe_the_run_cancelled_is_still_unprobed(coord, tmp_path): + """A cancelled probe is no evidence about the build, so the build gets another. + + The gate above narrows the window but cannot close it: a probe that fits when + it is opened can still be dropped before it is dispatched, and a probe row + cancelled that way owns this build's idempotency key for the rest of the + session. Both halves have to hold -- the build is routed again, and the key it + is routed on is a fresh generation rather than the cancelled row. + """ + build_tid = await _verified_build(coord, tmp_path / "attempt_again", gap_id="g_again") await Coordinator._maybe_route_build_outcomes(coord) + first = (await _queued_probes(coord))[0] + await coord.tasks.transition(first.task_id, "cancelled", evidence={"reason": "time_budget"}) + _restore_the_budget(coord) + await Coordinator._maybe_route_build_outcomes(coord) - probes = [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] - assert len(probes) == 1 # idempotent + probes = await _queued_probes(coord) + assert [p.task_id for p in probes] != [], "the build was left accounted for by a probe that never ran" + assert first.task_id not in {p.task_id for p in probes}, "the window resolved to the cancelled row" + record = Coordinator._build_routing_record(coord, build_tid) or {} + assert record.get("probe_task_id") == probes[0].task_id + assert int(record.get("probe_generation") or 0) == 1 + + +@pytest.mark.asyncio +async def test_a_build_whose_probe_ran_and_failed_is_not_probed_again(coord, tmp_path): + """A probe that ran said something about the build; only a cancel says nothing.""" + await _verified_build(coord, tmp_path / "attempt_failed", gap_id="g_failed") + await Coordinator._maybe_route_build_outcomes(coord) + probe = (await _queued_probes(coord))[0] + await coord.tasks.transition(probe.task_id, "running") + await coord.tasks.transition(probe.task_id, "failed") + + await Coordinator._maybe_route_build_outcomes(coord) + + assert await _queued_probes(coord) == [] @pytest.mark.asyncio diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index a95404bc24..ca17f52297 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -296,6 +296,13 @@ async def _record_obs(_source, _topic, payload): fake._maybe_enqueue_enablement_baseline_revalidation = types.MethodType( Coordinator._maybe_enqueue_enablement_baseline_revalidation, fake ) + fake._open_revalidation_row = types.MethodType(Coordinator._open_revalidation_row, fake) + fake._open_row_past_spent_generations = types.MethodType( + Coordinator._open_row_past_spent_generations, fake + ) + # Admission on the session wall-clock is exercised in test_coordinator_runtime + # against a real coordinator; here nothing is ever denied for want of budget. + fake._time_budget_denial_for_action = lambda _action: None from hyperloom.orchestrator.phases.framework import FrameworkPhase fake._enablement_in_flight = types.MethodType(FrameworkPhase._enablement_in_flight, fake) diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 8db389445d..c4a5901ab7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -25,8 +25,17 @@ canonical_fingerprint, ) from hyperloom.orchestrator.actions.executors._grid_runner import ( + _SESSION_KILL_GRACE_SEC, apply_compatibility_filter, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, +) +from hyperloom.orchestrator.actions.stop_attribution import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, +) from hyperloom.orchestrator.actions.executors._accuracy_gate import ( _RUN_EVAL_FALSE_VALUES as _RUN_EVAL_FALSE, ) @@ -1605,6 +1614,539 @@ def _fake_kill(cmd, *args, **kwargs): assert out["status"] == "succeeded" +@pytest.mark.asyncio +async def test_explore_variant_cap_is_clamped_to_the_session_budget( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """A granted cap never exceeds what is left of the session. + + explore derives the cap from the measured baseline (up to 4h) and never + consulted the budget, so a 3h session could hand a single variant more time + than the whole run was given. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 3.0 + sub.shared_state = state + # Read before the run: the budget only shrinks from here, so a cap granted + # later can only be smaller than what this allows. + usable_sec = state.session_budget_usable_sec() + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + granted: list[int] = [] + + def _fake_run(cmd, *args, **kwargs): + # Only benchmark rounds carry --output-dir; the interpreter probe does not, + # and it is module-memoized, so counting it would make this order-dependent. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + granted.append(int(kwargs["timeout"])) + slot = Path(cmd[cmd.index("--output-dir") + 1]) + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-clamp"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_fits", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-clamp", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert res.result["status"] == "succeeded" + assert granted, f"the variant should have been admitted (20s expected, ~{usable_sec:.0f}s left)" + # The hard cap is allowed to sit a grace window past the deadline so the + # in-process session watchdog reaps the tree first and the kill is attributed + # to the budget rather than to a slow variant. + assert all(t <= usable_sec + _SESSION_KILL_GRACE_SEC for t in granted), ( + f"caps must be clamped to the ~{usable_sec:.0f}s budget, got {granted}" + ) + assert all(t < 3600 for t in granted), f"the declared 3600s cap must not survive the budget, got {granted}" + + +@pytest.mark.asyncio +async def test_explore_skips_a_variant_the_budget_cannot_fit( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """Admission is judged on the expected runtime, and refused when it does not fit.""" + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 3.0 # ~60s usable + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + granted: list[int] = [] + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + granted.append(int(kwargs["timeout"])) + slot = Path(cmd[cmd.index("--output-dir") + 1]) + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-nofit"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_too_long", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 600.0, + }, + idempotency_key="ex-budget-nofit", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert granted == [], "a variant needing 600s must not start with ~60s left" + # Measuring nothing because the budget ran out is not the same as variants + # failing, and it must not be reported as a bare, unattributed failure. + assert res.result["error_class"] == "session_time_exhausted" + assert res.result["session_budget_untested"] == 1 + # Untested variants stay out of the ledger so a resume can retry them. + assert res.result["losers"] == [] + assert res.result["explore_search_update"]["tested"] == {} + + +@pytest.mark.asyncio +async def test_explore_leaves_a_variant_the_run_reaped_out_of_the_ledger( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """The common case: the budget expires while a variant is running, not before it. + + ``run_grid`` records such a variant as ``skipped`` because nothing was + measured. Explore has to consume that distinction: a variant written into + the KB-facing ``tested`` ledger as ``FAILED`` is one a resume will skip + forever, and one the KB learns is a bad idea, on the evidence of a clock. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 # admits both variants; the reap comes mid-round + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + ran: list[str] = [] + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + ran.append(slot.name) + if "v_reaped" not in str(slot): + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + # The session deadline elapsed while this round was running, so the + # reaper tore the tree down and named the cause. + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-reaped"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_measured", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + }, + { + "name": "v_reaped", + "extra_args": "--max-num-seqs 512", + "extra_envs": {}, + "provenance": "default_grid", + }, + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-reaped", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + tested = res.result["explore_search_update"]["tested"] + # A variant the run reaped was never measured; recording it keeps a resume + # from ever retrying it, and teaches the KB a clock's verdict. + assert [t["name"] for t in tested.values()] == ["v_measured"] + assert [lr["name"] for lr in res.result["losers"]] == [] + assert res.result["session_budget_untested"] == 1 + + +@pytest.mark.asyncio +async def test_explore_leaves_a_variant_out_when_the_run_reaped_its_grid_warmup( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """The stop has to survive ``run_grid``'s own discarded warmup round. + + With server-lifecycle reuse ineligible, explore's decision round is a plain + ``run_grid`` call, and ``run_grid`` runs its own warmup pass before it (on by + default outside pytest). Explore reads the stop off the result's + ``error_class``, so a warmup reap graded as ``warmup_round_failed`` reaches + this ledger as a measured verdict about the variant. + """ + _force_cold_decision(monkeypatch) + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "1") + monkeypatch.setattr( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + lambda _config_path: { + "eligible": True, + "framework": "sglang", + "port": 30000, + "reason": "", + }, + ) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + ran: list[str] = [] + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + ran.append(slot.name) + if slot.name == "warmup_round": + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-warmup-reaped"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_warmup_reaped", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-warmup-reaped", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert ran == ["warmup_round"], f"the decision round ran after the warmup was reaped: {ran}" + assert res.result["explore_search_update"]["tested"] == {} + assert res.result["losers"] == [] + assert res.result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert res.result["session_budget_untested"] == 1 + + +@pytest.mark.asyncio +async def test_explore_does_not_call_a_variant_unstable_when_the_run_reaped_its_rebench( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """A confirmation the run stopped is not a failed confirmation. + + The post-KEEP rebench is the second gate, so a reaped rebench used to evict + the variant as ``KEEP_UNSTABLE`` -- a stability verdict drawn from a round + that measured nothing. The decision round's own entry goes too, so a resume + re-measures the variant and its confirmation together. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + if "stack_rebench" in str(slot): + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + _fake_workspace(slot, tput=900.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-rebench-reaped"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_unconfirmed", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-rebench-reaped", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert res.result["explore_search_update"]["tested"] == {} + assert res.result["losers"] == [] + assert res.result["winners"] == [] + assert res.result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert res.result["session_budget_untested"] == 1 + + +@pytest.mark.asyncio +async def test_the_reaped_rebench_rollback_spares_a_prior_rounds_ledger_row( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """Rolling back this round's write must not delete an earlier round's. + + A fingerprint may be re-run across rounds, so the row a rerun overwrites is a + real measurement from a previous round. Undoing the rerun by deleting the key + takes that measurement out of the negative ledger with it, and the model is + free to re-propose a variant already measured and failed -- at the cost of a + full benchmark round. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + fp_rerun = canonical_fingerprint("--rerun-flag", {}) + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + # Match on path segments: ``tmp_path`` is named after the test, so a + # substring check would fire on every round. + parts = set(slot.parts) + rerun = "v01_v_ok" in parts + if rerun and "stack_rebench" in parts: + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + _fake_workspace(slot, tput=1000.0 if rerun else 900.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-rollback"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_measured", + "extra_args": "--other-flag", + "extra_envs": {}, + "provenance": "default_grid", + }, + { + "name": "v_ok", + "extra_args": "--rerun-flag", + "extra_envs": {}, + "provenance": "default_grid", + }, + ], + "explore_search": { + "tested": { + fp_rerun: { + "fingerprint": fp_rerun, + "name": "v_ok", + "extra_server_args": "--rerun-flag", + "extra_envs": {}, + "outcome": "FAILED", + "round_id": "explore-001", + } + }, + "rejected": [], + "name_index": {"v_ok": fp_rerun}, + }, + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + "enable_stack_rebench": True, + }, + idempotency_key="ex-rollback-prior-row", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + update = res.result["explore_search_update"] + prior = update["tested"].get(fp_rerun) + assert prior is not None, "prior round's ledger entry was deleted by the rollback" + assert prior["round_id"] == "explore-001" + assert prior["outcome"] == "FAILED" + assert update["name_index"]["v_ok"] == fp_rerun + # The round did measure something, so the update replaces the persisted + # ledger wholesale -- which is what makes a deletion here durable. + assert {t["name"] for t in update["tested"].values()} == {"v_measured", "v_ok"} + + +@pytest.mark.asyncio +async def test_explore_attributes_a_round_the_run_reaped_before_anything_measured( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """With nothing measured the round is ``failed``, and must say who stopped it.""" + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + return subprocess.CompletedProcess( + args=cmd, + returncode=ORCHESTRATOR_CANCELLED_RETURNCODE, + stdout="", + stderr="cancelled", + ) + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-cancelled"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_cancelled", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-cancelled", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert res.result["status"] == "failed" + assert res.result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + assert res.result["explore_search_update"]["tested"] == {} + + @pytest.mark.asyncio async def test_explore_executor_empty_grid_returns_failed(sub_agent_runner, tmp_path): sub, tr, _ = sub_agent_runner diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py index 387ab81830..023e575fd5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio import json import os import subprocess @@ -76,7 +77,7 @@ def _make_candidate( } -def _make_ctx(task_id: str, params: dict[str, Any]) -> RunnerContext: +def _make_ctx(task_id: str, params: dict[str, Any], extra: dict[str, Any] | None = None) -> RunnerContext: task = Task( task_id=task_id, kind="framework_agent", @@ -85,7 +86,7 @@ def _make_ctx(task_id: str, params: dict[str, Any]) -> RunnerContext: idempotency_key=task_id, requires_lanes=tuple(), ) - return RunnerContext(task=task, lease=None, extra={}) + return RunnerContext(task=task, lease=None, extra=extra if extra is not None else {}) def test_candidate_slug_prefers_repo_and_pr_number(): @@ -333,7 +334,7 @@ async def test_executor_keep_when_delta_above_threshold(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -360,6 +361,128 @@ async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 assert (repo / "src.py").read_text().endswith("return 2\n") +@pytest.mark.asyncio +async def test_bench_is_bounded_by_the_session_budget(tmp_path: Path): + """The candidate bench is handed the session budget, as the other arms are. + + Its declared cap answers "how long before this counts as hung", not "how much + budget is left", so without the session deadline a candidate benched near the + end of a run outlives the run itself. + """ + from unittest.mock import MagicMock + + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + captured: dict[str, Any] = {} + + async def fake_bench(self, *, params, output_root, slug, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return ( + {"status": "succeeded", "output_throughput": 1100.0}, + {"accuracy_pass": None}, + ) + + shared_state = MagicMock() + shared_state.grid_session_deadline_sec.return_value = 4242.0 + shared_state.baseline_runtime_sec = 600.0 + + ctx = _make_ctx( + "t-fp-budget", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + extra={"shared_state": shared_state}, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench): + result = await executor(ctx) + + assert result["status"] == "kept" + assert captured["session_deadline_sec"] == 4242.0 + # The expected runtime, not the backstop cap: admitting on the backstop + # abandons the tail of the budget. + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_bench_budget_is_unbounded_without_a_session(tmp_path: Path): + """No session context means no deadline, not a deadline of zero.""" + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + captured: dict[str, Any] = {} + + async def fake_bench(self, *, params, output_root, slug, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return ( + {"status": "succeeded", "output_throughput": 1100.0}, + {"accuracy_pass": None}, + ) + + ctx = _make_ctx( + "t-fp-nobudget", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench): + result = await executor(ctx) + + assert result["status"] == "kept" + assert captured["session_deadline_sec"] is None + assert captured["variant_expected_sec"] is None + + +@pytest.mark.asyncio +async def test_bench_candidate_forwards_the_session_budget_to_the_grid(tmp_path: Path): + session_dir = tmp_path / "session" + session_dir.mkdir() + config_path = tmp_path / "baseline.yaml" + config_path.write_text("benchmark: {}\n", encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + captured: dict[str, Any] = {} + + async def fake_run_grid(*args, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return [_mk_variant_result(tput=1100.0, status="succeeded")] + + from hyperloom.orchestrator.actions.executors import framework_agent as fp_mod + + with ( + patch.object(fp_mod, "run_grid", new=fake_run_grid), + patch.object(fp_mod, "materialize_config_with_envs", return_value=config_path), + ): + await executor._bench_candidate( + params={"config_path": str(config_path)}, + output_root=tmp_path / "out", + slug="budget", + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + @pytest.mark.asyncio async def test_executor_reverts_when_live_anchor_exceeds_queued_baseline(tmp_path: Path): session_dir = tmp_path / "session" @@ -372,7 +495,7 @@ async def test_executor_reverts_when_live_anchor_exceeds_queued_baseline(tmp_pat state.baseline_tput = 1000.0 state.current_best = {"tput": 1150.0} - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None} ctx = _make_ctx( @@ -412,7 +535,7 @@ async def test_executor_keep_writes_kb_lessons(tmp_path: Path, monkeypatch): cand = _make_candidate() cand["pr_url"] = "https://github.com/sgl-project/sglang/pull/1234" - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -458,7 +581,7 @@ async def test_executor_revert_writes_kb_lessons(tmp_path: Path, monkeypatch): cand = _make_candidate() cand["pr_url"] = "https://github.com/sgl-project/sglang/pull/1234" - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 980.0}, {"accuracy_pass": None}, @@ -500,7 +623,7 @@ async def test_executor_revert_when_delta_below_threshold(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 980.0}, {"accuracy_pass": None}, @@ -539,7 +662,7 @@ async def test_executor_revert_on_accuracy_regression(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": False}, @@ -576,7 +699,7 @@ async def test_executor_bench_exception_triggers_revert(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def boom(self, *, params, output_root, slug): # noqa: ARG001 + async def boom(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 raise RuntimeError("simulated bench crash") ctx = _make_ctx( @@ -596,6 +719,125 @@ async def boom(self, *, params, output_root, slug): # noqa: ARG001 assert (repo / "src.py").read_text().endswith("return 1\n") +@pytest.mark.asyncio +async def test_executor_cancelled_bench_reverts_and_re_raises(tmp_path: Path): + """A cancelled bench must not leave the candidate in the framework tree. + + The dispatcher cancels in-flight actions on shutdown and on a spent + wall-clock budget. ``CancelledError`` is not an ``Exception``, so the REVERT + handler beside it never sees the stop, and the candidate would stay applied + with the operator's auto-stash still on the stack. The cancel is re-raised + rather than graded as a REVERT: work the run stopped is not work that + failed, and SubAgentRunner records it as ``cancelled``. + """ + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + + async def cancelled(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 + raise asyncio.CancelledError + + ctx = _make_ctx( + "t-fp-cancel", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + }, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=cancelled): + with pytest.raises(asyncio.CancelledError): + await executor(ctx) + + assert (repo / "src.py").read_text().endswith("return 1\n"), ( + "the cancelled candidate was left applied in the framework tree" + ) + assert scratch.exists(), "user auto-stash was not restored after the cancel" + assert scratch.read_text(encoding="utf-8") == "user work in progress\n" + + +def _stash_list(repo: Path) -> str: + """What ``git stash list`` reports for ``repo``.""" + return subprocess.run( + ["git", "-C", str(repo), "stash", "list"], + capture_output=True, + text=True, + check=False, + ).stdout + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bench_tput,verdict", [(900.0, "revert"), (1100.0, "keep")]) +async def test_a_cancel_at_the_kb_writeback_still_hands_the_stash_back( + tmp_path: Path, + bench_tput: float, + verdict: str, +): + """The last await a candidate crosses is the KB writeback, not the bench. + + Both verdicts record their outcome after the verdict is decided and before + the stash restore that returns it. A cancel arrives at whatever await the + action happens to be at, and a spent wall-clock budget is exactly what makes + it arrive at an arbitrary one -- so if that window is unguarded the + operator's uncommitted work stays in ``git stash`` for the rest of the + session with nothing saying so. + """ + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 + return ( + {"status": "succeeded", "output_throughput": bench_tput}, + {"accuracy_pass": None}, + ) + + async def cancelled_writeback(self, **_kwargs): # noqa: ARG001 + raise asyncio.CancelledError + + ctx = _make_ctx( + f"t-fp-kb-cancel-{verdict}", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + ) + with ( + patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench), + patch.object(FrameworkAgentExecutor, "_write_kb_record", new=cancelled_writeback), + ): + with pytest.raises(asyncio.CancelledError): + await executor(ctx) + + assert scratch.read_text(encoding="utf-8") == "user work in progress\n", ( + "the user's uncommitted work was left in the stash" + ) + assert _stash_list(repo) == "", "the auto-stash was never popped" + # A KEEP that is cancelled before its result reaches the Coordinator is a + # KEEP the session does not record, so the commit must not survive either. + assert (repo / "src.py").read_text().endswith("return 1\n"), ( + "the ungraded candidate was left in the framework tree" + ) + + _PATCH_B_ADDS_FILE = """\ diff --git a/new.py b/new.py new file mode 100644 @@ -623,13 +865,13 @@ async def test_reject_after_keep_preserves_kept_changes(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) - async def keep_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def keep_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, ) - async def reject_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def reject_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 980.0}, {"accuracy_pass": None}, @@ -684,7 +926,7 @@ async def test_apply_failure_after_keep_preserves_kept_changes(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) - async def keep_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def keep_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -810,7 +1052,7 @@ async def test_executor_checkout_head_mode_applies_and_keeps(tmp_path: Path, mon "apply_mode": "checkout_head", } - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -877,7 +1119,7 @@ async def test_executor_keep_adds_new_file_pr(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -1078,7 +1320,7 @@ async def test_executor_require_accuracy_blocks_keep_when_unevaluated(tmp_path: executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -1117,7 +1359,7 @@ async def test_executor_require_accuracy_degrades_without_baseline(tmp_path: Pat executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index 4b73b955d5..0df03a1044 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -5,27 +5,35 @@ from __future__ import annotations +import asyncio import inspect import json import os import subprocess import time from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest import yaml from hyperloom.orchestrator.actions.executors import _grid_runner from hyperloom.orchestrator.actions.executors import _grid_runner as gr +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, +) from hyperloom.orchestrator.actions.executors._grid_runner import ( _MN_BACKENDS_PRIORITY, _MN_PARAMS_PRIORITY, + SESSION_TIME_EXHAUSTED_CLASS, GridVariant, VariantResult, _build_variant_yaml, _parse_skip_spec, _run_magpie, + _SESSION_KILL_GRACE_SEC, apply_runtime_benchmark_overrides, apply_user_skip_list, coerce_extra_envs, @@ -34,6 +42,8 @@ run_grid, ) +from .conftest import enable_multi_node, launches_by_round_slot + # Section 1: _write_variant_abort_marker @@ -1391,6 +1401,750 @@ def fake_run(cmd, *args, **kwargs): assert [r.status for r in results] == ["succeeded", "succeeded"] +def _capture_launches(recorded: list[dict]): + """A ``run_with_session_kill`` double that records how each round was launched. + + Only benchmark rounds are recorded: the interpreter probe carries no + ``--output-dir`` and is module-memoized, so counting it would make these + assertions depend on which test ran first. + + Args: + recorded: Appended to per launched round, each record carrying the + ``round_slot`` the round wrote into alongside the launch kwargs. + + Returns: + A callable usable as ``side_effect``. + """ + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + recorded.append({"round_slot": slot.name, **kwargs}) + _fake_workspace(slot) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + return fake_run + + +def _granted_timeouts(recorded: list[dict]) -> list[int]: + """The hard timeout each recorded round was granted, in launch order.""" + return [int(launch["timeout"]) for launch in recorded] + + +# Every output slot one variant launches a benchmark process into, in launch +# order: the discarded warmup, the multi-node client warmup, and the measured +# round. All three are full benchmark passes on the GPU. +_GRID_ROUND_SLOTS = ("warmup_round", "mn_warmup", "variant_00_v0") + + +async def _launch_every_pass_of_one_variant( + tmp_path, + monkeypatch, + *, + session_deadline_sec: float | None, +) -> list[dict]: + """Run one variant with every optional pass enabled, recording each launch. + + The passes are independently gated -- the discarded warmup on lifecycle + eligibility, the client warmup on multi-node -- so a test that wants to reach + every launch site the grid has must turn all of them on at once. + + Args: + tmp_path: Test-scoped directory for the config and the output root. + monkeypatch: Used to put the grid on the multi-node path. + session_deadline_sec: The session deadline handed to ``run_grid``. + + Returns: + list[dict]: One record per launched round, as ``_capture_launches`` makes + them. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + enable_multi_node(monkeypatch) + recorded: list[dict] = [] + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + return recorded + + +class TestSessionGridBounds: + """One definition of the two numbers every benching arm needs. + + A deadline derived one way in one executor and another way in the next + produces arms that abandon different amounts of the tail budget. + """ + + def test_no_session_means_no_bounds(self): + assert _grid_runner.session_grid_bounds(None) == (None, None) + + def test_reads_the_deadline_and_the_measured_baseline(self): + state = MagicMock() + state.grid_session_deadline_sec.return_value = 4242.0 + state.baseline_runtime_sec = 600.0 + assert _grid_runner.session_grid_bounds(state) == (4242.0, 600.0) + + def test_unmeasured_baseline_yields_no_estimate(self): + """Zero is "not measured yet", which must not read as "needs 0 seconds".""" + state = MagicMock() + state.grid_session_deadline_sec.return_value = 4242.0 + state.baseline_runtime_sec = 0.0 + assert _grid_runner.session_grid_bounds(state) == (4242.0, None) + + def test_unparseable_baseline_yields_no_estimate(self): + state = MagicMock() + state.grid_session_deadline_sec.return_value = None + state.baseline_runtime_sec = "not-a-number" + assert _grid_runner.session_grid_bounds(state) == (None, None) + + def test_state_without_the_deadline_accessor_is_tolerated(self): + state = SimpleNamespace(baseline_runtime_sec=600.0) + assert _grid_runner.session_grid_bounds(state) == (None, 600.0) + + def test_a_variant_is_priced_as_a_boot_and_a_benchmark(self): + """What a variant actually spends, rather than what the baseline spent. + + A variant cannot re-attach to anyone else's server -- its config differs + in the very knobs that decide how one comes up -- so it pays a boot and + then a benchmark. The baseline's 900s cold round is 350s of boot and 550s + of benchmarking that also paid the first request's kernel compile; the + variant pays that boot and the 400s a benchmark costs once the compile is + cached. Admitting on the 900s abandons 150s of every variant's worth of + tail budget. + """ + state = SimpleNamespace( + grid_session_deadline_sec=lambda: 4242.0, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + baseline_warm_runtime_sec=400.0, + ) + + deadline, variant_sec = _grid_runner.session_grid_bounds(state) + + assert deadline == 4242.0 + assert variant_sec == pytest.approx(750.0) + + def test_a_baseline_with_no_hot_pass_prices_the_benchmark_from_the_cold_one(self): + """The post-ready segment stands in, over-predicting by the compile.""" + state = SimpleNamespace( + grid_session_deadline_sec=lambda: None, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + ) + + assert _grid_runner.session_grid_bounds(state)[1] == pytest.approx(900.0) + + def test_a_baseline_that_never_reported_its_boot_falls_back_to_the_whole_round(self): + """A scriptable workload runs no server, so there is no split to read.""" + state = SimpleNamespace( + grid_session_deadline_sec=lambda: None, + baseline_runtime_sec=900.0, + baseline_warm_runtime_sec=400.0, + ) + + assert _grid_runner.session_grid_bounds(state)[1] == pytest.approx(900.0) + + +class TestSessionBudgetAdmission: + """A variant is admitted on what it is expected to need, not on its backstop. + + ``variant_timeout_sec`` is the catastrophic-hang cap (~baseline x 2 for + explore). Gating admission on it abandons the tail of the budget: with a + 20-minute baseline the grid refuses to start a round with 30 minutes left. + """ + + @pytest.mark.asyncio + async def test_variant_runs_when_budget_fits_expected_but_not_the_backstop(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=30.0, + ) + + assert [r.status for r in results] == ["succeeded"] + assert len(recorded) == 1 + + @pytest.mark.asyncio + async def test_variant_skipped_when_budget_cannot_fit_the_expected_runtime(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 10.0, + variant_expected_sec=300.0, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + + @pytest.mark.asyncio + async def test_without_an_estimate_the_stricter_backstop_check_is_kept(self, tmp_path): + """Callers that cannot estimate keep the pre-existing, stricter gate.""" + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=None, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + + +class TestSessionBudgetTimeoutClamp: + """A granted cap never exceeds what the session can still pay for. + + explore derives caps from the measured baseline (up to 4h) and never + consulted the budget, so a 3h session could hand a single variant more time + than the whole run was given. + """ + + @pytest.mark.asyncio + async def test_granted_cap_is_clamped_to_the_remaining_budget(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=30.0, + ) + + assert len(recorded) == 1 + granted = _granted_timeouts(recorded)[0] + # The cap is allowed a small grace past the deadline so the in-process + # session watchdog trips first and attributes the kill correctly. + assert 60 <= granted <= 120 + _SESSION_KILL_GRACE_SEC, ( + f"expected a cap clamped to the ~120s budget, got {granted}" + ) + + @pytest.mark.asyncio + async def test_declared_cap_is_kept_when_the_budget_is_larger(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 36000.0, + variant_expected_sec=30.0, + ) + + assert _granted_timeouts(recorded) == [600] + + @pytest.mark.asyncio + async def test_no_deadline_leaves_the_declared_cap_untouched(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=None, + variant_expected_sec=30.0, + ) + + assert _granted_timeouts(recorded) == [600] + + +class TestSessionKillAttribution: + """A round reaped for the session budget is not a verdict about the variant.""" + + @pytest.mark.asyncio + async def test_mid_round_budget_kill_is_recorded_as_skipped_not_failed(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + + def fake_run(cmd, *args, **kwargs): + return subprocess.CompletedProcess(cmd, SESSION_TIME_EXHAUSTED_RETURNCODE, "", "") + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=30.0, + ) + + assert [r.status for r in results] == ["skipped"] + assert results[0].error_class == "session_time_exhausted" + # Never the overtime label, which asserts the variant is abnormally slow. + assert not getattr(results[0], "killed_overtime", False) + assert results[0].output_throughput is None + + @pytest.mark.asyncio + async def test_the_hard_cap_leaves_room_for_the_session_watchdog_to_win(self, tmp_path): + """Both fire at the same instant, and the sentinel must get there first. + + The hard cap raises ``TimeoutExpired``, which the ledger reads as a variant + timeout, so it is granted a small grace past the session deadline. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 60.0, + variant_expected_sec=30.0, + ) + + assert len(recorded) == 1 + granted = _granted_timeouts(recorded)[0] + assert granted > 60, f"hard cap {granted}s must sit past the ~60s deadline, not on it" + assert granted <= 90, f"the grace must stay small, got {granted}s" + + @pytest.mark.parametrize("round_slot", _GRID_ROUND_SLOTS) + @pytest.mark.asyncio + async def test_the_session_deadline_reaches_the_subprocess_layer(self, tmp_path, monkeypatch, round_slot): + """Regression: the clamped cap alone bounds the round but mislabels the kill. + + Parameterized over every pass a variant costs, because each launch site + hands the deadline over on its own. A site that leaves it out is still + bounded by its clamped cap, so the round still ends -- as a + ``TimeoutExpired`` the ledger reads as a variant too slow to measure. + """ + deadline = time.monotonic() + 120.0 + recorded = await _launch_every_pass_of_one_variant( + tmp_path, + monkeypatch, + session_deadline_sec=deadline, + ) + + assert launches_by_round_slot(recorded)[round_slot]["session_deadline_sec"] == deadline + + @pytest.mark.asyncio + async def test_no_pass_of_a_variant_is_launched_without_the_deadline(self, tmp_path, monkeypatch): + """The net for a pass added later, which no per-slot test would know about.""" + deadline = time.monotonic() + 120.0 + recorded = await _launch_every_pass_of_one_variant( + tmp_path, + monkeypatch, + session_deadline_sec=deadline, + ) + + assert set(launches_by_round_slot(recorded)) >= set(_GRID_ROUND_SLOTS) + assert [c["round_slot"] for c in recorded if c.get("session_deadline_sec") != deadline] == [] + + +def _reaping_round(returncode: int, *, slot_name: str): + """A ``run_with_session_kill`` double that reaps one named round of a variant. + + Args: + returncode: The sentinel the reaped round comes back with. + slot_name: Output-slot directory name identifying the round to reap; + every other round succeeds with a valid report. + + Returns: + tuple: The ``side_effect`` callable, and the list of slot names it + appends to as rounds are launched. + """ + launched: list[str] = [] + + def fake_run(cmd, *args, **kwargs): + # The module-memoized interpreter probe is not a benchmark round. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + launched.append(slot.name) + if slot.name == slot_name: + return subprocess.CompletedProcess(cmd, returncode, "", "") + _fake_workspace(slot) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + return fake_run, launched + + +class TestEveryRoundCarriesTheStopThatEndedIt: + """A round the run stopped is a stop whichever round it was. + + The measured round is not the only full benchmark pass a variant costs: the + discarded warmup runs the same workload, and so does the multi-node client + warmup. A reap in either has exactly as much to say about the variant as one + in the measured round -- nothing -- so grading it as ``warmup_round_failed`` + files a verdict the run never reached, and ignoring the returncode entirely + keeps launching benchmark rounds after the orchestrator asked the action to + stop. + """ + + @pytest.mark.asyncio + async def test_a_warmup_reaped_by_the_budget_is_skipped_not_a_failed_variant(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + fake_run, launched = _reaping_round(SESSION_TIME_EXHAUSTED_RETURNCODE, slot_name="warmup_round") + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("cand0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 600.0, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + + assert launched == ["warmup_round"], "the measured round must not run after the warmup was reaped" + assert [r.status for r in results] == ["skipped"] + assert results[0].error_class == "session_time_exhausted" + assert _read_marker(tmp_path / "out" / "variant_00_cand0")["error_class"] == "session_time_exhausted" + + @pytest.mark.asyncio + async def test_a_cancelled_warmup_ends_the_grid_instead_of_booting_the_next_variant(self, tmp_path): + """Every remaining variant would boot its own server on the Ray path. + + ``run_session_kill`` re-``ensure()``s a lease whose actor the cancel just + killed, so a grid that keeps going after a cancel starts a fresh actor + and a fresh GPU server per remaining variant. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + fake_run, launched = _reaping_round(ORCHESTRATOR_CANCELLED_RETURNCODE, slot_name="warmup_round") + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("c0"), GridVariant("c1"), GridVariant("c2")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + keep_going_on_failure=True, + session_deadline_sec=time.monotonic() + 600.0, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + + assert launched == ["warmup_round"], f"a cancelled action kept launching rounds: {launched}" + assert [r.status for r in results] == ["skipped"] * 3 + assert [r.error_class for r in results] == ["orchestrator_cancelled"] * 3 + + @pytest.mark.asyncio + async def test_a_cancelled_multi_node_warmup_ends_the_grid(self, tmp_path, monkeypatch): + """The multi-node warmup discarded its returncode along with its report.""" + from hyperloom.orchestrator.actions.executors import _multi_node_env as mne + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnsl + + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + monkeypatch.setattr(mne, "is_multi_node", lambda: True) + monkeypatch.setattr(mne, "mn_bench_warmup_enabled", lambda: True) + + async def fake_restart_server_for_round(**_kwargs): + return None + + monkeypatch.setattr(mnsl, "restart_server_for_round", fake_restart_server_for_round) + fake_run, launched = _reaping_round(ORCHESTRATOR_CANCELLED_RETURNCODE, slot_name="mn_warmup") + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("c0"), GridVariant("c1")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + keep_going_on_failure=True, + session_deadline_sec=time.monotonic() + 600.0, + variant_expected_sec=30.0, + ) + + assert launched == ["mn_warmup"], f"a cancelled action kept launching rounds: {launched}" + assert [r.error_class for r in results] == ["orchestrator_cancelled"] * 2 + + +class TestSessionBudgetWarmupRounds: + """A warmup round costs a full pass, and the measured round is paid first.""" + + @pytest.mark.asyncio + async def test_admission_accounts_for_the_warmup_pass(self, tmp_path): + """Budget for one round is not budget for a warmup plus a measure round. + + Admitting on a single round's estimate would let a variant in and then + clamp its measured round to nothing, turning a budget shortfall into a + ledger full of spurious timeouts. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 40.0, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + + @pytest.mark.asyncio + async def test_the_budget_is_re_checked_after_the_uncapped_server_restart( + self, + tmp_path, + monkeypatch, + ): + """The admission gate is taken before the one launch it does not cover. + + The per-variant multi-node server restart sits between the gate at the top + of the loop and the first pass, and it is under no cap of its own: booting + a large model across nodes can take longer than a benchmark pass. So a + variant can be admitted on a budget that fits both its passes and reach the + warmup with a budget that fits neither -- and the grid has no skip there, + so it launches the warmup anyway, watches it get killed, swallows that as + best-effort, and finds the measured round no longer fits. + + Scaled down by a thousand from the field shape (1300s left, 2x600s + admitted, a 300s restart) so the restart's cost is real elapsed time + rather than a clock the test pretends about. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_env as mne + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnsl + + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + monkeypatch.setattr(mne, "is_multi_node", lambda: True) + monkeypatch.setattr(mne, "mn_bench_warmup_enabled", lambda: True) + restarts: list[float] = [] + + async def slow_restart(**_kwargs): + restarts.append(time.monotonic()) + await asyncio.sleep(0.5) + + monkeypatch.setattr(mnsl, "restart_server_for_round", slow_restart) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 1.3, + variant_expected_sec=0.6, + ) + + assert restarts, "the restart never ran, so this is not the case under test" + launched = [c["round_slot"] for c in recorded] + assert launched == [], f"a pass was launched into a budget the restart had spent: {launched}" + assert [r.status for r in results] == ["skipped"] + assert [r.error_class for r in results] == [SESSION_TIME_EXHAUSTED_CLASS] + + @pytest.mark.asyncio + async def test_warmup_cap_reserves_budget_for_the_measured_round(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 300.0, + variant_expected_sec=60.0, + warmup_before_measure=True, + ) + + by_round = launches_by_round_slot(recorded) + warmup = next(int(c["timeout"]) for c in recorded if "warmup" in c["round_slot"]) + measure = next(int(c["timeout"]) for c in recorded if "warmup" not in c["round_slot"]) + assert warmup >= 60, f"a warmup capped under the 60s a pass takes is launched to be killed, got {warmup}" + assert warmup <= 240 + _SESSION_KILL_GRACE_SEC, ( + f"warmup cap must hold back the measured round's 60s, got {warmup}" + ) + assert warmup <= measure, "the warmup is never granted more than the round it holds budget back for" + assert len(by_round) == 2, f"expected a warmup and a measured round, got {list(by_round)}" + + @pytest.mark.asyncio + async def test_a_warmup_killed_at_a_clamped_cap_is_logged_with_the_cap_it_got(self, tmp_path, caplog): + """The abort line is the only record of how long the round was allowed. + + The declared cap is a hang backstop; what the warmup was granted is that + cap minus the reserve, and a round killed after four minutes logged as a + two-hour timeout reads as a variant that hangs rather than a budget that + ran out. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + recorded.append({"round_slot": slot.name, **kwargs}) + raise subprocess.TimeoutExpired(cmd, float(kwargs.get("timeout") or 0.0)) + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + caplog.at_level("WARNING"), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 300.0, + variant_expected_sec=60.0, + warmup_before_measure=True, + ) + + granted = int(recorded[0]["timeout"]) + assert granted < 7800, f"this test needs a clamped cap to be about, got {granted}" + aborts = [r.message for r in caplog.records if "warmup timeout" in r.message] + assert aborts, f"the warmup abort was not logged: {[r.message for r in caplog.records]}" + assert f"timeout_sec={granted}" in aborts[0], f"the abort line reports a cap the round never had: {aborts[0]}" + assert results[0].error_class == "warmup_magpie_timeout" + + class TestCompactJsonServerArgs: """JSON-valued flags must be space-free to survive Magpie's unquoted ``$EXTRA_VLLM_ARGS`` splice (otherwise spec-decode / compilation-config diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py index 6549e7314e..a895860cf6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py @@ -17,6 +17,7 @@ import pytest import yaml +from hyperloom.orchestrator.actions.cancel_channel import CancelScope, use_cancel_scope from hyperloom.orchestrator.actions.executors import _grid_runner as gr from hyperloom.orchestrator.actions.executors import _multi_node_env as mne from hyperloom.orchestrator.actions.executors import ( @@ -27,6 +28,9 @@ GridVariant, run_grid, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, +) from hyperloom.orchestrator.trace.task_progress import progress_scope from .conftest import chatty_child, suppression_window_s @@ -107,59 +111,65 @@ def _invalid_rc0_workspace(slot: Path) -> Path: # --------------------------------------------------------------------------- +def _run_with_pulse_capture( + *, + multi_node, + run_side_effect, + base, + out, + restart=None, + keep_going=True, + grid_n=1, + scope=None, + notes=None, +): + pulse_calls: list = [] + + async def fake_pulse(**kwargs): + pulse_calls.append(kwargs) + + async def collect(**note): + notes.append(note) + + with ExitStack() as st: + st.enter_context(patch.object(mne, "is_multi_node", lambda: multi_node)) + st.enter_context(patch.object(gr, "_robustness_pulse", side_effect=fake_pulse)) + st.enter_context( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=run_side_effect, + ) + ) + if restart is not None: + st.enter_context(patch.object(mnsl, "restart_server_for_round", restart)) + # Entered outside ``asyncio.run`` on purpose: a task copies the context + # at creation, which is how the dispatcher's scope reaches the action it + # publishes it for. + if scope is not None: + st.enter_context(use_cancel_scope(scope)) + if notes is not None: + st.enter_context(progress_scope(collect)) + grid = [GridVariant(name=f"c{i}") for i in range(grid_n)] + results = asyncio.run( + run_grid( + base_yaml_path=base, + base_extra_args="", + grid=grid, + output_root=out, + magpie_python=sys.executable, + variant_timeout_sec=10, + gpu_type="mi300x", + keep_going_on_failure=keep_going, + ) + ) + return results, pulse_calls + + class TestPulseMatrix: """``_pulse_after_variant`` (progress note plus ``_robustness_pulse``) fires on every variant outcome, including the multi-node ``mn_server_restart_failed`` path that used to leave before reaching it.""" - def _run_with_pulse_capture( - self, - *, - multi_node, - run_side_effect, - base, - out, - restart=None, - keep_going=True, - grid_n=1, - notes=None, - ): - pulse_calls: list = [] - - async def fake_pulse(**kwargs): - pulse_calls.append(kwargs) - - async def collect(**note): - notes.append(note) - - with ExitStack() as st: - st.enter_context(patch.object(mne, "is_multi_node", lambda: multi_node)) - st.enter_context(patch.object(gr, "_robustness_pulse", side_effect=fake_pulse)) - st.enter_context( - patch( - "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=run_side_effect, - ) - ) - if notes is not None: - st.enter_context(progress_scope(collect)) - if restart is not None: - st.enter_context(patch.object(mnsl, "restart_server_for_round", restart)) - grid = [GridVariant(name=f"c{i}") for i in range(grid_n)] - results = asyncio.run( - run_grid( - base_yaml_path=base, - base_extra_args="", - grid=grid, - output_root=out, - magpie_python=sys.executable, - variant_timeout_sec=10, - gpu_type="mi300x", - keep_going_on_failure=keep_going, - ) - ) - return results, pulse_calls - def test_mn_server_restart_failed_reaches_the_variant_boundary(self, tmp_path, monkeypatch): """A variant whose remote server never came back still ends its own row. @@ -177,7 +187,7 @@ def test_mn_server_restart_failed_reaches_the_variant_boundary(self, tmp_path, m async def _restart_fail(**_kwargs): raise mnsl.ServerRestartFailed("server /health did not return 200") - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=True, run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 0, "ok", ""), base=base, @@ -200,7 +210,7 @@ def test_mn_server_restart_failed_reports_each_variant_it_ends(self, tmp_path, m async def _restart_fail(**_kwargs): raise mnsl.ServerRestartFailed("health probe timed out") - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=True, run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 0, "ok", ""), base=base, @@ -223,7 +233,7 @@ def test_no_benchmark_workspace_failure_pulses(self, tmp_path, monkeypatch): base = tmp_path / "base.yaml" _write_base_yaml(base) - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=False, run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 1, "stdout", "boom"), base=base, @@ -298,7 +308,7 @@ def _ok(cmd, *a, **k): _valid_workspace(Path(cmd[out_idx + 1])) return subprocess.CompletedProcess(cmd, 0, "ok", "") - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=False, run_side_effect=_ok, base=base, @@ -308,6 +318,78 @@ def _ok(cmd, *a, **k): assert len(pulse_calls) == 1 +class TestThePulseStaysOutOfACancelledActionsUnwind: + """A cancel is answered by unwinding, not by observing what was cancelled. + + A cooperative stop *returns* its sentinel, so ``run_grid`` walks its ordinary + stop path and would spend the pulse's whole budget between recording the row + and releasing the lease -- serially, inside the window the dispatcher gives + the action to finish. That window is derived from the terms of the unwind and + this is not one of them, so the pulse is skipped rather than budgeted for: + eight seconds of observing a tree the orchestrator just reaped is what the + rows already built are traded away for when the window expires. + + The gate is the cancel scope and not the sentinel returncode, because a + variant can be failing for its own reasons when the cancel lands -- its row + is a genuine failure and its pulse would run in the same window. + """ + + def _cancelled_scope(self) -> CancelScope: + scope = CancelScope() + scope.cancel(reason="session_time_exhausted") + return scope + + def test_the_round_the_run_stopped_is_not_pulsed(self, tmp_path, monkeypatch): + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "0") + base = tmp_path / "base.yaml" + _write_base_yaml(base) + + results, pulse_calls = _run_with_pulse_capture( + multi_node=False, + run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess( + cmd, ORCHESTRATOR_CANCELLED_RETURNCODE, "", "" + ), + base=base, + out=tmp_path / "out", + scope=self._cancelled_scope(), + ) + assert results[0].status == "skipped" + assert results[0].error_class == "orchestrator_cancelled" + assert pulse_calls == [], "the pulse must not run inside the cancel window" + + def test_a_variant_failing_on_its_own_when_the_cancel_lands_is_not_pulsed(self, tmp_path, monkeypatch): + """The row is a real failure; the eight seconds are still not affordable.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "0") + base = tmp_path / "base.yaml" + _write_base_yaml(base) + + results, pulse_calls = _run_with_pulse_capture( + multi_node=False, + run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 1, "stdout", "boom"), + base=base, + out=tmp_path / "out", + scope=self._cancelled_scope(), + ) + assert results[0].error_class == "no_benchmark_workspace" + assert pulse_calls == [] + + def test_a_variant_that_failed_under_a_live_scope_is_still_pulsed(self, tmp_path, monkeypatch): + """Only a cancel silences the tick, so #1177's terminal-row tick survives.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "0") + base = tmp_path / "base.yaml" + _write_base_yaml(base) + + results, pulse_calls = _run_with_pulse_capture( + multi_node=False, + run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 1, "stdout", "boom"), + base=base, + out=tmp_path / "out", + scope=CancelScope(), + ) + assert results[0].error_class == "no_benchmark_workspace" + assert len(pulse_calls) == 1 + + # --------------------------------------------------------------------------- # keep_going_on_failure asymmetry # --------------------------------------------------------------------------- diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py index 2f7c00ae9c..c83070f63d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio import json import os import subprocess @@ -238,6 +239,159 @@ def get_specialist_patch_verdict(self, tid): assert (repo / "src.py").read_text().endswith("return 1\n") +def _capture_bench(captured: dict, result: dict, gate: dict): + """A ``_bench_patch`` stub that records the kwargs it was handed.""" + + async def _b(self, **kwargs): + captured.update(kwargs) + return result, gate + + return _b + + +@pytest.mark.asyncio +async def test_bench_is_bounded_by_the_session_budget(tmp_path, monkeypatch): + """The patch bench is handed the session budget, as the other arms are. + + Its declared cap answers "how long before this counts as hung", not "how much + budget is left", so without the session deadline a patch benched near the end + of a run outlives the run itself. + """ + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + class _SS: + baseline_runtime_sec = 600.0 + + def get_specialist_patch_verdict(self, tid): + return "approve" + + def grid_session_deadline_sec(self, **_kwargs): + return 4242.0 + + captured: dict[str, Any] = {} + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr( + IntegratePatchExecutor, + "_bench_patch", + _capture_bench(captured, {"output_throughput": 200.0, "status": "succeeded"}, {"accuracy_pass": None}), + ) + res = await ex( + _make_ctx( + "t", + { + "specialist_task_id": "spec", + "framework_source_root": str(repo), + "base_tput": 100.0, + "enable_stack_rebench": False, + }, + extra={"shared_state": _SS()}, + ) + ) + + assert res["status"] == "kept" + assert captured["session_deadline_sec"] == 4242.0 + # The expected runtime, not the backstop cap: admitting on the backstop + # abandons the tail of the budget. + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_bench_budget_is_unbounded_without_a_session(tmp_path, monkeypatch): + """No session context means no deadline, not a deadline of zero.""" + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + captured: dict[str, Any] = {} + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr( + IntegratePatchExecutor, + "_bench_patch", + _capture_bench(captured, {"output_throughput": 200.0, "status": "succeeded"}, {"accuracy_pass": None}), + ) + res = await ex( + _make_ctx( + "t", + { + "specialist_task_id": "spec", + "framework_source_root": str(repo), + "base_tput": 100.0, + "enable_stack_rebench": False, + }, + ) + ) + + assert res["status"] == "kept" + assert captured["session_deadline_sec"] is None + assert captured["variant_expected_sec"] is None + + +@pytest.mark.asyncio +async def test_bench_patch_forwards_the_session_budget_to_the_grid(tmp_path, monkeypatch): + session = tmp_path / "s" + session.mkdir() + config_path = tmp_path / "baseline.yaml" + config_path.write_text("benchmark: {}\n", encoding="utf-8") + + from hyperloom.orchestrator.actions.executors import integrate_patch as ip_mod + + captured: dict[str, Any] = {} + + async def fake_run_grid(**kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr(ip_mod, "run_grid", fake_run_grid) + monkeypatch.setattr(ip_mod, "materialize_config_with_envs", lambda *a, **k: config_path) + + await IntegratePatchExecutor(session_dir=session)._bench_patch( + params={"config_path": str(config_path)}, + output_root=tmp_path / "out", + extra_server_args_applied="", + extra_envs_applied={}, + specialist_task_id="spec", + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_switch_off_parity_leg_is_bounded_by_the_session_budget(tmp_path, monkeypatch): + """The parity leg is a second full bench, so it needs the same bound.""" + session = tmp_path / "s" + session.mkdir() + captured: dict[str, Any] = {} + + async def _capture(**kwargs): + captured.update(kwargs) + return ({"output_throughput": 100.0, "status": "succeeded"}, {"accuracy_pass": None}) + + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr(ex, "_bench_patch", _capture) + + await ex._switch_off_parity( + params={}, + output_root=tmp_path, + specialist_task_id="spec", + switch_manifest=[{"switch": "HYPERLOOM_REWRITE_X"}], + base_tput=100.0, + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + @pytest.mark.asyncio async def test_keep_path(tmp_path, monkeypatch): session = tmp_path / "s" @@ -308,6 +462,72 @@ async def _measure(**kwargs): assert captured["stable_threshold_pct"] == pytest.approx(0.2) +@pytest.mark.asyncio +async def test_stack_rebench_is_bounded_by_the_session_budget(tmp_path, monkeypatch): + """The confirmation round must not outlive the run it is confirming for.""" + config_path = tmp_path / "baseline.yaml" + config_path.write_text("benchmark: {}\n", encoding="utf-8") + captured: dict[str, Any] = {} + + async def _measure(**kwargs): + captured.update(kwargs) + return StackRebenchResult(tput=None, workspace=None) + + monkeypatch.setattr(ip, "materialize_config_with_envs", lambda *a, **k: config_path) + monkeypatch.setattr(ip, "measure_stack_rebench", _measure) + + await IntegratePatchExecutor(session_dir=tmp_path)._confirm_stack_rebench( + params={"config_path": str(config_path)}, + output_root=tmp_path / "output", + extra_server_args_applied="", + extra_envs_applied={}, + specialist_task_id="spec", + base_tput=100.0, + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error_class", ["session_time_exhausted", "orchestrator_cancelled"]) +async def test_rebench_the_run_stopped_is_not_reported_as_a_failed_measurement(tmp_path, error_class): + """Not measuring a variant is not evidence that the variant is unstable.""" + from unittest.mock import patch + + from hyperloom.orchestrator.actions.executors._grid_runner import GridVariant, VariantResult + from hyperloom.orchestrator.actions.executors import _stack_rebench as sr + + skipped = VariantResult( + name="v", + extra_server_args="", + extra_envs={}, + status="skipped", + error="the run stopped this round before it measured anything", + error_class=error_class, + ) + + async def _fake_run_grid(**_kwargs): + return [skipped] + + with patch.object(sr, "run_grid", new=_fake_run_grid): + result = await sr.measure_stack_rebench( + config_path=tmp_path / "base.yaml", + base_extra_args="", + variant=GridVariant("v"), + base_tput=100.0, + stable_threshold_pct=0.5, + output_slot=tmp_path / "slot", + variant_timeout_sec=600, + ) + + assert result.error_class == error_class + assert result.warnings == [f"stack_rebench_skipped:{error_class}"] + assert not any("stack_rebench_failed" in w for w in result.warnings) + + @pytest.mark.asyncio async def test_keep_confirmed_by_rebench(tmp_path, monkeypatch): session = tmp_path / "s" @@ -990,3 +1210,117 @@ def _fake_apply(self, specs, *, backup_root): # in the stash. This is the regression the fix guards against. assert scratch.exists(), "user auto-stash was not restored after artifact_install_failed" assert scratch.read_text(encoding="utf-8") == "user work in progress\n" + + +@pytest.mark.asyncio +async def test_cancelled_gate_reverts_the_patch_and_re_raises(tmp_path, monkeypatch): + """A cancel unwinds the gate, so the tree it mutated must not outlive it. + + The dispatcher cancels in-flight actions when the run is shutting down or + the session wall-clock budget is spent. ``CancelledError`` is not an + ``Exception``, so the gate's own revert handlers never see it: the patch + would stay in the framework tree, the operator's auto-stash would stay + unpopped, and the session would run its CLOSE phase against a tree carrying + an ungraded patch. + + The cancel is re-raised rather than graded as a REVERT: SubAgentRunner + records a cancelled executor as ``cancelled``, and work the run stopped is + not work that failed. + """ + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + async def _cancel(self, **kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(IntegratePatchExecutor, "_bench_patch", _cancel) + ex = IntegratePatchExecutor(session_dir=session) + with pytest.raises(asyncio.CancelledError): + await ex( + _make_ctx( + "t", + {"specialist_task_id": "spec", "framework_source_root": str(repo)}, + ) + ) + + assert (repo / "src.py").read_text(encoding="utf-8").endswith("return 1\n"), ( + "the cancelled candidate was left applied in the framework tree" + ) + assert scratch.exists(), "user auto-stash was not restored after the cancel" + assert scratch.read_text(encoding="utf-8") == "user work in progress\n" + stash_list = subprocess.run( + ["git", "-C", str(repo), "stash", "list"], + check=True, + capture_output=True, + text=True, + ) + assert stash_list.stdout.strip() == "", "the auto-stash was left on the stack" + + +@pytest.mark.asyncio +async def test_a_cancel_in_the_apply_stage_still_hands_the_stash_back(tmp_path, monkeypatch): + """The apply stage stashes and mutates the tree, then awaits, same as the gate. + + Each of its failure verdicts writes a KB record before the stash restore that + returns it, and a cancel arrives at whatever await the action happens to be + at -- a spent wall-clock budget is what makes it arrive at an arbitrary one. + Only the gate was guarded, so this window left the operator's uncommitted + work in ``git stash`` for the rest of the session. + """ + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + def _fake_resolve(*args, **kwargs): + spec = ip._ArtifactSpec( + source=tmp_path / "tuned.json", + target=repo / "tuned.json", + rel_target="tuned.json", + kind="config_json", + ) + return [spec], [] + + async def _cancel(self, **kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(ip, "_resolve_artifact_specs", _fake_resolve) + monkeypatch.setattr( + IntegratePatchExecutor, + "_apply_artifacts", + lambda self, specs, *, backup_root: ([], [{"artifact": "tuned.json", "error": "disk full"}]), + ) + monkeypatch.setattr(IntegratePatchExecutor, "_maybe_write_framework_kb_record", _cancel) + + ex = IntegratePatchExecutor(session_dir=session) + with pytest.raises(asyncio.CancelledError): + await ex( + _make_ctx( + "t", + {"specialist_task_id": "spec", "framework_source_root": str(repo)}, + ) + ) + + assert scratch.read_text(encoding="utf-8") == "user work in progress\n", ( + "the user's uncommitted work was left in the stash" + ) + stash_list = subprocess.run( + ["git", "-C", str(repo), "stash", "list"], + check=True, + capture_output=True, + text=True, + ) + assert stash_list.stdout.strip() == "", "the auto-stash was left on the stack" + assert (repo / "src.py").read_text(encoding="utf-8").endswith("return 1\n"), ( + "the ungraded candidate was left applied in the framework tree" + ) diff --git a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py index 9a62476410..023f53dfda 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py +++ b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py @@ -19,10 +19,13 @@ import pytest +from hyperloom.orchestrator.actions.cancel_channel import CancelScope, use_cancel_scope from hyperloom.orchestrator.actions.executors._subprocess_kill import ( DETOKENIZER_STALL_RETURNCODE, + ORCHESTRATOR_CANCELLED_RETURNCODE, OVERTIME_KILL_RETURNCODE, SERVER_DEAD_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, _scan_logs_increment, _scan_server_log_increment, _server_log_shows_death, @@ -30,6 +33,8 @@ new_session_kwargs, run_with_session_kill, server_log_death_excerpt, + session_deadline_to_remaining_sec, + session_remaining_to_deadline_sec, ) @@ -298,6 +303,234 @@ def test_run_with_session_kill_soft_deadline_still_fires_without_eval_marker(tmp assert elapsed < 10.0, f"soft-deadline path took {elapsed:.2f}s" +class TestSessionDeadline: + """The session budget is a separate channel from the soft deadline. + + The soft deadline answers "is this variant abnormally slow", which is why it + retires when the accuracy eval starts. The session budget answers "is the run + out of time", which no phase boundary changes. + """ + + def test_expired_session_budget_reaps_the_tree_with_its_own_sentinel(self): + start = time.monotonic() + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + session_deadline_sec=time.monotonic() - 1.0, + ) + elapsed = time.monotonic() - start + assert cp.returncode == SESSION_TIME_EXHAUSTED_RETURNCODE + assert cp.returncode != OVERTIME_KILL_RETURNCODE, ( + "a budget kill must not share the overtime code, which asserts the variant is slow" + ) + assert elapsed < 10.0, f"session-deadline path took {elapsed:.2f}s" + + def test_eval_start_does_not_retire_the_session_budget(self, tmp_path): + """The marker that retires the soft deadline must not retire this one. + + An accuracy eval that starts one minute before the run is out of time + still has to stop; this is the whole reason the two are separate channels. + """ + log_path = tmp_path / "server.log" + log_path.write_text("Application startup complete\nHYPERLOOM_EVAL_START\n") + start = time.monotonic() + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + soft_deadline_sec=1.0, + server_log_path=str(log_path), + session_deadline_sec=time.monotonic() + 1.5, + ) + elapsed = time.monotonic() - start + assert cp.returncode == SESSION_TIME_EXHAUSTED_RETURNCODE + assert elapsed < 10.0, f"session budget was not enforced during eval ({elapsed:.2f}s)" + + def test_a_budget_with_room_left_leaves_the_child_alone(self): + start = time.monotonic() + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(2)"], + timeout=60, + session_deadline_sec=time.monotonic() + 3600.0, + ) + elapsed = time.monotonic() - start + assert cp.returncode == 0 + assert elapsed >= 1.5, f"child was cut short at {elapsed:.2f}s" + + def test_no_session_deadline_keeps_the_previous_behaviour(self): + cp = run_with_session_kill( + [sys.executable, "-c", "print('done')"], + timeout=30, + session_deadline_sec=None, + ) + assert cp.returncode == 0 + assert "done" in (cp.stdout or "") + + +class TestAnOrchestratorCancelReachesTheChild: + """The last defence has to stop the child, not just the coroutine above it. + + The executor blocks in a worker thread, so cancelling its task frees the + lanes and the GPU lease while the benchmark is still running. The cancel + scope is the channel the thread checks, at the poll it already runs. + """ + + def test_a_cancel_raised_before_the_call_reaps_the_tree(self): + scope = CancelScope() + scope.cancel(reason="shutdown_requested") + start = time.monotonic() + with use_cancel_scope(scope): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + ) + elapsed = time.monotonic() - start + assert cp.returncode == ORCHESTRATOR_CANCELLED_RETURNCODE + assert elapsed < 10.0, f"cancel path took {elapsed:.2f}s" + + def test_a_cancel_that_arrives_mid_run_still_reaches_it(self): + """The interesting case: nothing is wrong when the child is launched.""" + scope = CancelScope() + timer = threading.Timer(0.5, lambda: scope.cancel(reason="session_time_exhausted")) + timer.start() + start = time.monotonic() + try: + with use_cancel_scope(scope): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + ) + finally: + timer.cancel() + elapsed = time.monotonic() - start + assert cp.returncode == ORCHESTRATOR_CANCELLED_RETURNCODE + assert elapsed < 10.0, f"mid-run cancel took {elapsed:.2f}s" + + def test_a_scope_nobody_cancelled_leaves_the_child_alone(self): + """The channel must cost nothing on the path every healthy round takes.""" + with use_cancel_scope(CancelScope()): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(1); print('done')"], + timeout=60, + ) + assert cp.returncode == 0 + assert "done" in (cp.stdout or "") + + def test_a_spent_budget_keeps_its_own_attribution(self): + """Both are true at once whenever the budget is what triggered the cancel. + + The budget is a fact about the run and the cancel is only the dispatcher + acting on it, so the ledger gets the cause, not the mechanism. + """ + scope = CancelScope() + scope.cancel(reason="session_time_exhausted") + with use_cancel_scope(scope): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + session_deadline_sec=time.monotonic() - 1.0, + ) + assert cp.returncode == SESSION_TIME_EXHAUSTED_RETURNCODE + + def test_the_call_registers_as_a_listener_while_the_child_lives(self): + """The canceller waits only for work that can hear it, so this is load-bearing.""" + scope = CancelScope() + seen: list[bool] = [] + timer = threading.Timer( + 0.5, + lambda: (seen.append(scope.has_listeners), scope.cancel(reason="test")), + ) + timer.start() + try: + with use_cancel_scope(scope): + run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + ) + finally: + timer.cancel() + assert seen == [True] + assert not scope.has_listeners + + +class TestSessionDeadlineCrossesAProcessBoundary: + """A ``time.monotonic()`` instant is only meaningful in the process that read it. + + Handing the absolute deadline to a Ray worker would name an instant on the + worker's own clock, whose origin is unrelated -- an immediate kill or one + that never fires, both silently. Only a duration survives the trip. + """ + + def test_an_unbounded_budget_stays_unbounded_in_both_directions(self): + assert session_deadline_to_remaining_sec(None) is None + assert session_remaining_to_deadline_sec(None) is None + + def test_a_deadline_becomes_the_seconds_it_has_left(self, monkeypatch): + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + assert session_deadline_to_remaining_sec(1300.0) == pytest.approx(300.0) + + def test_a_spent_budget_crosses_as_a_non_positive_duration(self, monkeypatch): + """Not floored at zero: the receiver must reap, not read it as "no budget given".""" + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + assert session_deadline_to_remaining_sec(940.0) == pytest.approx(-60.0) + + def test_the_duration_re_anchors_onto_the_reading_clock(self, monkeypatch): + """The same duration names a different instant on each side; that is the point.""" + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + remaining = session_deadline_to_remaining_sec(1300.0) + monkeypatch.setattr(time, "monotonic", lambda: 5_000_000.0) + assert session_remaining_to_deadline_sec(remaining) == pytest.approx(5_000_300.0) + + def test_the_round_trip_is_the_identity_within_one_clock(self, monkeypatch): + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + assert session_remaining_to_deadline_sec(session_deadline_to_remaining_sec(1234.5)) == pytest.approx(1234.5) + + +def _sentinel_returncodes() -> dict[int, set[str]]: + """Map every sentinel returncode to the qualified names that claim it.""" + from hyperloom.orchestrator.actions.executors import _ray_serving, _subprocess_kill + + assigned: dict[int, set[str]] = {} + for module in (_subprocess_kill, _ray_serving): + short = module.__name__.rsplit(".", 1)[-1] + for name, value in vars(module).items(): + if not isinstance(value, int) or isinstance(value, bool): + continue + if not (name.endswith("_RETURNCODE") or name.endswith("_RC")): + continue + assigned.setdefault(value, set()).add(f"{short}.{name}") + return assigned + + +def test_every_sentinel_returncode_names_exactly_one_cause(): + """A sentinel shared by two causes makes attribution a coin flip. + + The codes are handed out in more than one module and all arrive at their + consumer as a plain ``returncode``, so a new one can quietly reuse a number + already taken. That is how the session-budget code first landed on the Ray + actor-died number, which would have had every actor death read as a spent + budget and taught the ledger the wrong thing about both. + """ + assigned = _sentinel_returncodes() + + collisions = {code: sorted(names) for code, names in assigned.items() if len(names) > 1} + assert not collisions, f"sentinel return codes collide: {collisions}" + assert SESSION_TIME_EXHAUSTED_RETURNCODE in assigned + + +def test_an_actor_timeout_is_not_recorded_as_a_failed_agentx_preflight(): + """The two causes that share ``_run_magpie``'s return channel stay apart. + + ``_run_magpie`` returns ``AGENTX_PREFLIGHT_RETURNCODE`` when the execution + boundary fails preflight and, a few lines on, whatever the serving lease's + actor returned -- including ``_ACTOR_TIMEOUT_RC``. Callers see one + ``returncode`` either way, so the two sharing a number (which they did) is + enough to have a hung actor blamed on a missing aiperf. + """ + from hyperloom.orchestrator.actions.executors import _ray_serving, _subprocess_kill + + assert _ray_serving._ACTOR_TIMEOUT_RC != _subprocess_kill.AGENTX_PREFLIGHT_RETURNCODE + + def test_run_with_session_kill_streams_child_output_to_parent(capsys): """Captured child output is also mirrored immediately to parent streams.""" code = "import sys\nprint('child-out', flush=True)\nprint('child-err', file=sys.stderr, flush=True)\n" diff --git a/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py b/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py index 294e79740e..80f2830185 100644 --- a/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py +++ b/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py @@ -460,7 +460,7 @@ async def test_on_enter_close_emits_report_end(session_dir, monkeypatch): report_task = Task( task_id="rpt-1", kind="report", - state="running", + state="queued", params={}, idempotency_key="internal-report-close", requires_lanes=[], @@ -468,7 +468,7 @@ async def test_on_enter_close_emits_report_end(session_dir, monkeypatch): bd_task = Task( task_id="bd-1", kind="session_breakdown", - state="running", + state="queued", params={}, idempotency_key="internal-breakdown-close", requires_lanes=[], @@ -532,7 +532,7 @@ async def test_on_enter_close_emits_report_error_for_failed_task( report_task = Task( task_id="rpt-1", kind="report", - state="running", + state="queued", params={}, idempotency_key="internal-report-close", requires_lanes=[], @@ -540,7 +540,7 @@ async def test_on_enter_close_emits_report_error_for_failed_task( bd_task = Task( task_id="bd-1", kind="session_breakdown", - state="running", + state="queued", params={}, idempotency_key="internal-breakdown-close", requires_lanes=[], @@ -597,7 +597,7 @@ async def test_on_enter_close_emits_report_error_for_exception( report_task = Task( task_id="rpt-1", kind="report", - state="running", + state="queued", params={}, idempotency_key="internal-report-close", requires_lanes=[], @@ -605,7 +605,7 @@ async def test_on_enter_close_emits_report_error_for_exception( bd_task = Task( task_id="bd-1", kind="session_breakdown", - state="running", + state="queued", params={}, idempotency_key="internal-breakdown-close", requires_lanes=[], diff --git a/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py b/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py index a0a5026c0e..46c5ccaa89 100644 --- a/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py +++ b/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py @@ -91,6 +91,55 @@ def test_sweep_closes_when_insufficient_remaining(): assert evidence["reloop_blocked"] == "insufficient_remaining" +def test_sweep_skip_to_close_does_not_override_a_settled_conc_sweep(): + """LLM skip_to_close after a refused conc_sweep must not become robustness_escalated.""" + st = _sweep_state(max_minutes=180, started_hours_ago=166 / 60.0) + st.last_sweep = {} + st.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "skip_reason": "session_time_budget", + } + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + nxt = ps.compute_next_phase(st, max_hours=3.0) + assert nxt is not None + target, reason, evidence = nxt + assert target == ps.PHASE_CLOSE + assert reason == "conc_sweep_done" + assert evidence.get("conc_sweep_status") == "skipped" + + +def test_sweep_skip_to_close_still_escalates_when_conc_sweep_never_settled(): + """skip_to_close remains a robustness abort when SWEEP has nothing to close on.""" + st = _sweep_state(max_minutes=180, started_hours_ago=1.0) + st.last_sweep = {} + st.last_conc_sweep = {} + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + nxt = ps.compute_next_phase(st, max_hours=3.0) + assert nxt is not None + target, reason, _evidence = nxt + assert target == ps.PHASE_CLOSE + assert reason == "robustness_escalated" + + +def test_sweep_skip_to_close_yields_to_reloop_when_conc_sweep_was_skipped(): + """A skipped conc_sweep with budget left must not be aborted by skip_to_close.""" + st = _sweep_state(macro_cycle=0, cumulative_gain=5.0, gain_at_cycle_start=0.0) + st.last_sweep = {} + st.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "skip_reason": "session_time_budget", + } + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + nxt = ps.compute_next_phase(st, max_hours=96.0) + assert nxt is not None + target, reason, evidence = nxt + assert target == ps.PHASE_EXPLORE + assert reason == "cycle_reloop" + assert evidence["loopback"] is True + + def test_short_bounded_run_reloops_when_budget_and_leverage_remain(): # 12h bounded run: macro-loop is available even though budget accounting # stays in short-run charge-back mode. @@ -342,6 +391,38 @@ async def test_coordinator_applies_loopback(cyclic_coordinator): assert loopback_row["cycle"] == 1 +@pytest.mark.asyncio +async def test_skip_to_close_is_consumed_when_sweep_already_settled( + cyclic_coordinator, + monkeypatch, +): + """A suppressed skip_to_close must not leak into the next phase.""" + c = cyclic_coordinator + st = c.shared_state + now = datetime.now(timezone.utc) + st.phase = ps.PHASE_SWEEP + st.start_ts = (now - timedelta(minutes=166)).isoformat() + st.max_minutes = 180 + st.macro_cycle = 0 + st.last_sweep = {} + st.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "skip_reason": "session_time_budget", + } + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + + async def _entered(*, from_phase, to_phase): + return None + + monkeypatch.setattr(c.phase_machine, "_on_phase_entered", _entered) + await c._advance_phase_if_needed() + + assert st.phase == ps.PHASE_CLOSE + assert st.pending_escalate_hint == "" + assert st.last_consumed_escalate_hint == ps.ESCALATE_HINT_SKIP_TO_CLOSE + + @pytest.mark.asyncio async def test_coordinator_converged_close_sets_stop_reason(cyclic_coordinator): c = cyclic_coordinator diff --git a/src/hyperloom/inference_optimizer/tests/test_objective.py b/src/hyperloom/inference_optimizer/tests/test_objective.py index 35a730b21c..8b2815c453 100644 --- a/src/hyperloom/inference_optimizer/tests/test_objective.py +++ b/src/hyperloom/inference_optimizer/tests/test_objective.py @@ -342,8 +342,9 @@ async def _enter_and_record(*, grace_sec: float) -> float: closing_grace_sec=5.0, tick_interval_sec=0.0, ) - assert spy.calls >= 1 assert calls_at_closing, "expected closing phase to be entered" + # A spent bound cancels phase-enter and skips reactors on the tick + # that trips CLOSE. CLOSE itself must still not add LLM turns. assert spy.calls == calls_at_closing[0] finally: await c.stop() diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py b/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py index c789b8d6eb..1b9048b435 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py @@ -153,6 +153,62 @@ def test_force_exit_unlimited_run_never_fires(): assert fired is False # Without max_minutes nothing is computable. assert "session_remaining_seconds" not in evidence + assert "hours_remaining_gate" not in evidence + + +def test_force_exit_hours_leavebehind_cannot_cover_the_session(): + """IR-6's 3h default on a 3h session must not skip EXPLORE at first tick. + + Remaining starts at max_hours, so remaining <= 3h is true as soon as any + time has been spent. CI's 3h smoke (and the 3h example) would otherwise + leave EXPLORE with 0 grids. + """ + state = _make_explore_state( + max_minutes=180, + started_hours_ago=0.0, + phase_started_hours_ago=0.0, + ) + fired, evidence = phase_state.should_force_exit_explore( + state, + hours_remaining_threshold=3.0, + budget_pct_threshold=0.20, + ) + assert fired is False + assert "session_remaining" not in evidence["fired_reasons"] + assert evidence["hours_remaining_gate"] == "disabled_leavebehind_covers_session" + + +def test_force_exit_hours_leavebehind_disabled_after_prelude_on_three_hour_session(): + """CI shape: ~67 min spent before EXPLORE on a 3h budget.""" + state = _make_explore_state( + max_minutes=180, + started_hours_ago=1.12, + phase_started_hours_ago=0.0, + ) + fired, evidence = phase_state.should_force_exit_explore( + state, + hours_remaining_threshold=3.0, + budget_pct_threshold=0.20, + ) + assert fired is False + assert "session_remaining" not in evidence["fired_reasons"] + assert evidence["hours_remaining_gate"] == "disabled_leavebehind_covers_session" + + +def test_force_exit_explicit_hours_leavebehind_still_fires_on_short_session(): + """An operator-set leave-behind smaller than the session still fires.""" + state = _make_explore_state( + max_minutes=180, + started_hours_ago=2.96, + phase_started_hours_ago=0.01, + ) + fired, evidence = phase_state.should_force_exit_explore( + state, + hours_remaining_threshold=0.05, + budget_pct_threshold=0.0, + ) + assert fired is True + assert "session_remaining" in evidence["fired_reasons"] def test_exit_normal_explore_force_exit_takes_priority_over_plateau(): diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py index 5b833e28c8..364976e7e6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py @@ -220,6 +220,214 @@ def test_exit_normal_prelude_blocked_while_warm_replay_in_flight(): assert out is not None and out[0] == "prelude_done" +def _prelude_state( + *, + max_minutes: int = 180, + spent_sec: float = 0.0, + usable_sec: float | None = None, + baseline_tput: float = 0.0, + baseline_runtime_sec: float = 0.0, + baseline_post_ready_runtime_sec: float = 0.0, + baseline_warm_runtime_sec: float = 0.0, + baseline_measure_round_dropped: bool = False, + baseline_double_run: bool = False, +) -> SimpleNamespace: + """A PRELUDE-phase state with an explicit clock, as the budget policy reads it.""" + return SimpleNamespace( + phase="PRELUDE", + max_minutes=max_minutes, + phase_elapsed_totals={"PRELUDE": spent_sec}, + phase_started_unix=0.0, + baseline_tput=baseline_tput, + baseline_runtime_sec=baseline_runtime_sec, + baseline_post_ready_runtime_sec=baseline_post_ready_runtime_sec, + baseline_warm_runtime_sec=baseline_warm_runtime_sec, + baseline_measure_round_dropped=baseline_measure_round_dropped, + baseline_double_run=baseline_double_run, + session_budget_usable_sec=lambda: usable_sec, + ) + + +def test_prelude_can_afford_an_arm_the_budget_still_covers(): + """The normal case must be untouched: a cheap arm early in a session runs.""" + state = _prelude_state(spent_sec=600.0, usable_sec=10_000.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=300.0) + assert affordable is True + # Half of 180 minutes is held for the optimization phases; the rest is PRELUDE's. + assert evidence["affordable_sec"] == pytest.approx(4600.0) + + +def test_prelude_refuses_an_arm_that_would_eat_the_optimization_reserve(): + """The Qwen3.5-397B shape: 51 minutes of baseline, then a roofline that costs another 45+.""" + state = _prelude_state(spent_sec=3090.0, usable_sec=7700.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=2706.0) + assert affordable is False + assert evidence["bound"] == "optimization_reserve" + # 7700s left, 5400s of it spoken for, so the arm may cost at most 2300s. + assert evidence["affordable_sec"] == pytest.approx(2300.0) + + +def test_a_resumed_prelude_is_not_charged_for_what_the_earlier_leg_spent(): + """Banked phase spend and the session clock answer to different origins. + + A resume that reanchors the budget restarts the session clock while the + phase ledger keeps every second the earlier leg banked. A bound read off + the ledger therefore declared preparation overspent on a session that had + its whole budget ahead of it, and the measured half of the baseline was + refused on every resumed run. Only the clock decides. + """ + state = _prelude_state(spent_sec=10_000.0, usable_sec=10_000.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=2706.0) + assert affordable is True + assert evidence["affordable_sec"] == pytest.approx(4600.0) + + +def test_prelude_budget_policy_is_inert_without_a_clock(): + """An unbounded run has no budget to protect, so nothing is refused.""" + state = _prelude_state(max_minutes=0, usable_sec=None) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=99_999.0) + assert affordable is True + assert evidence["reason"] == "unbounded_budget" + + +def test_time_exhausted_during_prelude_finally_has_a_producer(): + """The reason was in the vocabulary and in the report glossary with no code path to it.""" + state = _prelude_state(spent_sec=10_800.0, usable_sec=0.0) + out = phase_state.compute_next_phase(state, kernel_enabled=True) + assert out is not None + next_phase, reason, evidence = out + assert (next_phase, reason) == ("CLOSE", "time_exhausted_during_prelude") + assert evidence["terminal"] is True + assert phase_state.is_valid_stop_reason(reason) + + +def test_a_landed_baseline_outranks_the_exhausted_clock(): + """With a baseline in hand the run has something to optimize; the later phases judge for themselves.""" + state = _prelude_state(spent_sec=10_800.0, usable_sec=0.0, baseline_tput=1074.7) + out = phase_state.compute_next_phase(state, kernel_enabled=True) + assert out is not None + assert out[1] == "prelude_done" + + +def test_prelude_exit_states_whether_one_optimization_round_still_fits(): + """The plain statement neither field session ever got: preparation spent the run.""" + state = _prelude_state(baseline_tput=1074.7, baseline_runtime_sec=2705.7, usable_sec=2796.0) + out = phase_state.exit_normal_prelude(state) + assert out is not None + evidence = out[1] + assert evidence["fits_one_optimization_round"] is True + assert evidence["affordable_rounds"] == pytest.approx(1.03, abs=0.01) + + state.session_budget_usable_sec = lambda: 1200.0 + evidence = phase_state.exit_normal_prelude(state)[1] + assert evidence["fits_one_optimization_round"] is False + + +# The workload the cold-anchor cases below are priced against: a 900s cold round +# whose last 550s was the benchmark, so the boot took 350s, and a 400s hot pass. +# One further measured variant therefore costs 750s -- its own boot and a +# benchmark on a populated JIT cache -- while a double-run round costs the whole +# measured cold pass plus a second benchmark, 1300s. Together they are what a +# session must afford before measuring another baseline is worth doing. +_COLD_ANCHOR_WORKLOAD = { + "baseline_tput": 1074.7, + "baseline_runtime_sec": 900.0, + "baseline_post_ready_runtime_sec": 550.0, + "baseline_warm_runtime_sec": 400.0, + "baseline_double_run": True, +} +_RETRY_COST_SEC = 1300.0 + 750.0 + + +class TestAColdAnchorIsNotAFinishedPrelude: + """What happens to a session whose baseline could only keep its cold figure. + + The figure exists, so every rule that asks only whether a baseline landed + reads preparation as done. It is not: the number carries the boot, the first + request's compile and the graph capture, so every variant measured against it + reads as an improvement over a baseline that was never the baseline. + + Two outcomes are correct and the budget picks between them -- measure another + baseline, or stop and say why -- and neither is "optimize against it". + """ + + def test_a_dropped_hot_pass_does_not_finish_the_phase(self): + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=_RETRY_COST_SEC + 60.0, + ) + + assert phase_state.exit_normal_prelude(state) is None + + state.baseline_measure_round_dropped = False + assert phase_state.exit_normal_prelude(state)[0] == "prelude_done" + + def test_a_session_that_cannot_afford_another_baseline_closes(self): + """2050s buys a round and a variant to read against it; 1200s buys neither.""" + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=1200.0, + ) + + out = phase_state.compute_next_phase(state, kernel_enabled=True) + + assert out is not None + next_phase, reason, evidence = out + assert (next_phase, reason) == ("CLOSE", "prelude_cold_anchor_low_budget") + assert evidence["terminal"] is True + assert evidence["baseline_anchor"] == "cold" + assert evidence["retry_round_sec"] == pytest.approx(1300.0) + assert phase_state.is_valid_stop_reason(reason) + assert phase_state.is_valid_phase_exit_reason(reason) + + def test_a_session_resumed_with_a_fresh_clock_measures_another_baseline(self): + """The marker outlives the shortfall, so it must not decide on its own. + + A resume reanchors the session clock while the marker from the earlier leg + persists. Closing on the marker would end every resumed run before it + began, and no later baseline could clear the marker because none would + run. So the phase stays open with nothing to advance it but a new + baseline. + """ + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=_RETRY_COST_SEC + 60.0, + ) + + assert phase_state.exit_cold_anchor_prelude(state) is None + assert phase_state.compute_next_phase(state, kernel_enabled=True) is None + + def test_a_single_round_baseline_is_not_mistaken_for_a_dropped_one(self): + """A cold figure by configuration is consistent with what follows it. + + A session that never asked for a hot pass measures everything the same + way, so its comparisons hold. Only a pass that was *dropped* leaves a + denominator out of step with the numerators. + """ + state = _prelude_state( + baseline_tput=1074.7, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + usable_sec=1200.0, + ) + + assert phase_state.exit_cold_anchor_prelude(state) is None + assert phase_state.exit_normal_prelude(state)[0] == "prelude_done" + + def test_a_session_with_no_clock_is_not_closed_for_a_budget_it_does_not_have(self): + """An unbounded run cannot fail an affordability test, so it retries.""" + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=None, + ) + + assert phase_state.exit_cold_anchor_prelude(state) is None + + def test_exit_terminal_prelude_after_three_baseline_failures(): state = SimpleNamespace(baseline_failure_streak=2) assert phase_state.exit_terminal_prelude(state) is None diff --git a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py index 573bb99b16..75260e735d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py @@ -28,6 +28,9 @@ ServingLease, maybe_serving_lease, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + SESSION_TIME_EXHAUSTED_RETURNCODE, +) # ── flag gate ──────────────────────────────────────────────────────────────── @@ -397,8 +400,10 @@ def test_strip_visible_devices_noop_when_absent(tmp_path: Path): class _FakeMethod: def __init__(self, ret): self._ret = ret + self.calls: list[dict] = [] - def remote(self, *_a, **_k): + def remote(self, *_a, **kw): + self.calls.append(dict(kw)) return self._ret @@ -599,6 +604,113 @@ def _fake_build(*, python_exe, config_path, output_dir): assert lease.calls[0]["timeout"] == 10 +# ── the session budget across the Ray process boundary ─────────────────────── +class TestTheSessionBudgetReachesTheRayWorker: + """Production takes the Ray path on a single node; the local path is the test default. + + So the session reaper has to be carried across the boundary explicitly, and + as a duration: the absolute deadline is a ``time.monotonic()`` instant, and + the worker is another process whose clock starts somewhere else. Without it + the hard timeout is the only thing left, and a run that ran out of time gets + recorded as a variant that timed out. + """ + + def test_run_magpie_converts_the_deadline_before_handing_it_over( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + from hyperloom.orchestrator.actions.executors import _grid_runner as gr + + cfg = tmp_path / "config.yaml" + cfg.write_text("benchmark:\n framework: sglang\n envs:\n TP: 1\n", encoding="utf-8") + out_dir = tmp_path / "out" + out_dir.mkdir() + monkeypatch.setattr(gr, "build_benchmark_command", lambda **_kw: ["magpie"]) + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + + lease = _RecordingLease() + gr._run_magpie( + magpie_python="python3", + config_path=cfg, + output_dir=out_dir, + timeout_sec=10, + cwd=str(tmp_path), + serving_lease=lease, + session_deadline_sec=1250.0, + ) + + assert lease.calls[0]["session_remaining_sec"] == pytest.approx(250.0) + assert "session_deadline_sec" not in lease.calls[0], ( + "an absolute monotonic instant is meaningless in the actor's process" + ) + + def test_an_unbounded_budget_reaches_the_lease_as_none( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + from hyperloom.orchestrator.actions.executors import _grid_runner as gr + + cfg = tmp_path / "config.yaml" + cfg.write_text("benchmark:\n framework: sglang\n envs:\n TP: 1\n", encoding="utf-8") + out_dir = tmp_path / "out" + out_dir.mkdir() + monkeypatch.setattr(gr, "build_benchmark_command", lambda **_kw: ["magpie"]) + + lease = _RecordingLease() + gr._run_magpie( + magpie_python="python3", + config_path=cfg, + output_dir=out_dir, + timeout_sec=10, + cwd=str(tmp_path), + serving_lease=lease, + ) + + assert lease.calls[0]["session_remaining_sec"] is None + + def test_the_lease_forwards_the_budget_to_its_actor(self, monkeypatch: pytest.MonkeyPatch): + """The lease is the last in-process hop; dropping it here loses the reaper.""" + monkeypatch.setitem(sys.modules, "ray", _LeaseFakeRay()) + lease = ServingLease(num_gpus=1) + lease._actor = _FakeActor((0, "ok", "")) # pre-set so ensure() is a no-op + + lease.run_session_kill(["echo", "hi"], timeout=5, session_remaining_sec=42.0) + + assert lease._actor.run_blocking.calls[0]["session_remaining_sec"] == pytest.approx(42.0) + + def test_the_worker_reaps_a_child_whose_budget_is_already_spent(self): + """End of the chain: the duration becomes a deadline on the worker's own clock.""" + start = time.monotonic() + rc, _out, _err = rb._run_subprocess_worker( + cmd=[sys.executable, "-c", "import time; time.sleep(30)"], + env=None, + cwd=None, + timeout_s=60, + soft_deadline_sec=None, + server_log_path=None, + server_already_ready=False, + session_remaining_sec=-1.0, + ) + assert rc == SESSION_TIME_EXHAUSTED_RETURNCODE + assert time.monotonic() - start < 10.0 + + def test_the_worker_leaves_a_child_with_budget_left_alone(self): + rc, out, _err = rb._run_subprocess_worker( + cmd=["echo", "still-running"], + env=None, + cwd=None, + timeout_s=60, + soft_deadline_sec=None, + server_log_path=None, + server_already_ready=False, + session_remaining_sec=3600.0, + ) + assert rc == 0 + assert "still-running" in out + + # ── P2: ManagedServerProcess.exit_code (real subprocess) ───────────────────── def test_managed_process_exit_code_none_then_latched(): """exit_code is None before start / while alive, then the real return code.""" @@ -1535,6 +1647,27 @@ def test_serving_lease_close_swallows_kill_error(monkeypatch: pytest.MonkeyPatch assert lease._actor is None +def test_releasing_a_lease_reaps_the_served_process(serving_lease_on_a_ray_double): + """``ray.kill`` skips ``__ray_terminate__``, so the actor must be asked first. + + The served process is deliberately started in its own POSIX session, which + is exactly what a process-group teardown does not reach, so a lease released + without asking can leave a GPU held by a process nothing owns any more. + """ + lease = serving_lease_on_a_ray_double + lease.ensure() + pid = lease._actor.start.remote(["sleep", "60"]).result(timeout=10) + assert _pid_alive(pid) + + lease.close() + + deadline = time.time() + 10.0 + while time.time() < deadline and _pid_alive(pid): + time.sleep(0.05) + assert not _pid_alive(pid), "the served process outlived the lease that owned it" + assert lease._actor is None + + def test_managed_process_start_with_log_path(tmp_path: Path): """start(log_path=...) opens the log file (covers the log_path branch).""" log = tmp_path / "nested" / "server.log" diff --git a/src/hyperloom/inference_optimizer/tests/test_report.py b/src/hyperloom/inference_optimizer/tests/test_report.py index 0e5e3f3b24..d65fd9db09 100644 --- a/src/hyperloom/inference_optimizer/tests/test_report.py +++ b/src/hyperloom/inference_optimizer/tests/test_report.py @@ -274,6 +274,15 @@ def test_a_skip_with_no_recorded_reason_still_says_it_was_skipped(): assert "did not run" in rp._explain_stop_reason("conc_sweep_done", state) +def test_a_session_budget_skip_is_described_as_a_sweep_that_did_not_run(): + state = _SweepState( + {"status": "skipped", "was_skipped": True, "skip_reason": "session_time_budget"} + ) + msg = rp._explain_stop_reason("conc_sweep_done", state) + assert "did not run" in msg + assert "session_time_budget" in msg + + def test_a_sweep_that_spent_its_budget_is_not_reported_as_one_that_never_ran(tmp_path): """The budget path records was_skipped for a sweep that ran its whole ladder.""" from hyperloom.orchestrator.state.shared_state import SharedState diff --git a/src/hyperloom/inference_optimizer/tests/test_resume.py b/src/hyperloom/inference_optimizer/tests/test_resume.py index 7739648240..1a5c06c43c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_resume.py +++ b/src/hyperloom/inference_optimizer/tests/test_resume.py @@ -4,8 +4,8 @@ """Coordinator resume tests. Covers resume detection, ``replay_for_resume`` rebuilding undecided -pending_proposals, pruned_families preservation, and lazy replay on the first -``tick()``. +pending_proposals, pruned_families preservation, lazy replay on the first +``tick()``, and reopening the phase machine for a session that stopped in CLOSE. """ from __future__ import annotations @@ -62,6 +62,110 @@ async def test_existing_state_json_triggers_resume(session_dir): await c.stop() +class TestAClosedSessionIsReopenedOnResume: + """CLOSE has no way out, so a leg that loads it would tick in it to the end. + + The machine's only terminal phase, and the run loop stops on ``stop_reason`` + rather than on the phase. A resumed leg that keeps CLOSE therefore spends its + whole new clock in a phase admitting nothing but ``report``, + ``session_breakdown`` and ``recover``. Every design that stops early on the + promise of "resume with more budget" rests on this being reopened. + """ + + @pytest.mark.asyncio + async def test_a_session_stopped_in_close_starts_the_next_leg_at_the_entrance( + self, + session_dir, + ): + SharedState(session_id="closed", phase="CLOSE").save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.phase == "PRELUDE" + finally: + await coordinator.stop() + + @pytest.mark.asyncio + async def test_the_reopening_is_recorded_as_the_transition_it_is(self, session_dir): + """A phase the run did not reach by working its way there needs saying so.""" + SharedState(session_id="closed", phase="CLOSE").save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + latest = coordinator.shared_state.phase_history[-1] + finally: + await coordinator.stop() + + assert latest["from_phase"] == "CLOSE" + assert latest["to_phase"] == "PRELUDE" + assert latest["evidence"]["trigger"] == "resumed_from_close" + + @pytest.mark.asyncio + async def test_the_earlier_legs_close_sequence_does_not_count_for_this_one( + self, + session_dir, + ): + """The flag means "the sequencer already wrote the breakdown". + + Carried into a leg that then never reaches CLOSE, it silences the + end-of-run safety net that would have written one, and the leg finishes + with no breakdown at all. + """ + SharedState(session_id="closed", phase="CLOSE", close_sequence_done=True).save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.close_sequence_done is False + finally: + await coordinator.stop() + + @pytest.mark.asyncio + async def test_a_session_stopped_anywhere_else_resumes_where_it_stopped(self, session_dir): + """Only the phase with no exit is reopened; the rest can still make progress.""" + SharedState(session_id="mid", phase="EXPLORE").save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.phase == "EXPLORE" + assert coordinator.shared_state.phase_history == [] + finally: + await coordinator.stop() + + @pytest.mark.asyncio + async def test_the_reopened_leg_may_actually_measure_the_baseline_it_reopened_for( + self, + session_dir, + ): + """Reopening the phase is only half of it; the round has to be admissible. + + A cold anchor is a positive ``baseline_tput``, which is what the singleton + rule refuses repeats on -- so the leg would reopen at PRELUDE, decline to + finish while the mark is set, decline to close while the clock is healthy, + and have the one round that clears the mark denied on its way in. This is + the last link in the chain the whole cold-anchor design rests on, and + nothing above it can tell whether it holds. + """ + SharedState( + session_id="cold", + phase="CLOSE", + baseline_tput=1000.0, + baseline_measure_round_dropped=True, + ).save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.phase == "PRELUDE" + coordinator.policy.validate_intent( + "orchestration", + Intent( + type=IntentType.DELEGATE, + payload={"action_name": "baseline", "params": {}}, + ), + ) + finally: + await coordinator.stop() + + @pytest.mark.asyncio async def test_existing_events_triggers_resume(session_dir): c1 = Coordinator(session_dir, backends=_backends_full()) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py new file mode 100644 index 0000000000..94133eec74 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -0,0 +1,1363 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The session wall-clock budget defences that live in the orchestrator loop. + +Three of the defences are here, the ones outside the executors: + +* Admission -- an action whose expected cost cannot fit the budget that is left + never starts. Covers the pure fit decision, the SharedState accessor both it + and the grid deadline read, the dispatcher gate, the three intent paths that + share it, and the pre-dispatch backstop for a task that sat queued until its + budget drained. +* In-flight cancellation -- the backstop for work already running when the + budget goes or the process is asked to stop. Covers the handles the dispatcher + keeps, the cancellation itself, the closing-action carve-out, the task row + landing terminal instead of stranding at ``running``, which of those handles + the pump owns on its way out, and the pump and ``Coordinator.stop`` paths that + trigger it. +* Tick bound -- a reactor turn or phase-enter await that never returns is + cancelled when the session (or closing) bound elapses, so the tick can still + reach the wall-clock stop. + +The remaining two layers are enforced inside the executors and tested next to +them: the timeout clamp in ``test_explore_executor``, and the subprocess session +reaper in ``test_kill_spawned_server``. +""" + +from __future__ import annotations + +import asyncio +import sys +import threading +import time +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any + +import pytest + +from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE +from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType +from hyperloom.orchestrator.actions.cancel_channel import ( + CancelScope, + current_cancel_scope, + stop_was_asked_for, + use_cancel_scope, +) +from hyperloom.orchestrator.actions.executors._ray_serving import CANCEL_ROUND_GRACE_SEC +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + COOPERATIVE_REAP_BUDGET_SEC, + ORCHESTRATOR_CANCELLED_RETURNCODE, + STOP_GATE_POLL_SECONDS, + run_with_session_kill, +) +from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.loop.dispatcher import ( + _CANCEL_NOTICE_SEC, + _COOPERATIVE_CANCEL_GRACE_SEC, +) +from hyperloom.orchestrator.loop.coordinator_helpers import ( + TIME_BUDGET_EXEMPT_ACTIONS, + action_fits_time_budget, + expected_action_cost_minutes, + measured_baseline_runtime_sec, +) +from hyperloom.orchestrator.policy.gate import PolicyDenied +from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan +from hyperloom.orchestrator.roles.robustness_pulse import _PULSE_TIMEOUT_SEC +from hyperloom.orchestrator.state.shared_state import SharedState, effective_closing_grace_sec +from hyperloom.orchestrator.state.task_registry import Task + +# An action costing an hour at p50, so a short budget cannot fit it. +_EXPENSIVE_ACTION = "kernel_opt" +_EXPENSIVE_COST_MIN = 60.0 +# Cheap enough to fit anything but a nearly-spent budget. +_CHEAP_ACTION = "profile" +# An action the catalogue prices at five minutes, and what one of the two +# sessions that motivated the wall-clock work actually measured for it. +_BASELINE_ACTION = "baseline" +_MEASURED_BASELINE_SEC = 51 * 60.0 + + +def _heartbeat() -> Intent: + return Intent(type=IntentType.SEND_MESSAGE, payload={"topic": "heartbeat", "body_md": "ok"}) + + +def _backends() -> dict[str, Backend]: + silent = ScriptedPlan(turns=[], default_intent=_heartbeat()) + return {name: MockBackend(silent, name=name) for name in ("orchestration", "critic", "robustness")} + + +@pytest.fixture +def coord(session_dir) -> Coordinator: + c = Coordinator(session_dir, backends=_backends()) + # Past the baseline prerequisite so the sequence gate stays out of the way. + c.shared_state.baseline_tput = 800.0 + return c + + +def _budgeted_state( + *, + minutes: float, + elapsed_min: float = 0.0, + closing_grace_sec: float | None = None, +) -> SharedState: + """A standalone state with a finite budget and ``elapsed_min`` already spent.""" + state = SharedState(session_id="s", max_minutes=int(minutes), closing_grace_sec=closing_grace_sec) + state.elapsed_minutes = lambda **_kw: elapsed_min # type: ignore[method-assign] + return state + + +def _set_budget(coord: Coordinator, *, minutes: float, elapsed_min: float = 0.0) -> None: + """Give the session a finite budget with ``elapsed_min`` already spent.""" + coord.shared_state.max_minutes = int(minutes) + coord.shared_state.elapsed_minutes = lambda **_kw: elapsed_min # type: ignore[method-assign] + + +class TestTheCostTheGateJudgesOn: + """Where the expected cost comes from: the action catalogue, or nowhere.""" + + def test_a_catalogued_action_reads_its_expected_runtime(self): + assert expected_action_cost_minutes(ACTION_CATALOGUE[_EXPENSIVE_ACTION]) == pytest.approx(_EXPENSIVE_COST_MIN) + + def test_an_action_the_catalogue_does_not_carry_has_no_estimate(self): + assert expected_action_cost_minutes(None) == 0.0 + + def test_no_catalogued_action_reads_as_free(self): + """A zero cost admits an action on any budget, so a whole catalogue of + them is an admission gate that is not there — which is what reading a + renamed field through a ``getattr`` default silently produced.""" + free = sorted(name for name, meta in ACTION_CATALOGUE.items() if expected_action_cost_minutes(meta) <= 0.0) + assert free == [] + + +class TestTheCostIsAnchoredOnWhatThisSessionMeasured: + """The catalogue prices a baseline at five minutes; the field runs it in 51. + + Those estimates are calibrated on small models, so a gate anchored on them + admits arms a real model cannot pay for -- it would not have stopped either + of the two sessions that motivated the wall-clock work. PRELUDE's + affordability gate already anchors on the session's own baseline round; + admission now reads the same number through the same helper. + """ + + def test_a_measured_round_outprices_the_catalogue_for_an_action_that_benches(self): + cost = expected_action_cost_minutes( + ACTION_CATALOGUE["baseline"], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(51.0) + + def test_the_catalogue_wins_where_it_prices_more_than_one_round(self): + """The measurement is a floor, not a replacement: only the catalogue + knows an action benches a whole grid rather than a single variant.""" + cost = expected_action_cost_minutes( + ACTION_CATALOGUE[_EXPENSIVE_ACTION], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(_EXPENSIVE_COST_MIN) + + def test_an_action_that_never_benches_keeps_its_own_estimate(self): + """Writing the report costs what it costs; the model's size is not in it.""" + cost = expected_action_cost_minutes( + ACTION_CATALOGUE["report"], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(ACTION_CATALOGUE["report"].typical_runtime_min) + + def test_a_session_with_no_baseline_yet_falls_back_to_the_catalogue(self): + assert expected_action_cost_minutes(ACTION_CATALOGUE["baseline"]) == pytest.approx(5.0) + + def test_a_warm_replay_is_priced_as_the_baseline_round_it_is(self): + """Warm replay is not a cheap re-attach to a server that is already hot. + + ``replay_warm_recipe`` is dispatched to ``BaselineExecutor`` with the + recipe's ``extra_server_args``/``extra_envs``/``patches``, so it boots + its own server and runs the same benchmark the baseline ran; the recipe + changes what is measured, not how long measuring takes. Being refused + near the tail on a 51-minute price is therefore the gate working, not + the gate being timid — and if warm replay ever does learn to re-attach, + this is where the price stops being right. + """ + cost = expected_action_cost_minutes( + ACTION_CATALOGUE["replay_warm_recipe"], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(_MEASURED_BASELINE_SEC / 60.0) + assert ACTION_CATALOGUE["replay_warm_recipe"].requires_lanes == ACTION_CATALOGUE["baseline"].requires_lanes + + def test_a_measurement_that_is_not_a_number_is_not_a_cost(self): + assert measured_baseline_runtime_sec(None) == 0.0 + assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec="not-a-number")) == 0.0 + assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec=-1.0)) == 0.0 + assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec=_MEASURED_BASELINE_SEC)) == ( + pytest.approx(_MEASURED_BASELINE_SEC) + ) + + +class TestFitDecision: + """The pure fit rule, independent of any Coordinator.""" + + def test_an_unbounded_budget_fits_everything(self): + assert action_fits_time_budget(usable_sec=None, expected_cost_minutes=600.0) + + def test_an_action_with_no_cost_on_record_is_admitted(self): + assert action_fits_time_budget(usable_sec=60.0, expected_cost_minutes=0.0) + assert action_fits_time_budget(usable_sec=60.0, expected_cost_minutes=-1.0) + + def test_an_action_that_fits_is_admitted(self): + assert action_fits_time_budget(usable_sec=30 * 60.0, expected_cost_minutes=30.0) + + def test_an_action_that_does_not_fit_is_refused(self): + assert not action_fits_time_budget(usable_sec=30 * 60.0 - 1, expected_cost_minutes=30.0) + + def test_the_expected_cost_is_the_anchor_not_the_p75_backstop(self): + """A 90-minute budget admits a 60/120 action: the tail is not the bar. + + Judging fit on p75 would refuse work that finishes in the budget half the + time, abandoning usable minutes. The session reaper handles the overruns. + """ + assert action_fits_time_budget(usable_sec=90 * 60.0, expected_cost_minutes=60.0) + assert not action_fits_time_budget(usable_sec=90 * 60.0, expected_cost_minutes=120.0) + + +class TestUsableBudgetAccessor: + """``session_budget_usable_sec`` is the one number admission and the grid share.""" + + def test_an_unset_budget_reads_as_unbounded(self): + assert SharedState(session_id="s").session_budget_usable_sec() is None + + def test_the_closing_reserve_is_held_back(self): + state = _budgeted_state(minutes=60) + assert state.session_budget_usable_sec() == pytest.approx(3600.0 - 72.0) + + def test_a_budget_inside_the_reserve_reads_as_spent(self): + state = _budgeted_state(minutes=60, elapsed_min=59.9) + assert state.session_budget_usable_sec() == 0.0 + + def test_the_grid_deadline_is_derived_from_the_same_number(self, monkeypatch): + """Both wall-clock layers must agree on how much budget is left.""" + import time as _time + + state = _budgeted_state(minutes=60, elapsed_min=10.0) + monkeypatch.setattr(_time, "monotonic", lambda: 1000.0) + usable = state.session_budget_usable_sec() + assert state.grid_session_deadline_sec() == pytest.approx(1000.0 + usable) + + +class TestTheReserveIsTheClosingGraceWindow: + """The budget held back must be the budget the CLOSE phase actually gets. + + A fixed 120s reserve was only ever right for sessions of at least 100 + minutes: a shorter one was charged more than its closing phase can spend, + and an operator who passed ``--closing-grace-sec 0`` to disable that phase + paid 120 seconds for work that never runs. + """ + + @pytest.mark.parametrize( + ("minutes", "closing_grace_sec", "expected"), + [ + (120, None, 120.0), # the default session: unchanged by this fix + (60, None, 72.0), # min(120, 2% of the budget) + (60, 0.0, 0.0), # closing phase disabled: reserve nothing + (60, 600.0, 600.0), # an explicit window wins verbatim + (0, None, 0.0), # unbounded budget: nothing to reserve from + ], + ) + def test_the_reserve_tracks_the_resolved_grace_window(self, minutes, closing_grace_sec, expected): + state = SharedState(session_id="s", max_minutes=minutes, closing_grace_sec=closing_grace_sec) + assert state.closing_reserve_sec() == pytest.approx(expected) + assert state.closing_reserve_sec() == pytest.approx(effective_closing_grace_sec(minutes, closing_grace_sec)) + + @pytest.mark.parametrize("closing_grace_sec", [None, 0.0, 600.0]) + def test_admission_and_the_grid_deadline_agree_on_every_reserve(self, closing_grace_sec, monkeypatch): + import time as _time + + state = _budgeted_state(minutes=60, elapsed_min=20.0, closing_grace_sec=closing_grace_sec) + monkeypatch.setattr(_time, "monotonic", lambda: 1000.0) + usable = state.session_budget_usable_sec() + assert usable == pytest.approx(max(0.0, 2400.0 - state.closing_reserve_sec())) + assert state.grid_session_deadline_sec() == pytest.approx(1000.0 + usable) + + def test_a_disabled_closing_phase_leaves_the_last_minutes_spendable(self): + """The 120s a disabled phase used to cost is the difference here.""" + spent = _budgeted_state(minutes=60, elapsed_min=59.0) + kept = _budgeted_state(minutes=60, elapsed_min=59.0, closing_grace_sec=0.0) + assert spent.session_budget_usable_sec() == 0.0 + assert kept.session_budget_usable_sec() == pytest.approx(60.0) + + @pytest.mark.asyncio + async def test_the_coordinator_hands_the_operators_window_to_the_state(self, coord: Coordinator): + """The reserve lives on SharedState, but the flag arrives at the Coordinator.""" + try: + await coord.run(max_ticks=1, max_minutes=60, closing_grace_sec=0.0) + finally: + await coord.stop() + assert coord.shared_state.closing_grace_sec == 0.0 + assert coord.shared_state.closing_reserve_sec() == 0.0 + + +class TestTimeBudgetGate: + """The dispatcher gate that turns a fit failure into a refusal.""" + + def test_an_action_too_big_for_the_budget_is_denied(self, coord: Coordinator): + _set_budget(coord, minutes=20) + denied = coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) + assert isinstance(denied, PolicyDenied) + assert denied.rule == "time_budget" + assert f"{_EXPENSIVE_COST_MIN:.0f} min" in str(denied) + assert "report" in str(getattr(denied, "hint", "")) + + def test_an_action_that_fits_is_admitted(self, coord: Coordinator): + _set_budget(coord, minutes=20) + assert coord._time_budget_denial_for_action(_CHEAP_ACTION) is None + + def test_an_unbounded_budget_admits_the_most_expensive_action(self, coord: Coordinator): + coord.shared_state.max_minutes = 0 + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + + def test_an_action_with_no_registry_entry_is_admitted(self, coord: Coordinator): + _set_budget(coord, minutes=1) + assert coord._time_budget_denial_for_action("frobnicate") is None + + def test_only_the_closing_actions_are_exempt_from_the_budget(self): + """Recover restarts the server; it is not how a session ends.""" + assert TIME_BUDGET_EXEMPT_ACTIONS == frozenset({"report", "session_breakdown"}) + + def test_the_closing_actions_stay_startable_on_an_empty_budget(self, coord: Coordinator): + """Refusing these would strand the session with nothing to show.""" + _set_budget(coord, minutes=60, elapsed_min=60.0) + assert coord.shared_state.session_budget_usable_sec() == 0.0 + for action in TIME_BUDGET_EXEMPT_ACTIONS: + assert coord._time_budget_denial_for_action(action) is None, action + + def test_recover_is_refused_on_an_empty_budget(self, coord: Coordinator): + """A spent session that still starts recover cannot close.""" + _set_budget(coord, minutes=60, elapsed_min=60.0) + assert coord.shared_state.session_budget_usable_sec() == 0.0 + denied = coord._time_budget_denial_for_action("recover") + assert isinstance(denied, PolicyDenied) + assert denied.rule == "time_budget" + + def test_a_stopping_session_leaves_the_gate_to_the_stop_path(self, coord: Coordinator): + _set_budget(coord, minutes=1) + coord.shared_state.stop_reason = "time_exhausted" + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + + def test_this_session_s_own_baseline_changes_the_answer(self, coord: Coordinator): + """Half an hour left admits a baseline the catalogue prices at five + minutes -- until this session has measured one and knows better.""" + _set_budget(coord, minutes=30) + assert coord._time_budget_denial_for_action(_BASELINE_ACTION) is None + + coord.shared_state.baseline_runtime_sec = _MEASURED_BASELINE_SEC + denied = coord._time_budget_denial_for_action(_BASELINE_ACTION) + + assert isinstance(denied, PolicyDenied) + assert denied.rule == "time_budget" + assert "51 min" in str(denied) + + def test_the_budget_shrinks_the_gate_as_the_session_runs(self, coord: Coordinator): + _set_budget(coord, minutes=120, elapsed_min=0.0) + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + _set_budget(coord, minutes=120, elapsed_min=70.0) + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is not None + + +class TestAdmissionGateOrder: + """``_admission_denial_for_action`` chains the gates; the first one wins.""" + + def test_the_baseline_prerequisite_is_reported_before_the_budget(self, coord: Coordinator): + coord.shared_state.baseline_tput = 0.0 + _set_budget(coord, minutes=1) + denied = coord._admission_denial_for_action("explore") + assert denied is not None and denied.rule == "execution_order" + + def test_the_budget_gate_runs_once_the_sequence_gate_passes(self, coord: Coordinator): + _set_budget(coord, minutes=20) + denied = coord._admission_denial_for_action(_EXPENSIVE_ACTION) + assert denied is not None and denied.rule == "time_budget" + + def test_an_action_clearing_both_gates_is_admitted(self, coord: Coordinator): + _set_budget(coord, minutes=600) + assert coord._admission_denial_for_action(_EXPENSIVE_ACTION) is None + + +def _delegate(action_name: str, key: str) -> Intent: + return Intent( + type=IntentType.DELEGATE, + payload={"action_name": action_name, "params": {}, "idempotency_key": key}, + ) + + +class TestIntentPathsAreGated: + """A refusal must land before a task row exists, so no ledger sees it.""" + + @pytest.mark.asyncio + async def test_delegating_an_over_budget_action_queues_nothing( + self, + coord: Coordinator, + monkeypatch, + ): + _set_budget(coord, minutes=20) + recorded: list[PolicyDenied] = [] + + async def _rec(source, intent, denied, action_name=None): + recorded.append(denied) + + monkeypatch.setattr(coord.writeback, "_record_policy_denied", _rec) + await coord._handle_delegate("orchestration", _delegate(_EXPENSIVE_ACTION, "d-budget")) + assert [d.rule for d in recorded] == ["time_budget"] + assert [t for t in await coord.tasks.queued() if t.kind == _EXPENSIVE_ACTION] == [] + + @pytest.mark.asyncio + async def test_delegating_an_affordable_action_still_queues( + self, + coord: Coordinator, + monkeypatch, + ): + _set_budget(coord, minutes=600) + monkeypatch.setattr(coord.shared_state, "is_pruned", lambda a: False) + await coord._handle_delegate("orchestration", _delegate(_EXPENSIVE_ACTION, "d-ok")) + assert [t for t in await coord.tasks.queued() if t.kind == _EXPENSIVE_ACTION] + + @pytest.mark.asyncio + async def test_proposing_an_over_budget_action_never_reaches_the_critic( + self, + coord: Coordinator, + monkeypatch, + ): + _set_budget(coord, minutes=20) + recorded: list[PolicyDenied] = [] + + async def _rec(source, intent, denied, action_name=None): + recorded.append(denied) + + monkeypatch.setattr(coord.writeback, "_record_policy_denied", _rec) + intent = Intent( + type=IntentType.PROPOSE_ACTION, + payload={"action_name": _EXPENSIVE_ACTION, "predicted_gain_pct": 5.0}, + ) + await coord._handle_propose_action("orchestration", intent) + assert [d.rule for d in recorded] == ["time_budget"] + assert not coord.state.pending_proposals + + @pytest.mark.asyncio + async def test_the_inline_runner_reports_the_refusal(self, coord: Coordinator, monkeypatch): + _set_budget(coord, minutes=20) + + async def _rec(source, intent, denied, action_name=None): + return None + + monkeypatch.setattr(coord.writeback, "_record_policy_denied", _rec) + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + out = await coord._run_action_now(_EXPENSIVE_ACTION, {}) + assert "denied" in out + assert [t for t in await coord.tasks.queued() if t.kind == _EXPENSIVE_ACTION] == [] + + +class TestPreDispatchBackstop: + """A task can wait for a lane long enough for its budget to drain.""" + + @pytest.mark.asyncio + async def test_a_queued_task_the_budget_outlived_is_dropped_before_dispatch( + self, + coord: Coordinator, + ): + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind=_EXPENSIVE_ACTION, + params={}, + idempotency_key="q-drained", + ) + # The budget drains while the task waits in the queue. + _set_budget(coord, minutes=600, elapsed_min=590.0) + + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert [t.task_id for t, _, _ in spawned] == [] + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + + @pytest.mark.asyncio + async def test_the_drop_is_not_recorded_as_an_action_failure(self, coord: Coordinator): + """A task that never ran is not evidence about the action.""" + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind=_EXPENSIVE_ACTION, + params={}, + idempotency_key="q-no-failure", + ) + _set_budget(coord, minutes=600, elapsed_min=590.0) + + await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + failures = list(getattr(coord.shared_state, "last_action_failures", []) or []) + assert [f for f in failures if str(f.get("action") or "") == _EXPENSIVE_ACTION] == [] + + @pytest.mark.asyncio + async def test_a_queued_task_that_still_fits_is_left_alone(self, coord: Coordinator): + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind=_EXPENSIVE_ACTION, + params={}, + idempotency_key="q-fits", + ) + assert await coord.dispatcher._cancel_queued_task_over_budget(task) is False + assert (await coord.tasks.get(task.task_id)).state == "queued" + + @pytest.mark.asyncio + async def test_a_queued_recover_is_dropped_when_the_budget_is_spent( + self, + coord: Coordinator, + ): + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind="recover", + params={}, + idempotency_key="q-recover", + ) + _set_budget(coord, minutes=600, elapsed_min=600.0) + + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert [t.task_id for t, _, _ in spawned] == [] + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + + + @pytest.mark.asyncio + async def test_a_queued_conc_sweep_the_budget_outlived_is_recorded_as_skipped( + self, + coord: Coordinator, + ): + """Cancelling conc_sweep at dispatch must stamp last_conc_sweep so SWEEP can close.""" + from hyperloom.orchestrator.phases.machine_state import exit_normal_sweep + + _set_budget(coord, minutes=180) + task, _ = await coord.tasks.create_or_return_existing( + kind="conc_sweep", + params={}, + idempotency_key="q-conc-sweep", + ) + _set_budget(coord, minutes=180, elapsed_min=166.0) + + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert [t.task_id for t, _, _ in spawned] == [] + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + assert coord.shared_state.last_conc_sweep["status"] == "skipped" + assert coord.shared_state.last_conc_sweep["skip_reason"] == "session_time_budget" + assert coord.shared_state.last_conc_sweep["was_skipped"] is True + result = exit_normal_sweep(coord.shared_state) + assert result is not None + reason, evidence = result + assert reason == "conc_sweep_done" + assert evidence["conc_sweep_status"] == "skipped" + + @pytest.mark.asyncio + async def test_dropping_an_over_budget_conc_sweep_does_not_erase_a_prior_result( + self, + coord: Coordinator, + ): + """A later cancel must not overwrite a conc_sweep the session already measured.""" + _set_budget(coord, minutes=180) + coord.shared_state.record_conc_sweep( + {"status": "succeeded", "was_skipped": False, "summary": {"successful_pairs": 3}} + ) + task, _ = await coord.tasks.create_or_return_existing( + kind="conc_sweep", + params={}, + idempotency_key="q-conc-sweep-prior", + ) + _set_budget(coord, minutes=180, elapsed_min=166.0) + + await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + assert coord.shared_state.last_conc_sweep["status"] == "succeeded" + + +# One of the closing actions, exempt from the budget because the closing reserve +# is held back so it can run. +_CLOSING_ACTION = "report" +# The lane ``_CHEAP_ACTION`` holds while it runs, so a leak is observable. +_CHEAP_ACTION_LANE = "profile_lane" + + +def _never_finishes(started: asyncio.Event): + """Build an executor that only ever ends by being cancelled.""" + + async def _run(_ctx) -> dict: + started.set() + await asyncio.sleep(3600.0) + return {} + + return _run + + +async def _queue_action( + coord: Coordinator, + *, + kind: str, + key: str, + make_executor: Callable[[asyncio.Event], Any] = _never_finishes, +) -> tuple[Task, asyncio.Event]: + """Queue an action with its real lanes; ``make_executor`` shapes what it does.""" + started = asyncio.Event() + coord.sub.register_executor(kind, make_executor(started)) + lanes, ttl_sec = coord.dispatcher._registry_lanes_ttl(kind) + task, _ = await coord.tasks.create_or_return_existing( + kind=kind, + params={}, + idempotency_key=key, + requires_lanes=lanes, + lease_ttl_sec=ttl_sec, + ) + return task, started + + +async def _start_action( + coord: Coordinator, + *, + kind: str, + key: str, + make_executor: Callable[[asyncio.Event], Any] = _never_finishes, +) -> tuple[Task, asyncio.Task]: + """Dispatch the action with no pump running, for the pieces under it.""" + task, started = await _queue_action(coord, kind=kind, key=key, make_executor=make_executor) + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + assert [t.task_id for t, _, _ in spawned] == [task.task_id] + await asyncio.wait_for(started.wait(), timeout=5.0) + return task, spawned[0][1] + + +async def _start_action_under_pump( + coord: Coordinator, + *, + kind: str, + key: str, +) -> tuple[Task, asyncio.Task, asyncio.Task]: + """Let a running pump dispatch the action, the way a tick does. + + Returns ``(task, action task, pump task)``. The pump owns what it spawned, + so the triggers can only be tested against a pump that spawned the work. + """ + task, started = await _queue_action(coord, kind=kind, key=key) + pump = asyncio.create_task(coord._pump_dispatcher_once()) + await asyncio.wait_for(started.wait(), timeout=5.0) + return task, coord.dispatcher._inflight_actions[task.task_id][1], pump + + +async def _settle(atask: asyncio.Task) -> None: + """Wait for an action to finish unwinding, however it ended.""" + await asyncio.wait_for(asyncio.gather(atask, return_exceptions=True), timeout=5.0) + + +class TestInflightHandles: + """Something other than the pump has to be able to reach a running action.""" + + @pytest.mark.asyncio + async def test_a_running_action_is_reachable_by_task_id(self, coord: Coordinator): + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="h-live") + try: + entry = coord.dispatcher._inflight_actions[task.task_id] + assert (entry.kind, entry.atask) == (_CHEAP_ACTION, atask) + assert not entry.scope.cancelled + finally: + atask.cancel() + await _settle(atask) + + @pytest.mark.asyncio + async def test_the_handle_retires_itself_when_the_action_ends(self, coord: Coordinator): + """Self-removal is what keeps the set from outliving the work.""" + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="h-retire") + atask.cancel() + await _settle(atask) + assert task.task_id not in coord.dispatcher._inflight_actions + + @pytest.mark.asyncio + async def test_an_action_that_finishes_normally_leaves_no_handle(self, coord: Coordinator): + coord.sub.register_executor(_CHEAP_ACTION, lambda _ctx: _done({"ok": True})) + task, _ = await coord.tasks.create_or_return_existing( + kind=_CHEAP_ACTION, + params={}, + idempotency_key="h-quick", + ) + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + await _settle(spawned[0][1]) + assert task.task_id not in coord.dispatcher._inflight_actions + + +async def _done(payload: dict) -> dict: + return payload + + +class TestCancellingInflightActions: + """The cancellation itself, and who it spares.""" + + @pytest.mark.asyncio + async def test_it_stops_the_action_and_names_what_it_stopped(self, coord: Coordinator): + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="c-stop") + cancelled = await coord.dispatcher.cancel_inflight_actions(reason="test") + assert cancelled == [task.task_id] + assert atask.cancelled() + + @pytest.mark.asyncio + async def test_the_closing_actions_can_be_spared(self, coord: Coordinator): + """Cancelling the report to save time would leave nothing to show for the run.""" + _, atask = await _start_action(coord, kind=_CLOSING_ACTION, key="c-exempt") + try: + assert ( + await coord.dispatcher.cancel_inflight_actions( + reason="test", + exempt=TIME_BUDGET_EXEMPT_ACTIONS, + ) + == [] + ) + assert not atask.done() + finally: + atask.cancel() + await _settle(atask) + + @pytest.mark.asyncio + async def test_cancelling_with_nothing_running_is_a_no_op(self, coord: Coordinator): + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [] + + @pytest.mark.asyncio + async def test_the_lane_is_free_again_afterwards(self, coord: Coordinator): + """A cancelled action that kept its lane would wedge every later one.""" + await _start_action(coord, kind=_CHEAP_ACTION, key="c-lane") + assert (await coord.locks.lane_holders()).get(_CHEAP_ACTION_LANE, 0) == 1 + await coord.dispatcher.cancel_inflight_actions(reason="test") + assert (await coord.locks.lane_holders()).get(_CHEAP_ACTION_LANE, 0) == 0 + + +# Long enough that a round which ran to completion is unmistakable in the +# elapsed time, short enough that an abandoned thread cannot outlive the suite. +_BLOCKING_SEC = 30 + + +def _blocks_in_a_thread(started: asyncio.Event, *, outcome: dict[str, Any]): + """Build an executor shaped like every benchmark one: a subprocess in a thread. + + ``asyncio.to_thread`` is where all of them spend their time, and a thread + that has started cannot be cancelled, so this is the shape the last defence + actually has to stop. ``outcome`` is written after the thread returns, which + is what makes "the work is over" observable rather than inferred. + """ + + async def _run(_ctx) -> dict: + started.set() + proc = await asyncio.to_thread( + run_with_session_kill, + ["sleep", str(_BLOCKING_SEC)], + timeout=_BLOCKING_SEC * 4, + ) + outcome["returncode"] = proc.returncode + return {"returncode": proc.returncode} + + return _run + + +def _sleeps_in_a_thread(started: asyncio.Event, *, seconds: float = 2.0): + """Build an executor whose thread has no way to hear a cancel.""" + + async def _run(_ctx) -> dict: + started.set() + await asyncio.to_thread(time.sleep, seconds) + return {} + + return _run + + +class TestTheCooperativeStopWindowsCompose: + """Three waits on the same stop, which only mean anything together. + + Each was picked to look reasonable beside the others -- ten seconds at the + dispatcher, eight for a round in a Ray actor, five for the SIGTERM grace -- + and composed they said the dispatcher gives up before the work it is waiting + for can finish. A window a hair short of what stopping costs does not expire + occasionally: it expires every time, and what it discards is the attributed + sentinel the round was about to return. + + The components are spelled out here rather than re-derived from the constants + under test, so a change to one of them has to be argued for -- and the sum is + spelled out too, so a serial step the unwind takes and no term covers has to + be argued for as well, rather than quietly making the window short again. + """ + + def test_the_reap_budget_is_what_stopping_a_round_costs(self): + # Notice at the 0.5s poll, SIGTERM and wait out the 5s grace, collect the + # SIGKILL'd child for 1s, drain its pipes for 2s. + assert COOPERATIVE_REAP_BUDGET_SEC == 0.5 + 5.0 + 1.0 + 2.0 + + def test_the_ray_grace_outlasts_a_round_stopping_itself(self): + """A round in an actor stops the same way; the grace has to cover it.""" + assert CANCEL_ROUND_GRACE_SEC >= COOPERATIVE_REAP_BUDGET_SEC + + def test_the_dispatcher_outlasts_the_slowest_honest_stop(self): + # The Ray path is the long one: 8.5s for the round to stop itself, 0.25s + # for the answer to be seen, then up to 10s to release the lease it held. + assert _COOPERATIVE_CANCEL_GRACE_SEC >= 8.5 + 0.25 + 10.0 + + def test_reaping_a_server_and_dropping_its_lease_are_both_paid(self): + """The two release waits are a sequence, so the window has to cover both. + + A Ray round's unwind reaps the server it left behind and only then closes + the lease it ran in -- that order is a requirement, not an accident, so no + GPU process outlives the lease. Taking the longer of the two leaves the + window five seconds short of what that unwind costs, which is the same + shortfall these windows were derived to remove. + """ + assert _COOPERATIVE_CANCEL_GRACE_SEC >= 8.5 + 0.25 + 5.0 + 10.0 + + def test_the_window_is_exactly_the_terms_it_names(self): + """An upper bound, so a term the unwind pays and the sum omits is a bug. + + Spelled as a total and not only as a floor: a fifth serial step was found + in the unwind that no term covered, and a floor would have gone on passing + while the sum stayed short of what stopping costs. + """ + assert _COOPERATIVE_CANCEL_GRACE_SEC == 8.5 + 0.25 + 5.0 + 10.0 + + def test_the_variant_boundary_tick_is_skipped_rather_than_budgeted_for(self): + """The one step in the unwind this window deliberately does not cover. + + A cooperative stop returns its sentinel, so ``run_grid`` reaches its + variant boundary the ordinary way and would spend the robustness tick's + whole budget there -- between recording the stopped round's row and + releasing what the round held, which are the terms above. Counting it + would take the window past what an operator's own ``SIGTERM`` grace + allows, so the tick gives way instead: the scope it reads is cancelled for + the rest of the action's life, so once the cancel is out no boundary + spends it. + """ + assert _PULSE_TIMEOUT_SEC == 8.0 + assert 8.5 + 0.25 + 5.0 + 10.0 + _PULSE_TIMEOUT_SEC > _COOPERATIVE_CANCEL_GRACE_SEC + scope = CancelScope() + with use_cancel_scope(scope): + assert not stop_was_asked_for() + scope.cancel(reason="session_time_exhausted") + assert stop_was_asked_for() + + def test_work_outside_an_action_is_never_told_to_skip(self): + """No scope means no cancel, so a bare call keeps every step it had.""" + assert not stop_was_asked_for() + + def test_the_notice_window_is_the_poll_the_scope_is_checked_at(self): + """Nothing is listening yet is a claim about the poll, not about the work.""" + assert _CANCEL_NOTICE_SEC >= STOP_GATE_POLL_SECONDS + assert _CANCEL_NOTICE_SEC < COOPERATIVE_REAP_BUDGET_SEC + + +class TestTheCancelChannel: + """The channel itself: what it carries, and how far it reaches.""" + + @pytest.mark.asyncio + async def test_a_worker_thread_sees_the_scope_of_the_task_that_started_it(self): + """The whole idiom rests on ``to_thread`` copying the context.""" + scope = CancelScope() + with use_cancel_scope(scope): + seen = await asyncio.to_thread(current_cancel_scope) + assert seen is scope + + @pytest.mark.asyncio + async def test_code_outside_an_action_finds_no_scope(self): + """A Ray worker and a bare call are the same case: nothing to check.""" + assert await asyncio.to_thread(current_cancel_scope) is None + + def test_the_first_reason_is_the_one_kept(self): + """A blanket cancel arriving second must not overwrite the specific cause.""" + scope = CancelScope() + scope.cancel(reason="session_time_exhausted") + scope.cancel(reason="dispatcher_pump_exit") + assert scope.cancelled + assert scope.reason == "session_time_exhausted" + + def test_a_scope_reports_whether_anything_is_watching_it(self): + scope = CancelScope() + assert not scope.has_listeners + with scope.listening(): + assert scope.has_listeners + assert not scope.has_listeners + + +class TestCancellingWorkThatBlocksInAThread: + """Cancelling the coroutine does not stop the thread it is waiting on. + + The canceller gets a clean ``CancelledError`` off the ``await`` while the + subprocess runs on to its own hard timeout, so the lanes and the GPU lease + are released, and the database closed, with the benchmark still holding the + card. Stopping it takes a channel the thread itself checks. + """ + + @pytest.mark.asyncio + async def test_the_work_is_over_before_the_cancel_returns(self, coord: Coordinator): + outcome: dict[str, Any] = {} + task, atask = await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-thread", + make_executor=lambda started: _blocks_in_a_thread(started, outcome=outcome), + ) + began = time.monotonic() + + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task.task_id] + + assert outcome, "the cancel returned while the thread was still running" + assert time.monotonic() - began < _BLOCKING_SEC + await _settle(atask) + + @pytest.mark.asyncio + async def test_the_stop_is_attributed_to_the_orchestrator(self, coord: Coordinator): + """A cancel is not a timeout and not a slow variant; the ledger reads returncodes.""" + outcome: dict[str, Any] = {} + await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-thread-rc", + make_executor=lambda started: _blocks_in_a_thread(started, outcome=outcome), + ) + + await coord.dispatcher.cancel_inflight_actions(reason="test_reason") + + assert outcome["returncode"] == ORCHESTRATOR_CANCELLED_RETURNCODE + + @pytest.mark.asyncio + async def test_a_thread_with_nothing_listening_is_still_not_waited_for(self, coord: Coordinator): + """The channel is cooperative, so work that cannot hear it is left behind. + + Waiting on it anyway would trade a leaked thread for a shutdown that + hangs on one, which is the worse of the two. + """ + _task, atask = await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-deaf", + make_executor=_sleeps_in_a_thread, + ) + began = time.monotonic() + + await coord.dispatcher.cancel_inflight_actions(reason="test") + + assert time.monotonic() - began < _COOPERATIVE_CANCEL_GRACE_SEC + assert atask.cancelled() + + +def _runs_a_round_in_a_lease(started: asyncio.Event, *, outcome: dict[str, Any], lease: Any): + """An executor shaped like the production default: a round inside a Ray lease. + + ``_should_use_ray_backend`` is off under pytest and on by default on a single + node, so this is the branch every real run takes and no test did. + """ + + async def _run(_ctx) -> dict: + started.set() + rc, _out, _err = await asyncio.to_thread( + lease.run_session_kill, + [sys.executable, "-c", f"import time; time.sleep({_BLOCKING_SEC})"], + timeout=_BLOCKING_SEC * 4, + ) + outcome["returncode"] = rc + return {"returncode": rc} + + return _run + + +class TestCancellingARoundInsideARayLease: + """The production default routes rounds through a Ray actor, not a local child. + + The scope is a ContextVar, so it does not exist in the actor's process: the + lease has to notice the cancel on this side and forward it, or the four-layer + defence has no reach at all on the path every real single-node run takes. + """ + + @pytest.fixture + def lease(self, serving_lease_on_a_ray_double: Any) -> Any: + return serving_lease_on_a_ray_double + + @pytest.mark.asyncio + async def test_the_round_in_the_actor_stops_before_the_cancel_returns( + self, + coord: Coordinator, + lease: Any, + ): + outcome: dict[str, Any] = {} + task, atask = await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-ray", + make_executor=lambda started: _runs_a_round_in_a_lease(started, outcome=outcome, lease=lease), + ) + began = time.monotonic() + + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task.task_id] + + assert outcome, "the cancel returned while the round was still running in the actor" + assert time.monotonic() - began < _BLOCKING_SEC + await _settle(atask) + + @pytest.mark.asyncio + async def test_the_stop_is_attributed_to_the_orchestrator(self, coord: Coordinator, lease: Any): + """The actor reaps its own tree, so the sentinel is the same one the local path returns.""" + outcome: dict[str, Any] = {} + await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-ray-rc", + make_executor=lambda started: _runs_a_round_in_a_lease(started, outcome=outcome, lease=lease), + ) + + await coord.dispatcher.cancel_inflight_actions(reason="test_reason") + + assert outcome["returncode"] == ORCHESTRATOR_CANCELLED_RETURNCODE + + @pytest.mark.asyncio + async def test_an_actor_that_will_not_answer_is_killed_and_the_stop_still_named( + self, + coord: Coordinator, + lease: Any, + monkeypatch: pytest.MonkeyPatch, + ): + """A wedged actor must not hold the lease open, and must not go unattributed.""" + from hyperloom.orchestrator.actions.executors import _ray_serving as rs + + monkeypatch.setattr(rs, "CANCEL_ROUND_GRACE_SEC", 0.5) + monkeypatch.setattr(rs.ServingLease, "_ask_actor_to_cancel", lambda _self, _reason: False) + outcome: dict[str, Any] = {} + await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-ray-wedged", + make_executor=lambda started: _runs_a_round_in_a_lease(started, outcome=outcome, lease=lease), + ) + + await coord.dispatcher.cancel_inflight_actions(reason="test_reason") + + assert outcome["returncode"] == ORCHESTRATOR_CANCELLED_RETURNCODE + assert lease._actor is None, "the lease must be released when its actor is killed" + + +class TestTheRunnerRecordsACancellation: + """``CancelledError`` is not an ``Exception``, so the runner must name it.""" + + @pytest.mark.asyncio + async def test_a_cancelled_action_does_not_stay_running(self, coord: Coordinator): + """A row stuck at ``running`` reads as live work to every phase gate.""" + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="r-terminal") + atask.cancel() + await _settle(atask) + row = await coord.tasks.get(task.task_id) + assert row.state == "cancelled" + assert "cancelled_in_flight" in str(row.history) + + @pytest.mark.asyncio + async def test_the_cancellation_still_reaches_the_caller(self, coord: Coordinator): + """Recording it must not turn a cancellation into a normal return.""" + _task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="r-propagates") + atask.cancel() + await _settle(atask) + assert atask.cancelled() + + +def _quick_poll(coord: Coordinator) -> None: + """Shorten the pump's re-scan interval so a pump test is not a wall-clock test.""" + coord._dispatcher_poll_sec = 0.05 + + +class TestThePumpStopsWorkItCannotWaitFor: + """The trigger side: a spent budget, and a shutdown request.""" + + @pytest.mark.asyncio + async def test_a_budget_that_runs_out_stops_the_action(self, coord: Coordinator): + _quick_poll(coord) + _set_budget(coord, minutes=600) + task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-budget") + _set_budget(coord, minutes=600, elapsed_min=600.0) + + await asyncio.wait_for(pump, timeout=10.0) + + assert atask.cancelled() + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + + @pytest.mark.asyncio + async def test_the_closing_actions_keep_their_reserve(self, coord: Coordinator): + """The budget hits zero with the closing window still to spend.""" + _quick_poll(coord) + _set_budget(coord, minutes=600, elapsed_min=600.0) + _task, atask, pump = await _start_action_under_pump(coord, kind=_CLOSING_ACTION, key="p-closing") + await asyncio.sleep(0.3) + + assert not atask.done() + + pump.cancel() + await _settle(pump) + + @pytest.mark.asyncio + async def test_a_shutdown_request_stops_the_action(self, coord: Coordinator): + """SIGTERM sets the stop event; before this it only stopped the tick.""" + _quick_poll(coord) + _set_budget(coord, minutes=600) + _task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-signal") + coord._stop.set() + + await asyncio.wait_for(pump, timeout=10.0) + + assert atask.cancelled() + + @pytest.mark.asyncio + async def test_a_cancelled_pump_does_not_orphan_its_actions(self, coord: Coordinator): + """The handles live in the pump's frame; leaving must not drop them.""" + _quick_poll(coord) + _set_budget(coord, minutes=600) + _task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-orphan") + + pump.cancel() + await _settle(pump) + + assert atask.cancelled() + assert coord.dispatcher._inflight_actions == {} + + +def _allow_inline(coord: Coordinator, monkeypatch) -> asyncio.Event: + """Register a never-finishing executor and clear the gates around it.""" + started = asyncio.Event() + coord.sub.register_executor(_CHEAP_ACTION, _never_finishes(started)) + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + _set_budget(coord, minutes=600) + return started + + +async def _start_inline_action(coord: Coordinator, monkeypatch) -> asyncio.Task: + """Run an inline action and wait until it is registered and under way.""" + started = _allow_inline(coord, monkeypatch) + inline = asyncio.create_task(coord.dispatcher._run_action_now(_CHEAP_ACTION, {})) + await asyncio.wait_for(started.wait(), timeout=5.0) + return inline + + +class TestInlineActionsAreReachableToo: + """The inline path abandons its future, so it needs the same handle.""" + + @pytest.mark.asyncio + async def test_an_inline_action_that_outlived_its_caller_can_be_stopped( + self, + coord: Coordinator, + monkeypatch, + ): + """Before this, the only thing that ended it was the action itself.""" + inline = await _start_inline_action(coord, monkeypatch) + task_id = next(iter(coord.dispatcher._inflight_actions)) + + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task_id] + + await _settle(inline) + assert inline.cancelled() + assert (await coord.tasks.get(task_id)).state == "cancelled" + assert coord.dispatcher._inflight_actions == {} + + @pytest.mark.asyncio + async def test_an_inline_action_that_finishes_leaves_no_handle( + self, + coord: Coordinator, + monkeypatch, + ): + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + _set_budget(coord, minutes=600) + coord.sub.register_executor(_CHEAP_ACTION, lambda _ctx: _done({"ok": True})) + + await coord.dispatcher._run_action_now(_CHEAP_ACTION, {}) + + assert coord.dispatcher._inflight_actions == {} + + @pytest.mark.asyncio + async def test_the_sync_bridge_reports_the_cancellation_instead_of_raising( + self, + coord: Coordinator, + monkeypatch, + ): + """It runs on an agent's turn thread, which a ``CancelledError`` would end.""" + started = _allow_inline(coord, monkeypatch) + monkeypatch.setattr( + coord.dispatcher, + "_inline_action_whitelist", + lambda: frozenset({_CHEAP_ACTION}), + ) + coord._inline_fast_actions_enabled = True + coord._coordinator_loop = asyncio.get_running_loop() + + outcome: list[str] = [] + caller = threading.Thread( + target=lambda: outcome.append(coord.dispatcher._run_action_now_sync(_CHEAP_ACTION, {})), + daemon=True, + ) + caller.start() + try: + await asyncio.wait_for(started.wait(), timeout=5.0) + await coord.dispatcher.cancel_inflight_actions(reason="test") + await asyncio.to_thread(caller.join, 5.0) + finally: + caller.join(5.0) + + assert outcome and "was cancelled" in outcome[0] + + +class TestThePumpOnlyCancelsWhatItSpawned: + """The registry is dispatcher-wide; the pump's exit sweep is not. + + An inline action is registered by whoever ran it, not by the pump, and is + designed to keep going after that caller stops waiting. A tick with nothing + queued returns immediately, so an exit sweep over the whole registry would + make the emptiest possible pump the thing that kills it. + """ + + @pytest.mark.asyncio + async def test_a_tick_with_nothing_queued_leaves_an_inline_action_running( + self, + coord: Coordinator, + monkeypatch, + ): + inline = await _start_inline_action(coord, monkeypatch) + try: + await asyncio.wait_for(coord._pump_dispatcher_once(), timeout=10.0) + + assert not inline.done() + assert coord.dispatcher._inflight_actions + finally: + inline.cancel() + await _settle(inline) + + @pytest.mark.asyncio + async def test_a_cancelled_pump_takes_its_own_and_only_its_own( + self, + coord: Coordinator, + monkeypatch, + ): + """Narrowing the sweep must not cost the pump the actions it does own.""" + _quick_poll(coord) + inline = await _start_inline_action(coord, monkeypatch) + _task, spawned, pump = await _start_action_under_pump(coord, kind=_CLOSING_ACTION, key="own-spawn") + try: + pump.cancel() + await _settle(pump) + + assert spawned.cancelled() + assert not inline.done() + finally: + inline.cancel() + await _settle(inline) + + @pytest.mark.asyncio + async def test_a_shutdown_still_reaches_an_inline_action( + self, + coord: Coordinator, + monkeypatch, + ): + """The narrower sweep must not blunt the trigger that has to reach everything.""" + inline = await _start_inline_action(coord, monkeypatch) + coord._stop.set() + + await asyncio.wait_for(coord._pump_dispatcher_once(), timeout=10.0) + + await _settle(inline) + assert inline.cancelled() + + +class TestCoordinatorStop: + """Teardown closes the database, so it cannot leave actions using it.""" + + @pytest.mark.asyncio + async def test_stop_cancels_the_actions_still_running(self, coord: Coordinator): + _task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="s-stop") + + await coord.stop() + + assert atask.cancelled() + assert coord.dispatcher._inflight_actions == {} + + +async def _idle(*_args, **_kwargs) -> None: + return None + + +async def _hang_forever(*_args, **_kwargs) -> None: + await asyncio.sleep(3600) + + +class TestATickCannotOutliveTheSessionBound: + """A step that never returns used to skip the wall-clock stop at tick end.""" + + @pytest.mark.asyncio + async def test_a_hanging_reactor_still_stops_when_the_budget_ends( + self, + coord: Coordinator, + monkeypatch, + ): + monkeypatch.setattr(coord, "_advance_phase_if_needed", _idle) + monkeypatch.setattr(coord, "_reactor_pass", _hang_forever) + monkeypatch.setattr(coord, "_pump_dispatcher_once", _idle) + started = time.monotonic() + try: + reason = await asyncio.wait_for( + coord.run(max_minutes=0.05, closing_grace_sec=0.0, tick_interval_sec=0.0), + timeout=15.0, + ) + finally: + await coord.stop() + assert reason == "time_exhausted" + assert time.monotonic() - started < 10.0 + + @pytest.mark.asyncio + async def test_a_hanging_phase_enter_still_stops_when_the_budget_ends( + self, + coord: Coordinator, + monkeypatch, + ): + monkeypatch.setattr(coord, "_advance_phase_if_needed", _hang_forever) + monkeypatch.setattr(coord, "_reactor_pass", _idle) + monkeypatch.setattr(coord, "_pump_dispatcher_once", _idle) + started = time.monotonic() + try: + reason = await asyncio.wait_for( + coord.run(max_minutes=0.05, closing_grace_sec=0.0, tick_interval_sec=0.0), + timeout=15.0, + ) + finally: + await coord.stop() + assert reason == "time_exhausted" + assert time.monotonic() - started < 10.0 + + @pytest.mark.asyncio + async def test_a_spent_bound_does_not_start_the_next_step(self, coord: Coordinator): + coord._run_deadline = time.monotonic() - 1.0 + started: list[bool] = [] + + async def _must_not_run() -> None: + started.append(True) + + await coord._await_within_session_bound(_must_not_run, stage="test") + assert started == [] + + @pytest.mark.asyncio + async def test_no_deadline_still_runs_the_step(self, coord: Coordinator): + started: list[bool] = [] + + async def _ok() -> None: + started.append(True) + + await coord._await_within_session_bound(_ok, stage="test") + assert started == [True] + + @pytest.mark.asyncio + async def test_closing_uses_the_grace_bound_not_the_session_deadline(self, coord: Coordinator): + coord._run_deadline = time.monotonic() - 10.0 + coord._closing_deadline = time.monotonic() + 60.0 + coord.shared_state.closing_phase = True + started: list[bool] = [] + + async def _ok() -> None: + started.append(True) + + await coord._await_within_session_bound(_ok, stage="close") + assert started == [True] diff --git a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py index f6f702046c..2c3d163fe2 100644 --- a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py @@ -122,8 +122,8 @@ def test_future_deadline_when_ample_budget(self, monkeypatch): s.max_minutes = 60.0 monkeypatch.setattr(type(s), "remaining_minutes", lambda self, **_: 10.0) monkeypatch.setattr(ss_mod.time, "monotonic", lambda: 1000.0) - # 10 min remaining, 120s reserve -> now + (600 - 120). - assert s.grid_session_deadline_sec() == pytest.approx(1000.0 + 480.0) + # 10 min remaining, 72s closing reserve -> now + (600 - 72). + assert s.grid_session_deadline_sec() == pytest.approx(1000.0 + 528.0) def test_deadline_is_now_when_under_reserve(self, monkeypatch): import hyperloom.orchestrator.state.shared_state as ss_mod @@ -132,7 +132,7 @@ def test_deadline_is_now_when_under_reserve(self, monkeypatch): s.max_minutes = 60.0 monkeypatch.setattr(type(s), "remaining_minutes", lambda self, **_: 1.0) monkeypatch.setattr(ss_mod.time, "monotonic", lambda: 500.0) - # 60s remaining < 120s reserve -> deadline == now (already exhausted). + # 60s remaining < 72s closing reserve -> deadline == now (already exhausted). assert s.grid_session_deadline_sec() == pytest.approx(500.0) diff --git a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py index db92727dbe..64185af8a1 100644 --- a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py +++ b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py @@ -1,12 +1,17 @@ # SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Tests for the from-ready soft-deadline caliber. +"""Tests for the two things measured from the server-ready marker. When a ``server.log`` is available the explore overtime soft deadline measures only the post-ready phase: the clock starts at the server-ready marker, excluding pre-ready boot / weight load / first-request recompile. Opt out via ``INFERENCE_OPTIMIZER_SOFT_DEADLINE_FROM_READY=0``. + +The same marker is recorded so a round can be *priced* by its two parts rather +than its total, which is what lets later work be charged for what it will +actually spend: a variant boots its own server and pays both parts, a pass that +re-attaches pays only the second. """ from __future__ import annotations @@ -14,9 +19,14 @@ import sys import time +import pytest + from hyperloom.orchestrator.actions.executors._subprocess_kill import ( OVERTIME_KILL_RETURNCODE, + clear_server_ready_stamp, + post_ready_runtime_sec, run_with_session_kill, + server_ready_unix, ) # Long stall grace so the detok-stall watchdog never interferes here. @@ -115,3 +125,271 @@ def test_no_server_log_uses_from_spawn(tmp_path): elapsed = time.monotonic() - start assert cp.returncode == OVERTIME_KILL_RETURNCODE assert elapsed < 15.0, f"from-spawn (no server.log) took {elapsed:.2f}s" + + +class TestARoundIsPricedByItsTwoParts: + """What a round spent booting and what it spent benchmarking, told apart. + + The whole point of separating them is that they are spent by different + things. Charging a re-attaching pass for a boot it never pays is what makes + a budget gate refuse work that fits. + """ + + def test_the_boot_is_not_charged_to_the_benchmark(self, tmp_path): + """A round that boots for 3s and benchmarks for 1s reports 1s, not 4s.""" + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(3)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + runtime_sec = time.time() - started_unix + assert cp.returncode == 0 + + post_ready = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + assert post_ready is not None, "the round reported ready but nothing recorded when" + # The benchmark's own second, found without the three the boot took. The + # windows are wide because the poll interval and process spawn are inside + # them; what is being pinned is that the two parts are told apart at all. + assert 0.5 <= post_ready <= 2.5, f"benchmark share read as {post_ready:.2f}s, expected ~1s" + boot_sec = runtime_sec - post_ready + assert 2.5 <= boot_sec <= 4.5, f"boot share read as {boot_sec:.2f}s, expected ~3s" + + def test_a_previous_attempts_log_does_not_time_this_rounds_boot(self, tmp_path): + """Only bytes this round writes may say when its server came up. + + Magpie writes into a ``benchmark_*/`` workspace under the round's output + dir, and a reused dir can still hold one from an earlier attempt. Scanned + from byte zero, its ready line latches on the first poll -- seconds after + spawn -- and the round reports a boot that took minutes as one that took + none. Every variant is then admitted at a benchmark's price and reaped. + """ + stale = tmp_path / "benchmark_vllm_20200101" / "server.log" + stale.parent.mkdir(parents=True) + stale.write_text("Application startup complete\n", encoding="utf-8") + + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(3)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + runtime_sec = time.time() - started_unix + assert cp.returncode == 0 + + post_ready = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + assert post_ready is not None + boot_sec = runtime_sec - post_ready + assert boot_sec >= 2.5, ( + f"the stale log timed the boot: read {boot_sec:.2f}s of boot for a round " + f"that spent 3s on it" + ) + + def test_the_split_does_not_depend_on_an_unrelated_watchdog(self, tmp_path): + """The stall grace is a hang backstop, not a switch for the cost model. + + Turning it off used to withdraw the ready timestamp with it, and a session + run that way prices every round at its whole cold wall-clock without + anything saying so. + """ + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(2)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=0.0, + session_deadline_sec=time.monotonic() + 30.0, + ) + assert cp.returncode == 0 + assert server_ready_unix(str(log_path)) is not None, ( + "the boot/benchmark split was withdrawn along with the stall watchdog" + ) + + def test_a_reader_on_another_clock_gets_the_same_boot(self, tmp_path): + """The split survives being read where it was not written. + + On the Ray path the round runs inside an actor, possibly on another host, + and the caller subtracting its own clock from the actor's would charge the + boot for whatever the two disagree by -- inflating it, and making the + budget gates refuse rounds that fit. The boot is a duration taken on one + clock, so a reader whose clock is five seconds off reads the same figure. + """ + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(3)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + runtime_sec = time.time() - started_unix + assert cp.returncode == 0 + + agreed = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + # A caller whose clock runs five seconds behind the writer's. The instant + # is still this round's, so the stamp is accepted, and the boot it + # reports does not move. + skewed = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix - 5.0, + runtime_sec=runtime_sec, + ) + assert agreed is not None + assert skewed == pytest.approx(agreed), ( + f"a five-second clock disagreement moved the boot: {skewed} vs {agreed}" + ) + + def test_a_round_that_never_came_up_is_not_priced(self, tmp_path): + """No ready marker means no split, reported as unknown rather than guessed.""" + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(1)\n" + ) + started_unix = time.time() + run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + assert server_ready_unix(str(log_path)) is None + assert ( + post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=time.time() - started_unix, + ) + is None + ) + + def test_an_earlier_rounds_stamp_is_not_read_as_this_ones(self, tmp_path): + """A stamp predating the round is unknown, not "it never booted". + + The clamp alone would report such a stamp as a whole-round benchmark, + which is the reading that would price a cold round as a warm one. + """ + log_path = tmp_path / "server.log" + log_path.write_text("INFO loading weights\n", encoding="utf-8") + # A complete stamp, so what rejects it can only be its instant. + (tmp_path / "server_ready_at").write_text(f"{time.time() - 600.0:.3f} 30.000\n", encoding="utf-8") + + assert ( + post_ready_runtime_sec( + str(log_path), + started_unix=time.time(), + runtime_sec=90.0, + ) + is None + ) + + def test_a_stamp_is_cleared_so_the_next_round_starts_blind(self, tmp_path): + """Clearing is what keeps the case above from arising in a reused dir.""" + log_path = tmp_path / "server.log" + stamp = tmp_path / "server_ready_at" + stamp.write_text(f"{time.time():.3f} 30.000\n", encoding="utf-8") + assert server_ready_unix(str(log_path)) is not None + + clear_server_ready_stamp(str(log_path)) + + assert not stamp.exists() + assert server_ready_unix(str(log_path)) is None + # Idempotent: a round whose dir never had one must not fail to start. + clear_server_ready_stamp(str(log_path)) + + def test_a_boot_longer_than_the_round_cannot_produce_a_negative_price(self, tmp_path): + """A corrupt stamp must cost a measurement, never produce a nonsense one. + + The two figures come from two places -- the boot from inside the round, + the total from the caller around it -- so nothing structurally forbids a + boot larger than the total. It is floored, and the total caps the other + end. + """ + log_path = tmp_path / "server.log" + started_unix = time.time() + (tmp_path / "server_ready_at").write_text(f"{started_unix + 10.0:.3f} 500.000\n", encoding="utf-8") + + priced = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=100.0, + ) + + assert priced == 0.0 + + def test_a_stamp_with_no_boot_recorded_is_no_stamp_at_all(self, tmp_path): + """A missing boot must not read as a round that never booted. + + Zero is a legitimate boot, so it cannot also mean "not recorded" -- + reading it that way hands the whole round to the benchmark, which is the + figure every later variant is then admitted on, and every one of them is + reaped. Unmeasured is reported as unmeasured; the gates know what to do + with that. + """ + log_path = tmp_path / "server.log" + started_unix = time.time() + (tmp_path / "server_ready_at").write_text(f"{started_unix + 10.0:.3f}\n", encoding="utf-8") + + assert ( + post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=100.0, + ) + is None + ) diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py index 38a92680a0..0266178e15 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py @@ -48,6 +48,11 @@ class _BareState: conc_sweep_total_budget_sec: int = 60 conc_sweep_variant_timeout_sec: int = 30 save_count: int = 0 + stop_reason: str = "" + usable_sec: float | None = None + + def session_budget_usable_sec(self, *, reserve_sec=None) -> float | None: + return self.usable_sec def save(self, _session_dir: Path | None) -> None: self.save_count += 1 @@ -947,6 +952,34 @@ async def test_on_enter_sweep_skips_when_no_validated_gain_since_last_conc_sweep assert coord.shared_state.save_count >= 1 +@pytest.mark.asyncio +async def test_on_enter_sweep_skips_when_the_session_budget_cannot_fit_conc_sweep(coord): + """A conc_sweep the clock cannot pay for must not be enqueued, or SWEEP idles.""" + coord.shared_state.usable_sec = 14 * 60.0 + coord.shared_state.phase_history = [ + {"to_phase": "SWEEP", "reason": "plateau_kernel", "evidence": {}}, + ] + await coord._on_enter_sweep(from_phase="KERNEL") + assert coord.tasks._tasks == {} + evidence = coord.shared_state.phase_history[-1]["evidence"] + assert evidence["auto_conc_sweep_skipped"] == "session_time_budget" + assert coord.shared_state.last_conc_sweep["status"] == "skipped" + assert coord.shared_state.last_conc_sweep["skip_reason"] == "session_time_budget" + assert coord.shared_state.last_conc_sweep["was_skipped"] is True + + +@pytest.mark.asyncio +async def test_on_enter_sweep_still_enqueues_when_the_session_budget_fits(coord): + """The session-budget skip must not fire when the catalogue cost still fits.""" + coord.shared_state.usable_sec = 60 * 60.0 + coord.shared_state.phase_history = [ + {"to_phase": "SWEEP", "reason": "plateau_kernel", "evidence": {}}, + ] + await coord._on_enter_sweep(from_phase="KERNEL") + assert "internal-conc_sweep-phase_entry" in coord.tasks._tasks + assert coord.shared_state.last_conc_sweep == {} + + @pytest.mark.asyncio async def test_on_enter_sweep_runs_when_validated_gain_improved(coord): """A new validated gain after the last conc_sweep watermark dispatches conc_sweep.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_warm_replay.py b/src/hyperloom/inference_optimizer/tests/test_warm_replay.py index b1054c1852..8dd1fe525d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_warm_replay.py +++ b/src/hyperloom/inference_optimizer/tests/test_warm_replay.py @@ -1436,6 +1436,51 @@ async def test_prelude_initial_analysis_enqueued_after_warm_replay_finishes( assert coord.shared_state.auto_roofline_pending_task_id +@pytest.mark.asyncio +async def test_prelude_initial_analysis_dropped_when_it_would_cost_the_optimization_phases( + tmp_path, +): + """A roofline is worth an hour only if the session can still use what it finds. + + The Qwen3.5-397B shape: 51 minutes of baseline, then an 81-minute TraceLens + arm that left FRAMEWORK_AGENT 46 minutes against its 108-minute threshold. + """ + coord = _make_coord(tmp_path) + state = coord.shared_state + state.baseline_tput = 600.0 + state.max_minutes = 180 + state.baseline_runtime_sec = 2705.7 + state.phase_elapsed_totals = {"PRELUDE": 3090.0} + state.phase_history = [{"to_phase": "PRELUDE", "evidence": {}}] + state.session_budget_usable_sec = lambda: 7700.0 + + await coord._maybe_enqueue_prelude_initial_analysis_after_baseline() + + assert coord.tasks.calls == [] + assert not coord.shared_state.auto_roofline_pending_task_id + dropped = state.phase_history[-1]["evidence"]["budget_dropped_arms"] + assert dropped[0]["arm"] == "initial_analysis" + assert dropped[0]["expected_cost_sec"] == pytest.approx(2705.7) + + +@pytest.mark.asyncio +async def test_prelude_initial_analysis_runs_when_the_budget_covers_it(tmp_path): + """Same wiring, ordinary session: the arm is not dropped just because the guard exists.""" + coord = _make_coord(tmp_path) + state = coord.shared_state + state.baseline_tput = 600.0 + state.max_minutes = 180 + state.baseline_runtime_sec = 300.0 + state.phase_elapsed_totals = {"PRELUDE": 320.0} + state.phase_history = [{"to_phase": "PRELUDE", "evidence": {}}] + state.session_budget_usable_sec = lambda: 10_300.0 + + await coord._maybe_enqueue_prelude_initial_analysis_after_baseline() + + assert len(coord.tasks.calls) == 1 + assert coord.shared_state.auto_roofline_pending_task_id + + def test_prelude_bootstrap_runs_on_positive_baseline(tmp_path): coord = _make_coord(tmp_path) assert coord._should_run_prelude_bootstrap(600.0) is True diff --git a/src/hyperloom/orchestrator/actions/cancel_channel.py b/src/hyperloom/orchestrator/actions/cancel_channel.py new file mode 100644 index 0000000000..1c132b9317 --- /dev/null +++ b/src/hyperloom/orchestrator/actions/cancel_channel.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Cooperative cancellation channel between the dispatcher and blocking work. + +Every benchmark executor spends its time inside ``asyncio.to_thread``, and a +thread that has already started cannot be cancelled. Cancelling the coroutine +returns a clean ``CancelledError`` to the canceller while the subprocess it was +waiting on keeps running -- long enough for the lanes and the GPU lease to be +released, and the database closed, under a benchmark that still owns the card. + +So the thread needs something it can check. A :class:`CancelScope` carries a +:class:`threading.Event` on a :class:`contextvars.ContextVar`: the dispatcher +publishes one per action, and ``asyncio.to_thread`` copies the context into the +worker thread, so code many frames down finds it without every executor +signature growing a parameter. Blocking code checks the scope at whatever +interval it already polls at and stops itself. + +Cooperative means exactly that: work that never looks at the scope cannot be +stopped through it, which is why a scope also counts its listeners -- the +canceller waits only for work that can hear it. + +The channel is in-process: this ContextVar is unset inside a Ray actor, so a +round running there cannot read the scope. What crosses is the request, not the +channel -- :class:`..executors._ray_serving.ServingLease` watches the scope on +the submitter's side and forwards a cancel to the actor, which publishes a scope +of its own around the round so the same reaper stops it. Work put on Ray by +anything that does not do that forwarding is out of reach, and stops only on the +session deadline it was handed. +""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +__all__ = [ + "CancelScope", + "cancel_scope_listener", + "current_cancel_scope", + "stop_was_asked_for", + "use_cancel_scope", +] + + +class CancelScope: + """One action's cancel channel: the flag, why it was raised, who watches it. + + Every method is safe to call from any thread: the flag is a + :class:`threading.Event` and the listener count is taken under a lock, since + the writer is the event loop and the readers are worker threads. + """ + + def __init__(self) -> None: + """Create an uncancelled scope with no listeners.""" + self._event = threading.Event() + self._lock = threading.Lock() + self._reason = "" + self._listeners = 0 + + def cancel(self, *, reason: str) -> None: + """Ask the work running in this scope to stop. + + Returns as soon as the flag is raised -- whoever is watching it decides + how to stop, and how long that takes. + + Args: + reason (str): Short cause, kept for the log line the stopped work + writes. The first reason wins, so a later blanket cancel cannot + overwrite the specific one that got there first. + """ + with self._lock: + if not self._reason: + self._reason = str(reason) + self._event.set() + + @property + def cancelled(self) -> bool: + """bool: Whether this scope has been cancelled.""" + return self._event.is_set() + + @property + def reason(self) -> str: + """str: Why the scope was cancelled; empty while it is not.""" + with self._lock: + return self._reason + + @property + def has_listeners(self) -> bool: + """bool: Whether any blocking call is currently watching this scope.""" + with self._lock: + return self._listeners > 0 + + @contextmanager + def listening(self) -> Iterator["CancelScope"]: + """Count the caller as a watcher for the duration of the block. + + Yields: + CancelScope: This scope, so the block can check it. + """ + with self._lock: + self._listeners += 1 + try: + yield self + finally: + with self._lock: + self._listeners = max(0, self._listeners - 1) + + +_CURRENT_SCOPE: ContextVar[CancelScope | None] = ContextVar( + "hyperloom_cancel_scope", + default=None, +) + + +def current_cancel_scope() -> CancelScope | None: + """Return the cancel scope of the action running in this context. + + Returns: + CancelScope | None: The published scope, or ``None`` when the caller is + not running under one -- a Ray worker, a unit test, or any code the + dispatcher did not start. + """ + return _CURRENT_SCOPE.get() + + +def stop_was_asked_for() -> bool: + """Whether the action running in this context has already been asked to stop. + + For code deciding whether a step is still worth taking, rather than for code + stopping something in flight: no scope, or an uncancelled one, both read as + "carry on", so a caller outside an action behaves exactly as it always did. + + Returns: + bool: ``True`` when a scope is published and cancelled. + """ + scope = _CURRENT_SCOPE.get() + return scope is not None and scope.cancelled + + +@contextmanager +def use_cancel_scope(scope: CancelScope | None) -> Iterator[CancelScope | None]: + """Publish ``scope`` for the duration of the block. + + Must be entered inside the task that runs the action: a task copies the + context at creation, so a value set afterwards from outside never reaches + it. + + Args: + scope (CancelScope | None): The scope to publish; ``None`` leaves the + context untouched, for callers with nothing to cancel. + + Yields: + CancelScope | None: The published scope, unchanged. + """ + if scope is None: + yield None + return + token = _CURRENT_SCOPE.set(scope) + try: + yield scope + finally: + _CURRENT_SCOPE.reset(token) + + +@contextmanager +def cancel_scope_listener() -> Iterator[CancelScope | None]: + """Watch the published scope, if there is one, for the duration of the block. + + Registering is what tells the canceller this work can be asked to stop, so + the window has to cover everything the cancel is meant to reach -- from the + moment the child is spawned, not from the first poll. + + Yields: + CancelScope | None: The scope to check, or ``None`` when none is + published, in which case the block is a plain no-op. + """ + scope = _CURRENT_SCOPE.get() + if scope is None: + yield None + return + with scope.listening(): + yield scope diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 2de6d32297..491a15c916 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -32,17 +32,28 @@ scrub_benchmark_process_env, ) +from ...phases import machine_state as _phase_state from ...roles.robustness_pulse import pulse as _robustness_pulse from ...trace.task_progress import heartbeat_while_output_flows, report_progress +from ..cancel_channel import stop_was_asked_for +from ..stop_attribution import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) from ._accuracy_gate import materialized_run_eval_disabled from ._subprocess_kill import ( AGENTX_PREFLIGHT_RETURNCODE, DETOKENIZER_STALL_RETURNCODE, EVAL_PROBE_UNPATCHABLE_RETURNCODE, + ORCHESTRATOR_CANCELLED_RETURNCODE, OVERTIME_KILL_RETURNCODE, SERVER_DEAD_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, run_with_session_kill, server_log_death_excerpt, + session_deadline_to_remaining_sec, ) from .benchmark_result import ( estimate_killed_variant_throughput, @@ -938,6 +949,7 @@ def _run_magpie( server_already_ready: bool = False, serving_lease: Any = None, on_output: Callable[[], None] | None = None, + session_deadline_sec: float | None = None, ) -> tuple[int, str, str]: """Blocking subprocess wrapper. Returns (rc, stdout, stderr). @@ -971,6 +983,12 @@ def _run_magpie( line the benchmark emits, so the caller's heartbeat can keep reporting across a run that blocks for hours. Ignored on the ``serving_lease`` path — see the note there. + session_deadline_sec (float | None): Absolute ``time.monotonic()`` instant + at which the session budget expires. Reaps the tree and returns + ``SESSION_TIME_EXHAUSTED_RETURNCODE``. Enforced in every phase, + including the accuracy eval that retires ``soft_deadline_sec``. The + lease path sends it across as a remaining-seconds duration, the only + form that means the same thing in the actor's process. Returns: tuple[int, str, str]: ``(returncode, stdout, stderr)``. @@ -1079,6 +1097,9 @@ def _run_magpie( soft_deadline_sec=soft_deadline_sec, server_log_path=str(output_dir / "server.log"), server_already_ready=server_already_ready, + # Converted here, at the last moment before the process boundary: + # the actor cannot read this process's monotonic clock. + session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), ) cmd = build_benchmark_command( @@ -1097,6 +1118,7 @@ def _run_magpie( server_log_path=str(output_dir / "server.log"), server_already_ready=server_already_ready, on_output=on_output, + session_deadline_sec=session_deadline_sec, ) return proc.returncode, proc.stdout or "", proc.stderr or "" @@ -1224,6 +1246,123 @@ def _variant_progress_note( } +# Seconds by which a round's hard cap is allowed to sit past the session deadline, +# so the in-process session watchdog (which attributes the kill correctly) trips +# before the hard cap (which the ledger reads as a variant timeout). Small enough +# that it cannot meaningfully eat the close-window reserve. +_SESSION_KILL_GRACE_SEC: int = 15 + +# The returncode side of :mod:`...stop_attribution`: the same two causes, keyed +# by the sentinel a subprocess comes back with. The classes themselves live in +# that leaf because the ledgers downstream of here carry the class, not the +# returncode. +_STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_RETURNCODE: STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + ORCHESTRATOR_CANCELLED_RETURNCODE: STOPPED_BY_THE_RUN[ORCHESTRATOR_CANCELLED_CLASS], +} + + +def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: + """Return how to record a round the run itself stopped, if it did. + + Args: + returncode: The round's returncode. + + Returns: + StoppedByTheRun | None: How to record it, or ``None`` when the + returncode says something about the round rather than about the run. + """ + if returncode is None: + return None + return _STOPPED_BY_THE_RUN.get(int(returncode)) + + +def session_clamped_timeout_sec( + cap: int, + session_deadline_sec: float | None, + *, + reserve_sec: float = 0.0, +) -> int: + """Reduce a hard timeout to what the session budget can still pay for. + + ``session_deadline_sec`` already excludes the close-window reserve, so no + further margin is taken for the session itself. Never returns less than 1 -- + a non-positive cap reads as "no timeout" to the subprocess layer, which is + the opposite of what an exhausted budget means. Whether the round should + start at all is the caller's decision, not this one's. + + A small grace is added past the session deadline so the in-process session + watchdog trips first: both fire at the same instant, and the hard cap raises + ``TimeoutExpired``, which the ledger records as a timeout of the thing being + measured. The watchdog's sentinel says the run ran out of time, which is what + actually happened. + + A non-zero ``reserve_sec`` is the only thing that can move the returned cap + earlier than the session deadline: with no reserve the grace puts the cap at + least ``_SESSION_KILL_GRACE_SEC`` past it, so the watchdog always gets there + first. Every caller that reserves therefore re-opens the spurious-timeout + window the grace exists to close, and owes a reason its round cannot be reaped + by its own cap: :func:`run_grid` refuses a variant whose rounds do not fit, + both before the per-variant server restart and again after it, so what it + reserves for is a round it has just re-checked there is room for. + + Args: + cap: The timeout the caller would grant with an unbounded budget. + session_deadline_sec: Monotonic-clock session deadline, or ``None`` when + the budget is unbounded, which leaves ``cap`` untouched. + reserve_sec: Seconds held back for rounds that must still follow this + one, so an early round cannot spend the budget a later one needs. + + Returns: + int: The hard timeout to grant this round, in seconds. + """ + if session_deadline_sec is None: + return int(cap) + usable = int(session_deadline_sec - time.monotonic() - max(0.0, reserve_sec)) + _SESSION_KILL_GRACE_SEC + return int(cap) if usable >= int(cap) else max(1, usable) + + +def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: + """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call. + + Every arm that benches on the GPU needs the same two numbers, and they have + to agree: a deadline derived one way in one executor and another way in the + next produces arms that abandon different amounts of the tail budget. Both + are read here so there is one definition. + + ``variant_expected_sec`` is what a normally-behaving variant needs -- its own + server boot and then its benchmark + (:func:`~...phases.machine_state.one_more_measurement_sec`) -- and is + deliberately not the declared timeout, which is a catastrophic-hang backstop + roughly twice as large. Admitting on the backstop would refuse to start a + 20-minute round with 30 minutes left. + + Nor is it the baseline's cold wall-clock, which is the fallback only for a + session whose baseline reported no boot/benchmark split. That figure also + carries the first request's kernel compile on a cold JIT cache, which a + variant on the now-populated cache does not pay again, so admitting on it + abandons the tail of the budget to variants that would have finished. + + Args: + shared_state: The session ``SharedState``, or ``None`` when the caller + has no session context (direct executor invocation, tests). + + Returns: + tuple[float | None, float | None]: The monotonic-clock session deadline + and the expected per-variant runtime. Either is ``None`` when + unknown, which leaves the corresponding check disabled rather than + guessing a bound. + """ + if shared_state is None: + return (None, None) + deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) + deadline = deadline_fn() if callable(deadline_fn) else None + variant_sec = _phase_state.one_more_measurement_sec(shared_state) + if variant_sec is None: + variant_sec = _phase_state.measured_seconds(shared_state, "baseline_runtime_sec") + return (deadline, variant_sec) + + async def run_grid( *, base_yaml_path: Path, @@ -1245,6 +1384,7 @@ async def run_grid( server_already_ready: bool = False, serving_lease: Any = None, session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> list[VariantResult]: """Execute each grid variant and return all per-variant results. @@ -1257,9 +1397,22 @@ async def run_grid( decision). ``session_deadline_sec`` is a ``time.monotonic()`` deadline for the whole - session budget; a variant is skipped once the remaining budget cannot fit - another ``variant_timeout_sec`` worst-case run, so a wall-clock timeout stops - the grid mid-way and the last variant never overruns the close window. + session budget, already excluding the close-window reserve. It does two + things, deliberately split: + + * **Whether to start.** A variant is skipped, and every variant after it, + once the remaining budget cannot fit ``variant_expected_sec`` -- what a + normally-behaving variant needs. Judging by ``variant_timeout_sec`` + instead means judging by the catastrophic backstop (``baseline x 2`` for + explore), which abandons the tail of the budget to variants that would + have finished comfortably. ``None`` falls back to ``variant_timeout_sec``, + preserving the stricter behaviour for callers that cannot estimate. + * **How long it may run.** The cap handed to each round is clamped to what + the session can still pay for, recomputed per round because a warmup + consumes budget too. Without this a variant admitted near the end can + outlive the whole session: explore derives caps up to 4h from the measured + baseline and never consulted the budget, so a 3h run could grant one + variant more time than the run was given. Every pass runs with ``output_root`` as its working directory, the way the baseline arm anchors Magpie to its own output dir. That is what marks the @@ -1375,37 +1528,235 @@ async def _pulse_after_variant(idx: int) -> None: """Report the finished variant and run a best-effort robustness pulse. Called once the variant's result has been appended, so the progress - note carries what actually landed. Exceptions from the pulse are - swallowed (logged at debug) so a pulse failure never aborts the grid. + note carries what actually landed. The robustness tick is skipped once + the action has been asked to stop: a cooperative stop returns its + sentinel rather than raising, so this is reached on the ordinary path + with the cancel already outstanding, and the tick is a subprocess + spawned and waited out for its whole timeout -- serially, between + recording the row and releasing what the round held, inside the one + window the canceller allows the entire unwind. What it would observe + is the reap the orchestrator just ordered, and what waiting for it costs + is every row this grid has already built. + + The gate is the cancel scope rather than the round's returncode because a + variant can be failing for its own reasons when the cancel lands: that + row is a genuine failure and its tick is just as unaffordable. + + Exceptions from the pulse are swallowed (logged at debug) so a pulse + failure never aborts the grid. Args: idx (int): Zero-based index of the just-finished variant, passed through as the pulse ``tick_index``. """ await report_progress(**_variant_progress_note(grid, results, idx)) + if stop_was_asked_for(): + log.info( + "grid_runner: variant %d/%d robustness pulse skipped; the orchestrator cancelled this action", + idx + 1, + len(grid), + ) + return try: await _robustness_pulse(tick_index=idx) except Exception as exc: # noqa: BLE001 log.debug("robustness pulse swallowed: %r", exc) + def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: float = 0.0) -> int: + """``variant_timeout_sec`` capped at what the session can still pay for. + + Recomputed per round rather than once per variant: a warmup round spends + budget the measure round would otherwise still count on. + + ``reserve_sec`` holds back what the variant's remaining rounds still + need, so a slow warmup cannot consume the budget belonging to the + measured round. The measured round is the only one that yields a usable + data point, so it is the round the budget is kept for; a discarded + warmup that overruns is the cheaper thing to cut short. + + The clamp itself is :func:`session_clamped_timeout_sec`; this adds the + grid's own log line. + + Args: + idx (int): Zero-based variant index, for the log line. + name (str): Variant name, for the log line. + round_label (str): Which round is being capped (warmup / measure). + reserve_sec (float): Seconds held back for this variant's later + rounds. Zero for the last round. + + Returns: + int: The hard timeout to grant this round, in seconds. + """ + cap = int(variant_timeout_sec) + clamped = session_clamped_timeout_sec(cap, session_deadline_sec, reserve_sec=reserve_sec) + if clamped == cap: + return cap + log.info( + "grid_runner: variant %d/%d name=%s %s cap clamped %ds -> %ds by the session budget", + idx + 1, + len(grid), + name, + round_label, + cap, + clamped, + ) + return clamped + + def _record_round_stop( + stopped: StoppedByTheRun, + *, + idx: int, + variant: GridVariant, + slot: Path, + round_label: str, + returncode: int | None, + started_unix: float, + server_log: Path, + ) -> bool: + """Record a round the run stopped and say whether the grid is over. + + Every round a variant runs is a full benchmark pass -- the discarded + warmup and the multi-node client warmup as much as the measured one -- so + any of them can be reaped by the session deadline or by an orchestrator + cancel, and none of them says anything about the variant when it is. + Hence one place decides what such a round means, called after each launch + and before any grading: the row is ``skipped``, exactly like a variant the + budget refused to start, because nothing was measured and there is no + verdict to record. Grading it as a failure -- or worse as + ``killed_overtime``, which asserts the variant is abnormally slow -- puts + a conclusion the run never reached into the ledger and the KB. + + A cause that ``ends_the_batch`` also fills in every variant after this + one: nothing new may start under it, and their rows have to say why they + were never tested rather than be absent. + + Args: + stopped (StoppedByTheRun): How to record the cause. + idx (int): Zero-based index of the variant whose round was stopped. + variant (GridVariant): That variant. + slot (Path): Its slot directory, where the abort marker is written. + round_label (str): Which round was stopped, for the log line. + returncode (int | None): The round's returncode, kept on the row. + started_unix (float): ``time.time()`` when the round was launched. + server_log (Path): The stopped round's ``server.log``, if it wrote one. + + Returns: + bool: ``True`` when the caller must stop testing variants. + """ + runtime_sec = round(max(0.0, time.time() - started_unix), 2) + log.warning( + "grid_runner: variant %d/%d name=%s %s round reaped after %.1fs: %s; recorded as skipped, not failed", + idx + 1, + len(grid), + variant.name, + round_label, + runtime_sec, + stopped.interrupted, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=stopped.error_class, + error_summary=f"{stopped.interrupted}; tree reaped", + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + returncode=returncode, + runtime_sec=runtime_sec, + error=stopped.interrupted, + error_class=stopped.error_class, + server_log_path=_existing_log_path(server_log), + note=variant.note, + ) + ) + if stopped.ends_the_batch: + results.extend(_not_run_skip_result(rest, stopped) for rest in grid[idx + 1 :]) + return True + return not keep_going_on_failure + + # How many full benchmark passes one variant costs. Both warmups run the same + # workload as the measured pass -- neither is a reduced one -- so a variant + # that warms up costs twice what its measured round does. Admitting on a + # single round's estimate would systematically let in variants that then get + # their measured round clamped to nothing, turning a budget shortfall into a + # ledger full of spurious timeouts. Both flags are run-level, so this is known + # before the loop; the per-variant ``auto_warmup`` is only ever narrower. + _mn_warmup_rounds = 0 + if variant_expected_sec is not None: + from ._multi_node_env import ( + is_multi_node as _mn_is_multi_node, + mn_bench_warmup_enabled as _mn_bench_warmup_enabled, + ) + + _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 + variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds + + def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant_rounds) -> bool: + """Skip variant ``idx`` and every one after it, or say the budget still fits. + + Checked twice per variant, because the two things that spend the budget + between the check and the launch are not under any cap: the per-variant + multi-node server restart, and the variant before this one overrunning. + Once a variant does not fit, none after it does either -- the clock only + shrinks -- so this ends the batch rather than trying the next name. + + ``rounds_left`` is what makes the second check a different question from + the first rather than the same one asked with less clock. A pass this + variant has already run is paid for, and charging for it again refuses a + variant that fits and throws away the GPU time already spent on it. + + Args: + idx: Zero-based index of the variant about to run. + spent_on: What has been spent since the last check, for the log line. + rounds_left: Passes of this variant still to launch. Defaults to all + of them, which is right for the check before any has run. + + Returns: + bool: ``True`` when the batch is over and nothing more was launched. + """ + if session_deadline_sec is None: + return False + remaining_sec = session_deadline_sec - time.monotonic() + # Falls back to a single ``variant_timeout_sec`` when no estimate was + # given, which is what callers that cannot estimate already got. + required_sec = ( + float(variant_expected_sec) * rounds_left + if variant_expected_sec is not None + else float(variant_timeout_sec) + ) + if remaining_sec >= required_sec: + return False + log.warning( + "grid_runner: %.0fs left cannot fit this variant's %d remaining round(s) " + "of %.0fs (spent on: %s); skipping %d remaining variant(s) rather than " + "launching a pass whose measured round cannot follow it", + max(0.0, remaining_sec), + rounds_left, + required_sec, + spent_on, + len(grid) - idx, + ) + for skipped_variant in grid[idx:]: + results.append( + _not_run_skip_result( + skipped_variant, + _STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_RETURNCODE], + ) + ) + return True + for i, variant in enumerate(grid): # Session-budget stop: skip the remaining variants once the wall-clock - # deadline is reached or the remaining budget cannot fit another - # variant's worst-case runtime, so a timeout halts the grid instead of - # draining it (and the last variant cannot overrun the close window). - if session_deadline_sec is not None: - remaining_sec = session_deadline_sec - time.monotonic() - if remaining_sec < float(variant_timeout_sec): - log.warning( - "grid_runner: session budget exhausted (%.0fs left < variant cap %ds); " - "skipping %d remaining variant(s)", - max(0.0, remaining_sec), - variant_timeout_sec, - len(grid) - i, - ) - for skipped_variant in grid[i:]: - results.append(_session_deadline_skip_result(skipped_variant)) - break + # deadline is reached or the remaining budget cannot fit another variant, + # so a timeout halts the grid instead of draining it (and the last variant + # cannot overrun the close window). + if _skip_rest_for_budget(i, spent_on="the variants before this one"): + break await _unit_started(i, "variant") slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" server_log = slot / "server.log" @@ -1584,6 +1935,19 @@ async def _pulse_after_variant(idx: int) -> None: warmup_workspaces_before = snapshot_workspaces(warmup_slot) warmup_started_unix = time.time() + # Held in a local because the abort line below has to name the cap the + # round was actually granted: the declared one is a hang backstop, and + # a round killed at the reserved cap logged as a two-hour timeout reads + # as a variant that hangs rather than a budget that ran out. The + # admission gate above is what keeps this reserve from starving the + # round -- it refuses a variant whose passes do not fit before any of + # them is reserved for. + warmup_cap_sec = _round_timeout_sec( + i, + variant.name, + round_label="warmup", + reserve_sec=float(variant_expected_sec or 0.0) * (1 + _mn_warmup_rounds), + ) try: warmup_rc, warmup_stdout, warmup_stderr = await _reported_magpie( i, @@ -1591,27 +1955,22 @@ async def _pulse_after_variant(idx: int) -> None: magpie_python=magpie_python, config_path=warmup_cfg_path, output_dir=warmup_slot, - timeout_sec=variant_timeout_sec, + timeout_sec=warmup_cap_sec, cwd=cwd, result_dir=result_dir, soft_deadline_sec=None, preclean=True, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, ) except subprocess.TimeoutExpired as exc: - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) log.warning( "grid_runner: variant %d/%d name=%s aborted: warmup timeout (timeout_sec=%d): %s", i + 1, len(grid), variant.name, - variant_timeout_sec, + warmup_cap_sec, exc, ) _write_variant_abort_marker( @@ -1640,6 +1999,24 @@ async def _pulse_after_variant(idx: int) -> None: break continue + warmup_stopped = stopped_by_the_run(warmup_rc) + if warmup_stopped is not None: + _teardown_variant_server(slot, lifecycle) + grid_is_over = _record_round_stop( + warmup_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="warmup", + returncode=warmup_rc, + started_unix=warmup_started_unix, + server_log=warmup_server_log, + ) + await _pulse_after_variant(i) + if grid_is_over: + break + continue + warmup_run_ws = select_run_workspace(warmup_slot, known_before=warmup_workspaces_before) warmup_workspace = warmup_run_ws if warmup_run_ws is not None else warmup_slot warmup_harvested = harvest_leaked_artifacts( @@ -1659,13 +2036,7 @@ async def _pulse_after_variant(idx: int) -> None: subprocess_started_unix=warmup_started_unix, ) if warmup_rc != 0 or not warmup_measurement.get("valid_measurement"): - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) warmup_error = ( server_log_death_excerpt(str(warmup_server_log)) or redact_secret_values((warmup_stderr or warmup_stdout)[-2000:]) @@ -1793,6 +2164,21 @@ async def _pulse_after_variant(idx: int) -> None: break continue + # The restart above is a launch of its own and is under no cap, so the + # budget admitted at the top of the loop may no longer be there. Booting + # a large model can take longer than a benchmark pass. Only the passes + # still ahead are charged for: the auto warmup above this point has + # already run, and re-charging it here would end the batch over time + # that is spent either way. + if _skip_rest_for_budget( + i, + spent_on="this variant's server restart", + rounds_left=1 + _mn_warmup_rounds, + ): + if auto_warmup: + _teardown_variant_server(slot, lifecycle) + break + # Multi-node client warmup: one discarded benchmark pass against the # just-restarted, persistent remote server to warm JIT / steady-state # before the measured pass. No lifecycle / no restart between; best- @@ -1805,25 +2191,37 @@ async def _pulse_after_variant(idx: int) -> None: if _mn_imn() and _mn_warm(): _mn_warm_slot = slot / "mn_warmup" + _mn_warm_started_unix = time.time() + # The measurement is discarded, but the returncode is not: a warmup + # the run stopped is the same stop as one in the measured round, and + # discarding it launches the measured round after the cancel. + _mn_warm_rc: int | None = None try: - await _reported_magpie( + _mn_warm_rc, _, _ = await _reported_magpie( i, "mn_warmup", magpie_python=magpie_python, config_path=cfg_path, output_dir=_mn_warm_slot, - timeout_sec=variant_timeout_sec, + timeout_sec=_round_timeout_sec( + i, + variant.name, + round_label="mn_warmup", + reserve_sec=float(variant_expected_sec or 0.0), + ), cwd=cwd, result_dir=None, soft_deadline_sec=None, preclean=False, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, ) log.info( - "grid_runner: MN warmup pass done (discarded) %d/%d name=%s", + "grid_runner: MN warmup pass done (discarded) %d/%d name=%s rc=%s", i + 1, len(grid), variant.name, + _mn_warm_rc, ) except Exception as exc: # noqa: BLE001 - warmup is best-effort log.warning( @@ -1831,6 +2229,22 @@ async def _pulse_after_variant(idx: int) -> None: variant.name, exc, ) + _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) + if _mn_warm_stopped is not None: + grid_is_over = _record_round_stop( + _mn_warm_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="mn_warmup", + returncode=_mn_warm_rc, + started_unix=_mn_warm_started_unix, + server_log=_mn_warm_slot / "server.log", + ) + await _pulse_after_variant(i) + if grid_is_over: + break + continue # Snapshot wall-clock before launch so the salvage path can mtime-gate # leak destinations per-variant. @@ -1843,13 +2257,14 @@ async def _pulse_after_variant(idx: int) -> None: magpie_python=magpie_python, config_path=cfg_path, output_dir=slot, - timeout_sec=variant_timeout_sec, + timeout_sec=_round_timeout_sec(i, variant.name, round_label="measure"), cwd=cwd, result_dir=result_dir, soft_deadline_sec=soft_deadline_sec, preclean=(False if auto_warmup else preclean_before_run), server_already_ready=(server_already_ready or auto_warmup), serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, ) except subprocess.TimeoutExpired as exc: # Harvest pre-timeout leaks. @@ -1896,13 +2311,7 @@ async def _pulse_after_variant(idx: int) -> None: continue finally: if auto_warmup: - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) # Eval bounds could not be installed for a variant that runs eval, so # nothing launched. Labelled rather than left to the generic path, which @@ -2048,6 +2457,23 @@ async def _pulse_after_variant(idx: int) -> None: break continue + stopped = stopped_by_the_run(rc) + if stopped is not None: + grid_is_over = _record_round_stop( + stopped, + idx=i, + variant=variant, + slot=slot, + round_label="measured", + returncode=rc, + started_unix=variant_started_unix, + server_log=server_log, + ) + await _pulse_after_variant(i) + if grid_is_over: + break + continue + # Soft overtime gate fired: record a ``killed_overtime=True`` result with # no tput and still harvest leaks for post-mortem. if rc == OVERTIME_KILL_RETURNCODE: @@ -2267,15 +2693,46 @@ async def _pulse_after_variant(idx: int) -> None: return results -def _session_deadline_skip_result(variant: GridVariant) -> VariantResult: - """Synthetic ``skipped`` result for a variant dropped when the session budget ran out.""" +def _teardown_variant_server(slot: Path, lifecycle: dict[str, Any]) -> None: + """Stop the server this variant booted for its own rounds. + + Every path out of a variant that booted one owes this call, including the + ones that leave over the budget rather than over a result: the process + outlives the loop iteration that started it, and the next variant boots its + own. Kept in one function so a new exit path is one line rather than a + four-line block someone can leave out. + + Args: + slot: The variant's workspace, which is also its pid directory. + lifecycle: The variant's resolved lifecycle, carrying framework and port. + """ + from ._server_lifecycle import teardown_lifecycle_server + + teardown_lifecycle_server( + pid_dir=slot, + framework=str(lifecycle.get("framework") or ""), + port=int(lifecycle.get("port") or 0), + ) + + +def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: + """Build the ``skipped`` result for a variant the run never got to. + + Args: + variant: The variant that was dropped. + stopped: Why the run stopped, which is the whole content of the result: + nothing was measured, so there is nothing else to report. + + Returns: + VariantResult: A synthetic ``skipped`` result carrying that cause. + """ return VariantResult( name=variant.name, extra_server_args=variant.extra_server_args, extra_envs=dict(variant.extra_envs), status="skipped", - error="session wall-clock budget exhausted before this variant ran", - error_class="session_time_exhausted", + error=stopped.never_started, + error_class=stopped.error_class, note=variant.note, ) @@ -2412,6 +2869,9 @@ def _write_variant_abort_marker_impl( "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", "GridVariant", "MULTI_NODE_DEFAULT_KEEP_THRESHOLD_PCT", + "ORCHESTRATOR_CANCELLED_CLASS", + "SESSION_TIME_EXHAUSTED_CLASS", + "StoppedByTheRun", "SGLANG_WATCHDOG_TIMEOUT_ENV", "SINGLE_NODE_DEFAULT_KEEP_THRESHOLD_PCT", "VariantResult", @@ -2429,6 +2889,9 @@ def _write_variant_abort_marker_impl( "sanitize_result_dir", "sanitize_script_name", "server_args_env_name", + "session_clamped_timeout_sec", + "session_grid_bounds", + "stopped_by_the_run", # Re-exported from the sibling modules to keep the namespace intact. "coerce_extra_envs", "compact_json_server_args", diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_backend.py b/src/hyperloom/orchestrator/actions/executors/_ray_backend.py index 991be587e4..2b61d767b0 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_backend.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_backend.py @@ -184,12 +184,15 @@ def _run_subprocess_worker( soft_deadline_sec: float | None, server_log_path: str | None, server_already_ready: bool, + session_remaining_sec: float | None = None, ) -> tuple[int, str, str]: """Ray worker body: run the subprocess under session-kill semantics. Executes on a Ray worker where ``*_VISIBLE_DEVICES`` are already set by Ray. Reuses :func:`run_with_session_kill` so kill/soft-deadline behaviour matches - the local path exactly. + the local path exactly -- including the session reaper, which is the only + defence that attributes running out of time to the run rather than to the + variant that happened to be in flight. Args: cmd: The command to execute. @@ -199,12 +202,18 @@ def _run_subprocess_worker( soft_deadline_sec: Overtime soft deadline. server_log_path: Path to the server log for watchdog markers. server_already_ready: Start the soft clock from spawn (warm reuse). + session_remaining_sec: Seconds left on the session budget when the + submitter made the call. A duration rather than the in-process + absolute deadline because this body runs in a Ray worker, whose + ``time.monotonic()`` origin is its own; it is re-anchored here onto + this process's clock. Returns: ``(returncode, stdout, stderr)``. """ from hyperloom.orchestrator.actions.executors._subprocess_kill import ( run_with_session_kill, + session_remaining_to_deadline_sec, ) worker_env = _merge_worker_env(env) @@ -216,6 +225,7 @@ def _run_subprocess_worker( soft_deadline_sec=soft_deadline_sec, server_log_path=server_log_path, server_already_ready=server_already_ready, + session_deadline_sec=session_remaining_to_deadline_sec(session_remaining_sec), ) return proc.returncode, proc.stdout or "", proc.stderr or "" diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py index 892d291c13..635d9558eb 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py @@ -14,16 +14,47 @@ from hyperloom.common.env_safety import scrub_benchmark_process_env +from ._subprocess_kill import COOPERATIVE_REAP_BUDGET_SEC + log = logging.getLogger(__name__) -# Ray-side sentinel returncodes. -912 is also used by -# ``_subprocess_kill.AGENTX_PREFLIGHT_RETURNCODE``. -_ACTOR_TIMEOUT_RC: int = -912 +# Ray-side sentinel returncodes, allocated out of the same space as +# ``_subprocess_kill``'s -- read the note there before claiming a new one. Both +# of these leave ``_grid_runner._run_magpie`` through the very return channel +# that carries ``AGENTX_PREFLIGHT_RETURNCODE``, so an overlap would have an +# actor timeout recorded as a failed AgentX preflight. +_ACTOR_TIMEOUT_RC: int = -916 _RAY_ACTOR_DIED_RC: int = -913 # Timeout for ray.get probes on specialist actor methods (is_alive/exit_code/stop). _LEASE_PROBE_TIMEOUT_SEC: float = 30.0 +# How often the submitter of a round looks up from ``ray.wait`` to see whether +# the action it belongs to has been cancelled. Short enough that the hop through +# Ray costs the cancel almost nothing on top of what stopping the round costs +# anyway, long enough not to spin. +_CANCEL_POLL_SEC: float = 0.25 + +# How long the submitter waits for a cancelled round to come back on its own +# before killing the actor out from under it. The round in the actor stops itself +# exactly the way a local child does, so this is that cost plus the poll the +# answer is seen at -- derived, not picked, because a grace even slightly short of +# the reap expires every single time and throws away the sentinel the round was +# about to hand back. The kill is what a wedged actor gets, not what a working one +# gets for being ordinary. +CANCEL_ROUND_GRACE_SEC: float = COOPERATIVE_REAP_BUDGET_SEC + _CANCEL_POLL_SEC + +# How long releasing a lease waits for the actor to reap its served process +# before killing the actor anyway. Sized on what that reap costs -- SIGTERM, the +# grace, SIGKILL -- and deliberately short: teardown often runs inside the +# closing window, which is reserved for the report, not for waiting on a server. +CLOSE_STOP_TIMEOUT_SEC: float = 10.0 + +# Method slots the serving actor runs at once: the round, plus room for the +# cancel that has to reach it. A single-slot actor would queue the cancel behind +# the very round it is meant to stop. +_SERVING_ACTOR_CONCURRENCY: int = 2 + _VISIBLE_DEVICE_ENV_KEYS: tuple[str, ...] = ( "ROCR_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", @@ -218,6 +249,11 @@ class ServingActor: def __init__(self) -> None: self._mgr = ManagedServerProcess() + # The cancel scope of the round currently in flight, if any. The + # dispatcher's scope is a ContextVar in the submitter's process and + # cannot cross into this one, so the actor keeps its own and + # :meth:`cancel_round` is the wire between them. + self._round_scope: Any = None def start( self, @@ -274,24 +310,64 @@ def run_blocking( soft_deadline_sec=None, server_log_path=None, server_already_ready=False, + session_remaining_sec=None, ): - """Run one benchmark round to completion; return ``(rc, stdout, stderr)``.""" + """Run one benchmark round to completion; return ``(rc, stdout, stderr)``. + + ``session_remaining_sec`` is a duration, not the submitter's absolute + session deadline: this actor is a separate process with its own + ``time.monotonic()`` origin, so only a duration survives the trip. + + The round runs under a cancel scope published in this process, which + is what gives :meth:`cancel_round` something to raise: the reaper + inside ``run_with_session_kill`` then stops the tree and names the + stop exactly as it does on the local path. + """ import subprocess as _sp # noqa: PLC0415 + from ..cancel_channel import CancelScope, use_cancel_scope # noqa: PLC0415 from ._ray_backend import _run_subprocess_worker # noqa: PLC0415 + scope = CancelScope() + self._round_scope = scope try: - return _run_subprocess_worker( - cmd=cmd, - env=env, - cwd=cwd, - timeout_s=timeout, - soft_deadline_sec=soft_deadline_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - ) + with use_cancel_scope(scope): + return _run_subprocess_worker( + cmd=cmd, + env=env, + cwd=cwd, + timeout_s=timeout, + soft_deadline_sec=soft_deadline_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + session_remaining_sec=session_remaining_sec, + ) except _sp.TimeoutExpired as exc: return _ACTOR_TIMEOUT_RC, "", f"TimeoutExpired: {exc}" + finally: + self._round_scope = None + + def cancel_round(self, reason: str) -> bool: + """Ask the round in flight to stop itself; return whether there was one. + + Runs in a second method slot (see ``_SERVING_ACTOR_CONCURRENCY``) so + it is not queued behind the round it is cancelling. Returns as soon + as the flag is raised: the round is what decides how to stop, and the + submitter waits for it to come back. + + Args: + reason: Short cause from the canceller, carried into the stopped + round's message. + + Returns: + ``True`` when a round was asked to stop, ``False`` when the actor + was idle -- which the submitter reads as "nothing to wait for". + """ + scope = self._round_scope + if scope is None: + return False + scope.cancel(reason=reason) + return True def is_alive(self) -> bool: """Return whether the serving process is still up. @@ -332,10 +408,19 @@ def __ray_terminate__(self) -> None: # pragma: no cover - Ray teardown hook def make_serving_actor(num_gpus: float, *, serving_slot: bool = True): - """Create a ServingActor handle holding ``num_gpus`` (+ optional ``serving_slot``).""" + """Create a ServingActor handle holding ``num_gpus`` (+ optional ``serving_slot``). + + Given more than one method slot so ``cancel_round`` can reach a round that is + already running; with the default single slot it would wait for the round to + finish, which is the one thing a cancel cannot do. + """ actor_cls: Any = _serving_actor_body() resources = {"serving_slot": 1} if serving_slot else None - return actor_cls.options(num_gpus=num_gpus, resources=resources).remote() + return actor_cls.options( + num_gpus=num_gpus, + resources=resources, + max_concurrency=_SERVING_ACTOR_CONCURRENCY, + ).remote() def make_gpu_specialist_actor(num_gpus: float, *, serving_slot: bool = False): @@ -389,31 +474,86 @@ def run_session_kill( soft_deadline_sec: float | None = None, server_log_path: str | None = None, server_already_ready: bool = False, + session_remaining_sec: float | None = None, ) -> tuple[int, str, str]: """Run one benchmark round inside the lease's actor; return ``(rc, stdout, stderr)``. Drop-in for ``run_with_session_kill``; re-raises ``subprocess.TimeoutExpired`` on hard timeout. Cluster-ensure failures and Ray worker errors degrade to a benchmark failure (rc=1) rather than crashing the session. - """ - import subprocess as _sp # noqa: PLC0415 - import ray # noqa: PLC0415 + The cancel scope published by the dispatcher is watched for as long as + the round is in flight, the same as on the local path -- the difference + is that the round is in another process, so the scope cannot be read + there and the cancel is forwarded to the actor instead. + + Args: + cmd: The benchmark command to run inside the actor. + env: Caller env for the subprocess. + cwd: Working directory for the subprocess. + timeout: Hard timeout in seconds. + soft_deadline_sec: Overtime soft deadline. + server_log_path: Path to the server log for the watchdogs. + server_already_ready: Warm reuse round (soft clock from spawn). + session_remaining_sec: Seconds left on the session budget, as + produced by ``session_deadline_to_remaining_sec``. The absolute + deadline the local path uses cannot cross into the actor: it is + a ``time.monotonic()`` instant, and the actor's clock has its own + origin. The actor re-anchors this duration onto its own clock. + + Returns: + ``(returncode, stdout, stderr)`` from the round. + """ + from ..cancel_channel import cancel_scope_listener # noqa: PLC0415 try: self.ensure() except (RayInfeasibleError, RuntimeError) as exc: log.warning("ServingLease.run_session_kill: cluster ensure failed: %r", exc) return 1, "", f"ray_ensure_error: {exc}"[:2000] - ref = self._actor.run_blocking.remote( - cmd, - env=env, - cwd=cwd, - timeout=timeout, - soft_deadline_sec=soft_deadline_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - ) + # Registered before the round is submitted, so a cancel that arrives + # while Ray is still scheduling it is one this call is counted as able + # to hear -- the same window the local path opens around its spawn. + with cancel_scope_listener() as cancel_scope: + ref = self._actor.run_blocking.remote( + cmd, + env=env, + cwd=cwd, + timeout=timeout, + soft_deadline_sec=soft_deadline_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + session_remaining_sec=session_remaining_sec, + ) + return self._collect_round(ref, cmd=cmd, timeout=timeout, cancel_scope=cancel_scope) + + def _collect_round( + self, + ref: Any, + *, + cmd: list[str], + timeout: int | float | None, + cancel_scope: Any, + ) -> tuple[int, str, str]: + """Wait for a submitted round, forwarding a cancel to the actor if one comes. + + Args: + ref: The ``ObjectRef`` for the round in flight. + cmd: The round's command, for the ``TimeoutExpired`` it may raise. + timeout: The round's hard timeout, for the same reason. + cancel_scope: The scope to watch, or ``None`` when the caller is not + running under one, in which case this is a plain blocking wait. + + Returns: + ``(returncode, stdout, stderr)`` from the round. + + Raises: + subprocess.TimeoutExpired: When the actor reports a hard timeout. + """ + import subprocess as _sp # noqa: PLC0415 + + import ray # noqa: PLC0415 + # Resolve Ray's exception classes defensively. Real ray always exposes # both, but this is a failure hot-path: a partial test double or a future # ray rename must never turn a benchmark failure into an AttributeError @@ -423,7 +563,10 @@ def run_session_kill( _actor_err: Any = getattr(_ray_exc, "RayActorError", ()) if _ray_exc else () _task_err: Any = getattr(_ray_exc, "RayTaskError", ()) if _ray_exc else () try: - rc, out, err = ray.get(ref) + if cancel_scope is None: + rc, out, err = ray.get(ref) + else: + rc, out, err = self._await_or_cancel(ref, cancel_scope=cancel_scope) except _actor_err as exc: # type: ignore[misc] # The actor (worker) itself died — e.g. its server OOM-killed the # worker, or raylet reaped it. Drop the dead handle so the NEXT round @@ -450,8 +593,106 @@ def run_session_kill( raise _sp.TimeoutExpired(cmd, timeout or 0, output=out or None, stderr=err or None) return rc, out, err + def _await_or_cancel(self, ref: Any, *, cancel_scope: Any) -> tuple[int, str, str]: + """Block on a round, asking the actor to stop it if the scope is cancelled. + + The round attributes its own stop, exactly as the local path does, so + the sentinel this returns is the actor's whenever the actor answers. + Only a wedged actor -- one that has not come back within + ``CANCEL_ROUND_GRACE_SEC`` of being asked -- is killed, and only then + does the submitter attribute the stop on its behalf, because otherwise + an unattributed failure is what the ledger would read. + + Args: + ref: The ``ObjectRef`` for the round in flight. + cancel_scope: The scope this call is listening on. + + Returns: + ``(returncode, stdout, stderr)`` from the round, or an + ``ORCHESTRATOR_CANCELLED_RETURNCODE`` triple when the actor had to + be killed. + """ + import ray # noqa: PLC0415 + + from ._subprocess_kill import ORCHESTRATOR_CANCELLED_RETURNCODE # noqa: PLC0415 + + asked_at: float | None = None + while True: + ready, _ = ray.wait([ref], num_returns=1, timeout=_CANCEL_POLL_SEC) + if ready: + return ray.get(ref) + if asked_at is None: + if not cancel_scope.cancelled: + continue + reason = cancel_scope.reason or "orchestrator_cancelled" + asked_at = time.monotonic() + log.warning( + "ServingLease: asking the actor to stop the round in flight (%s)", + reason, + ) + if not self._ask_actor_to_cancel(reason): + # The actor never took the round, or cannot be reached to be + # told about it. Either way nothing in there will stop on its + # own, so go straight to the kill. + asked_at -= CANCEL_ROUND_GRACE_SEC + elif time.monotonic() - asked_at >= CANCEL_ROUND_GRACE_SEC: + log.warning( + "ServingLease: the actor did not return its cancelled round within %.0fs; " + "killing it to release the lease", + CANCEL_ROUND_GRACE_SEC, + ) + # Straight to the kill: an actor that has not answered is not + # going to answer a graceful stop either, and waiting for one + # would spend the rest of the window the caller is owed. + self._kill_actor() + return ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + "", + "the orchestrator cancelled this action; its Ray actor was killed after " + f"{CANCEL_ROUND_GRACE_SEC:.0f}s without returning the round", + ) + + def _ask_actor_to_cancel(self, reason: str) -> bool: + """Tell the actor to stop the round it is running. Never raises. + + Args: + reason: Short cause, carried into the stopped round's message. + + Returns: + ``True`` when the actor confirmed it had a round to stop. + """ + import ray # noqa: PLC0415 + + actor = self._actor + if actor is None: + return False + try: + return bool(ray.get(actor.cancel_round.remote(reason), timeout=_LEASE_PROBE_TIMEOUT_SEC)) + except Exception as exc: # noqa: BLE001 — an unreachable actor gets killed instead + log.warning("ServingLease: could not reach the actor to cancel its round: %r", exc) + return False + def close(self) -> None: - """Kill the actor, releasing the GPU lease. Idempotent, never raises.""" + """Release the GPU lease: stop the server, then kill the actor. Idempotent. + + The stop comes first because ``ray.kill`` skips ``__ray_terminate__``, + so the actor's own reaper never runs on that path; the served process is + deliberately in its own POSIX session, which is exactly what a + process-group teardown does not reach. Never raises, and the kill still + happens when the stop does not. + """ + if self._actor is None: + return + try: + import ray # noqa: PLC0415 + + ray.get(self._actor.stop.remote(), timeout=CLOSE_STOP_TIMEOUT_SEC) + except Exception as exc: # noqa: BLE001 — the kill below is the backstop + log.warning("ServingLease.close: the actor did not stop its server: %r", exc) + self._kill_actor() + + def _kill_actor(self) -> None: + """Kill the actor handle without waiting for it. Idempotent, never raises.""" if self._actor is None: return try: diff --git a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py index 32ce3342ad..6e2a38e1a1 100644 --- a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py +++ b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py @@ -24,7 +24,7 @@ import yaml -from ._subprocess_kill import _process_group_alive, _signal_group +from ._subprocess_kill import TERM_GRACE_SECONDS, _process_group_alive, _signal_group log = logging.getLogger(__name__) @@ -278,7 +278,7 @@ def teardown_lifecycle_server( # Server is setsid'd, so pgid == pid unless the pid file gave one. pgid = server_pgid if server_pgid is not None else server_pid _signal_group(pgid, signal.SIGTERM) - deadline = time.monotonic() + 5.0 + deadline = time.monotonic() + TERM_GRACE_SECONDS while time.monotonic() < deadline: if not _process_group_alive(pgid): break diff --git a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py index dbe72fe402..922203e82f 100644 --- a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py +++ b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Any +from ..stop_attribution import stopped_by_the_run_class from ._grid_runner import GridVariant, run_grid @@ -37,6 +38,11 @@ class StackRebenchResult: workspace: str | None warnings: list[str] = field(default_factory=list) stable_floor: float = 0.0 + # Set to the ledger class from :mod:`..stop_attribution` when the run itself + # stopped the round, empty otherwise. Lets a caller tell "the confirmation + # did not happen" apart from "the confirmation failed": :attr:`stable` is + # ``False`` for both, and only one of them is evidence about the variant. + error_class: str = "" @property def stable(self) -> bool: @@ -64,6 +70,8 @@ async def measure_stack_rebench( soft_deadline_sec: float | None = None, server_already_ready: bool = False, serving_lease: Any = None, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> StackRebenchResult: """Run ``variant`` once on the stack and grade it against the floor. @@ -76,6 +84,12 @@ async def measure_stack_rebench( execution, §12 T1) routes the round through the caller's held Ray lease so the rebench shares the same lease as the warmup/decision rounds it reuses the hot server from; ``None`` keeps the local path. + + ``session_deadline_sec`` bounds the rebench by the session wall-clock, so a + confirmation round cannot outlive the run it is confirming for. + A rebench dropped for lack of budget is reported as its own warning rather + than as a failed measurement: not measuring a variant is not evidence that + the variant is unstable, and the caller grades on the distinction. """ output_slot.mkdir(parents=True, exist_ok=True) results = await run_grid( @@ -95,21 +109,33 @@ async def measure_stack_rebench( soft_deadline_sec=soft_deadline_sec, server_already_ready=server_already_ready, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) rb = results[0] if results else None tput: float | None = None workspace: str | None = None warnings: list[str] = [] + error_class = "" if rb is not None and rb.status == "succeeded": tput = rb.output_throughput workspace = rb.workspace warnings = list(rb.nonfatal_warnings) + elif rb is not None and stopped_by_the_run_class(getattr(rb, "error_class", "")) is not None: + error_class = rb.error_class + warnings.append(f"stack_rebench_skipped:{error_class}") elif rb is not None: warnings.append(f"stack_rebench_failed:{(rb.error or '')[-120:]}") else: warnings.append("stack_rebench_no_result") stable_floor = base_tput * (1.0 + stable_threshold_pct / 100.0) - return StackRebenchResult(tput=tput, workspace=workspace, warnings=warnings, stable_floor=stable_floor) + return StackRebenchResult( + tput=tput, + workspace=workspace, + warnings=warnings, + stable_floor=stable_floor, + error_class=error_class, + ) __all__ = ["DEFAULT_STACK_STABLE_PCT", "StackRebenchResult", "measure_stack_rebench"] diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 90f1b71640..9f80e573d1 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -28,11 +28,40 @@ from .bypass_analysis import parse_server_log_throughput +from ..cancel_channel import CancelScope, cancel_scope_listener + log = logging.getLogger(__name__) -# Grace window between SIGTERM and SIGKILL. -_TERM_GRACE_SECONDS = 5.0 +# Grace window between SIGTERM and SIGKILL, here and for a driver-side teardown +# of a server a round left behind: the same signal, the same thing being waited +# for. +TERM_GRACE_SECONDS: float = 5.0 + +# How long the reaper waits to collect the SIGKILL'd child before giving up on it. +_REAP_COLLECT_SECONDS: float = 1.0 + +# How long draining a reaped child's capture threads is given. +_CAPTURE_DRAIN_SECONDS: float = 2.0 + +# How often the blocking side looks up from the child to check its stop gates -- +# the session deadline and the cancel scope among them. Short enough that the +# check costs a stop almost nothing on top of the reap, long enough not to spin. +STOP_GATE_POLL_SECONDS: float = 0.5 + +# What stopping a running round costs, end to end, from the moment something asks +# it to: noticing at the poll, SIGTERM'ing the tree, waiting out the grace before +# SIGKILL, collecting the child, and draining its pipes. +# +# Every window that waits for a round to stop itself is derived from this rather +# than picked next to it -- the Ray submitter's grace and the dispatcher's +# cooperative window both are. A window shorter than what stopping costs looks +# generous in isolation and still expires every time, and what it discards is the +# honest sentinel the round was about to return, replacing it with a hard kill +# and an unattributed failure. +COOPERATIVE_REAP_BUDGET_SEC: float = ( + STOP_GATE_POLL_SECONDS + TERM_GRACE_SECONDS + _REAP_COLLECT_SECONDS + _CAPTURE_DRAIN_SECONDS +) def new_session_kwargs() -> dict: @@ -97,7 +126,7 @@ def _signal_group(pgid: int, sig: int) -> None: def kill_my_spawned_server( proc: subprocess.Popen | None, *, - grace_seconds: float = _TERM_GRACE_SECONDS, + grace_seconds: float = TERM_GRACE_SECONDS, ) -> None: """Tear down the entire process tree rooted at ``proc``. @@ -169,12 +198,13 @@ def kill_my_spawned_server( _signal_group(pgid, signal.SIGKILL) try: - proc.wait(timeout=1.0) + proc.wait(timeout=_REAP_COLLECT_SECONDS) except subprocess.TimeoutExpired: log.warning( - "_subprocess_kill: proc.wait() did not return within 1s " + "_subprocess_kill: proc.wait() did not return within %.0fs " "after SIGKILL'ing pgid=%d (pid=%d). The reaper may be " "wedged; leaving the zombie for init to collect.", + _REAP_COLLECT_SECONDS, pgid, proc.pid, ) @@ -182,6 +212,13 @@ def kill_my_spawned_server( pass +# Sentinel ``returncode`` allocation. This module is not the only owner of the +# space -- ``_ray_serving`` hands out Ray-actor codes from it too, and both +# arrive at their consumer as a bare ``returncode`` carrying no other tag. A +# number claimed by a second cause therefore makes attribution a coin flip, so +# allocate an unused one; ``test_every_sentinel_returncode_names_exactly_one_cause`` +# enumerates both modules and fails on reuse. + # Sentinel ``returncode`` when ``run_with_session_kill`` reaps a child for an # elapsed ``soft_deadline_sec`` (vs the ``timeout=`` hard cap, which raises # ``TimeoutExpired``). Chosen not to collide with a real signal-based returncode. @@ -253,6 +290,25 @@ def kill_my_spawned_server( # already fails with, so a bounds gap reads identically on both arms. EVAL_PROBE_UNPATCHABLE_RETURNCODE: int = -914 +# Sentinel ``returncode`` when the session wall-clock budget ran out mid-round and +# the tree was reaped. Deliberately distinct from ``OVERTIME_KILL_RETURNCODE``: +# that one says "this variant ran far longer than the baseline", which is a +# judgement about the variant, while this one says "the run was out of time", +# which says nothing about the variant at all. Sharing a code would teach the KB +# that a variant is slow whenever a session happened to end during it. +SESSION_TIME_EXHAUSTED_RETURNCODE: int = -915 + +# -916 is ``_ray_serving._ACTOR_TIMEOUT_RC``. + +# Sentinel ``returncode`` when the orchestrator cancelled the action this child +# was launched for -- a shutdown, or a budget that is spent. Distinct from +# ``SESSION_TIME_EXHAUSTED_RETURNCODE`` because the two causes are told apart at +# the source: that one is the round's own deadline elapsing where it runs, this +# one is a decision taken outside it, and only one of them is still true when the +# same session resumes. Distinct from ``OVERTIME_KILL_RETURNCODE`` for the reason +# that one already carries: neither says anything about the variant. +ORCHESTRATOR_CANCELLED_RETURNCODE: int = -917 + # Server-ready markers: their appearance in ``server.log`` means the server has # finished startup and is accepting traffic. Only after one is observed does the # detokenizer-stall clock start. Covers the uvicorn frontend (vLLM + sglang) and @@ -533,6 +589,144 @@ def server_log_death_excerpt(path: str, *, max_chars: int = 1200) -> str | None: return None +# Name of the stamp written beside the caller's ``server.log`` the moment the +# server first reports ready. +_READY_STAMP_NAME = "server_ready_at" + + +def _ready_stamp_path(server_log_path: str) -> Path: + """Return where a round's ready stamp lives, given its ``server.log`` path.""" + return Path(server_log_path).parent / _READY_STAMP_NAME + + +def _stamp_server_ready(server_log_path: str, boot_sec: float) -> None: + """Record, beside ``server_log_path``, that the server just reported ready. + + Two numbers, because they answer two questions and one clock cannot answer + both. ``boot_sec`` is how long the round took to come up, measured from spawn + to this moment on one ``time.monotonic()`` reading in the process that + spawned the child. The wall-clock instant beside it only ever says *which + round* the stamp belongs to. + + Keeping the boot a duration is what makes it safe to read across a process + boundary. On the Ray path the round runs inside an actor, possibly on another + host; subtracting the actor's wall-clock from the driver's would charge the + boot for whatever the two clocks disagree by, and a positive disagreement + inflates the boot and makes the budget gates refuse rounds that fit. A + duration crosses the boundary meaning the same thing on both sides -- the + same reason ``session_remaining_sec`` is passed to the actor as a duration + rather than as a deadline. + + A file is used because it crosses that boundary without widening the round's + return value, and the round's output directory is already how post-mortem + evidence gets back (the caller reads the same directory's ``server.log`` to + classify server deaths). + + Best effort: a round whose stamp cannot be written loses a measurement, which + callers already have to handle, and must not lose the round. + + Args: + server_log_path: The ``/server.log`` path from the caller. + boot_sec: Seconds from spawn to this moment, on the spawning process's + monotonic clock. Required rather than defaulted: a caller that + omitted it would write a well-formed stamp claiming the round booted + instantly, which reads as a whole round of benchmark and is the one + wrong answer the two-field format exists to make impossible. + """ + try: + _ready_stamp_path(server_log_path).write_text( + f"{time.time():.3f} {max(0.0, float(boot_sec)):.3f}\n", + encoding="utf-8", + ) + except OSError as exc: + log.warning("_subprocess_kill: could not stamp server-ready time (%s)", exc) + + +def clear_server_ready_stamp(server_log_path: str) -> None: + """Drop any ready stamp an earlier round left in this output directory. + + Args: + server_log_path: The ``/server.log`` path from the caller. + """ + try: + _ready_stamp_path(server_log_path).unlink(missing_ok=True) + except OSError as exc: + log.warning("_subprocess_kill: could not clear stale server-ready stamp (%s)", exc) + + +def _read_ready_stamp(server_log_path: str) -> tuple[float, float] | None: + """Return a round's ``(ready_unix, boot_sec)``, or ``None`` when unrecorded. + + Both fields are required. A stamp missing its boot is reported as no stamp at + all rather than as a boot of zero: zero is a legitimate boot, so it cannot + also stand for "not recorded", and reading it that way would hand the whole + round to the benchmark -- the figure every later variant is then admitted on. + + Args: + server_log_path: The ``/server.log`` path from the caller. + + Returns: + tuple[float, float] | None: The wall-clock instant the stamp was written + and the boot it measured, or ``None`` when no readable stamp exists. + """ + try: + fields = _ready_stamp_path(server_log_path).read_text(encoding="utf-8").split() + ready_unix = float(fields[0]) + boot_sec = float(fields[1]) + except (OSError, ValueError, IndexError): + return None + return (ready_unix, max(0.0, boot_sec)) if ready_unix > 0.0 else None + + +def server_ready_unix(server_log_path: str) -> float | None: + """Return when the server reported ready, or ``None`` when nothing recorded it. + + Args: + server_log_path: The ``/server.log`` path from the caller. + + Returns: + float | None: The wall-clock instant, or ``None`` when no stamp exists or + it is unreadable. + """ + stamp = _read_ready_stamp(server_log_path) + return None if stamp is None else stamp[0] + + +def post_ready_runtime_sec( + server_log_path: str, + *, + started_unix: float, + runtime_sec: float, +) -> float | None: + """Return how long a round ran *after* its server was ready. + + This is the part of a round's wall-clock that is the benchmark itself, with + boot, weight load, compile and graph capture excluded. It is what makes two + rounds comparable when one paid for a cold start and the other re-attached to + a server already up. + + The round's wall-clock less the boot the stamp measured. The boot is a + duration taken on one clock, so this holds however far apart the writer and + the reader are; ``started_unix`` is only compared against the stamp's own + instant, to tell this round's stamp from one an earlier round left behind. + + Args: + server_log_path: The ``/server.log`` path from the caller. + started_unix: When the round was spawned. + runtime_sec: The round's full wall-clock. + + Returns: + float | None: Seconds after ready, bounded by the round's own runtime, or + ``None`` when the round never reported ready or the only stamp present + predates it (an earlier round's, left behind by a failed clear -- reading + it would report a cold round as though it had never booted). + """ + stamp = _read_ready_stamp(server_log_path) + if stamp is None or stamp[0] < started_unix: + return None + return max(0.0, min(float(runtime_sec), float(runtime_sec) - stamp[1])) + + def _resolve_scan_logs(server_log_path: str) -> list[str]: """Return the log files to scan for markers, newest-nesting first. @@ -586,6 +780,40 @@ class _LogScan(NamedTuple): child_spoke: bool +def _stale_scan_log_sizes(server_log_path: str) -> dict[str, int]: + """Current byte length of each nested log that already exists at spawn. + + Seeded as starting offsets so such a log contributes only what it grows by, + never what a previous attempt left in it. + + Only the nested ``benchmark_*/`` workspaces, not the path the caller named. + That one the caller owns: it clears it when it means this round to start + clean, and a caller that means to attach to a server already up says so with + ``server_already_ready``. The nested ones are found by globbing a directory + the caller reuses, so nothing in them was asserted by anyone -- and a stale + ready line there would otherwise latch on the first poll and report a boot + that took minutes as one that took none. + + Args: + server_log_path: The ``/server.log`` path from the caller. + + Returns: + dict[str, int]: Per-path byte lengths; a path whose size cannot be read + is left out, so it is scanned from the start as an absent one would be. + """ + owned_dir = Path(server_log_path).parent + sizes: dict[str, int] = {} + for path in _resolve_scan_logs(server_log_path): + candidate = Path(path) + if candidate.parent == owned_dir: + continue + try: + sizes[path] = candidate.stat().st_size + except OSError: + continue + return sizes + + def _scan_logs_increment(server_log_path: str, offsets: dict[str, int]) -> _LogScan: """Scan every resolved log for markers, advancing ``offsets`` in place. @@ -666,6 +894,51 @@ def _scan_server_log_increment(path: str, from_offset: int) -> tuple[int, bool, return size, saw_ready, saw_progress, saw_eval_start +def session_deadline_to_remaining_sec(session_deadline_sec: float | None) -> float | None: + """Convert an in-process session deadline into seconds still left on it. + + The pair of this and :func:`session_remaining_to_deadline_sec` is how a + session deadline crosses a process boundary. ``time.monotonic()`` has an + unspecified, per-process origin, so the absolute instant means nothing to a + reader in another process; a duration means the same thing everywhere. + + Args: + session_deadline_sec: Absolute ``time.monotonic()`` instant at which the + session budget expires, or ``None`` when the budget is unbounded. + + Returns: + Seconds left on the budget, or ``None`` when unbounded. Non-positive + when the deadline has already passed, which the receiving side is meant + to act on immediately rather than treat as "no budget given". + """ + if session_deadline_sec is None: + return None + return float(session_deadline_sec) - time.monotonic() + + +def session_remaining_to_deadline_sec(session_remaining_sec: float | None) -> float | None: + """Re-anchor a remaining session budget onto this process's monotonic clock. + + The inverse of :func:`session_deadline_to_remaining_sec`. Whatever the trip + itself cost is forgiven: no clock is shared across the boundary to measure it + with, so the receiver starts a fresh window of the full duration. The + receiver can therefore run marginally past the sender's deadline, which is + the safe direction -- the alternative is guessing at the transit and charging + a round for time it never had. + + Args: + session_remaining_sec: Seconds left on the session budget as measured by + the sender, or ``None`` when the budget is unbounded. + + Returns: + An absolute ``time.monotonic()`` deadline usable in this process, or + ``None`` when unbounded. + """ + if session_remaining_sec is None: + return None + return time.monotonic() + float(session_remaining_sec) + + def run_with_session_kill( cmd: list[str], *, @@ -679,13 +952,32 @@ def run_with_session_kill( detok_stall_grace_sec: float | None = None, server_already_ready: bool = False, on_output: Callable[[], None] | None = None, + session_deadline_sec: float | None = None, ) -> subprocess.CompletedProcess: """Run a subprocess in its own session and reap descendants on every exit path. + ``session_deadline_sec`` is an absolute ``time.monotonic()`` instant, unlike + ``soft_deadline_sec`` which is a relative duration. It is the session's own + wall-clock budget and is enforced in every phase, including the accuracy + eval -- the phase that retires ``soft_deadline_sec``. That distinction is the + reason it is a separate channel rather than a reuse of the soft deadline: the + eval-start boundary is meaningful for "is this variant abnormally slow" and + meaningless for "is the run out of time". + + The cancel scope published by the dispatcher (:mod:`..cancel_channel`) is + watched for as long as the child lives, so an orchestrator that cancels the + action reaches the tree rather than just the coroutine awaiting this call. + Every cause that reaps the tree is reported as its own sentinel + ``returncode``, which is all a caller of a subprocess gets to tell them apart + by. + Args: on_output: Called from a reader thread each time the child produces output, so a caller can report a long step alive on the child's own activity rather than on a timer. + session_deadline_sec: Absolute ``time.monotonic()`` instant at which the + session budget expires. Reaps the tree and returns + ``SESSION_TIME_EXHAUSTED_RETURNCODE``. """ if server_dead_grace_sec is None: try: @@ -710,105 +1002,147 @@ def run_with_session_kill( proc: subprocess.Popen | None = None capture: _StreamCapture | None = None try: - proc = subprocess.Popen( # noqa: S603 — cmd is caller's responsibility - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=text, - env=env, - cwd=cwd, - **new_session_kwargs(), - ) - capture = _StreamCapture(proc, text=text, on_output=on_output) - capture.start() - try: - stdout, stderr = _communicate_with_soft_deadline( - proc, - hard_timeout=timeout, - soft_deadline_sec=soft_deadline_sec, - server_log_path=server_log_path, - server_dead_grace_sec=server_dead_grace_sec, - detok_stall_grace_sec=detok_stall_grace_sec, - capture=capture, - server_already_ready=server_already_ready, - ) - except subprocess.TimeoutExpired: - kill_my_spawned_server(proc) - if capture is not None: - capture.finish(timeout=2.0) - raise - except _ServerDeadDetected as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.warning( - "_subprocess_kill: server-liveness watchdog reaped tree — " - "engine/worker init died but parent hung (marker=%r, " - "grace=%.1fs, elapsed=%.1fs); returncode=%d.", - exc.marker, - exc.grace_sec, - exc.elapsed_sec, - SERVER_DEAD_RETURNCODE, - ) - return subprocess.CompletedProcess( - args=cmd, - returncode=SERVER_DEAD_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) - except _ServerStalledDetected as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.warning( - "_subprocess_kill: detokenizer-stall watchdog reaped tree — " - "server reported ready but emitted no log output (grace=%.1fs, " - "elapsed=%.1fs); returncode=%d.", - exc.grace_sec, - exc.elapsed_sec, - DETOKENIZER_STALL_RETURNCODE, - ) - return subprocess.CompletedProcess( - args=cmd, - returncode=DETOKENIZER_STALL_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) - except _SoftDeadlineExceeded as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.info( - "_subprocess_kill: soft_deadline_sec=%.1fs exceeded " - "(elapsed=%.1fs); reaped tree with sentinel returncode=%d.", - exc.deadline_sec, - exc.elapsed_sec, - OVERTIME_KILL_RETURNCODE, + with cancel_scope_listener() as cancel_scope: + proc = subprocess.Popen( # noqa: S603 — cmd is caller's responsibility + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=text, + env=env, + cwd=cwd, + **new_session_kwargs(), ) + capture = _StreamCapture(proc, text=text, on_output=on_output) + capture.start() + try: + stdout, stderr = _communicate_with_soft_deadline( + proc, + hard_timeout=timeout, + soft_deadline_sec=soft_deadline_sec, + server_log_path=server_log_path, + server_dead_grace_sec=server_dead_grace_sec, + detok_stall_grace_sec=detok_stall_grace_sec, + capture=capture, + server_already_ready=server_already_ready, + session_deadline_sec=session_deadline_sec, + cancel_scope=cancel_scope, + ) + except subprocess.TimeoutExpired: + kill_my_spawned_server(proc) + if capture is not None: + capture.finish(timeout=_CAPTURE_DRAIN_SECONDS) + raise + except _ReapedByWatchdog as exc: + kill_my_spawned_server(proc) + stdout, stderr = _finish_capture(capture, text=text) + log.log( + exc.log_level, + "_subprocess_kill: %s; reaped the tree with sentinel returncode=%d.", + exc, + exc.returncode, + ) + return subprocess.CompletedProcess( + args=cmd, + returncode=exc.returncode, + stdout=stdout, + stderr=stderr, + ) + empty: str | bytes = "" if text else b"" return subprocess.CompletedProcess( args=cmd, - returncode=OVERTIME_KILL_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), + returncode=proc.returncode, + stdout=stdout if stdout is not None else empty, + stderr=stderr if stderr is not None else empty, ) - return subprocess.CompletedProcess( - args=cmd, - returncode=proc.returncode, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) finally: kill_my_spawned_server(proc) -class _SoftDeadlineExceeded(Exception): - """Internal sentinel for an elapsed soft deadline. Never bubbles - past :func:`run_with_session_kill` (converted to a ``CompletedProcess``). +def _finish_capture(capture: _StreamCapture | None, *, text: bool) -> tuple[str | bytes, str | bytes]: + """Drain the capture threads of a reaped child, never returning ``None``. + + Args: + capture: The stream capture to finish, or ``None`` when the child's + output was not captured. + text: Whether the streams are ``str`` (``True``) or ``bytes``, which + decides what an absent stream reads as. + + Returns: + tuple[str | bytes, str | bytes]: The captured ``(stdout, stderr)``. + """ + empty: str | bytes = "" if text else b"" + if capture is None: + return empty, empty + stdout, stderr = capture.finish(timeout=_CAPTURE_DRAIN_SECONDS) + return ( + stdout if stdout is not None else empty, + stderr if stderr is not None else empty, + ) + + +class _ReapedByWatchdog(Exception): + """Internal base for a cause that reaps the tree and names itself. + + None of these bubble past :func:`run_with_session_kill`: each is converted + to a ``CompletedProcess`` carrying the subclass's own ``returncode``, since + a returncode is the only thing that survives the trip back to a caller of a + subprocess. Subclasses build their own message and declare how much the + event is worth logging. """ + returncode: int = -1 + log_level: int = logging.WARNING + + +class _SessionDeadlineExceeded(_ReapedByWatchdog): + """Internal sentinel: the session wall-clock budget ran out mid-round.""" + + returncode = SESSION_TIME_EXHAUSTED_RETURNCODE + + def __init__(self, *, overrun_sec: float, elapsed_sec: float) -> None: + """Record how far past the session deadline the round got. + + Args: + overrun_sec (float): Seconds past the session deadline at trip time. + elapsed_sec (float): Wall-clock elapsed for this round at trip time. + """ + super().__init__( + f"the session wall-clock budget was exhausted {overrun_sec:.1f}s ago (round elapsed={elapsed_sec:.1f}s)" + ) + self.overrun_sec = float(overrun_sec) + self.elapsed_sec = float(elapsed_sec) + + +class _OrchestratorCancelled(_ReapedByWatchdog): + """Internal sentinel: the orchestrator cancelled the action this child serves. + + The cause is a decision taken outside the round -- a shutdown, or a budget + the dispatcher found spent -- which is why it carries the caller's reason + rather than a measurement of its own. + """ + + returncode = ORCHESTRATOR_CANCELLED_RETURNCODE + + def __init__(self, *, reason: str, elapsed_sec: float) -> None: + """Record who asked for the stop and how far the round had got. + + Args: + reason (str): Short cause from the canceller, e.g. ``shutdown_requested``. + elapsed_sec (float): Wall-clock elapsed for this round at trip time. + """ + super().__init__( + f"the orchestrator cancelled this action ({reason or 'no reason given'}; round elapsed={elapsed_sec:.1f}s)" + ) + self.reason = str(reason) + self.elapsed_sec = float(elapsed_sec) + + +class _SoftDeadlineExceeded(_ReapedByWatchdog): + """Internal sentinel for an elapsed soft deadline.""" + + returncode = OVERTIME_KILL_RETURNCODE + log_level = logging.INFO + def __init__(self, *, deadline_sec: float, elapsed_sec: float) -> None: """Record the deadline and actual elapsed time on the sentinel. @@ -816,18 +1150,18 @@ def __init__(self, *, deadline_sec: float, elapsed_sec: float) -> None: deadline_sec (float): The soft deadline that was exceeded. elapsed_sec (float): The actual wall-clock elapsed at trip time. """ - super().__init__(f"soft deadline {deadline_sec:.1f}s elapsed (actual={elapsed_sec:.1f}s)") + super().__init__(f"soft_deadline_sec={deadline_sec:.1f}s elapsed (actual={elapsed_sec:.1f}s)") self.deadline_sec = float(deadline_sec) self.elapsed_sec = float(elapsed_sec) -class _ServerDeadDetected(Exception): +class _ServerDeadDetected(_ReapedByWatchdog): """Internal sentinel: the server-liveness watchdog saw a terminal engine / - worker init marker that persisted past the grace window. Never bubbles past - :func:`run_with_session_kill` (converted to a ``CompletedProcess`` carrying - ``SERVER_DEAD_RETURNCODE``). + worker init marker that persisted past the grace window. """ + returncode = SERVER_DEAD_RETURNCODE + def __init__( self, *, @@ -843,7 +1177,7 @@ def __init__( elapsed_sec: Actual wall-clock elapsed at trip time. """ super().__init__( - f"server init died (marker={marker!r}) and parent hung past " + f"server init died (marker={marker!r}) and the parent hung past " f"grace {grace_sec:.1f}s (elapsed={elapsed_sec:.1f}s)" ) self.marker = marker @@ -851,13 +1185,13 @@ def __init__( self.elapsed_sec = float(elapsed_sec) -class _ServerStalledDetected(Exception): +class _ServerStalledDetected(_ReapedByWatchdog): """Internal sentinel: the detokenizer-stall watchdog saw the server report - ready and then produce no generation progress for the grace window. Never - bubbles past :func:`run_with_session_kill` (converted to a - ``CompletedProcess`` carrying ``DETOKENIZER_STALL_RETURNCODE``). + ready and then produce no generation progress for the grace window. """ + returncode = DETOKENIZER_STALL_RETURNCODE + def __init__( self, *, @@ -890,6 +1224,8 @@ def _communicate_with_soft_deadline( detok_stall_grace_sec: float | None = None, capture: _StreamCapture | None = None, server_already_ready: bool = False, + session_deadline_sec: float | None = None, + cancel_scope: CancelScope | None = None, ) -> tuple[str | bytes, str | bytes]: """Communicate with a child while enforcing soft and server-log watchdogs.""" watchdog_active = bool(server_log_path) and ( @@ -897,9 +1233,14 @@ def _communicate_with_soft_deadline( ) stall_active = bool(server_log_path) and (detok_stall_grace_sec is not None and float(detok_stall_grace_sec) > 0.0) soft_active = soft_deadline_sec is not None and float(soft_deadline_sec) > 0.0 - if capture is None and not soft_active and not watchdog_active and not stall_active: + session_active = session_deadline_sec is not None + # A cancel scope is polled like any other gate, so its presence rules out the + # single-wait fast paths below: a call that blocks until the child exits + # cannot notice a cancel that arrives while it is blocked. + gated = soft_active or watchdog_active or stall_active or session_active or cancel_scope is not None + if capture is None and not gated: return proc.communicate(timeout=hard_timeout) - if capture is not None and not soft_active and not watchdog_active and not stall_active: + if capture is not None and not gated: proc.wait(timeout=hard_timeout) return capture.finish() @@ -921,13 +1262,15 @@ def _communicate_with_soft_deadline( not in {"0", "false", "no", "off"} ) # The log increment scan feeds the stall watchdog, the from-ready - # soft-deadline anchor and the eval-start boundary; run it once per slice - # when any of them needs it. Broader than ``soft_from_ready``: a warm-reuse - # round (``server_already_ready``) still needs the eval-start boundary, so - # any soft deadline with a log present scans. - soft_watches_log = soft_active and bool(server_log_path) - scan_active = stall_active or soft_watches_log - poll_interval = 0.5 + # soft-deadline anchor, the eval-start boundary and the ready timestamp the + # caller prices later work off; run it once per slice whenever a log is + # present. Not narrowed to the watchdogs that consume it: tying it to + # ``stall_active`` let an unrelated knob + # (``INFERENCE_OPTIMIZER_DETOK_STALL_GRACE_SEC=0``) silently withdraw the + # boot/benchmark split for a whole session. It costs nothing where it did not + # already run, since without a gate this call never enters the loop at all. + scan_active = bool(server_log_path) and gated + poll_interval = STOP_GATE_POLL_SECONDS start = time.monotonic() dead_marker_since: float | None = None # Detokenizer-stall watchdog state: per-log byte offsets consumed so far, @@ -935,6 +1278,12 @@ def _communicate_with_soft_deadline( # new output (seeded to the ready time). The gate only arms once # ``server_ready_since`` is set. scan_offsets: dict[str, int] = {} + if scan_active: + # A reused output_dir can still hold a prior attempt's nested workspace, + # whose markers are not this round's. The workspace this round creates + # does not exist yet, so it is discovered later at offset zero, which is + # correct: all of its bytes are this round's. + scan_offsets.update(_stale_scan_log_sizes(server_log_path)) # type: ignore[arg-type] server_ready_since: float | None = None last_activity_at: float | None = None # Latched once the accuracy eval starts: the soft deadline bounds the @@ -943,6 +1292,25 @@ def _communicate_with_soft_deadline( while True: now = time.monotonic() elapsed = now - start + # Session budget. Checked before every other gate and never suspended: + # unlike the soft deadline it makes no claim about the variant, so the + # eval-start boundary that retires the soft deadline does not apply. An + # accuracy eval that starts one minute before the run is out of time still + # has to stop. + if session_active and session_deadline_sec is not None and now >= session_deadline_sec: + raise _SessionDeadlineExceeded( + overrun_sec=now - float(session_deadline_sec), + elapsed_sec=elapsed, + ) + # Orchestrator cancellation. Checked after the session deadline so a + # round that was already out of time keeps that reason: the budget is a + # fact about the run, while the cancel is only the dispatcher acting on + # it, and the more specific of the two is the one worth recording. + if cancel_scope is not None and cancel_scope.cancelled: + raise _OrchestratorCancelled( + reason=cancel_scope.reason, + elapsed_sec=elapsed, + ) # Advance the log scan, latching the server-ready, last-activity # and eval-start signals. if scan_active: @@ -953,6 +1321,12 @@ def _communicate_with_soft_deadline( if scan.saw_ready and server_ready_since is None: server_ready_since = now last_activity_at = now # start the silence clock at ready + # Recorded for the caller, which prices later work off the + # post-ready segment rather than the whole round: a pass that + # re-attaches to this server pays none of the boot. Taken as + # ``now - start`` so the boot is measured end to end on this + # process's own clock, whatever host the caller reads it on. + _stamp_server_ready(server_log_path, now - start) # type: ignore[arg-type] if scan.saw_eval_start and not soft_deadline_suspended: soft_deadline_suspended = True log.info( @@ -1016,6 +1390,8 @@ def _communicate_with_soft_deadline( # Slice bounded by every active remaining window so the right gate # fires first; the child can still finish inside any slice. slice_sec = poll_interval + if session_active and session_deadline_sec is not None: + slice_sec = min(slice_sec, float(session_deadline_sec) - now) if soft_active and deadline_sec is not None and not soft_deadline_suspended: if soft_from_ready: if server_ready_since is not None: @@ -1039,12 +1415,22 @@ def _communicate_with_soft_deadline( __all__ = [ "AGENTX_PREFLIGHT_RETURNCODE", + "COOPERATIVE_REAP_BUDGET_SEC", "DETOKENIZER_STALL_RETURNCODE", "EVAL_PROBE_UNPATCHABLE_RETURNCODE", + "ORCHESTRATOR_CANCELLED_RETURNCODE", "OVERTIME_KILL_RETURNCODE", "SERVER_DEAD_RETURNCODE", + "SESSION_TIME_EXHAUSTED_RETURNCODE", + "STOP_GATE_POLL_SECONDS", + "TERM_GRACE_SECONDS", + "clear_server_ready_stamp", "kill_my_spawned_server", "new_session_kwargs", + "post_ready_runtime_sec", "run_with_session_kill", "server_log_death_excerpt", + "server_ready_unix", + "session_deadline_to_remaining_sec", + "session_remaining_to_deadline_sec", ] diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index a48de07d13..ba469f752d 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -36,6 +36,12 @@ from ...framework.paths import resolve_session_framework_root from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import heartbeat_while_output_flows, report_progress +from ...phases import machine_state as _phase_state +from ..stop_attribution import ( + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) from . import _server_lifecycle as _lifecycle from ._file_lock import best_effort_file_lock from ._aiter_jit import ( @@ -45,16 +51,28 @@ probe_aiter_jit_cache as _probe_aiter_jit_cache, sweep_stale_aiter_locks_if_dead, ) +# The grid module is the namespace the helpers both benching arms share ended up +# in: how a sentinel returncode reads back, how a round's cap is clamped to the +# budget, how the two session bounds are resolved, and the hygiene every launch +# needs. Imported rather than restated here so a baseline round and a grid round +# are priced the same way. The returncode decoder's class-side sibling is in +# ``..stop_attribution``, which says why the two sides sit where they do. from ._grid_runner import ( _kill_stale_servers, sanitize_result_dir, sanitize_script_name, + session_clamped_timeout_sec, + session_grid_bounds, + stopped_by_the_run, ) from ._subprocess_kill import ( DETOKENIZER_STALL_RETURNCODE, SERVER_DEAD_RETURNCODE, + clear_server_ready_stamp, + post_ready_runtime_sec, run_with_session_kill, server_log_death_excerpt, + session_deadline_to_remaining_sec, ) from ._accuracy_gate import ( _RUN_EVAL_FALSE_VALUES, @@ -96,6 +114,15 @@ ) # Bounded per-file read so log scanning never slurps a multi-GB server.log. _LOG_SCAN_MAX_BYTES = 262_144 +# The measured pass ran as the first traffic against a freshly restarted server, +# because the warmup that exists to drive it did not. Its throughput is a cold +# number, and the session anchors every later gain on it. +_MN_WARMUP_DID_NOT_WARM_WARNING = "baseline_mn_warmup_did_not_run" +# The round kept its warmup pass as the baseline because the budget could not +# pay for the measured pass after it. Same consequence as the warning above -- +# a cold anchor -- reached from the other direction, and carried on the result +# so a reader of the session's gains knows the denominator is depressed. +MEASURE_ROUND_DROPPED_WARNING = "baseline_measure_round_dropped_low_budget" # The cold-start guard's two round directories. The warmup round is the only one # that measures accuracy (``RUN_EVAL=true``); the measured round is hot @@ -264,6 +291,240 @@ def _watchdog_server_log_path(output_dir: Path, framework: str) -> str | None: return str(output_dir / "server.log") +def _round_post_ready_sec( + server_log_path: str | None, + *, + started_unix: float, + runtime_sec: float, +) -> float | None: + """How much of a round's wall-clock was the benchmark rather than the boot. + + Splitting the two is what lets later work be priced on what it will actually + cost: an explore variant boots its own server and pays both, while a pass + that re-attaches to a server already up pays only the second. + + Args: + server_log_path: The round's ``server.log``, or ``None`` for a scriptable + framework, which runs no server and therefore has no split to make. + started_unix: When the round was spawned. + runtime_sec: The round's full wall-clock. + + Returns: + float | None: Seconds after the server reported ready, or ``None`` when + the round has no such boundary or nothing recorded one. + """ + if not server_log_path: + return None + return post_ready_runtime_sec( + server_log_path, + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + + +def _logged_session_clamp(timeout_sec: int, clamped: int, *, output_dir: Path) -> int: + """Announce a round's cap being cut by the session budget, and return the cut. + + Every pass of a baseline round derives its cap from the same budget but on its + own terms, so the line names the round it belongs to. + + Args: + timeout_sec: The cap the round would have had on an unbounded budget. + clamped: The cap the budget leaves it; equal to ``timeout_sec`` when the + budget was never the binding constraint, which logs nothing. + output_dir: The round's workspace, whose name identifies the pass. + + Returns: + int: ``clamped``, unchanged. + """ + if clamped != timeout_sec: + log.info( + "baseline_executor: timeout clamped %ds -> %ds by the session budget (round=%s)", + timeout_sec, + clamped, + output_dir.name, + ) + return clamped + + +def _stopped_round_result( + stopped: StoppedByTheRun, + *, + round_label: str, + returncode: int | None, + runtime_sec: float, + output_dir: Path, + capture_meta: dict[str, Any], + started: bool = True, +) -> dict[str, Any]: + """Build the result for a round the run itself stopped. + + The session budget elapsed mid-round, or the orchestrator cancelled the + action. Classified apart from every measurement failure -- and checked before + them, because the reap leaves exactly the evidence a broken server does (no + workspace, no report, a non-zero returncode) and being graded as + ``server_init_dead`` or ``subprocess_nonzero`` would put a verdict on the + model that this round never reached. Every round the baseline runs goes + through here, the discarded multi-node warmup pass included: a stop in the + round that warms the server means the round is over, and going on to the + measured pass would spend GPU time the run has been told to stop spending. + Nothing here arms a retry either: the cause is the run, and a resume meets it + again. + + A round the budget stopped *before* it booted anything is the same cause and + carries the same class; only the wording and the absent returncode differ, so + a reader is told whether GPU time was spent. + + Args: + stopped: How to record the cause. + round_label: Which round was stopped, for the log line. + returncode: The stopped round's returncode, or ``None`` when nothing ran. + runtime_sec: Wall-clock seconds the round had run for. + output_dir: The task workspace, echoed onto the result. + capture_meta: Config/eval-contract facts every failure result carries. + started: Whether the round had begun. ``False`` selects the wording for + work that never launched. + + Returns: + dict[str, Any]: The failed result carrying the stop's own error class. + """ + detail = stopped.interrupted if started else stopped.never_started + if started: + log.warning( + "baseline_executor: %s reaped after %.1fs: %s; error_class=%s.", + round_label, + runtime_sec, + detail, + stopped.error_class, + ) + else: + log.warning( + "baseline_executor: %s not launched: %s; error_class=%s.", + round_label, + detail, + stopped.error_class, + ) + return { + "status": "failed", + "error_class": stopped.error_class, + "returncode": returncode, + "error": detail, + "subprocess_runtime_sec": round(runtime_sec, 2), + "output_dir": str(output_dir), + **capture_meta, + } + + +def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple[float | None, dict[str, Any]]: + """Seconds this round's budget may still spend, and the numbers behind it. + + The session's own usable remainder, which is what every other admission + decision reads, so a round cannot be judged against a figure the rest of the + run disagrees with. Not a share of it: the share held back for the + optimization phases + (:func:`~...phases.machine_state.prelude_affordable_seconds`) sizes the + *optional* arms of preparation, where the question is proportion. A round is + a feasibility question, and answering it with a percentage refuses rounds + that fit -- a session with ninety minutes left and a twelve-minute pass to + run is told it has six. + + Falls back to the session deadline for a caller with no session state at all, + and to no bound when there is neither. + + Args: + state: The session ``SharedState``, or ``None`` when the caller has no + session context (direct executor invocation, tests). + session_deadline_sec: Monotonic-clock session deadline, or ``None``. + + Returns: + tuple[float | None, dict[str, Any]]: The headroom, or ``None`` when the + round is under no budget at all, plus the evidence behind it. + """ + if state is not None: + usable_sec = _phase_state.session_usable_seconds(state) + if usable_sec is not None: + return usable_sec, {"bound": "session_usable", "affordable_sec": round(usable_sec, 1)} + outside: dict[str, Any] = {"reason": "unbounded_session_budget"} + else: + outside = {"reason": "no_session_state"} + if session_deadline_sec is None: + return None, outside + remaining_sec = max(0.0, session_deadline_sec - time.monotonic()) + return remaining_sec, {**outside, "bound": "session_deadline", "affordable_sec": round(remaining_sec, 1)} + + +def _cold_anchor_from_warmup( + warmup_result: dict[str, Any], + *, + dropped: dict[str, Any], +) -> dict[str, Any]: + """Keep the warmup's cold figure as the anchor, marked as the cold one. + + Two things end a round once its warmup has already paid for the boot, the + compile and the capture: the budget cannot cover a second pass, and the + session's budget reaped the second pass mid-flight. Either way a number + exists and the GPU time behind it is spent, so it is kept rather than + discarded -- a marked cold anchor beats no anchor. The marker is what tells a + reader of the session's later gains that their denominator is depressed. + + Args: + warmup_result: The succeeded warmup pass's result, mutated in place. + dropped: Why the measured round did not produce the figure, recorded so + the decision is legible in the result and the session record. + + Returns: + dict[str, Any]: ``warmup_result``, marked. + """ + warnings = warmup_result.setdefault("nonfatal_warnings", []) + if MEASURE_ROUND_DROPPED_WARNING not in warnings: + warnings.append(MEASURE_ROUND_DROPPED_WARNING) + warmup_result["measure_round_dropped"] = dropped + return warmup_result + + +def _a_use_must_follow_the_round(state: Any) -> bool: + """Whether this round is only worth running if something can be measured after it. + + A PRELUDE baseline is not a result. It is the denominator later results are + read against and the anchor their overtime kill uses, so a session that + cannot afford one variant after it would spend the wall-clock on a number + nothing ever reads. + + A re-baseline in a later phase is the opposite: it re-measures the stack the + session has assembled, and that measurement is the deliverable. Requiring a + successor would refuse exactly the round that validates the run's own answer, + at the point in the budget where it is most likely to be the last thing left. + + Args: + state: The session ``SharedState``, or ``None``. + + Returns: + bool: ``True`` while the round's worth depends on a successor. + """ + phase = str(getattr(state, "phase", "") or "").strip().upper() + return phase == _phase_state.PHASE_PRELUDE + + +def _positive_seconds(value: Any) -> float | None: + """Coerce a duration a round reported to seconds, or ``None`` when it did not. + + Absent, unparseable and zero are one answer: nothing was measured. None of + them may read as work that took no time, which is what a plain + ``float(... or 0.0)`` would make of them. + + Args: + value: The reported duration, from a round's result. + + Returns: + float | None: The seconds, or ``None`` when there is no measurement. + """ + try: + seconds = float(value or 0.0) + except (TypeError, ValueError): + return None + return seconds if seconds > 0.0 else None + + def _disable_cuda_graph_flag(framework: str) -> str: """Return the framework-correct flag that disables cuda-graph capture. @@ -1624,6 +1885,10 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: ``INFERENCE_OPTIMIZER_COLD_START_TIMEOUT_SEC``) → warm default. Every path emits one log line for greppability. + All three are hang backstops sized in hours, with no reference to the + session budget; :meth:`_session_capped_timeout` reduces the result to + what is actually left, per round, at the point each round launches. + Args: params: Task params; an explicit ``timeout_sec`` overrides the probe-based selection. @@ -1700,6 +1965,43 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: ) return self.default_timeout_sec + @staticmethod + def _session_capped_timeout( + timeout_sec: int, + session_deadline_sec: float | None, + *, + output_dir: Path, + ) -> int: + """``timeout_sec`` reduced to what the session can still pay for. + + The baseline's own timeout is a catastrophic-hang backstop -- two hours + by default, four for a cold start -- chosen with no reference to how much + of the session is left. A round granted more than the budget has runs + past the end of the session and takes the closing phase with it. + + Nothing is held back here, and no pass of a baseline round holds anything + back either. That is what keeps every cap sitting past the session + deadline, so the watchdog reaches a round before the round's own timeout + does and the kill is attributed to the budget rather than to the model. + Whether a round should start, and whether its measured pass should follow + its warmup, are decided by the gates that price those questions -- not by + shortening a cap until the round dies of it. + + Args: + timeout_sec: The timeout this round would get on an unbounded budget. + session_deadline_sec: Monotonic-clock session deadline, or ``None`` + when there is no budget to respect. + output_dir: The round's workspace, for the log line. + + Returns: + int: The hard timeout to grant this round, in seconds. + """ + return _logged_session_clamp( + timeout_sec, + session_clamped_timeout_sec(timeout_sec, session_deadline_sec), + output_dir=output_dir, + ) + @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: """Resolve the InferenceX checkout the subprocess will ``cd`` into. @@ -2763,6 +3065,50 @@ async def _run_once( materialized_config_path ) + # Asked before the lease, because a round that will not be run should not + # hold a GPU while being refused. + ignitable, ignition_evidence = self._round_affordable_before_ignition( + double_run=double_run, + ctx_extra=extra, + ) + if not ignitable: + stopped_result = _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + round_label="baseline round", + returncode=None, + runtime_sec=0.0, + output_dir=output_dir, + capture_meta={ + "materialized_config": str(materialized_config_path), + "run_eval_disabled": bool(run_eval_disabled), + }, + started=False, + ) + stopped_result["budget_shortfall"] = ignition_evidence + if ignition_evidence.get("one_more_measurement_sec"): + log.warning( + "baseline_executor: this round (%.0fs) and one variant to read " + "against it (%.0fs) need %.0fs, and %.0fs is left (bound=%s), so " + "nothing is booted. A baseline no variant can follow is a " + "denominator with no numerator; the anchor this session already " + "measured stands.", + ignition_evidence.get("round_sec", 0.0), + ignition_evidence.get("one_more_measurement_sec", 0.0), + ignition_evidence.get("expected_cost_sec", 0.0), + ignition_evidence.get("affordable_sec", 0.0), + ignition_evidence.get("bound", ""), + ) + else: + log.warning( + "baseline_executor: this round needs %.0fs and only %.0fs is left " + "(bound=%s), so nothing is booted. The anchor this session already " + "measured stands.", + ignition_evidence.get("expected_cost_sec", 0.0), + ignition_evidence.get("affordable_sec", 0.0), + ignition_evidence.get("bound", ""), + ) + return stopped_result + before_apply_sha = _git_head_sha(patch_target) def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if live_shared_state is None: @@ -2902,7 +3248,6 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: len(applied_patches), [p["patch_file"] for p in applied_patches], ) - # Ray-managed GPU execution (§12 T1): one held Ray lease (``num_gpus=TP``) # spans this baseline's benchmark rounds — a double-run's warmup + # measure reuse one persistent server, so both must run under the same @@ -2992,6 +3337,11 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: "baseline_executor: cold-start guard — warmup round (discarded, boots persistent server) in %s", warmup_dir, ) + # The warmup runs under the round's own cap, which the session clamp + # leaves sitting past the session deadline so the watchdog reaches it + # first and a budget kill is recorded as one. Whether the measured + # round can follow is asked after this pass, priced with what it + # actually cost rather than a prediction of what it would. warmup_result = await self._run_reported_round( label="warmup", config_path=warmup_cfg, @@ -3025,6 +3375,7 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: return warmup_result warmup_tput = warmup_result.get("output_throughput") warmup_runtime = warmup_result.get("subprocess_runtime_sec") + warmup_post_ready = warmup_result.get("post_ready_runtime_sec") await report_progress( unit="baseline_round", label="warmup", @@ -3035,6 +3386,41 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: runtime_sec=warmup_runtime, ) + if not defer_accuracy_until_after_measure: + affordable, gate_evidence = self._measure_round_affordable( + warmup_runtime_sec=warmup_runtime, + warmup_post_ready_sec=warmup_post_ready, + ctx_extra=extra, + ) + if not affordable: + if gate_evidence.get("one_more_measurement_sec"): + why = ( + "a hot pass (%.0fs) and one variant to read against it " + "(%.0fs) need %.0fs" % ( + gate_evidence.get("measure_round_sec", 0.0), + gate_evidence.get("one_more_measurement_sec", 0.0), + gate_evidence.get("expected_cost_sec", 0.0), + ) + ) + else: + why = "a hot pass needs %.0fs" % ( + gate_evidence.get("expected_cost_sec", 0.0), + ) + log.warning( + "baseline_executor: %s, and %.0fs is left (bound=%s), so the hot " + "pass is not run. It would have bought a denominator nothing " + "could then be compared to, and its own overtime anchor would " + "have gone unused. Keeping the warmup as the baseline; it is the " + "cold anchor a single-round baseline would have produced, and " + "the GPU time it cost is already spent. The marker below says " + "the figure is cold so the session's later gains can be read " + "against a known-depressed denominator.", + why, + gate_evidence.get("affordable_sec", 0.0), + gate_evidence.get("bound", ""), + ) + return _cold_anchor_from_warmup(warmup_result, dropped=gate_evidence) + # Round 2 (measured): re-attach to the hot server (client only). # Warm re-attach is intentional — all comparison points (baseline, # explore decision, stack_rebench, and their grading anchor) are @@ -3079,6 +3465,31 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: result["warm_kernel_apply_results"] = list( params.get("warm_kernel_apply_results") or [] ) + if ( + result.get("status") != "succeeded" + and result.get("error_class") == SESSION_TIME_EXHAUSTED_CLASS + ): + # The gate before this pass admitted it and the run's clock took + # it anyway -- the pass overran what it was priced at. Reporting + # the round as failed would throw away the warmup's figure too, + # leaving the session with nothing from GPU time it has already + # spent, so the warmup is kept and marked exactly as a refusal + # keeps it. Only a reap is handled this way: a pass that failed + # for a reason of its own is a failure worth surfacing, and the + # warmup having succeeded does not make it comparable. + log.warning( + "baseline_executor: the run's clock stopped the measured " + "round mid-flight, so the warmup stands as the baseline. It " + "is the cold anchor a single-round baseline would have " + "produced, and the marker below says so.", + ) + return _cold_anchor_from_warmup( + warmup_result, + dropped={ + "reason": "measure_round_reaped_by_the_run", + "measure_round_error": result.get("error"), + }, + ) if result.get("status") == "succeeded": result.setdefault("nonfatal_warnings", []) result["nonfatal_warnings"].append( @@ -3097,6 +3508,14 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: float(warmup_runtime), 2, ) + # The split belongs to round 1 for the same reason its + # wall-clock does: round 2 re-attached, so it has no boot to + # separate and its own reading says nothing about what booting + # this workload costs. Assigned with that wall-clock and not + # beside it, so the total and the part of it reported here can + # never come from different rounds -- their difference is + # published as this workload's boot. + result["post_ready_runtime_sec"] = warmup_result.get("post_ready_runtime_sec") _hot = result.get("output_throughput") or 0.0 _cold = warmup_tput or 0.0 log.info( @@ -3201,6 +3620,185 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() + def _round_affordable_before_ignition( + self, + *, + double_run: bool, + ctx_extra: dict[str, Any] | None = None, + ) -> tuple[bool, dict[str, Any]]: + """Whether the budget holds a whole round *and a use for it*, before anything boots. + + The companion to :meth:`_measure_round_affordable`, and the two divide + the session's baselines between them. A session's first round is not + asked this question, because there is nothing to ask it with: the + measured runtimes are written only once an anchor lands, so a gate here + would either refuse every first baseline or wave every one through. That + round runs, and whether its second pass may follow is settled afterwards + against what the first actually cost. + + Every later round is asked, and a resumed session's first round is the + case that most needs it: it holds the earlier leg's measurements, so a + round that cannot lead anywhere can be refused before a single second of + GPU time is spent on it, and the refusal names a number an operator can + act on. + + What is required in PRELUDE is the round *plus one further measured + variant*, not the round alone. A baseline is not a result there; it is the + denominator later results are read against, and one that nothing is ever + compared to is wall-clock spent on a number no one uses. A variant's own + measurement is a result, which is why the variant gates ask only whether + the variant itself fits -- and why a re-baseline in a later phase is asked + the same narrower question by :func:`_a_use_must_follow_the_round`. + + The round is priced by + :func:`~...phases.machine_state.baseline_round_cost_sec`, which the phase + machine also reads to decide whether a session stopped for this reason may + try again on a fresh clock. Two definitions would let the executor refuse + rounds the phase machine had just decided were affordable. + + Args: + double_run: Whether this round will run both passes. + ctx_extra: The runner context extras carrying ``shared_state``. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. + """ + state = (ctx_extra or {}).get("shared_state") or self.shared_state + return self._round_affordable( + state, + round_sec=_phase_state.baseline_round_cost_sec(state, double_run=double_run), + ) + + @staticmethod + def _round_affordable(state: Any, *, round_sec: float | None) -> tuple[bool, dict[str, Any]]: + """Whether the budget holds a round costing ``round_sec`` and a use for it. + + The shared half of every before-ignition gate, so the two round shapes a + baseline has -- the single-node cold-then-hot double run and the + multi-node pair of client passes -- differ only in what they cost, never + in what they are asked. Each supplies its own price and this decides. + + Args: + state: The session ``SharedState``, or ``None``. + round_sec: What this round is expected to cost, or ``None`` when the + session has measured nothing to price it from. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. + """ + cold_sec = _phase_state.measured_seconds(state, "baseline_runtime_sec") + if cold_sec is None or round_sec is None: + return True, {"reason": "no_measured_round_to_predict_from"} + use_sec = 0.0 + if _a_use_must_follow_the_round(state): + # Without the split, a variant is priced at a whole cold round, which + # is what it is: a boot and a benchmark that pays the compile. The + # same fallback the phase machine uses, so the two agree on when a + # stopped session may try again. + # + # Known gap: a multi-node variant runs two client passes, not one + # (``_grid_runner`` reserves for it as ``x (1 + _mn_warmup_rounds)``), + # so one pass is left unreserved here. It needs a multi-node round to + # reach PRELUDE with an earlier one already measured, which takes an + # enablement round holding the phase open past an anchor that would + # otherwise finish it -- narrow enough not to be worth teaching the + # phase machine's pricing what shape the cluster is. + use_sec = _phase_state.one_more_measurement_sec(state) or cold_sec + headroom_sec, evidence = _round_headroom_sec(state, None) + if headroom_sec is None: + return True, evidence + cost = round_sec + use_sec + priced = { + "expected_cost_sec": round(cost, 1), + "round_sec": round(round_sec, 1), + "one_more_measurement_sec": round(use_sec, 1), + **evidence, + } + return headroom_sec >= cost, priced + + def _measure_round_affordable( + self, + *, + warmup_runtime_sec: Any, + warmup_post_ready_sec: Any = None, + ctx_extra: dict[str, Any] | None = None, + ) -> tuple[bool, dict[str, Any]]: + """Whether the budget covers the measured round *and a use for it*. + + Asked after the warmup rather than before it, and priced from what that + pass just measured rather than from a prediction. This is the gate every + round faces, including the first, which is the one + :meth:`_round_affordable_before_ignition` cannot judge. + + The measured round re-attaches to the server the warmup left running, so + it costs a benchmark and no boot. A hot pass this session already measured + prices it best, being exactly that; before one has ever run, the warmup's + post-ready segment stands in -- the same pass, with its boot taken off. The + segment over-predicts, since it also paid the first request's kernel + compile, but it is far tighter than the warmup's whole wall-clock, which + prices a client-only pass as though it booted a server. + + Reading the session's hot figure first is also what keeps this gate and + :meth:`_round_affordable_before_ignition` on one ruler. Both price the + same second pass, so a round admitted before ignition would otherwise meet + a stricter question here and be refused for certain, having spent a whole + cold pass to find out. + + In PRELUDE, covering the pass is not enough to justify running it. A hot + baseline is an input there, not a result: it is the denominator later + variants are read against, and it is what anchors their overtime kill. + Neither buys anything if no variant can follow, so what is required is the + pass plus one further measured variant -- a boot and a benchmark, because + a variant's config differs in the very knobs that decide how a server + comes up and it cannot re-attach to anyone else's. A re-baseline in a + later phase is its own deliverable and is asked only to cover its pass; + :func:`_a_use_must_follow_the_round` draws that line. + + A round that fails here has still produced a number. It is the cold one + the double run exists to discard, so the caller keeps it as the anchor and + marks it -- the GPU time is spent either way, and a marked cold anchor + beats no anchor. + + Args: + warmup_runtime_sec: Wall-clock the warmup round took. + warmup_post_ready_sec: The part of it that ran after the server was + ready. Only a stamp that could not be written leaves this unset, + since this path runs a server by definition -- the workloads with + no ready boundary to record never reach a double run at all -- so + it is a defect rather than a shape, and the gate waves the round + through rather than guess. Guessing high is what a gate before + ignition may safely do; here a refusal ends the session, and a + session ended by a missing timestamp is the worse error. + ctx_extra: The runner context extras carrying ``shared_state``. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. + """ + state = (ctx_extra or {}).get("shared_state") or self.shared_state + headroom_sec, evidence = _round_headroom_sec(state, None) + if headroom_sec is None: + return True, evidence + warmup_sec = _positive_seconds(warmup_runtime_sec) + priced_by = "session_hot_pass" + benchmark_sec = _phase_state.measured_seconds(state, "baseline_warm_runtime_sec") + if benchmark_sec is None: + priced_by = "warmup_post_ready" + benchmark_sec = _positive_seconds(warmup_post_ready_sec) + if benchmark_sec is None or warmup_sec is None: + return True, {"reason": "no_measured_benchmark_to_predict_from", **evidence} + use_sec = 0.0 + if _a_use_must_follow_the_round(state): + use_sec = _phase_state.one_more_measurement_sec(state) or warmup_sec + cost = benchmark_sec + use_sec + priced = { + "expected_cost_sec": round(cost, 1), + "priced_by": priced_by, + "measure_round_sec": round(benchmark_sec, 1), + "one_more_measurement_sec": round(use_sec, 1), + **evidence, + } + return headroom_sec >= cost, priced + def _double_run_enabled( self, *, @@ -3434,6 +4032,131 @@ async def _run_reported_round( **common, ) + async def _mn_warmup_pass( + self, + *, + cmd: list[str], + env: dict[str, str], + output_dir: Path, + framework: str, + timeout_sec: int, + session_deadline_sec: float | None, + capture_meta: dict[str, Any], + round_warnings: list[str], + ctx_extra: dict[str, Any] | None = None, + ) -> dict[str, Any] | None: + """Run the discarded multi-node client warmup against the restarted server. + + The pass exists because the restart just above it left a cold server and + this is the only thing that drives it before the measured pass. So the + two are one round: skipping the warmup does not save the round's cost, + it moves the round's measurement onto a cold server and anchors the + session's every later gain on it. + + Both passes run under the round's own cap, which the session clamp leaves + past the session deadline so a budget kill arrives as the watchdog's + sentinel rather than as this pass timing out. Nothing is held back for + the measured pass: holding back moves the cap in front of the deadline + and puts the round back in reach of its own timeout. + + Args: + cmd: The measured pass's command, re-pointed at the warmup slot. + env: The measured pass's environment, re-pointed the same way. + output_dir: The round's workspace; the warmup runs in a slot under it. + framework: Framework name, for the watchdog's server log. + timeout_sec: The round's cap after the session clamp. + session_deadline_sec: Monotonic-clock session deadline, or ``None``. + capture_meta: Config/eval-contract facts every failure result carries. + round_warnings: Collected onto the round's result, for a warmup that + did not run and did not end the round. + ctx_extra: The runner context extras carrying ``shared_state``, read + to price the pair of passes against what is left of the budget. + + Returns: + dict[str, Any] | None: The round's result when the round is over, + else ``None`` to go on to the measured pass. + """ + # A multi-node round is two client passes of the same shape against a + # server the round did not boot, and the session's measured figure is one + # of them -- the round's wall-clock is taken after this pass -- so the pair + # costs twice it. Asked before the warmup because that is the pass whose + # number nothing may use: spending it and then meeting the deadline in the + # measured pass leaves the round with no anchor and the session with the + # GPU time gone. + state = (ctx_extra or {}).get("shared_state") or self.shared_state + one_pass_sec = _phase_state.measured_seconds(state, "baseline_runtime_sec") + affordable, evidence = self._round_affordable( + state, + round_sec=None if one_pass_sec is None else one_pass_sec * 2.0, + ) + if not affordable: + log.warning( + "baseline_executor: a multi-node round is two passes needing %.0fs " + "and %.0fs is left (bound=%s), so neither is launched. The anchor " + "this session already measured stands.", + evidence.get("expected_cost_sec", 0.0), + evidence.get("affordable_sec", 0.0), + evidence.get("bound", ""), + ) + refused = _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + round_label="multi-node round", + returncode=None, + runtime_sec=0.0, + output_dir=output_dir, + capture_meta=capture_meta, + started=False, + ) + refused["budget_shortfall"] = evidence + return refused + warm_dir = output_dir / "mn_warmup" + started_unix = time.time() + # The measurement is discarded, but the returncode is not: this pass is a + # full benchmark round, so a stop here ends the baseline round. Going on + # to the measured pass would spend a second round of GPU time the run has + # already been told to stop spending. + warm_rc: int | None = None + try: + warm_dir.mkdir(parents=True, exist_ok=True) + warm_cmd = [str(warm_dir) if c == str(output_dir) else c for c in cmd] + warm_env = dict(env) + warm_env["RESULT_DIR"] = str(warm_dir) + warm_env["EVAL_RESULT_DIR"] = str(warm_dir / "eval_output") + warm_env["SERVER_LOG"] = str(warm_dir / "server.log") + warm_env["GPU_METRICS_CSV"] = str(warm_dir / "gpu_metrics.csv") + async with heartbeat_while_output_flows( + unit="baseline_round", + label="mn_warmup", + ) as warm_activity: + warm_proc = await asyncio.to_thread( + run_with_session_kill, + warm_cmd, + env=warm_env, + cwd=str(warm_dir), + timeout=timeout_sec, + server_log_path=_watchdog_server_log_path(warm_dir, framework), + on_output=warm_activity.note, + session_deadline_sec=session_deadline_sec, + ) + warm_rc = warm_proc.returncode + log.info("baseline_executor: MN warmup pass done (discarded) rc=%s", warm_rc) + except subprocess.TimeoutExpired as exc: + log.warning("baseline_executor: MN warmup pass hit its own hang backstop (ignored): %r", exc) + round_warnings.append(_MN_WARMUP_DID_NOT_WARM_WARNING) + except Exception as exc: # noqa: BLE001 - a warmup that fails on its own is best-effort + log.warning("baseline_executor: MN warmup pass failed (ignored): %r", exc) + round_warnings.append(_MN_WARMUP_DID_NOT_WARM_WARNING) + warm_stopped = stopped_by_the_run(warm_rc) + if warm_stopped is not None: + return _stopped_round_result( + warm_stopped, + round_label="multi-node warmup pass", + returncode=warm_rc, + runtime_sec=max(0.0, time.time() - started_unix), + output_dir=output_dir, + capture_meta=capture_meta, + ) + return None async def _run_single_benchmark( self, *, @@ -3551,6 +4274,12 @@ async def _run_single_benchmark( ) ctx_extra = getattr(ctx, "extra", None) or {} + # The session's wall-clock budget, resolved per round rather than once + # per task: a baseline runs up to three of them, and a warmup that + # overran has already spent budget the ones after it were counting on. + _session_state = ctx_extra.get("shared_state") or self.shared_state + session_deadline_sec, _ = session_grid_bounds(_session_state) + timeout_sec = self._session_capped_timeout(timeout_sec, session_deadline_sec, output_dir=output_dir) if not ctx_extra.get("mn_round_restarted"): try: # Merge the reference base UNDER the per-task args (last-wins) so @@ -3614,32 +4343,21 @@ async def _run_single_benchmark( mn_bench_warmup_enabled as _mn_warm, ) + round_warnings: list[str] = [] if _mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted"): - _mn_warm_dir = output_dir / "mn_warmup" - try: - _mn_warm_dir.mkdir(parents=True, exist_ok=True) - _mn_warm_cmd = [str(_mn_warm_dir) if c == str(output_dir) else c for c in cmd] - _mn_warm_env = dict(env) - _mn_warm_env["RESULT_DIR"] = str(_mn_warm_dir) - _mn_warm_env["EVAL_RESULT_DIR"] = str(_mn_warm_dir / "eval_output") - _mn_warm_env["SERVER_LOG"] = str(_mn_warm_dir / "server.log") - _mn_warm_env["GPU_METRICS_CSV"] = str(_mn_warm_dir / "gpu_metrics.csv") - async with heartbeat_while_output_flows( - unit="baseline_round", - label="mn_warmup", - ) as _mn_warm_activity: - await asyncio.to_thread( - run_with_session_kill, - _mn_warm_cmd, - env=_mn_warm_env, - cwd=str(_mn_warm_dir), - timeout=timeout_sec, - server_log_path=_watchdog_server_log_path(_mn_warm_dir, framework), - on_output=_mn_warm_activity.note, - ) - log.info("baseline_executor: MN warmup pass done (discarded)") - except Exception as exc: # noqa: BLE001 - warmup is best-effort - log.warning("baseline_executor: MN warmup pass failed (ignored): %r", exc) + _mn_warm_result = await self._mn_warmup_pass( + cmd=cmd, + env=env, + output_dir=output_dir, + framework=framework, + timeout_sec=timeout_sec, + session_deadline_sec=session_deadline_sec, + capture_meta=capture_meta, + round_warnings=round_warnings, + ctx_extra=ctx_extra, + ) + if _mn_warm_result is not None: + return _mn_warm_result workspaces_before = snapshot_workspaces(output_dir) subprocess_started_unix = time.time() @@ -3664,6 +4382,9 @@ async def _run_single_benchmark( stale_server_log, exc, ) + # And the ready stamp beside it, for the same reason: a prior attempt's + # would make this attempt's boot look like it never happened. + clear_server_ready_stamp(str(stale_server_log)) try: if serving_lease is not None: # Ray-managed GPU execution (§12 T1): run inside the lease's @@ -3690,6 +4411,7 @@ async def _run_single_benchmark( cwd=str(output_dir), timeout=timeout_sec, server_log_path=watchdog_server_log, + session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), ) subprocess_runtime_sec = max(0.0, time.time() - subprocess_started_unix) else: @@ -3705,6 +4427,7 @@ async def _run_single_benchmark( timeout=timeout_sec, server_log_path=watchdog_server_log, on_output=activity.note, + session_deadline_sec=session_deadline_sec, ) subprocess_runtime_sec = max( 0.0, @@ -3729,6 +4452,17 @@ async def _run_single_benchmark( **capture_meta, } + stopped = stopped_by_the_run(proc_returncode) + if stopped is not None: + return _stopped_round_result( + stopped, + round_label="measured round", + returncode=proc_returncode, + runtime_sec=subprocess_runtime_sec, + output_dir=output_dir, + capture_meta=capture_meta, + ) + # Detokenizer-stall watchdog reap: the server came up healthy but went # silent for the stall grace window (hung engine / wedged detokenizer). # A stall reap leaves no benchmark_* workspace; a distinct error_class @@ -3882,7 +4616,7 @@ async def _run_single_benchmark( workspace=workspace, subprocess_started_unix=subprocess_started_unix, ) - warnings = list(measurement.pop("nonfatal_warnings", []) or []) + warnings = round_warnings + list(measurement.pop("nonfatal_warnings", []) or []) if proc_returncode != 0: warnings.append("magpie_nonzero_after_valid_measurement") for leak_src, _ in harvested: @@ -3942,6 +4676,16 @@ async def _run_single_benchmark( # promotes into ``SharedState.baseline_runtime_sec``, the explore # overtime-kill anchor. Omitted on failure paths. "subprocess_runtime_sec": round(subprocess_runtime_sec, 2), + # The benchmark's own share of that wall-clock, boot excluded. Kept + # separate rather than folded in because the two are spent by + # different things: every later variant boots again, so it is the + # sum that prices one, while only this part prices a pass that + # re-attaches. ``None`` when nothing recorded a ready boundary. + "post_ready_runtime_sec": _round_post_ready_sec( + watchdog_server_log, + started_unix=subprocess_started_unix, + runtime_sec=subprocess_runtime_sec, + ), # Authoritative (materialized-config) view of whether the serving # lm-eval ran this run. The accuracy-stop decision reads this rather # than re-deriving from params, so a YAML/reference-env RUN_EVAL=false diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index c5826da120..1f2b08ff7c 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -47,6 +47,12 @@ tail_excerpt, ) from ...state.shared_state import first_positive_tput, resolve_grading_anchor_tput, stack_base_params +from ..stop_attribution import ( + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, + stopped_by_the_run_class, +) from ._accuracy_gate import ( accuracy_passed, is_high_accuracy_risk, @@ -66,6 +72,7 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from ._grid_server_args import compose_server_args, server_args_env_name from ._ray_serving import maybe_serving_lease @@ -978,7 +985,7 @@ async def __call__(self, ctx) -> dict[str, Any]: search.setdefault(key, default) tested_dict = search.get("tested") or {} - name_index = dict(search.get("name_index") or {}) + inherited_name_index: dict[str, Any] = dict(search.get("name_index") or {}) # Attach the per-variant fingerprint as an attribute so the result # loop needn't recompute. @@ -1057,7 +1064,13 @@ async def __call__(self, ctx) -> dict[str, Any]: winners: list[dict[str, Any]] = [] losers: list[dict[str, Any]] = [] keep_unstable: list[dict[str, Any]] = [] - tested_update: dict[str, dict[str, Any]] = dict(tested_dict) + # This round's own ledger writes, kept apart from the ledger it inherited + # and merged over it once the loop is done. A variant whose confirmation + # round the run reaps has to be rolled back, and a fingerprint may be + # re-run across rounds -- so rolling back against a merged dict deletes + # the earlier round's measured row along with this round's write. + round_tested: dict[str, dict[str, Any]] = {} + round_name_index: dict[str, Any] = {} rejected_update: list[dict[str, Any]] = list(search.get("rejected") or []) winners_history_update: list[dict[str, Any]] = list(search.get("winners_history") or []) @@ -1112,13 +1125,88 @@ async def __call__(self, ctx) -> dict[str, Any]: round_serving_lease = maybe_serving_lease(num_gpus=_num_gpus_for_config(config_path)) if runnable else None # Stop testing further variants once the session wall-clock budget runs # out; untested variants stay out of the ledger so a resume can retry them. - _ss = extra.get("shared_state") or extra.get("state") - session_deadline_sec = _ss.grid_session_deadline_sec() if _ss is not None else None + # + # What a normally-behaving round needs, as opposed to ``timeout_sec``, + # which is the catastrophic backstop (``baseline x (kill_ratio + margin)`` + # ~= baseline x 2). Gating on the backstop abandons the tail of the budget + # to variants that would have finished comfortably: with a 20-min baseline + # it refuses to start with 30 min left, for a round that needs ~20. The + # params values win over the session's because an operator may override + # them per task; ``None`` when neither is known, which leaves the stricter + # backstop check in place rather than guessing. + # + # The two rounds are estimated separately because they cost different + # amounts: the warmup pass pays a cold server boot and is discarded, while + # the decision round is client-only against the hot server -- which is + # exactly the split ``decision_anchor_sec`` already draws for the overtime + # kill. + session_deadline_sec, session_expected_sec = session_grid_bounds( + extra.get("shared_state") or extra.get("state") + ) + warmup_expected_sec = (baseline_runtime_sec if baseline_runtime_sec > 0 else None) or session_expected_sec + decision_expected_sec = (decision_anchor_sec if decision_anchor_sec > 0 else None) or session_expected_sec + # Set when the loop stops because the run stopped it -- the budget ran + # out, or the orchestrator cancelled the action -- so the round can say + # so instead of reporting a bare, unattributed failure: a variant that + # never ran is not a variant that failed. ``run_stop_detail`` is the + # lead clause, which differs by whether a round was already under way. + run_stop: StoppedByTheRun | None = None + run_stop_detail = "" + session_budget_untested = 0 + + def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_label: str) -> bool: + """Whether the run stopped this round, and record it if it did. + + A reaped round measured nothing, so the variant is left out of every + ledger exactly as an unadmitted one is: writing it as ``FAILED`` + would make a resume skip a variant nothing ever measured, and would + teach the KB that these knobs are bad because a clock ran out. + + Args: + result: The round's :class:`VariantResult`, or ``None``. + variant: The variant the round was measuring, for the log line. + idx: Its index in ``runnable``, for the untested count. + round_label: Which round was stopped, for the log line. + + Returns: + bool: ``True`` when the caller must stop testing variants. + """ + nonlocal run_stop, run_stop_detail, session_budget_untested + stopped = stopped_by_the_run_class(getattr(result, "error_class", "") if result is not None else "") + if stopped is None: + return False + run_stop = stopped + run_stop_detail = stopped.interrupted + session_budget_untested = len(runnable) - idx + log.warning( + "explore: the %s round of variant %s was stopped by the run (%s); it and the " + "%d variant(s) after it stay out of the ledger so a resume can retry them", + round_label, + variant.name, + stopped.error_class, + session_budget_untested - 1, + ) + return True + try: for idx, gv in enumerate(runnable): - if session_deadline_sec is not None and (session_deadline_sec - time.monotonic()) < float(timeout_sec): + # A warm-decision variant pays for both rounds, so admitting it on + # the decision round alone would let it in and then strand it + # mid-variant with a discarded warmup and no measurement. + if decision_expected_sec is not None: + fit_required_sec = float(decision_expected_sec) + ( + float(warmup_expected_sec or 0.0) if use_warm_decision else 0.0 + ) + else: + fit_required_sec = float(timeout_sec) + if session_deadline_sec is not None and (session_deadline_sec - time.monotonic()) < fit_required_sec: + run_stop = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS] + run_stop_detail = run_stop.never_started + session_budget_untested = len(runnable) - idx log.warning( - "explore: session budget cannot fit another variant; stopping after %d/%d variant(s)", + "explore: session budget cannot fit another variant " + "(needs %.0fs); stopping after %d/%d variant(s)", + fit_required_sec, idx, len(runnable), ) @@ -1205,8 +1293,12 @@ async def __call__(self, ctx) -> dict[str, Any]: server_lifecycle=round1_lifecycle, base_args_mode=stack_base_args_mode, serving_lease=variant_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=warmup_expected_sec, ) w = warmup_results[0] if warmup_results else None + if _stopped_by_the_run(w, variant=gv, idx=idx, round_label="warmup"): + break if w is None or getattr(w, "status", "") != "succeeded": werr = (getattr(w, "error", "") or "")[-200:] if w is not None else "no_result" log.warning( @@ -1214,7 +1306,7 @@ async def __call__(self, ctx) -> dict[str, Any]: gv.name, werr, ) - tested_update[fp] = { + round_tested[fp] = { "fingerprint": fp, "name": gv.name, "extra_server_args": gv.extra_server_args, @@ -1240,7 +1332,7 @@ async def __call__(self, ctx) -> dict[str, Any]: "raw_result_path": w.raw_result_path if w is not None else None, } if gv.name: - name_index[gv.name] = fp + round_name_index[gv.name] = fp rejected_update.append( { "fingerprint": fp, @@ -1296,6 +1388,8 @@ async def __call__(self, ctx) -> dict[str, Any]: preclean_before_run=not use_warm_decision, server_already_ready=use_warm_decision, serving_lease=variant_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=decision_expected_sec, ) if not results: # run_grid returns one result per grid entry. @@ -1305,6 +1399,8 @@ async def __call__(self, ctx) -> dict[str, Any]: ) continue r = results[0] + if _stopped_by_the_run(r, variant=gv, idx=idx, round_label="decision"): + break # Overtime gate fired: record a ``KILLED_OVERTIME`` row (no # faked tput/gain), skip downstream gates, leave the stack @@ -1318,7 +1414,7 @@ async def __call__(self, ctx) -> dict[str, Any]: # Informational only: ``tput`` stays None so this never # enters winner selection or gain math. est_tput = getattr(r, "estimated_output_throughput", None) - tested_update[fp] = { + round_tested[fp] = { "fingerprint": fp, "name": gv.name, "extra_server_args": gv.extra_server_args, @@ -1354,7 +1450,7 @@ async def __call__(self, ctx) -> dict[str, Any]: "error_class": "killed_overtime", } if gv.name: - name_index[gv.name] = fp + round_name_index[gv.name] = fp rejected_update.append( { "fingerprint": fp, @@ -1466,7 +1562,7 @@ async def __call__(self, ctx) -> dict[str, Any]: outcome = "KEEP" decision_tput = r.output_throughput - tested_update[fp] = { + round_tested[fp] = { "fingerprint": fp, "name": gv.name, "extra_server_args": gv.extra_server_args, @@ -1491,7 +1587,7 @@ async def __call__(self, ctx) -> dict[str, Any]: "stage": FAILURE_STAGE_DECISION, } if gv.name: - name_index[gv.name] = fp + round_name_index[gv.name] = fp # ---- KEEP path (with warm round-2 rebench) ---- if outcome == "KEEP": @@ -1606,7 +1702,21 @@ async def __call__(self, ctx) -> dict[str, Any]: soft_deadline_sec=decision_deadline_sec, server_already_ready=lifecycle_eligible, serving_lease=variant_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=decision_expected_sec, ) + # A confirmation the run stopped is not a failed + # confirmation: grading it would evict a variant as + # unstable on the strength of a round that never + # measured it. Undo the stack fold and drop the + # decision-round entry too, so a resume re-measures + # the variant and its confirmation together. + if _stopped_by_the_run(rebench, variant=gv, idx=idx, round_label="stack rebench"): + in_batch_keeps.pop() + round_tested.pop(fp, None) + if gv.name: + round_name_index.pop(gv.name, None) + break stack_rebench_tput = rebench.tput stack_rebench_workspace = rebench.workspace stack_rebench_warnings = rebench.warnings @@ -1623,10 +1733,10 @@ async def __call__(self, ctx) -> dict[str, Any]: running_base_tput, stack_stable_threshold_pct, ) - tested_update[fp]["outcome"] = "KEEP_UNSTABLE" - tested_update[fp]["stack_rebench_tput"] = stack_rebench_tput - tested_update[fp]["stack_rebench_workspace"] = stack_rebench_workspace - tested_update[fp]["stack_rebench_warnings"] = stack_rebench_warnings + round_tested[fp]["outcome"] = "KEEP_UNSTABLE" + round_tested[fp]["stack_rebench_tput"] = stack_rebench_tput + round_tested[fp]["stack_rebench_workspace"] = stack_rebench_workspace + round_tested[fp]["stack_rebench_warnings"] = stack_rebench_warnings keep_unstable.append( { **keep_entry, @@ -1671,11 +1781,11 @@ async def __call__(self, ctx) -> dict[str, Any]: keep_entry["stack_rebench_tput"] = stack_rebench_tput keep_entry["stack_rebench_workspace"] = stack_rebench_workspace keep_entry["stack_rebench_warnings"] = stack_rebench_warnings - tested_update[fp]["tput"] = stack_rebench_tput - tested_update[fp]["gain_pct"] = gain - tested_update[fp]["stack_rebench_tput"] = stack_rebench_tput - tested_update[fp]["stack_rebench_workspace"] = stack_rebench_workspace - tested_update[fp]["stack_rebench_warnings"] = stack_rebench_warnings + round_tested[fp]["tput"] = stack_rebench_tput + round_tested[fp]["gain_pct"] = gain + round_tested[fp]["stack_rebench_tput"] = stack_rebench_tput + round_tested[fp]["stack_rebench_workspace"] = stack_rebench_workspace + round_tested[fp]["stack_rebench_warnings"] = stack_rebench_warnings else: # Round 2 disabled — KEEP on the cold round-1 # measurement, advance the running baseline naively. @@ -1757,6 +1867,11 @@ async def __call__(self, ctx) -> dict[str, Any]: round_serving_lease.close() # ----- Ledger compaction (per-fingerprint last-wins) ---------------- + # This round's writes over the ledger it inherited: a re-run fingerprint + # replaces its earlier row, which is what a fresh measurement means, and a + # variant this round rolled back leaves the earlier row standing. + tested_update: dict[str, dict[str, Any]] = {**tested_dict, **round_tested} + name_index: dict[str, Any] = {**inherited_name_index, **round_name_index} rejected_dedup: dict[str, dict[str, Any]] = {} for entry in rejected_update: fp = str(entry.get("fingerprint") or "") @@ -1914,9 +2029,24 @@ async def __call__(self, ctx) -> dict[str, Any]: if t.get("round_id") == round_id ) status = "succeeded" if produced_measurement or winners else "failed" + # A round that measured nothing because the run stopped it is not the + # same as one whose variants failed, and it used to be reported as a bare + # ``failed`` with no error_class at all -- nothing downstream could tell + # the two apart, so the KB could learn that these variants are bad. + budget_error: dict[str, Any] = {} + if status == "failed" and run_stop is not None: + budget_error = { + "error_class": run_stop.error_class, + "error": ( + f"{run_stop_detail}; {session_budget_untested} variant(s) went unmeasured " + "and stay out of the ledger so a resume can retry them" + ), + } return { "status": status, + **budget_error, + "session_budget_untested": session_budget_untested, "base_tput": base_tput, "running_base_tput": running_base_tput, "output_throughput": output_throughput, diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 0d41d5361e..44d2c556cb 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -28,6 +28,7 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from ._workload_envs import ( FrameworkScriptMismatchError, @@ -40,6 +41,7 @@ _accuracy_delta_pct, _git_apply_collect_feedback, _git_stash_if_dirty, + _restore_stash_logged, _with_stash_restore, _resolve_framework_root, ) @@ -710,6 +712,27 @@ async def __call__(self, ctx) -> dict[str, Any]: applied: list[Path] = [] apply_errors: list[dict[str, str]] = [] apply_feedbacks: list[ApplyFeedback] = [] + + def _undo_candidate() -> None: + """Take the candidate back out of the tree and hand the stash back. + + What a stop owes, as opposed to a verdict. The dispatcher cancels + in-flight actions on shutdown and on a spent wall-clock budget, and + ``CancelledError`` is not an ``Exception``, so none of the REVERT + handlers below see one. Unhandled it leaves the candidate applied and + the operator's uncommitted work in ``git stash`` indefinitely -- the + budget case does not end the process, so CLOSE would go on to report + against a tree carrying a patch nothing ever graded. + + Reverting past a KEEP that was already committed is deliberate: the + result carrying that KEEP never reaches the Coordinator, so leaving + the commit would leave the tree claiming a win the session does not + record. Every step is synchronous, so no second cancel can be + delivered part-way through the undo. + """ + self._revert_patches(framework_root, applied, pre_apply_sha=pre_apply_sha) + _restore_stash_logged(framework_root, stash_state, stash_note) + # Structural safety gate on the (remote / untrusted) diff before it is # applied to the live framework tree: reject non-diff blobs and any # header path that escapes the tree (absolute / ``..``). Stale / @@ -796,12 +819,21 @@ async def __call__(self, ctx) -> dict[str, Any]: }, ) - # Bench via run_grid (size=1). + # Bench via run_grid (size=1). Bound it by the session wall-clock, as the + # sweep and explore arms already are: without it the declared cap is the + # only limit, and that cap answers "how long before this counts as hung", + # not "how much budget is left" -- so a candidate benched near the end of + # a run could outlive the run itself. + session_deadline_sec, variant_expected_sec = session_grid_bounds( + extra.get("shared_state") or extra.get("state") + ) try: bench_result, gate_evidence = await self._bench_candidate( params=params, output_root=output_root, slug=slug, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) except FrameworkScriptMismatchError as exc: reverted = self._revert_patches( @@ -847,6 +879,12 @@ async def __call__(self, ctx) -> dict[str, Any]: "workspace": str(output_root), }, ) + except BaseException: + # A stop, not a verdict: let it through rather than grading it, since + # as a REVERT it would read as the patch having failed a bench that + # never ran. See :func:`_undo_candidate` for what the stop owes. + _undo_candidate() + raise # KEEP / REVERT decision. base_tput = float(params.get("base_tput") or 0.0) @@ -881,6 +919,34 @@ async def __call__(self, ctx) -> dict[str, Any]: gate_pass = tput_ok and not acc_block acc_delta_pct = _accuracy_delta_pct(gate_evidence.get("accuracy"), params.get("accuracy_baseline")) + async def _record_outcome(outcome: str) -> None: + """Write this candidate's KB record, undoing it if the write is stopped. + + Both verdicts record the same measurements and differ only in the + outcome label, and both record them after the verdict is decided and + before the ``_with_stash_restore`` that returns it. That await is the + last one the candidate crosses while the auto-stash is still on the + stack, and no handler stands between it and the caller: a cancel + delivered here -- which is what a spent budget delivers, at whatever + await the action happens to be at -- would strand the operator's work + in the stash with nothing in the session saying so. + + Args: + outcome: The KB outcome label for the verdict just decided. + """ + try: + await self._write_kb_record( + candidate=candidate, + outcome=outcome, + tps_delta_pct=float(delta_pct or 0.0), + patch_path=str(applied[0]) if applied else "", + extra=extra, + accuracy_delta_pct=acc_delta_pct, + ) + except BaseException: + _undo_candidate() + raise + if not gate_pass: reverted = self._revert_patches( framework_root, @@ -899,14 +965,7 @@ async def __call__(self, ctx) -> dict[str, Any]: revert_status = ( "accuracy_unavailable_reject" if (acc_block and accuracy_pass is None and tput_ok) else "reverted" ) - await self._write_kb_record( - candidate=candidate, - outcome=OUTCOME_REVERTED_SMOKE_FAIL, - tps_delta_pct=float(delta_pct or 0.0), - patch_path=str(applied[0]) if applied else "", - extra=extra, - accuracy_delta_pct=acc_delta_pct, - ) + await _record_outcome(OUTCOME_REVERTED_SMOKE_FAIL) return _with_stash_restore( framework_root, stash_state, @@ -966,14 +1025,7 @@ async def __call__(self, ctx) -> dict[str, Any]: }, ) - await self._write_kb_record( - candidate=candidate, - outcome=OUTCOME_INTEGRATED, - tps_delta_pct=float(delta_pct or 0.0), - patch_path=str(applied[0]) if applied else "", - extra=extra, - accuracy_delta_pct=acc_delta_pct, - ) + await _record_outcome(OUTCOME_INTEGRATED) return _with_stash_restore( framework_root, stash_state, @@ -1127,6 +1179,8 @@ async def _bench_candidate( params: dict[str, Any], output_root: Path, slug: str, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Run a 1-variant Magpie bench under the patched server + accuracy gate. Mirrors :meth:`IntegratePatchExecutor._bench_patch`. @@ -1135,6 +1189,11 @@ async def _bench_candidate( params: The task params (config / model / bench knobs). output_root: The per-task workspace root for the bench. slug: The candidate slug used to name the variant. + session_deadline_sec: Monotonic-clock session budget deadline, or + ``None`` when unbounded. Resolved by the caller, which owns the + session context. + variant_expected_sec: Expected bench runtime used to decide whether + the remaining budget can fit this bench at all. Returns: A ``(bench, gate_evidence)`` tuple: the bench result dict and a @@ -1205,6 +1264,8 @@ async def _bench_candidate( benchmark_script=override_script, result_dir=override_result_dir, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) finally: if serving_lease is not None: diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 58089ca0f9..dbdb8925f5 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -55,10 +55,15 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from . import _framework_switch_manifest as _switch_manifest from ._grid_server_args import compose_server_args -from ._stack_rebench import DEFAULT_STACK_STABLE_PCT, measure_stack_rebench +from ._stack_rebench import ( + DEFAULT_STACK_STABLE_PCT, + StackRebenchResult, + measure_stack_rebench, +) from ._workload_envs import ( FrameworkScriptMismatchError, default_baseline_config, @@ -979,6 +984,27 @@ def _git_restore_stash_if_needed( return f"git stash pop {ref} rc={cp.returncode}: {detail}; user changes remain in git stash" +def _restore_stash_logged( + framework_root: Path, + stash_state: str, + stash_ref: str, +) -> str: + """Restore a pre-candidate stash, reporting a failure to do so. + + Args: + framework_root (Path): The tree the stash was taken from. + stash_state (str): The state :func:`_git_stash_if_dirty` returned. + stash_ref (str): The stash ref to pop. + + Returns: + str: The failure note, or ``""`` when nothing was left in the stash. + """ + note = _git_restore_stash_if_needed(framework_root, stash_state, stash_ref) + if note: + log.warning("integrate_patch: user-change stash restore failed: %s", note) + return note + + def _with_stash_restore( framework_root: Path, stash_state: str, @@ -986,10 +1012,9 @@ def _with_stash_restore( result: dict[str, Any], ) -> dict[str, Any]: """Restore a pre-candidate stash before returning an executor result.""" - note = _git_restore_stash_if_needed(framework_root, stash_state, stash_ref) + note = _restore_stash_logged(framework_root, stash_state, stash_ref) if not note: return result - log.warning("integrate_patch: user-change stash restore failed: %s", note) out = dict(result) out["stash_restore_error"] = note return out @@ -1651,40 +1676,54 @@ async def __call__(self, ctx) -> dict[str, Any]: if localize_early is not None: return localize_early - apply_result = await self._stage_apply(ctx, params, extra, specialist_task_id, shared_state, done_payload) - if apply_result is not None: - return apply_result - - # _stage_apply populates these. - output_root: Path = ctx._ip_output_root # type: ignore[attr-defined] - framework_root: Path | None = ctx._ip_framework_root # type: ignore[attr-defined] - stash_state: str = ctx._ip_stash_state # type: ignore[attr-defined] - stash_note: str = ctx._ip_stash_note # type: ignore[attr-defined] - applied: list[Path] = ctx._ip_applied # type: ignore[attr-defined] - applied_artifacts: list[dict[str, Any]] = ctx._ip_applied_artifacts # type: ignore[attr-defined] - config_changes_applied: dict[str, str] = ctx._ip_config_changes_applied # type: ignore[attr-defined] - extra_server_args_applied: str = ctx._ip_extra_server_args_applied # type: ignore[attr-defined] - extra_envs_applied: dict[str, str] = ctx._ip_extra_envs_applied # type: ignore[attr-defined] - setup_result: dict[str, Any] = ctx._ip_setup_result # type: ignore[attr-defined] - - return await self._stage_gate( - ctx, - params, - extra, - specialist_task_id=specialist_task_id, - shared_state=shared_state, - done_payload=done_payload, - output_root=output_root, - framework_root=framework_root, - stash_state=stash_state, - stash_note=stash_note, - applied=applied, - applied_artifacts=applied_artifacts, - config_changes_applied=config_changes_applied, - extra_server_args_applied=extra_server_args_applied, - extra_envs_applied=extra_envs_applied, - setup_result=setup_result, - ) + # The apply and the gate both mutate the framework tree behind the + # operator's auto-stash, and both cross awaits while it is on the stack -- + # the apply stage writes a KB record on each of its failure verdicts. So + # the guard spans both: whichever stage was running, the candidate is + # taken back out and the stash handed back, and the stop is re-raised + # rather than graded. Each stage publishes its tree-mutation bookkeeping + # to ``ctx`` as it becomes real, because that is all the handler can see + # when the stop arrives mid-stage. + try: + apply_result = await self._stage_apply( + ctx, params, extra, specialist_task_id, shared_state, done_payload + ) + if apply_result is not None: + return apply_result + + # _stage_apply populates these. + output_root: Path = ctx._ip_output_root # type: ignore[attr-defined] + framework_root: Path | None = ctx._ip_framework_root # type: ignore[attr-defined] + stash_state: str = ctx._ip_stash_state # type: ignore[attr-defined] + stash_note: str = ctx._ip_stash_note # type: ignore[attr-defined] + applied: list[Path] = ctx._ip_applied # type: ignore[attr-defined] + applied_artifacts: list[dict[str, Any]] = ctx._ip_applied_artifacts # type: ignore[attr-defined] + config_changes_applied: dict[str, str] = ctx._ip_config_changes_applied # type: ignore[attr-defined] + extra_server_args_applied: str = ctx._ip_extra_server_args_applied # type: ignore[attr-defined] + extra_envs_applied: dict[str, str] = ctx._ip_extra_envs_applied # type: ignore[attr-defined] + setup_result: dict[str, Any] = ctx._ip_setup_result # type: ignore[attr-defined] + + return await self._stage_gate( + ctx, + params, + extra, + specialist_task_id=specialist_task_id, + shared_state=shared_state, + done_payload=done_payload, + output_root=output_root, + framework_root=framework_root, + stash_state=stash_state, + stash_note=stash_note, + applied=applied, + applied_artifacts=applied_artifacts, + config_changes_applied=config_changes_applied, + extra_server_args_applied=extra_server_args_applied, + extra_envs_applied=extra_envs_applied, + setup_result=setup_result, + ) + except BaseException: + self._undo_ungraded_candidate(ctx) + raise # --------------------------------------------------------------------------- # Stage helpers (called sequentially by __call__) @@ -2357,6 +2396,14 @@ async def _stage_apply( applied_artifacts: list[dict[str, Any]] = [] apply_errors: list[dict[str, str]] = [] apply_feedbacks: list[ApplyFeedback] = [] + # From here on the tree is mutable and the stash is on the stack, so + # ``__call__``'s undo has to be able to see both before this stage + # returns: ``applied`` is published by identity and appended to in place. + ctx._ip_framework_root = framework_root # type: ignore[attr-defined] + ctx._ip_stash_state = stash_state # type: ignore[attr-defined] + ctx._ip_stash_note = stash_note # type: ignore[attr-defined] + ctx._ip_applied = applied # type: ignore[attr-defined] + ctx._ip_applied_artifacts = applied_artifacts # type: ignore[attr-defined] for patch in patch_paths: if git_tree: ok, err, fb = _git_apply_collect_feedback(framework_root, patch, three_way=False) @@ -2412,6 +2459,7 @@ async def _stage_apply( artifact_specs, backup_root=output_root / "artifact_backups", ) + ctx._ip_applied_artifacts = applied_artifacts # type: ignore[attr-defined] if artifact_apply_errors: self._revert_artifacts(applied_artifacts) reverted = self._revert_patches(framework_root, applied) @@ -2461,12 +2509,9 @@ async def _stage_apply( }, ) + # The tree-mutation values are already published above, as the tree took + # them; what is left is what only the gate reads. ctx._ip_output_root = output_root # type: ignore[attr-defined] - ctx._ip_framework_root = framework_root # type: ignore[attr-defined] - ctx._ip_stash_state = stash_state # type: ignore[attr-defined] - ctx._ip_stash_note = stash_note # type: ignore[attr-defined] - ctx._ip_applied = applied # type: ignore[attr-defined] - ctx._ip_applied_artifacts = applied_artifacts # type: ignore[attr-defined] ctx._ip_config_changes_applied = config_changes_applied # type: ignore[attr-defined] ctx._ip_extra_server_args_applied = extra_server_args_applied # type: ignore[attr-defined] ctx._ip_extra_envs_applied = extra_envs_applied # type: ignore[attr-defined] @@ -2505,6 +2550,13 @@ async def _stage_gate( params = dict(params) params["runtime_override"] = provision_result.runtime.to_runtime_override() + # Bound both bench legs by the session wall-clock, as the sweep and explore + # arms already are: the declared cap answers "how long before this counts + # as hung", not "how much budget is left", so without this a patch benched + # near the end of a run could outlive the run itself. Resolved once here + # and reused by the parity leg -- the deadline is an absolute monotonic + # timestamp, so it stays correct as the gate progresses. + session_deadline_sec, variant_expected_sec = session_grid_bounds(shared_state) try: bench_result, gate_evidence = await self._bench_patch( params=params, @@ -2512,6 +2564,8 @@ async def _stage_gate( extra_server_args_applied=extra_server_args_applied, extra_envs_applied=extra_envs_applied, specialist_task_id=specialist_task_id, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) except FrameworkScriptMismatchError as exc: artifacts_reverted = self._revert_artifacts(applied_artifacts) @@ -2988,6 +3042,13 @@ async def _gate_perf( # is genuinely unchanged with every switch unset, which is exactly what this # leg measures. An unswitched patch has no "off" state to fall back to, so # it still reverts without spending the leg. + # Both the parity leg and the stack rebench below are additional full + # benches, so they need the same session bound the first bench got. + # Resolved here rather than threaded from the caller because the deadline + # is an absolute monotonic timestamp: the budget the first bench spent is + # already reflected in it. + session_deadline_sec, variant_expected_sec = session_grid_bounds(shared_state) + parity: dict[str, Any] = {"ran": False, "ok": True, "reason": ""} if switch_manifest: parity = await self._switch_off_parity( @@ -2996,6 +3057,8 @@ async def _gate_perf( specialist_task_id=specialist_task_id, switch_manifest=switch_manifest, base_tput=base_tput, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) if not parity.get("ok"): # An unmeasurable parity leg reverts under its own verdict: the patch @@ -3149,6 +3212,8 @@ async def _gate_perf( extra_envs_applied=extra_envs_applied, specialist_task_id=specialist_task_id, base_tput=base_tput, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) rb_acc_block, rb_acc_reason, _rb_degraded = accuracy_keep_block( confirm["accuracy_pass"], @@ -3340,6 +3405,8 @@ async def _switch_off_parity( specialist_task_id: str, switch_manifest: list[dict[str, Any]], base_tput: float, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> dict[str, Any]: """Verify the patch is genuinely inert with every rewrite switch unset. @@ -3363,6 +3430,10 @@ async def _switch_off_parity( specialist_task_id: The originating specialist. switch_manifest: Parsed switch manifest. base_tput: Pre-patch throughput to compare against. + session_deadline_sec: Monotonic-clock session budget deadline for the + parity bench, or ``None`` when unbounded. + variant_expected_sec: Expected bench runtime, used to decide whether + the remaining budget can fit the parity leg at all. Returns: ``{"ran", "ok", "tput", "delta_pct", "band_pct", "accuracy_pass", @@ -3389,6 +3460,8 @@ async def _switch_off_parity( specialist_task_id=specialist_task_id, unset_envs=switch_names, variant_suffix="-parity", + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) except Exception as exc: # noqa: BLE001 — a failed probe must not read as a pass return { @@ -3728,6 +3801,44 @@ async def _maybe_write_framework_kb_record( exc, ) + def _undo_ungraded_candidate(self, ctx: Any) -> None: + """Take the candidate back out when a stage unwound instead of returning. + + Every REVERT the stages themselves decide hangs off an ``except + Exception``, and the stop that matters most here is not one of those: the + dispatcher cancels in-flight actions on shutdown and on a spent + wall-clock budget, and ``CancelledError`` derives from ``BaseException``. + Unhandled, it leaves the patch in the framework tree and the operator's + auto-stash on the stack — and the budget case does not end the process, + so CLOSE would report against a tree carrying a patch nothing ever + graded. + + The cancel itself is re-raised by the caller rather than turned into a + REVERT verdict, so the run records it the way + :mod:`..stop_attribution` requires: work the run stopped, not work that + failed. Every step here is synchronous, so no second cancel can be + delivered part-way through the undo. + + Read from ``ctx`` rather than from arguments because a stop can arrive + mid-stage, before the stage has returned anything to the caller: what the + undo owes is exactly what the tree has already been given, and each stage + publishes that as it happens. A stage that has not stashed yet leaves + ``clean`` behind, which makes the whole undo a no-op. + + Args: + ctx: The runner context the stages publish their ``_ip_*`` + tree-mutation bookkeeping onto. + """ + framework_root: Path | None = getattr(ctx, "_ip_framework_root", None) + self._revert_artifacts(list(getattr(ctx, "_ip_applied_artifacts", None) or [])) + self._revert_patches(framework_root, list(getattr(ctx, "_ip_applied", None) or [])) + if framework_root is not None: + _restore_stash_logged( + framework_root, + str(getattr(ctx, "_ip_stash_state", "") or "clean"), + str(getattr(ctx, "_ip_stash_note", "") or ""), + ) + def _revert_patches( self, framework_root: Path | None, @@ -3865,6 +3976,8 @@ async def _bench_patch( specialist_task_id: str, unset_envs: "list[str] | None" = None, variant_suffix: str = "", + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Run a 1-variant Magpie bench under the patched server + accuracy gate. @@ -3881,6 +3994,11 @@ async def _bench_patch( an earlier KEEP put them into the base configuration. variant_suffix: Appended to the variant name so a second leg does not collide with the first one's grid slot. + session_deadline_sec: Monotonic-clock session budget deadline, or + ``None`` when unbounded. Resolved by the caller, which owns the + session context. + variant_expected_sec: Expected bench runtime used to decide whether + the remaining budget can fit this bench at all. Returns: A ``(bench_result_dict, gate_evidence)`` tuple where @@ -3969,6 +4087,8 @@ async def _bench_patch( result_dir=override_result_dir, base_args_mode=str(params.get("base_args_mode") or "append"), serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) finally: if serving_lease is not None: @@ -4141,12 +4261,18 @@ async def _confirm_stack_rebench( extra_envs_applied: dict[str, str], specialist_task_id: str, base_tput: float, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> dict[str, Any]: """Re-bench the patched stack once more and re-grade throughput + accuracy. Mirrors the explore ledger's post-KEEP confirmation: a patch only KEEPs if a second full-stack run still clears the stability floor and the accuracy gate. Returns ``stable`` / ``tput`` / ``accuracy_pass`` / etc. + + ``session_deadline_sec`` / ``variant_expected_sec`` bound the + confirmation round by the session wall-clock, so it cannot outlive the + run it is confirming for. """ config_path = Path(params.get("config_path") or self.default_config_path or default_baseline_config()) resolved_model = str(params.get("model_path") or "").strip() or os.environ.get("MODEL_PATH", "").strip() @@ -4180,24 +4306,12 @@ async def _confirm_stack_rebench( _rt_rb = params.get("runtime_override") if isinstance(_rt_rb, dict) and _rt_rb: variant.runtime_override = dict(_rt_rb) - # A confirmation rebench must remain a stability check, not become a - # stricter second discovery gate as the per-cycle KEEP threshold decays. - # An explicit lower per-task floor remains valid, but it cannot exceed - # half of the threshold that admitted this patch. - keep_threshold_pct = float(params.get("keep_threshold_pct", self.keep_threshold_pct)) - requested_stable_threshold_pct = float( - params.get("rebench_stable_threshold_pct", DEFAULT_STACK_STABLE_PCT) - ) - stable_threshold_pct = min( - requested_stable_threshold_pct, - max(0.0, keep_threshold_pct / 2.0), - ) rebench = await measure_stack_rebench( config_path=config_path, base_extra_args=base_extra_args, variant=variant, base_tput=base_tput, - stable_threshold_pct=stable_threshold_pct, + stable_threshold_pct=self._rebench_stable_threshold_pct(params), output_slot=output_root / "stack_rebench", variant_timeout_sec=int(params.get("variant_timeout_sec", self.variant_timeout_sec)), model_path=resolved_model or None, @@ -4206,7 +4320,49 @@ async def _confirm_stack_rebench( result_dir=override_result_dir, magpie_python=params.get("magpie_python") or None, base_args_mode=str(params.get("base_args_mode") or "append"), + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) + return self._graded_rebench(rebench, params=params, override_result_dir=override_result_dir) + + def _rebench_stable_threshold_pct(self, params: dict[str, Any]) -> float: + """The floor a confirmation rebench's throughput is graded against. + + A confirmation must remain a stability check rather than become a + stricter second discovery gate as the per-cycle KEEP threshold decays. An + explicit lower per-task floor stays valid, but it cannot exceed half of + the threshold that admitted this patch in the first place. + + Args: + params: The task parameters, read for the per-task overrides. + + Returns: + float: The stability floor as a percentage over the base throughput. + """ + keep_threshold_pct = float(params.get("keep_threshold_pct", self.keep_threshold_pct)) + requested_stable_threshold_pct = float(params.get("rebench_stable_threshold_pct", DEFAULT_STACK_STABLE_PCT)) + return min(requested_stable_threshold_pct, max(0.0, keep_threshold_pct / 2.0)) + + def _graded_rebench( + self, + rebench: StackRebenchResult, + *, + params: dict[str, Any], + override_result_dir: str | None, + ) -> dict[str, Any]: + """Grade a finished confirmation round and shape its verdict for the caller. + + Args: + rebench: The confirmation round's measurement. + params: The task parameters, read for the accuracy baseline and + framework. + override_result_dir: An explicit ``$RESULT_DIR``, which wins over the + round's own workspace when grading accuracy. + + Returns: + dict[str, Any]: ``stable`` / ``tput`` / ``workspace`` / ``warnings`` / + ``stable_floor`` / ``accuracy_pass``. + """ # See ``_bench_patch``: lm-eval writes to the grid slot (the parent of # ``rebench.workspace``), so grade from there, honoring ``result_dir``. rebench_eval_root = override_result_dir or (str(Path(rebench.workspace).parent) if rebench.workspace else "") diff --git a/src/hyperloom/orchestrator/actions/executors/report.py b/src/hyperloom/orchestrator/actions/executors/report.py index 6259fe1b3c..5888d56492 100644 --- a/src/hyperloom/orchestrator/actions/executors/report.py +++ b/src/hyperloom/orchestrator/actions/executors/report.py @@ -377,7 +377,18 @@ def _build_failure_summary( # PRELUDE-phase early exits (before optimization begins). "prelude_baseline_failed": "PRELUDE baseline failed before optimization could start; see the baseline failure summary.", "prelude_policy_loop": "The policy gate detected a decision loop during PRELUDE and stopped.", - "time_exhausted_during_prelude": "The wall-clock budget was exhausted while still in PRELUDE, before optimization began.", + "time_exhausted_during_prelude": ( + "The session's wall-clock budget ran out during preparation, before optimization began. Whatever PRELUDE " + "was doing when the clock reached zero — measuring the baseline, bringing up the framework agent, taking " + "the roofline — is where the time went; the phase record shows which arms ran and what each cost." + ), + "prelude_cold_anchor_low_budget": ( + "The baseline's hot pass was skipped because the clock could not cover it together with one variant to " + "measure against it, so the only figure available is the cold pass's — depressed by the server boot, the " + "first request's kernel compile and the graph capture. Optimizing against it would report every variant as " + "an improvement over a baseline that was never the baseline, so the run stopped with the figure kept and " + "marked. Resume with more budget to measure a comparable baseline." + ), # Recipe KB knowledge-plane bootstrap failures. "recipe_kb_t0_failed": "Recipe KB knowledge-plane bootstrap (t0) failed; the run stopped early.", "recipe_kb_drain_failed": "Recipe KB knowledge-plane drain failed; the run stopped early.", diff --git a/src/hyperloom/orchestrator/actions/executors/sweep.py b/src/hyperloom/orchestrator/actions/executors/sweep.py index 0a12642e3b..311f20f58c 100644 --- a/src/hyperloom/orchestrator/actions/executors/sweep.py +++ b/src/hyperloom/orchestrator/actions/executors/sweep.py @@ -43,6 +43,7 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from ._ray_serving import maybe_serving_lease from ._workload_envs import ( @@ -310,9 +311,13 @@ async def __call__(self, ctx) -> dict[str, Any]: # outlives its GPU lease. ``None`` on the local path (multi-node / # RAY_EXEC off / tests) keeps the legacy behaviour. sweep_lease = maybe_serving_lease(num_gpus=_num_gpus_for_config(config_path)) - # Stop the sweep grid mid-way when the session wall-clock budget runs out. - _ss = extra.get("shared_state") or extra.get("state") - session_deadline_sec = _ss.grid_session_deadline_sec() if _ss is not None else None + # Stop the sweep grid mid-way when the session wall-clock budget runs out, + # and cap each variant at what the budget can still pay for. Admission is + # judged on the expected runtime rather than ``timeout_sec``, which is the + # catastrophic-hang backstop and would abandon the tail of the budget. + session_deadline_sec, variant_expected_sec = session_grid_bounds( + extra.get("shared_state") or extra.get("state") + ) try: # Pass resolved_model / resolved_gpu so variant servers inherit TP/precision. results = await run_grid( @@ -327,6 +332,7 @@ async def __call__(self, ctx) -> dict[str, Any]: result_dir=override_result_dir, serving_lease=sweep_lease, session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) finally: if sweep_lease is not None: diff --git a/src/hyperloom/orchestrator/actions/stop_attribution.py b/src/hyperloom/orchestrator/actions/stop_attribution.py new file mode 100644 index 0000000000..c207f63b33 --- /dev/null +++ b/src/hyperloom/orchestrator/actions/stop_attribution.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""How work the run itself stopped is told apart from work that failed. + +Two causes stop a unit of work without saying anything about what it was +measuring: the session's wall-clock budget running out, and the orchestrator +cancelling the action (a shutdown, or a budget the dispatcher found spent). +Neither is evidence about the model under test, so nothing that ends this way +may be graded as a failed measurement. + +The distinction has to survive into every ledger, not just the one nearest the +subprocess. A grid records the variant, ``explore`` records the fingerprint the +KB reads, and the baseline arm records a failure streak that stops the session +-- three places that each decide what a round *meant*, and each of which would +otherwise file "the run ran out of time" as a verdict about the model. So the +notion lives here, in a leaf every one of them can import, rather than as three +local judgements that can drift apart. + +This module is the error-class side, which is what the ledgers carry. The +returncode side of the same distinction is deliberately not here, and is itself +in two pieces: the two sentinel codes are allocated in +:mod:`..executors._subprocess_kill`, beside every other code that space hands +out so that a collision is visible in one file, and ``stopped_by_the_run``, the +function that reads a code back, sits with the session-budget helpers in +``..executors._grid_runner`` that both benching arms already share. Neither +piece can move here: this leaf is imported by the executors package, so naming +a returncode would close an import cycle. A reader following a sentinel +returncode therefore starts there and arrives at the classes below. +""" + +from __future__ import annotations + +from typing import NamedTuple + +__all__ = [ + "ORCHESTRATOR_CANCELLED_CLASS", + "SESSION_TIME_EXHAUSTED_CLASS", + "STOPPED_BY_THE_RUN", + "StoppedByTheRun", + "stopped_by_the_run_class", +] + +# Labels work that never ran, or did not finish, because the session wall-clock +# budget was spent. Distinct from any measurement failure: work that was not run +# is not evidence about what it would have measured. +SESSION_TIME_EXHAUSTED_CLASS = "session_time_exhausted" + +# Labels work the orchestrator stopped from outside -- a shutdown, or a budget +# the dispatcher found spent. Kept apart from the class above for the same +# reason their returncodes are: a resume faces the spent budget again and does +# not face the shutdown. +ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" + + +class StoppedByTheRun(NamedTuple): + """How work that the run stopped from outside is recorded. + + Attributes: + error_class: The ledger class for the cause. + interrupted: What to report when the work was already running. + never_started: What to report when it never began. + ends_the_batch: Whether the caller should stop launching further work on + its own, rather than leave that to a budget check it may not have. + """ + + error_class: str + interrupted: str + never_started: str + ends_the_batch: bool + + +# The two causes, keyed by the class the ledgers carry. The budget leaves the +# rest of a batch to the fit check its caller runs, which sees a deadline in the +# past and skips them all under the same label; a cancel has no such check, and +# nothing new should start under one. +STOPPED_BY_THE_RUN: dict[str, StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_CLASS: StoppedByTheRun( + error_class=SESSION_TIME_EXHAUSTED_CLASS, + interrupted="session wall-clock budget exhausted while this round was running", + never_started="session wall-clock budget exhausted before this round ran", + ends_the_batch=False, + ), + ORCHESTRATOR_CANCELLED_CLASS: StoppedByTheRun( + error_class=ORCHESTRATOR_CANCELLED_CLASS, + interrupted="the orchestrator cancelled this action while this round was running", + never_started="the orchestrator cancelled this action before this round ran", + ends_the_batch=True, + ), +} + + +def stopped_by_the_run_class(error_class: str | None) -> StoppedByTheRun | None: + """Return how to record work the run itself stopped, if it did. + + Args: + error_class: The ``error_class`` a result carries; anything that is not + one of the two causes (including ``None`` and ``""``) reads as work + that has something to say about what it measured. + + Returns: + StoppedByTheRun | None: How to record it, or ``None`` when the class + names a failure of the thing under test rather than of the run. + """ + if not error_class: + return None + return STOPPED_BY_THE_RUN.get(str(error_class)) diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index b78f962bbc..6b55f540ca 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -77,7 +77,7 @@ ResourceLockManager, SqliteLeaseBackend, ) -from ..state.shared_state import SharedState +from ..state.shared_state import SharedState, effective_closing_grace_sec from .intent_router import IntentRouter from .sub_agent_runner import SubAgentRunner from ..state.task_registry import TaskRegistry @@ -89,7 +89,6 @@ ) from .coordinator_helpers import ( _infer_model_class_from_config, - effective_closing_grace_sec, format_exc_brief, serialize_verdict_advisory, ) @@ -865,6 +864,9 @@ def _ckpt_fraction(env_key: str, default: float) -> float: # Wall-clock budget tracking for per-tick Time-budget prompt injection. self._run_deadline: float | None = None self._run_started_monotonic: float | None = None + # Closing-grace bound; used only while ``closing_phase`` is set so CLOSE + # work is not skipped just because the session deadline has passed. + self._closing_deadline: float | None = None # Latest objective wired by run(); refreshes target_gap_pct each tick. None outside a run. self._current_objective: Objective | None = None @@ -928,6 +930,8 @@ def router(self) -> IntentRouter: "_reseed_orch_prompt_for_phase": "phase_machine", "_record_phase_entry_evidence": "phase_machine", "_internal_analysis_kind": "phase_prelude", + "_measured_analysis_cost_sec": "phase_prelude", + "_record_prelude_arm_dropped": "phase_prelude", "_warm_recipe_proven_items": "phase_prelude", "_inject_warm_recipe_history_into_ledger": "phase_prelude", "_filter_warm_patches_with_kg": "phase_prelude", @@ -939,12 +943,16 @@ def router(self) -> IntentRouter: "_enqueue_internal_conc_sweep_task": "phase_sweep", "_enqueue_internal_sweep_task": "phase_sweep", "_build_sweep_params_from_recipe": "phase_sweep", + "_record_session_budget_conc_sweep_skip": "phase_sweep", + "_record_terminal_conc_sweep_skip": "phase_sweep", "_derive_close_stop_reason": "phase_close", "_session_integrated_kernel_patch": "phase_close", "_maybe_run_close_post_opt_roofline": "phase_close", "_on_enter_close": "phase_close", + "_enqueue_runnable_internal_task": "phase_close", "_enqueue_internal_report_task": "phase_close", "_enqueue_internal_session_breakdown_task": "phase_close", + "_run_close_task": "phase_close", "_record_close_step": "phase_close", "_enter_closing_phase": "phase_close", "_closing_report_terminal": "phase_close", @@ -1051,6 +1059,10 @@ def router(self) -> IntentRouter: "_maybe_escalate_to_targeted_build": "phase_framework", "_maybe_enqueue_specialist_requested_build": "phase_framework", "_maybe_route_build_outcomes": "phase_framework", + "_route_succeeded_build": "phase_framework", + "_build_routing_record": "phase_framework", + "_note_build_routed": "phase_framework", + "_build_probe_was_cancelled": "phase_framework", "_enqueue_build_launch_probe": "phase_framework", "_maybe_rearm_authored_lane": "phase_framework", "_enqueue_author_specialist": "phase_framework", @@ -1083,6 +1095,8 @@ def router(self) -> IntentRouter: "_pump_framework_agent_phase_safely": "phase_framework", "_pump_enablement_safely": "phase_framework", "_maybe_enqueue_enablement_baseline_revalidation": "phase_framework", + "_open_revalidation_row": "phase_framework", + "_open_row_past_spent_generations": "phase_framework", "_record_framework_agent_authored_outcome": "phase_framework", "_recover_framework_agent_authoring_outcome": "phase_framework", "_record_framework_agent_authoring_empty_outcome": "phase_framework", @@ -1128,7 +1142,6 @@ def router(self) -> IntentRouter: "_current_primary_gap": "conversation", "_recent_proposed_variants": "conversation", "_priors_match_advisory_block": "conversation", - "_resolve_issue_canonical": "proposals", "_workload_canonical_id": "proposals", "_read_local_recipe_row": "proposals", "_extract_kept_best_config": "proposals", @@ -1140,7 +1153,6 @@ def router(self) -> IntentRouter: "_record_proposal_task_map": "proposals", "_registry_lanes_ttl": "dispatcher", "_cycle_idem_suffix": "dispatcher", - "_wait_for_task_terminal": "dispatcher", "_cursor_advance_to_latest": "dispatcher", "_dispatch_paused_for_phase_budget": "dispatcher", "_pump_dispatcher_once": "dispatcher", @@ -1154,6 +1166,8 @@ def router(self) -> IntentRouter: "_account_dead_holder_failures": "dispatcher", "_lanes_fit": "dispatcher", "_sequence_denial_for_action": "dispatcher", + "_time_budget_denial_for_action": "dispatcher", + "_admission_denial_for_action": "dispatcher", "_sequence_denial_for_request": "dispatcher", "_skip_gemm_tuning": "dispatcher", "_gemm_tuning_required_before_kernel_opt": "dispatcher", @@ -1396,15 +1410,24 @@ def _reap_orphaned_servers_best_effort(self) -> None: # Lifecycle async def stop(self) -> None: - """Signal shutdown, cancel reactor tasks, finalize, and close the DB. - - Sets the stop event, cancels and awaits every running reactor task, - runs the Recipe KB T4 safety-net finalize hook when CLOSE never reached - a terminal publication status or its earlier attempt failed, then closes - the SQLite connection. Exceptions raised by reactor tasks during - teardown are logged, not propagated. + """Signal shutdown, cancel in-flight work, finalize, and close the DB. + + Sets the stop event, cancels and awaits the dispatched actions still + running plus every running reactor task, runs the Recipe KB T4 + safety-net finalize hook when CLOSE never reached a terminal + publication status or its earlier attempt failed, then closes the + SQLite connection. Exceptions raised by reactor tasks during teardown + are logged, not propagated. + + Dispatched actions are cancelled first and awaited: the stop event alone + only asks the loop to stop between ticks, so a teardown that skipped + them would close the database out from under work still using it. """ self._stop.set() + try: + await self.dispatcher.cancel_inflight_actions(reason="coordinator_stop") + except Exception: # noqa: BLE001 — teardown proceeds even if cancellation misbehaves + log.exception("Coordinator.stop: cancelling in-flight actions raised") for t in self._tasks_running: if not t.done(): t.cancel() @@ -1501,18 +1524,30 @@ async def tick(self, n: int = 1) -> None: # hint (for example current GEAK returning no_gain -> skip_to_sweep). # Consume that before prompting agents again so stale phase prompts # cannot enqueue legacy work. - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_pre_reactor", + ) if str(getattr(self.shared_state, "pending_escalate_hint", "") or "").strip(): - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_hint", + ) for name in self._tick_roles: - await self._reactor_pass(name) + await self._await_within_session_bound( + lambda n=name: self._reactor_pass(n), + stage=f"reactor:{name}", + ) await self._pump_dispatcher_once() # FRAMEWORK_AGENT phase pump: enqueue next candidate / fetch next batch. await self._pump_framework_agent_phase_safely(caller="tick") # Phase-independent enablement pump: repair a non-runnable combo. await self._pump_enablement_safely(caller="tick") # phase machine advance at tick boundary. - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase", + ) def _record_coordinator_exception( self, @@ -1545,6 +1580,53 @@ def _record_coordinator_exception( except Exception: # noqa: BLE001 log.exception("failed to persist Coordinator exception metadata") + def _seconds_until_session_bound(self) -> float | None: + """Seconds left on the active run or closing bound; ``None`` if unbounded. + + During CLOSE the session deadline has already passed, so the bound + switches to ``_closing_deadline`` and CLOSE work is not skipped. + + Returns: + Remaining seconds, or ``None`` when no bound is armed. + """ + if bool(getattr(self.shared_state, "closing_phase", False)): + bound = self._closing_deadline + else: + bound = self._run_deadline + if bound is None: + return None + return float(bound) - time.monotonic() + + async def _await_within_session_bound( + self, + factory: Callable[[], Awaitable[Any]], + *, + stage: str, + ) -> None: + """Run one tick step, cancelling it when the session/closing bound elapses. + + The wall-clock stop lives at the end of each tick. A conversational + reactor turn or a long phase-enter await that never returns would skip + that stop. Cancelling the step lets the tick finish and enter CLOSE. + + Args: + factory: Builds the awaitable so a skipped step is never started. + stage: Label for the warning log. + """ + remaining = self._seconds_until_session_bound() + if remaining is not None and remaining <= 0.0: + log.warning("Coordinator: skipping %s; session bound already elapsed", stage) + return + try: + # ``timeout=None`` waits until the step finishes (unbounded run). + await asyncio.wait_for(factory(), timeout=remaining) + except asyncio.TimeoutError: + log.warning( + "Coordinator: %s hit the session bound after %.1fs; cancelled so the tick can close", + stage, + remaining, + ) + # Long-run interface async def run( self, @@ -1594,6 +1676,10 @@ async def run( self._coordinator_loop = asyncio.get_running_loop() except RuntimeError: self._coordinator_loop = None + # The reserve every unit of work holds back IS this window, so the + # admission gate has to see the operator's choice too, not only the + # closing phase that spends it. + self.shared_state.closing_grace_sec = closing_grace_sec max_minutes_value = max_minutes if max_minutes is not None else 0 # Persist budget so prompts and Resume can see it. if max_minutes is not None: @@ -1627,9 +1713,15 @@ async def run( # Bump the persistent tick counter — drives phase/plateau math. self.shared_state.increment_tick() try: - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_pre_reactor", + ) if str(getattr(self.shared_state, "pending_escalate_hint", "") or "").strip(): - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_hint", + ) except Exception as exc: # noqa: BLE001 log.exception("phase advance before reactors (run) failed") self._record_coordinator_exception( @@ -1643,7 +1735,10 @@ async def run( for name in self._tick_roles: if self._stop.is_set(): break - await self._reactor_pass(name) + await self._await_within_session_bound( + lambda n=name: self._reactor_pass(n), + stage=f"reactor:{name}", + ) # Orchestration checkpoint/compaction; cadence-based. if not self._stop.is_set(): try: @@ -1668,7 +1763,10 @@ async def run( log.exception("targeted-build tick raised") # phase machine advance; runs even in_closing so CLOSE is recorded. try: - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase", + ) except Exception as exc: # noqa: BLE001 log.exception("phase advance (run) failed") self._record_coordinator_exception( @@ -1709,6 +1807,7 @@ async def run( closing_deadline = await self._enter_closing_phase( grace_sec=grace_sec, ) + self._closing_deadline = closing_deadline continue if in_closing: report_terminal = await self._closing_report_terminal() @@ -2080,7 +2179,7 @@ async def _track_backend_error_streak( "CoordinatorState", "PendingProposal", "SharedState", - # Re-exported from coordinator_helpers for callers/tests. + # Re-exported from coordinator_helpers / state.shared_state for callers/tests. "_infer_model_class_from_config", "effective_closing_grace_sec", # Re-exported from policy.gate; referenced via ``coordinator.`` in tests. diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index e26ac35bc8..a332f1339b 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -31,9 +31,13 @@ # Constants below are read from other modules; listed here to mark them as # intentionally exported. __all__ = [ + "TIME_BUDGET_EXEMPT_ACTIONS", "_GEAK_MEASUREMENT_DIVERGENCE_WARN_PCT", "_MIN_KERNEL_ENGAGED_GAIN_PCT", + "action_fits_time_budget", "coerce_needs_gpu", + "expected_action_cost_minutes", + "measured_baseline_runtime_sec", ] @@ -216,26 +220,142 @@ def _positive_int(*keys: str) -> bool: _MAX_ROOFLINE_FAILURE_RETRIES: int = 3 -def effective_closing_grace_sec( - max_minutes: float | None, - closing_grace_sec: float | None, +# Actions that must stay startable no matter how little budget is left: they +# are how a session ends cleanly, so a time gate that refused them would +# strand the run with nothing to show. ``recover`` is not among them — it +# takes the server-lifecycle lane, prices at five catalogue minutes, and +# holds a twenty-minute lease; treating it as a closing action is what let a +# spent session keep working past the wall clock. +TIME_BUDGET_EXEMPT_ACTIONS: frozenset[str] = frozenset( + { + "report", + "session_breakdown", + } +) + +# The lanes that serialize GPU work: an action requiring one of them spends its +# time running a benchmark round, so what this session measured says more about +# it than a catalogue estimate does. +_GPU_BENCH_LANES: frozenset[str] = frozenset( + { + "benchmark_lane", + "profile_lane", + } +) + + +def measured_baseline_runtime_sec(shared_state: Any | None) -> float: + """Read this session's own measured baseline round, in seconds. + + Args: + shared_state (Any | None): The session ``SharedState``, or ``None`` when + the caller has no session context. + + Returns: + float: The measured baseline runtime; ``0.0`` when the session has not + landed a baseline yet, which every caller reads as "no measurement". + """ + try: + return max(0.0, float(getattr(shared_state, "baseline_runtime_sec", 0.0) or 0.0)) + except (TypeError, ValueError): + return 0.0 + + +def _action_benches_on_gpu(meta: Any | None) -> bool: + """Whether an action's cost is dominated by a benchmark round on the GPU. + + Read off the lanes the action must hold rather than off a list of names, so + an action added to the catalogue is classified by what it does. The + benchmark and profile lanes are exactly the two that serialize GPU work; an + action holding neither (``report``, ``target_analysis``, ``specialist``) + costs what its own bookkeeping costs and has nothing to do with model size. + + Args: + meta (Any | None): The action's catalogue metadata. + + Returns: + bool: ``True`` when the action runs at least one benchmark round. + """ + lanes = getattr(meta, "requires_lanes", ()) or () + try: + return any(str(lane) in _GPU_BENCH_LANES for lane in lanes) + except TypeError: + return False + + +def expected_action_cost_minutes( + meta: Any | None, + *, + measured_baseline_sec: float = 0.0, ) -> float: - """Resolve the closing-phase grace window after the wall-clock deadline. + """Read an action's expected cost, preferring what this session measured. + + Every budget guard goes through here so the field is named once. Reading it + inline with a ``getattr`` default turned the catalogue's move off YAML — + which renamed the field — into a gate that admitted everything without a + word, because "no estimate on record" and "the field moved" look the same + from a default. + + The catalogue's estimates are calibrated on small models (``baseline`` 5 + min, ``roofline`` 10 min) while the two sessions that motivated the + wall-clock work measured 51 and 125 minutes of baseline and an 81-minute + roofline. A guard anchored on the catalogue alone therefore admits arms the + session cannot pay for — it would not have stopped either field run. So one + measured baseline round is taken as a *floor* on any action that runs a + benchmark round of its own: it is this model on this GPU under this + workload, which is what those actions spend their time doing. It is a floor + rather than a replacement because an action that benches several variants + costs more than one round, never less, and the catalogue is the only thing + that knows how many. - Explicit ``closing_grace_sec`` (including ``0`` to disable) wins; - otherwise default to ``min(120, max_minutes * 60 * 0.02)``. + Args: + meta (Any | None): The action's catalogue metadata, or ``None`` for an + action the catalogue does not carry. + measured_baseline_sec (float): This session's measured baseline runtime + in seconds, from :func:`measured_baseline_runtime_sec`; ``0.0`` + before a baseline lands, which leaves the catalogue in charge. + + Returns: + float: The expected cost in minutes; ``0.0`` when nothing is on record. + """ + try: + catalogue_min = float(getattr(meta, "typical_runtime_min", 0.0) or 0.0) + except (TypeError, ValueError): + catalogue_min = 0.0 + if measured_baseline_sec <= 0.0 or not _action_benches_on_gpu(meta): + return catalogue_min + return max(catalogue_min, measured_baseline_sec / 60.0) + + +def action_fits_time_budget( + *, + usable_sec: float | None, + expected_cost_minutes: float, +) -> bool: + """Decide whether an action's expected cost still fits the remaining budget. + + The anchor is the action's *expected* cost (its typical runtime), not its + pessimistic tail. Judging fit on the pessimistic tail would abandon usable + budget — with 90 minutes left we would refuse an action that finishes in 60 + minutes half the time — and the session already has a wall-clock reaper for + the overruns, so the optimistic anchor is the one that keeps the tail of a + session productive. This mirrors how the grid admits variants. Args: - max_minutes: The wall-clock budget in minutes (used for the default). - closing_grace_sec: Explicit grace window in seconds; when not - ``None`` it is used verbatim. + usable_sec: Budget left after the closing reserve, from + ``SharedState.session_budget_usable_sec``; ``None`` means unbounded. + expected_cost_minutes: The action's expected cost in minutes; values at + or below zero mean "no estimate on record". Returns: - The closing-phase grace window in seconds. + ``True`` when the action may start: the budget is unbounded, no estimate + is on record, or the expected cost fits what is left. """ - if closing_grace_sec is not None: - return float(closing_grace_sec) - return min(120.0, (max_minutes or 0.0) * 60.0 * 0.02) + if usable_sec is None: + return True + if expected_cost_minutes <= 0.0: + return True + return usable_sec >= expected_cost_minutes * 60.0 def _parse_iso_unix(ts: str) -> float: diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 37d053246a..3cd71f870a 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -8,12 +8,24 @@ import hashlib import json import os +from collections.abc import Collection +from concurrent.futures import CancelledError as FuturesCancelledError from concurrent.futures import TimeoutError as FuturesTimeoutError -from typing import Any +from typing import Any, NamedTuple from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType from hyperloom.inference_optimizer.protocol.action_surfaces import ( KERNEL_AGENT_OWNED_ACTIONS, ) +from ..actions.cancel_channel import CancelScope, use_cancel_scope +from ..actions.executors._ray_serving import ( + CANCEL_ROUND_GRACE_SEC, + CLOSE_STOP_TIMEOUT_SEC, +) +from ..actions.executors._subprocess_kill import ( + COOPERATIVE_REAP_BUDGET_SEC, + STOP_GATE_POLL_SECONDS, + TERM_GRACE_SECONDS, +) from ..phases import machine_state as _phase_state from ..bus.message_bus import Message from ..kernel.request_handlers import get_handler @@ -31,7 +43,13 @@ ) from .sub_agent_runner import SubAgentResult from ..state.task_registry import Task -from .coordinator_helpers import coerce_needs_gpu +from .coordinator_helpers import ( + TIME_BUDGET_EXEMPT_ACTIONS, + action_fits_time_budget, + coerce_needs_gpu, + expected_action_cost_minutes, + measured_baseline_runtime_sec, +) from .coordinator import ( _format_inbox_event, @@ -40,6 +58,70 @@ log = _logging.getLogger(__name__) +# How long a cancel waits for work that is listening on its cancel scope to stop +# itself, composed from the two things an action still has to do after it is +# asked: +# +# * stop the round in flight -- locally that is +# :data:`COOPERATIVE_REAP_BUDGET_SEC`, through a Ray actor it is +# :data:`CANCEL_ROUND_GRACE_SEC`, and whichever path this action took, only the +# longer of the two bounds it; +# * release what the round held -- the driver-side teardown of the server it left +# behind (:data:`TERM_GRACE_SECONDS`) and then the release of the Ray lease it +# ran in (:data:`CLOSE_STOP_TIMEOUT_SEC`). A sequence, not alternatives: the +# server is reaped BEFORE the lease is dropped so that no GPU process outlives +# it (§4.2), and on the Ray path a single unwind pays both -- in the explore +# executor the per-variant ``finally`` tears the server down and the enclosing +# one then closes the round's lease; the baseline executor does the same two +# calls in one ``finally``. +# +# Derived rather than picked, because these three windows only mean anything +# together. Each was plausible on its own at ten, eight and five seconds, and +# composed they said the dispatcher gives up a good five seconds before the work +# it is waiting for can finish -- so the honest sentinel the round was about to +# return was discarded for a hard ``CancelledError`` every time, which is exactly +# what the cooperative channel exists to avoid. Taking the longer of the two +# release terms instead of both reproduced that shortfall exactly, on the path +# that pays the most: the teardown a Ray round owes is not an alternative to +# closing its lease, it is what it does first. +# +# The enumeration above has to stay exhaustive, so a serial step the unwind takes +# and this sum does not name has to be removed from the unwind instead. One is: +# ``run_grid`` fires a robustness tick at each variant boundary, and a cooperative +# stop reaches that boundary the ordinary way, so the tick would sit between +# recording the round's row and releasing what the round held -- eight more +# seconds, spent observing the reap this cancel just ordered. It is skipped +# whenever the scope is already cancelled rather than budgeted for, because the +# rows the grid has already built are what a window short by those eight seconds +# throws away, and they are worth more than one tick. +# +# Past this window the coroutine is cancelled anyway, and the window is only ever +# spent when work is still unwinding: the wait ends the moment the last victim is +# done, so covering the teardown term costs a round that stops promptly nothing. +# What the budget case pays for it is five more seconds before the run crosses its +# deadline, because this wait runs inside the reserve the admission gate holds +# back. It does not come out of the closing phase, whose grace window is measured +# from the moment it starts, and five seconds of overshoot is cheaper than the +# attributed sentinel a hard cancel destroys. +_COOPERATIVE_CANCEL_GRACE_SEC: float = ( + max(COOPERATIVE_REAP_BUDGET_SEC, CANCEL_ROUND_GRACE_SEC) + TERM_GRACE_SECONDS + CLOSE_STOP_TIMEOUT_SEC +) + +# How long a cancel waits for anything to start listening before deciding +# nothing will. Work that is already blocking is listening before the cancel is +# raised; the window is for the gap between an action starting and reaching the +# call that watches the scope, so it is the poll that call checks the scope at +# rather than anything about how long the work takes. +_CANCEL_NOTICE_SEC: float = STOP_GATE_POLL_SECONDS + + +class _InflightAction(NamedTuple): + """A running action's handle: what it is, its task, and how to ask it to stop.""" + + kind: str + atask: asyncio.Task[Any] + scope: CancelScope + class DispatcherCollaborator: """Extracted collaborator; delegates unknown attrs to its Coordinator.""" @@ -49,6 +131,13 @@ def __init__(self, coordinator) -> None: # Task ids already charged a failure by the dead-holder reclaim path, so # a late normal result for the same task cannot double-count it. self._dead_holder_accounted: set[str] = set() + # Handles on the actions currently running, ``task_id -> _InflightAction``. + # Kept on the collaborator and not only in the pump's frame: an action + # whose handle lives in a frame can only be stopped by the frame that is + # already blocked awaiting it, which is precisely the situation shutdown + # and an exhausted wall-clock budget have to break. Entries remove + # themselves in :meth:`_run_dispatched_with_gpu_release`. + self._inflight_actions: dict[str, _InflightAction] = {} def __getattr__(self, name: str): return getattr(object.__getattribute__(self, "_coord"), name) @@ -116,24 +205,148 @@ def _dispatch_paused_for_phase_budget(self) -> bool: return False return remaining is not None and remaining <= 0.0 - async def _pump_dispatcher_once(self) -> None: - """Dispatch queued tasks respecting per-lane capacity, re-scanning for - newly-fittable tasks while in-flight tasks run. + async def cancel_inflight_actions( + self, + *, + reason: str, + exempt: frozenset[str] = frozenset(), + only_task_ids: Collection[str] | None = None, + ) -> list[str]: + """Stop the running dispatched actions and wait for them to unwind. + + The last of the wall-clock defences, and the only one that reaches work + already under way: admission refuses what cannot fit, the timeout clamp + bounds what does, and the subprocess reaper stops the child trees that + were handed a session deadline. + + Cancelling the action's task is not by itself enough to stop it. Every + benchmark executor spends its time inside ``asyncio.to_thread``, and a + thread that has started cannot be cancelled: the ``await`` raises here + while the subprocess runs on, so the lanes and the GPU lease would be + released, and the database closed, under a benchmark still holding the + card. So the cancel goes out on the action's :class:`CancelScope` first + -- the channel the blocking side polls -- and work that is listening on + it is given :data:`_COOPERATIVE_CANCEL_GRACE_SEC` to stop itself and + return through its own ``finally`` blocks. Whatever is still running + after that is cancelled the old way, which is no worse than not having + asked. + + Every scope is cancelled before the first await, so a caller that is + itself being cancelled still leaves no action running unattended: the + work that can hear the channel stops on it even if this coroutine never + reaches the wait. - Re-scans the queue whenever an in-flight task completes - (FIRST_COMPLETED) or a short poll elapses, so a queued GPU task starts - the moment its lane frees. The pump still fully drains all currently - dispatchable work before returning. Each GPU lease is bound to its - task_id and released by the runner. + Args: + reason: Short cause, logged, used as evidence, and carried to the + blocking side so it can attribute its own stop. + exempt: Action kinds to leave running -- the closing actions, when + the trigger is a budget that already reserved time for them. + only_task_ids: Restrict the cancel to these ids, for a caller that + owns part of the registry rather than all of it. ``None`` reaches + every action that is not exempt, which is what a shutdown or a + spent budget needs. - Budget guard: once the phase's cyclic budget is spent - (:meth:`_dispatch_paused_for_phase_budget`), stop spawning NEW - phase-scoped variants — drain in-flight, then return so the tick can - advance the phase. + Returns: + list[str]: Task ids that were stopped (empty when nothing ran). + """ + victims = [ + (task_id, entry) + for task_id, entry in self._inflight_actions.items() + if entry.kind not in exempt + and not entry.atask.done() + and (only_task_ids is None or task_id in only_task_ids) + ] + if not victims: + return [] + log.warning( + "dispatcher: cancelling %d in-flight action(s) [%s]: %s", + len(victims), + reason, + ", ".join(f"{entry.kind}/{task_id[:12]}" for task_id, entry in victims), + ) + for _task_id, entry in victims: + entry.scope.cancel(reason=reason) + try: + await self._wait_for_cooperative_stop(victims) + finally: + for _task_id, entry in victims: + if not entry.atask.done(): + entry.atask.cancel() + await asyncio.gather(*(entry.atask for _task_id, entry in victims), return_exceptions=True) + return [task_id for task_id, _entry in victims] + + async def _wait_for_cooperative_stop(self, victims: list[tuple[str, _InflightAction]]) -> None: + """Give already-cancelled work the chance to stop itself and return. + + Only work watching its scope can be waited for: waiting on the rest + would trade a thread that outlives the cancel for a shutdown that blocks + on one, and the second is the worse failure. So the wait ends at + whichever comes first -- every victim unwound, the grace spent, or the + notice window closing with nothing listening. + + Args: + victims: The ``(task_id, handle)`` pairs whose scopes were just + cancelled. + """ + loop = asyncio.get_running_loop() + notice_deadline = loop.time() + _CANCEL_NOTICE_SEC + grace_deadline = loop.time() + _COOPERATIVE_CANCEL_GRACE_SEC + listening = False + while True: + alive = [entry.atask for _task_id, entry in victims if not entry.atask.done()] + if not alive: + return + listening = listening or any(entry.scope.has_listeners for _task_id, entry in victims) + now = loop.time() + deadline = grace_deadline if listening else notice_deadline + if now >= deadline: + return + await asyncio.wait(alive, timeout=deadline - now, return_when=asyncio.FIRST_COMPLETED) + + async def _cancel_inflight_that_outlived_the_session(self) -> bool: + """Stop in-flight actions the session can no longer wait for. + + Two causes, both of which mean no result is coming: the process was + asked to shut down, or the wall-clock budget is spent. The budget case + spares :data:`TIME_BUDGET_EXEMPT_ACTIONS` because the closing reserve + this gate trips on exists precisely so those actions can run. + + Returns: + bool: ``True`` when the pump must not start anything new. Only a + shutdown says that; a spent budget does not, because the queue scan + is what cancels the rows it can no longer fit, and the closing + actions it exempts still have their reserve to run in. + """ + stop_event = getattr(self, "_stop", None) + if stop_event is not None and stop_event.is_set(): + await self.cancel_inflight_actions(reason="shutdown_requested") + return True + usable_sec = self.shared_state.session_budget_usable_sec() + if usable_sec is not None and usable_sec <= 0.0: + await self.cancel_inflight_actions( + reason="session_time_exhausted", + exempt=TIME_BUDGET_EXEMPT_ACTIONS, + ) + return False + + async def _reclaim_stale_dispatch_state(self) -> None: + """Free rows and leases a previous tick left stuck, before scanning the queue. + + Runs every tick and is idempotent. Four independent claims a crashed or + vanished worker can leave behind, each reclaimed on its own so one + failure does not hide the next -- and every one of them best-effort, + because a self-heal that raises would stop the pump it exists to keep + running: + + * a running row whose holder PID is dead, so its lanes free and the task + fails while it is still retry-eligible, this same tick; + * the failure that reclaim implies, charged once; + * the lane leases that holder still held; + * a running row past its TTL, covering a recycled holder PID or a + missing holder record, which the dead-PID check cannot see; + * an ``integrate_patch`` row cancelled at dispatch whose critic verdict + was restored afterwards by a resume. """ - # Dead-holder self-heal (runs every tick, before scanning the queue): - # detect a crashed worker's dead PID so its leased lanes free and the - # stuck task fails (retry-eligible) this same tick. dead_tasks: list[str] = [] try: dead_tasks = await self.tasks.reclaim_dead_running(reason="dead_holder_pump") @@ -154,8 +367,6 @@ async def _pump_dispatcher_once(self) -> None: await self.locks.reap_dead_holders() except Exception: # noqa: BLE001 log.exception("dispatcher: dead-holder lease reap failed") - # TTL-expiry self-heal (runs every tick): covers tasks whose holder PID - # was recycled or whose holder record is missing. Idempotent. try: expired_tasks = await self.tasks.reclaim_expired_running(reason="pump_watchdog") if expired_tasks: @@ -170,43 +381,76 @@ async def _pump_dispatcher_once(self) -> None: await self._reconcile_cancelled_policy_denied_integrate_tasks() except Exception: # noqa: BLE001 — reconcile must not abort the pump log.exception("dispatcher: cancelled policy-denied integrate_patch reconcile failed") + + async def _pump_dispatcher_once(self) -> None: + """Dispatch queued tasks respecting per-lane capacity, re-scanning for + newly-fittable tasks while in-flight tasks run. + + Re-scans the queue whenever an in-flight task completes + (FIRST_COMPLETED) or a short poll elapses, so a queued GPU task starts + the moment its lane frees. The pump still fully drains all currently + dispatchable work before returning. Each GPU lease is bound to its + task_id and released by the runner. + + Budget guard: once the phase's cyclic budget is spent + (:meth:`_dispatch_paused_for_phase_budget`), stop spawning NEW + phase-scoped variants — drain in-flight, then return so the tick can + advance the phase. + """ + await self._reclaim_stale_dispatch_state() inflight: list[tuple[Task, asyncio.Task[SubAgentResult], Any]] = [] # Cumulative across the whole pump, not just the live in-flight set, so a # fast task reaped before its queued->running transition is visible is # not re-dispatched. A task is dispatched at most once per pump. dispatched_ids: set[str] = set() - while True: - # Budget guard: stop launching NEW phase-scoped variants once the - # phase's cyclic budget is spent; drain in-flight then return. - if not self._dispatch_paused_for_phase_budget(): - spawned = await self._spawn_fitting_queued(exclude_ids=dispatched_ids) - dispatched_ids.update(t.task_id for t, _, _ in spawned) - inflight.extend(spawned) - if not inflight: - return - done, _pending = await asyncio.wait( - [atask for _, atask, _ in inflight], - timeout=self._dispatcher_poll_sec, - return_when=asyncio.FIRST_COMPLETED, + try: + while True: + # Wall-clock guard: a spent session budget (or a shutdown + # request) stops the actions already running, because waiting + # for them is what the budget no longer allows. + shutting_down = await self._cancel_inflight_that_outlived_the_session() + # Budget guard: stop launching NEW phase-scoped variants once the + # phase's cyclic budget is spent; drain in-flight then return. + if not shutting_down and not self._dispatch_paused_for_phase_budget(): + spawned = await self._spawn_fitting_queued(exclude_ids=dispatched_ids) + dispatched_ids.update(t.task_id for t, _, _ in spawned) + inflight.extend(spawned) + if not inflight: + return + done, _pending = await asyncio.wait( + [atask for _, atask, _ in inflight], + timeout=self._dispatcher_poll_sec, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + # Poll elapsed with no completion; re-scan in case a lane freed. + continue + remaining: list[tuple[Task, asyncio.Task[SubAgentResult], Any]] = [] + completed: list[tuple[Task, Any, Any]] = [] + for entry in inflight: + task, atask, gpu_lease = entry + if atask in done: + try: + maybe_result: Any = atask.result() + except (Exception, asyncio.CancelledError) as exc: # noqa: BLE001 — mirror gather(return_exceptions=True); capture task error + cancellation, never KeyboardInterrupt/SystemExit + maybe_result = exc + completed.append((task, maybe_result, gpu_lease)) + else: + remaining.append(entry) + inflight = remaining + for task, maybe_result, gpu_lease in completed: + await self._reap_dispatched_task(task, maybe_result, gpu_lease) + finally: + # ``_inflight_actions`` is dispatcher-wide: the inline path registers + # a handle there too, and that action is meant to outlive the caller + # that started it. The pump owns exactly the entries still in its own + # ``inflight``, so leaving by any door other than the drained one -- + # cancelled at shutdown, or a raise from the bookkeeping -- takes + # those and nothing else. A drained pump has nothing left to cancel. + await self.cancel_inflight_actions( + reason="dispatcher_pump_exit", + only_task_ids={task.task_id for task, _atask, _gpu_lease in inflight}, ) - if not done: - # Poll elapsed with no completion; re-scan in case a lane freed. - continue - remaining: list[tuple[Task, asyncio.Task[SubAgentResult], Any]] = [] - completed: list[tuple[Task, Any, Any]] = [] - for entry in inflight: - task, atask, gpu_lease = entry - if atask in done: - try: - maybe_result: Any = atask.result() - except (Exception, asyncio.CancelledError) as exc: # noqa: BLE001 — mirror gather(return_exceptions=True); capture task error + cancellation, never KeyboardInterrupt/SystemExit - maybe_result = exc - completed.append((task, maybe_result, gpu_lease)) - else: - remaining.append(entry) - inflight = remaining - for task, maybe_result, gpu_lease in completed: - await self._reap_dispatched_task(task, maybe_result, gpu_lease) async def _reconcile_cancelled_policy_denied_integrate_tasks(self) -> list[str]: """Re-queue integrate_patch rows cancelled at dispatch when policy now passes. @@ -358,6 +602,8 @@ async def _spawn_fitting_queued( # Off-loop builds run in their own process group and are pumped/ # reaped by BuildLifecycleCollaborator; never drain them here. continue + if await self._cancel_queued_task_over_budget(task): + continue lanes_needed = list(task.requires_lanes or []) if lanes_needed: # SQLite lane gate. Under single-node Ray execution the @@ -566,21 +812,19 @@ async def _spawn_fitting_queued( ) except Exception: # noqa: BLE001 - audit must never affect dispatch pass - spawned.append( - ( + cancel_scope = CancelScope() + atask = asyncio.create_task( + self._run_dispatched_with_gpu_release( task, - asyncio.create_task( - self._run_dispatched_with_gpu_release( - task, - prebound_lease=lease, - extra_context=extra_context, - gpu_lease=gpu_lease, - gpu_specialist_lease=gpu_specialist_lease, - ), - ), - gpu_lease, - ) + prebound_lease=lease, + extra_context=extra_context, + gpu_lease=gpu_lease, + gpu_specialist_lease=gpu_specialist_lease, + cancel_scope=cancel_scope, + ), ) + self._inflight_actions[task.task_id] = _InflightAction(task.kind, atask, cancel_scope) + spawned.append((task, atask, gpu_lease)) return spawned async def _run_dispatched_with_gpu_release( @@ -591,6 +835,7 @@ async def _run_dispatched_with_gpu_release( extra_context: dict[str, Any], gpu_lease: Any, gpu_specialist_lease: Any = None, + cancel_scope: CancelScope | None = None, ) -> "SubAgentResult": """Run a dispatched task, releasing its GPU lease in a structured finally. @@ -600,7 +845,9 @@ async def _run_dispatched_with_gpu_release( ``release`` is idempotent, so the release in :meth:`_reap_dispatched_task` remains harmless. When a Ray ``GpuSpecialistLease`` was acquired it is closed here too so - the ``num_gpus`` lease is released on every exit path. + the ``num_gpus`` lease is released on every exit path. The same finally + retires this task's ``_inflight_actions`` handle, so the set cannot + outlive the work it points at. Args: task: The dispatched task. @@ -611,17 +858,22 @@ async def _run_dispatched_with_gpu_release( gpu_specialist_lease: The Ray ``GpuSpecialistLease`` to close (release the ``num_gpus`` actor lease), or None on the local path. + cancel_scope: The action's cancel channel, published here rather + than at the call site because a task copies its context when it + is created and never sees a value set afterwards. Returns: SubAgentResult: The result from ``sub.run_task``. """ try: - return await self.sub.run_task( - task, - prebound_lease=prebound_lease, - extra_context=extra_context, - ) + with use_cancel_scope(cancel_scope): + return await self.sub.run_task( + task, + prebound_lease=prebound_lease, + extra_context=extra_context, + ) finally: + self._inflight_actions.pop(task.task_id, None) if gpu_lease is not None: try: await self.gpu_specialist_pool.release(gpu_lease) @@ -858,6 +1110,16 @@ async def _reap_dispatched_task( "dispatcher: failed to release GPU specialist lease for task=%s", task.task_id, ) + if isinstance(maybe_result, asyncio.CancelledError): + # Asked for, not gone wrong: the wall-clock defences stop + # in-flight actions on purpose. Logged as the deliberate act it + # is so a shutdown does not read as a crash. + log.warning( + "dispatcher: in-flight action task=%s kind=%s was cancelled", + task.task_id, + task.kind, + ) + continue if isinstance(maybe_result, BaseException): log.exception( "dispatcher: spawned task %s raised: %r", @@ -1157,6 +1419,149 @@ def _sequence_denial_for_action( ) return None + def _time_budget_denial_for_action( + self, + action_name: str, + ) -> PolicyDenied | None: + """Refuse an action whose expected cost cannot fit the remaining session budget. + + The first of the wall-clock defences: cheaper to never start a 60-minute + action with 20 minutes left than to reap it half-done, because a reaped + action spends the budget and yields no measurement. Denying here also + keeps the refusal out of the failure ledgers — no task row is created, so + nothing teaches the KB that the action failed. + + What the action is expected to cost is anchored on this session's own + baseline round once one exists, the way PRELUDE's affordability gate + already is; see :func:`expected_action_cost_minutes` for why a + catalogue-anchored gate admits arms a real model cannot pay for. + + Args: + action_name: The proposed/delegated/inline action name. + + Returns: + A :class:`PolicyDenied` when the budget cannot fit the action, else + ``None`` (unbounded budget, exempt action, or no cost on record). + """ + action = str(action_name or "").strip() + if not action or action in TIME_BUDGET_EXEMPT_ACTIONS: + return None + if self.shared_state.stop_reason: + return None + reg = getattr(self, "action_registry", None) + meta = reg.get(action) if reg is not None else None + if meta is None: + return None + expected_min = expected_action_cost_minutes( + meta, + measured_baseline_sec=measured_baseline_runtime_sec(self.shared_state), + ) + usable_sec = self.shared_state.session_budget_usable_sec() + if action_fits_time_budget( + usable_sec=usable_sec, + expected_cost_minutes=expected_min, + ): + return None + remaining_min = (usable_sec or 0.0) / 60.0 + return PolicyDenied( + f"action={action!r} denied: needs ~{expected_min:.0f} min but only " + f"{remaining_min:.0f} min of the session budget is left", + rule="time_budget", + hint=( + "the wall-clock budget cannot fit this action; delegate `report` " + "to close the session, or pick an action that fits the time left" + ), + ) + + def _admission_denial_for_action( + self, + action_name: str, + ) -> PolicyDenied | None: + """Run every pre-dispatch gate for an action name, first denial wins. + + The single entry point the intent handlers and the inline runner share, + so a new gate reaches all three paths at once. + + Args: + action_name: The proposed/delegated/inline action name. + + Returns: + The first :class:`PolicyDenied` that fires, else ``None``. + """ + denied = self._sequence_denial_for_action(action_name) + if denied is not None: + return denied + return self._time_budget_denial_for_action(action_name) + + async def _cancel_queued_task_over_budget(self, task: Task) -> bool: + """Drop a queued task the budget can no longer fit, before it takes a lane. + + The admission gate runs when the action is proposed, but a task can wait + for a busy lane long enough for the budget to drain underneath it. This + is the same gate re-applied at the last moment it is still free to say + no. The row is cancelled rather than left queued so the pump does not + re-examine it every tick, and it stays out of the failure ledgers: a task + that never ran is not evidence about the action. + + Args: + task: The queued task about to be considered for dispatch. + + Returns: + ``True`` when the task was cancelled and must be skipped this pass. + """ + denied = self._time_budget_denial_for_action(task.kind) + if denied is None: + return False + try: + await self.tasks.transition( + task.task_id, + "cancelled", + evidence={"reason": "time_budget", "error": str(denied)}, + ) + except Exception: # noqa: BLE001 — a lost row must not abort the pump + log.exception( + "dispatcher: could not cancel over-budget task=%s kind=%s", + task.task_id, + task.kind, + ) + return True + log.warning( + "dispatcher: dropped queued task=%s kind=%s before dispatch: %s", + task.task_id, + task.kind, + denied, + ) + try: + await self._record_observation( + "coordinator", + "observation", + { + "kind": "dispatch_denied_time_budget", + "task_id": task.task_id, + "action": task.kind, + "error": str(denied), + "hint": getattr(denied, "hint", ""), + }, + ) + except Exception: # noqa: BLE001 — observability must not block dispatch + log.exception( + "dispatcher: could not record time-budget denial for task=%s", + task.task_id, + ) + # A cancelled conc_sweep never writes last_conc_sweep on its own, so + # SWEEP would idle until the LLM emits skip_to_close and CI would read + # that as robustness_escalated. Stamp the skip here so the phase + # machine can close on conc_sweep_done. + if str(task.kind or "") == "conc_sweep": + try: + self._record_session_budget_conc_sweep_skip(denied=denied) + except Exception: # noqa: BLE001 — a stamp miss must not abort the pump + log.exception( + "dispatcher: could not record conc_sweep time-budget skip for task=%s", + task.task_id, + ) + return True + def _sequence_denial_for_request( self, target_agent: str, @@ -1348,27 +1753,42 @@ def _run_action_now_sync( "get_recent_outcomes or the next-tick inbox for its " "delegated_result)" ) + except (FuturesCancelledError, asyncio.CancelledError): + # The wall-clock defences stopped it on purpose. Named before the + # generic handler, which would file a deliberate stop as ``errored`` + # and read as a fault in the action. Both classes are caught because + # whether the two are the same one varies by Python version, and on + # the versions where they are, it is a ``BaseException`` that would + # escape this bridge entirely and take the agent's turn down with it. + return ( + f"(run_action_now: {name!r} was cancelled — the session is shutting down or out of wall-clock budget)" + ) except Exception as exc: # noqa: BLE001 — never crash the turn log.exception("run_action_now: inline run of %r failed", name) return f"(run_action_now: {name!r} errored: {exc!r})" - async def _run_action_now( + async def _inline_action_denial( self, action_name: str, params: dict[str, Any], - ) -> str: - """Coordinator-loop coroutine that runs a whitelisted action inline through PolicyGate + SubAgentRunner, publishing a delegated_result for audit/inbox parity. + ) -> str | None: + """Run the admission gates an inline action shares with a dispatched one. + + The synthetic delegate goes through PolicyGate so the phase, role and + path gates reach an inline call exactly as they reach one that went + through the queue; the sequence gate is asked separately because it is + about what the run has already done rather than about the intent. Either + denial is recorded before it is reported, so an inline call that never + ran is still auditable. Args: - action_name: Name of the action to execute. - params: Parameter mapping forwarded to the task/executor. + action_name: Name of the action about to run. + params: Parameters it would run with. Returns: - A status string: a policy/sequence denial message, an - already-in-flight notice, or the rendered delegated_result line. + str | None: The message for the caller when the action is denied, or + ``None`` when it may run. """ - - # PolicyGate parity: validate the synthetic delegate so phase/role/path gates apply. intent = Intent( type=IntentType.DELEGATE, payload={"action_name": action_name, "params": dict(params or {})}, @@ -1382,15 +1802,35 @@ async def _run_action_now( f"{getattr(denied, 'rule', '')!s} — " f"{str(getattr(denied, 'hint', denied))[:200]})" ) - seq_denied = self._sequence_denial_for_action(action_name) - if seq_denied is not None: - await self._record_policy_denied( - "orchestration", - intent, - seq_denied, - action_name=action_name, - ) - return f"(run_action_now: {action_name!r} denied: {str(getattr(seq_denied, 'hint', seq_denied))[:200]})" + seq_denied = self._admission_denial_for_action(action_name) + if seq_denied is None: + return None + await self._record_policy_denied( + "orchestration", + intent, + seq_denied, + action_name=action_name, + ) + return f"(run_action_now: {action_name!r} denied: {str(getattr(seq_denied, 'hint', seq_denied))[:200]})" + + async def _run_action_now( + self, + action_name: str, + params: dict[str, Any], + ) -> str: + """Coordinator-loop coroutine that runs a whitelisted action inline through PolicyGate + SubAgentRunner, publishing a delegated_result for audit/inbox parity. + + Args: + action_name: Name of the action to execute. + params: Parameter mapping forwarded to the task/executor. + + Returns: + A status string: a policy/sequence denial message, an + already-in-flight notice, or the rendered delegated_result line. + """ + denial = await self._inline_action_denial(action_name, params) + if denial is not None: + return denial lanes, ttl = self._registry_lanes_ttl(action_name) content_fp = hashlib.sha1( json.dumps(params or {}, sort_keys=True, default=str).encode(), @@ -1414,7 +1854,21 @@ async def _run_action_now( f"(run_action_now: an identical {action_name!r} task is " f"already {task.state!r}; wait for its delegated_result)" ) - result = await self.sub.run_task(task) + # Publish a handle for as long as the action runs. This path abandons its + # future once the caller's inline wait elapses -- the action keeps going + # by design, so without a handle nothing could stop it, including a + # teardown that is about to close the database underneath it. The handle + # is this coroutine's own task: cancelling it drops the audit publication + # below, which the runner's terminal transition already accounts for. + inline_handle = asyncio.current_task() + cancel_scope = CancelScope() + if inline_handle is not None: + self._inflight_actions[task.task_id] = _InflightAction(task.kind, inline_handle, cancel_scope) + try: + with use_cancel_scope(cancel_scope): + result = await self.sub.run_task(task) + finally: + self._inflight_actions.pop(task.task_id, None) result_payload = { "task_id": task.task_id, "kind": task.kind, diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 7ba9685686..dd6ef0cd2b 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -168,7 +168,7 @@ async def _handle_propose_action(self, source: str, intent: Intent) -> None: ), }, ) - denied = self._sequence_denial_for_action(action_name) + denied = self._admission_denial_for_action(action_name) if denied is not None: await self._record_policy_denied(source, intent, denied) return @@ -506,7 +506,7 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: ), }, ) - denied = self._sequence_denial_for_action(action_name) + denied = self._admission_denial_for_action(action_name) if denied is not None: await self._record_policy_denied( source, diff --git a/src/hyperloom/orchestrator/loop/sub_agent_runner.py b/src/hyperloom/orchestrator/loop/sub_agent_runner.py index de1bef9ebc..4fe85fd4be 100644 --- a/src/hyperloom/orchestrator/loop/sub_agent_runner.py +++ b/src/hyperloom/orchestrator/loop/sub_agent_runner.py @@ -11,6 +11,7 @@ from __future__ import annotations +import asyncio from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, Awaitable, Callable @@ -293,6 +294,22 @@ async def run_task( try: with progress_scope(self._progress_reporter(task.task_id)): result_payload = await runner(ctx) + except asyncio.CancelledError: + # Stopped from outside -- shutdown, or a wall-clock budget that + # ran out while this was running. ``CancelledError`` is not an + # ``Exception``, so it skips the handler below and nothing else + # would move the row off ``running``: it would hold its lanes + # and read as live work to every phase gate until the TTL sweep + # noticed. Recorded as ``cancelled`` rather than ``failed`` + # because the action was never given the chance to fail. + await self._transition_resilient( + task.task_id, + "cancelled", + evidence={"reason": "cancelled_in_flight"}, + context="executor_cancelled", + allow_terminal=True, + ) + raise except Exception as exc: # noqa: BLE001 — surface to task.history await self._transition_resilient( task.task_id, diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 924c45f41a..1759b4011e 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -25,6 +25,7 @@ summarize_change, ) from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON +from ..actions.stop_attribution import stopped_by_the_run_class from ..state.shared_state import SharedState, resolve_grading_anchor_tput from hyperloom.inference_optimizer.protocol.intent import Intent from ..bus.message_bus import Message @@ -761,6 +762,84 @@ def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: state.enablement.baseline_eval_evidence = evidence[:4000] state.enablement.launch_log = evidence + def _reopen_revalidation_window(self) -> None: + """Leave an enablement revalidation window open for a round the run stopped. + + A round the run stopped measured nothing, so it says nothing about whether + the KEEP'd patch still revalidates. The window therefore stays open -- + only an eval-origin KEEP ever opens one, and closing it here would strand + a patch nothing revalidated -- and the stall streak is not charged, + because reaching the ``enablement_stalled`` cap on the evidence of a clock + is exactly what the baseline failure streak already exempts this round + from. + + The generation advances for the same reason opening a window does: the + next enqueue's idempotency key must not resolve to the row the run + stopped. The tracked id goes with it, since the row it names is spent and + the next enqueue records its own. + + Two callers reach the same round by different routes: the writeback, when + the reaped row's result is routed, and the resume recovery, when the row + was cancelled at dispatch and so produced no result to route at all. + """ + state = self.shared_state + state.enablement.revalidation_generation = int(state.enablement.revalidation_generation or 0) + 1 + state.enablement.revalidation_task_id = "" + state.enablement.inflight_task_id = "" + + def _record_revalidation_not_promoted( + self, + *, + task: Task, + result_payload: dict[str, Any], + err_class: str, + stopped_by_the_run: bool, + ) -> None: + """Close out an enablement revalidation baseline that did not promote. + + A genuine failure -- boot, OOM, timeout, eval -- is a no-progress round: + it closes the revalidation window, reopens the authoring loop, and counts + toward the ``enablement_stalled`` cap so repeated KEEP-then-fail cycles + terminate. + + A round the run stopped is none of those things; what it gets instead, and + why, is :meth:`_reopen_revalidation_window`. + + Args: + task: The revalidation baseline task that came back unpromotable. + result_payload: Its result, read for the launch/traceback text. + err_class: Its ``error_class``, for the log line. + stopped_by_the_run: Whether the run stopped the round rather than the + round saying anything about the baseline. + """ + state = self.shared_state + if stopped_by_the_run: + self._reopen_revalidation_window() + else: + state.enablement.revalidation_task_id = "" + state.enablement.validation_pending = False + state.enablement.stall_streak = int(state.enablement.stall_streak or 0) + 1 + try: + from ..phases.framework import _ENABLEMENT_MAX_STALL as _max_stall + except ImportError: + _max_stall = 5 + if state.enablement.stall_streak >= _max_stall and not state.stop_reason: + state.set_stop_reason("enablement_stalled") + else: + state.enablement.inflight_task_id = "" + launch_log = _extract_enablement_launch_log(result_payload) + if launch_log: + state.enablement.launch_log = launch_log + log.warning( + "enablement revalidation task %s %s (error_class=%s); stall_streak=%d pending=%s rearm=%s", + task.task_id, + "was stopped by the run" if stopped_by_the_run else "failed", + err_class, + int(state.enablement.stall_streak or 0), + bool(state.enablement.validation_pending), + not bool(state.stop_reason), + ) + async def _handle_unpromotable_result( self, task: Task, @@ -891,6 +970,13 @@ async def _handle_unpromotable_result( # Only arm/streak while no baseline has succeeded yet (tput <= 0). if task.kind == "baseline" and self.shared_state.baseline_tput <= 0: err_class = result_payload.get("error_class", "") + # A round the run itself stopped -- the session budget reaped it, or + # the orchestrator cancelled the action -- measured nothing, so it + # says nothing about whether this baseline boots. Charging it to the + # streaks would let three stops the run chose end the session as + # ``baseline_failed``, blaming the model for the clock. The executor + # already refuses to grade such a round; the ledger has to agree. + stopped_by_the_run = stopped_by_the_run_class(err_class) is not None # While a serial enablement is actively engaged, baseline boots # re-fail on purpose (each round clears a deeper gap), so the # ``baseline_failed`` fast-fail must NOT fire here; the @@ -908,29 +994,11 @@ async def _handle_unpromotable_result( or (reval_tid and reval_tid == str(task.task_id or "")) ) if is_revalidation and bool(getattr(self.shared_state.enablement, "validation_pending", False)): - self.shared_state.enablement.validation_pending = False - self.shared_state.enablement.revalidation_task_id = "" - self.shared_state.enablement.stall_streak = ( - int(getattr(self.shared_state.enablement, "stall_streak", 0) or 0) + 1 - ) - try: - from ..phases.framework import _ENABLEMENT_MAX_STALL as _max_stall - except ImportError: - _max_stall = 5 - if self.shared_state.enablement.stall_streak >= _max_stall and not self.shared_state.stop_reason: - self.shared_state.set_stop_reason("enablement_stalled") - else: - self.shared_state.enablement.inflight_task_id = "" - launch_log = _extract_enablement_launch_log(result_payload) - if launch_log: - self.shared_state.enablement.launch_log = launch_log - log.warning( - "enablement revalidation task %s failed (error_class=%s); " - "stall_streak=%d rearm=%s", - task.task_id, - err_class, - self.shared_state.enablement.stall_streak, - not bool(self.shared_state.stop_reason), + self._record_revalidation_not_promoted( + task=task, + result_payload=result_payload, + err_class=err_class, + stopped_by_the_run=stopped_by_the_run, ) any_changed = True from ..actions.executors._accuracy_gate import eval_enablement_allowed # noqa: PLC0415 @@ -946,7 +1014,15 @@ async def _handle_unpromotable_result( ) if eval_failed: self._persist_eval_failure(result_payload) - if err_class == "fast_exit_arg_error": + if stopped_by_the_run: + log.warning( + "baseline %s was stopped by the run (%s); the failure streak stays at %d " + "because nothing about the baseline was measured", + task.task_id, + err_class, + self.shared_state.baseline_failure_streak, + ) + elif err_class == "fast_exit_arg_error": self.shared_state.baseline_arg_error_streak += 1 if self.shared_state.baseline_arg_error_streak >= 2: self.shared_state.set_stop_reason("baseline_arg_error") @@ -957,7 +1033,8 @@ async def _handle_unpromotable_result( self.shared_state.set_stop_reason("baseline_failed") # Combined backstop: count ALL baseline failures so mixed # error_classes that split the per-class streaks still fast-fail. - self.shared_state.baseline_total_failures += 1 + if not stopped_by_the_run: + self.shared_state.baseline_total_failures += 1 if ( self.shared_state.baseline_total_failures >= _BASELINE_MAX_TOTAL_FAILURES and not self.shared_state.stop_reason @@ -2861,10 +2938,31 @@ async def _promote_baseline( # revalidation is exempt: the enablement patch changed the stack, so the # prior anchor no longer describes anything reproducible. prior_anchor = float(self.shared_state.baseline_tput or 0.0) + # A hot pass measured against a recorded COLD anchor is a correction, not + # a regression, so it lands even when the number is lower. The two are not + # comparable -- that is what the cold marker says -- and a cold figure can + # read higher than the hot one that replaces it whenever the "cold" pass + # was not really cold: weights in page cache and a JIT cache a prior run + # populated leave it paying none of the startup its depressed reputation + # assumes. Without this the session cannot escape the marker. PRELUDE + # refuses to finish while it is set, the retry that would clear it is + # rejected for measuring lower, and the run re-measures whole baseline + # rounds until the clock kills it. + hot_pass_ran = result.get("measure_round_runtime_sec") + corrects_a_cold_anchor = ( + bool(getattr(self.shared_state, "baseline_measure_round_dropped", False)) + and isinstance(hot_pass_ran, (int, float)) + and hot_pass_ran > 0 + ) anchor_accepted = bool( isinstance(tput, (int, float)) and tput > 0 - and (prior_anchor <= 0.0 or float(tput) > prior_anchor or is_revalidation) + and ( + prior_anchor <= 0.0 + or float(tput) > prior_anchor + or is_revalidation + or corrects_a_cold_anchor + ) ) if isinstance(tput, (int, float)) and tput > 0: if anchor_accepted: @@ -2969,6 +3067,30 @@ async def _promote_baseline( elif float(getattr(self.shared_state, "baseline_warm_runtime_sec", 0.0) or 0.0) != 0.0: self.shared_state.baseline_warm_runtime_sec = 0.0 changed = True + # Whether this baseline had to keep its cold figure because the budget + # could not fund a hot pass and a variant to use it. Carried on the + # session because the decision it drives is the session's: PRELUDE + # routes to CLOSE rather than optimizing against a denominator that + # was never the baseline. Cleared by a baseline that does land a hot + # figure, so a resumed session with a fresh clock is not held to the + # earlier leg's shortfall. + measure_round_dropped = bool(result.get("measure_round_dropped")) + if measure_round_dropped != bool( + getattr(self.shared_state, "baseline_measure_round_dropped", False) + ): + self.shared_state.baseline_measure_round_dropped = measure_round_dropped + changed = True + # Promote the cold round's boot/benchmark split. Cleared the same way + # the warm figure is when a later baseline does not carry one, so a + # stale split can never be subtracted from a fresh total and reported + # as this workload's boot. + post_ready_raw = result.get("post_ready_runtime_sec") + if isinstance(post_ready_raw, (int, float)) and post_ready_raw > 0: + self.shared_state.baseline_post_ready_runtime_sec = float(post_ready_raw) + changed = True + elif float(getattr(self.shared_state, "baseline_post_ready_runtime_sec", 0.0) or 0.0) != 0.0: + self.shared_state.baseline_post_ready_runtime_sec = 0.0 + changed = True # current_best.tput follows the same hot baseline contract so the # gain numerator and denominator stay aligned. Once the stack carries a # validated layer, current_best belongs to the stack top and a baseline @@ -4362,8 +4484,9 @@ async def _resume_consistency_pass(self) -> dict[str, Any]: # restart, so restore both Recipe and Kernel trees before continuing. await self._resume_recover_pending_warm_replay(report) # (1d) Orphaned revalidation tasks: if enablement_validation_pending is set - # but the tracked revalidation task is already terminal, clear the pending - # flag and rearm the stall counter so a fresh revalidation can be enqueued. + # but the tracked revalidation task is already terminal, unstick the window + # so a fresh revalidation can be enqueued -- closed and charged to the + # stall counter, or reopened uncharged when the run cancelled the row. await self._resume_recover_pending_revalidation(report) # (2) Orphaned KEEPs: replay integrate_patch KEEPs # that crashed before the append landed; surface ambiguous ones loudly. @@ -4814,12 +4937,20 @@ async def _resume_recover_pending_targeted_build(self, report: dict[str, Any]) - report["fixes"].append(summary) async def _resume_recover_pending_revalidation(self, report: dict[str, Any]) -> None: - """Clear stale enablement_validation_pending when the tracked revalidation task is terminal. + """Unstick enablement_validation_pending when the tracked revalidation task is terminal. If the coordinator died while a revalidation baseline was running, the task row may already be in a terminal state on resume. Without this recovery the pending flag stays set indefinitely and the next revalidation cannot be enqueued (tracked_tid is still the old row). + + Which recovery depends on how the row ended, not on this being the resume + path. A row the run cancelled -- which is what the queue scan does to a + revalidation the wall-clock budget can no longer fit -- measured nothing + and produced no result to route, so it is no evidence about the baseline + and gets :meth:`_reopen_revalidation_window`, the same verdict the + writeback reaches for the same round. Anything else ended having had its + chance: the window closes and the round is charged to the stall streak. """ state = self.shared_state if not bool(state.enablement.validation_pending): @@ -4830,25 +4961,38 @@ async def _resume_recover_pending_revalidation(self, report: dict[str, Any]) -> try: from ..state.task_registry import TERMINAL_STATES, TaskNotFound + row_state = "" try: row = await self.tasks.get(tracked_tid) - is_terminal = row.state in TERMINAL_STATES + row_state = str(getattr(row, "state", "") or "") + is_terminal = row_state in TERMINAL_STATES except TaskNotFound: is_terminal = True - if is_terminal: - state.enablement.validation_pending = False - state.enablement.revalidation_task_id = "" - state.enablement.stall_streak = ( - int(state.enablement.stall_streak or 0) + 1 - ) - state.enablement.inflight_task_id = "" + if not is_terminal: + return + if row_state == "cancelled": + self._reopen_revalidation_window() report["fixes"].append( - {"kind": "cleared_orphaned_revalidation_pending", "task_id": tracked_tid} + {"kind": "reopened_revalidation_the_run_cancelled", "task_id": tracked_tid} ) log.info( - "resume: cleared stale enablement_validation_pending for terminal revalidation task %s", + "resume: revalidation task %s was cancelled by the run; window left open " + "at generation %d without charging the stall streak", tracked_tid, + int(state.enablement.revalidation_generation or 0), ) + return + state.enablement.validation_pending = False + state.enablement.revalidation_task_id = "" + state.enablement.stall_streak = int(state.enablement.stall_streak or 0) + 1 + state.enablement.inflight_task_id = "" + report["fixes"].append( + {"kind": "cleared_orphaned_revalidation_pending", "task_id": tracked_tid} + ) + log.info( + "resume: cleared stale enablement_validation_pending for terminal revalidation task %s", + tracked_tid, + ) except Exception: # noqa: BLE001 — best-effort log.debug("resume: revalidation pending recovery check failed", exc_info=True) diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index 9f8917f357..9133861d22 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -18,6 +18,50 @@ log = _logging.getLogger(__name__) +# Terminal task states, split by what the CLOSE sequencer can still do with a +# task in one. Both are dead ends for ``run_task``: ``enter_running`` refuses +# any terminal row, so handing one over takes the close step down with it. +# ``succeeded`` means the step's artifact is already on disk (skip it); +# ``cancelled``/``failed`` mean the work never happened and the sequencer needs +# a fresh row (re-enqueue under a distinct idempotency key). +_TASK_STATE_DONE: str = "succeeded" +_DEAD_TASK_STATES: frozenset[str] = frozenset({"cancelled", "failed"}) +# A row the wall-clock deadline path already dispatched. Not terminal, and not +# runnable either: the registry allows ``running`` only into a terminal state. +_TASK_STATE_RUNNING: str = "running" +# Appended to a close step's idempotency key when its first row is dead, so +# ``create_or_return_existing`` mints a new task instead of returning the +# corpse. +_RETRY_KEY_SUFFIX: str = "retry" +# Fallback registry poll interval, for a caller with no dispatcher poll set. +_DEFAULT_TASK_POLL_SEC: float = 10.0 + +# Floor on how long CLOSE waits for a step it found already running, for a step +# the catalogue prices at almost nothing or does not carry at all. Long enough +# that a step which is merely slow to be written is not abandoned one poll in. +_CLOSE_STEP_WAIT_FLOOR_SEC: float = 60.0 + +# Ceiling on the same wait. The step is the last thing standing between the run +# and having nothing to show for itself, so the wait is generous -- but a task +# wedged forever must not hold the process open, and a report that has taken +# five times its typical runtime is not about to land. +_CLOSE_STEP_WAIT_CEILING_SEC: float = 600.0 + + +def _task_is_dead(task: Task | None) -> bool: + """True when ``task`` reached a terminal state without producing its artifact. + + Args: + task: The task to inspect; ``None`` reads as not dead (there is nothing + to reuse, which the caller handles as a fresh enqueue anyway). + + Returns: + ``True`` when the task is ``cancelled`` or ``failed``. + """ + if task is None: + return False + return str(getattr(task, "state", "") or "") in _DEAD_TASK_STATES + class ClosePhase(PhaseHandler): """Extracted phase handler; delegates unknown attrs to its Coordinator.""" @@ -189,8 +233,7 @@ async def _on_enter_close(self, *, from_phase: str) -> None: report_task = await self._enqueue_internal_report_task( reason="close_phase_entry", ) - report_result = await self.sub.run_task(report_task) - terminal_state = report_result.state + terminal_state = await self._run_close_task(report_task, step="1 (report)") if terminal_state in {"succeeded", None}: await self._record_close_step( "report", @@ -243,8 +286,7 @@ async def _on_enter_close(self, *, from_phase: str) -> None: bd_task = await self._enqueue_internal_session_breakdown_task( reason="close_phase_entry", ) - bd_result = await self.sub.run_task(bd_task) - terminal_state = bd_result.state + terminal_state = await self._run_close_task(bd_task, step="2 (session_breakdown)") if terminal_state in {"succeeded", None}: await self._record_close_step( "session_breakdown", @@ -340,6 +382,61 @@ async def _on_enter_close(self, *, from_phase: str) -> None: await self._record_close_step("done", status="done") log.info("CLOSE 7-step sequencer complete") + async def _enqueue_runnable_internal_task( + self, + *, + kind: str, + params: dict[str, Any], + idempotency_key: str, + ) -> Task: + """Enqueue a Coordinator-internal close-step task the sequencer can still run. + + Idempotency is what lets the wall-clock deadline path and the CLOSE + sequencer reach for the same task instead of writing the artifact + twice. Its cost is that the key can resolve to a row that is already + terminal — most often ``cancelled``, because the deadline path that + enqueued the task is also the path that cancels in-flight work. Such a + row cannot be run, so one retry under a suffixed key mints a fresh one. + + Args: + kind: Task kind (``report`` / ``session_breakdown``). + params: Task parameters, identical across attempts. + idempotency_key: The step's key; the retry appends a suffix. + + Returns: + The created or reused :class:`Task`. Still terminal only when the + retry also resolved to a dead row, which + :meth:`_run_close_task` reports rather than runs. + """ + task: Task | None = None + for key in (idempotency_key, f"{idempotency_key}-{_RETRY_KEY_SUFFIX}"): + task, was_existing = await self.tasks.create_or_return_existing( + kind=kind, + params=params, + idempotency_key=key, + requires_lanes=[], + allowed_tools=["Read"], + side_effects=["writes_results"], + lease_ttl_sec=120, + ) + if not was_existing: + return task + if not _task_is_dead(task): + log.info( + "internal-%s task reused (idempotent: task_id=%s, state=%s)", + kind, + task.task_id, + task.state, + ) + return task + log.warning( + "internal-%s task %s is %s and cannot be run; re-enqueueing under a fresh key", + kind, + task.task_id, + task.state, + ) + return task # type: ignore[return-value] # loop body always binds it + async def _enqueue_internal_report_task( self, *, @@ -357,8 +454,12 @@ async def _enqueue_internal_report_task( """ existing_id = (self.shared_state.closing_report_task_id or "").strip() if existing_id: + task = None try: task = await self.tasks.get(existing_id) + except Exception: # noqa: BLE001 — TaskNotFound + friends + pass # Stale id; fall through to fresh enqueue. + if task is not None and not _task_is_dead(task): log.info( "internal-report task already enqueued by wall-clock " "deadline path (task_id=%s, state=%s); sequencer will " @@ -367,9 +468,15 @@ async def _enqueue_internal_report_task( task.state, ) return task - except Exception: # noqa: BLE001 — TaskNotFound + friends - # Stale id; fall through to fresh enqueue. - pass + # Dead or vanished: the id names a report that will never be + # written, so drop it before the fresh enqueue mirrors its own. + if task is not None: + log.warning( + "internal-report task %s recorded on closing_report_task_id is %s; re-enqueueing", + task.task_id, + task.state, + ) + self.shared_state.closing_report_task_id = "" params: dict[str, Any] = { "source": "coordinator_internal", @@ -377,14 +484,10 @@ async def _enqueue_internal_report_task( "session_dir": str(self.session_dir), "max_highlights": 50, } - task, was_existing = await self.tasks.create_or_return_existing( + task = await self._enqueue_runnable_internal_task( kind="report", params=params, idempotency_key=f"internal-report-{reason}", - requires_lanes=[], - allowed_tools=["Read"], - side_effects=["writes_results"], - lease_ttl_sec=120, ) # Mirror onto closing_report_task_id. if not self.shared_state.closing_report_task_id: @@ -393,12 +496,6 @@ async def _enqueue_internal_report_task( self.shared_state.save(self.session_dir) except Exception: # noqa: BLE001 log.exception("internal-report: closing_report_task_id save failed") - if was_existing: - log.info( - "internal-report task reused (idempotent: task_id=%s, state=%s)", - task.task_id, - task.state, - ) return task async def _enqueue_internal_session_breakdown_task( @@ -420,22 +517,140 @@ async def _enqueue_internal_session_breakdown_task( "reason": str(reason), "session_dir": str(self.session_dir), } - task, was_existing = await self.tasks.create_or_return_existing( + return await self._enqueue_runnable_internal_task( kind="session_breakdown", params=params, idempotency_key=f"internal-session_breakdown-{reason}", - requires_lanes=[], - allowed_tools=["Read"], - side_effects=["writes_results"], - lease_ttl_sec=120, ) - if was_existing: + + def _close_step_wait_sec(self, task: Task) -> float: + """How long to wait for a close-step task that is already running. + + The bound is the step's own expected runtime, clamped into + ``[_CLOSE_STEP_WAIT_FLOOR_SEC, _CLOSE_STEP_WAIT_CEILING_SEC]``. This is + deliberately not the closing reserve: the reserve answers "how much of + the session do we hold back for CLOSE", which scales with the session + and is a handful of seconds for a short one, while this answers "how + long is it reasonable to wait for work that is already under way", + which scales with the work. Bounding a two-minute report by a + twelve-second reserve is a wait only on paper. + + Args: + task: The close-step task found in ``running``. + + Returns: + The bound in seconds. + """ + from ..loop.coordinator_helpers import expected_action_cost_minutes + + registry = getattr(self, "action_registry", None) + kind = str(getattr(task, "kind", "") or "") + meta = registry.get(kind) if registry is not None else None + typical_sec = expected_action_cost_minutes(meta) * 60.0 + return min(_CLOSE_STEP_WAIT_CEILING_SEC, max(_CLOSE_STEP_WAIT_FLOOR_SEC, typical_sec)) + + async def _await_running_close_task(self, task: Task, *, step: str) -> str: + """Wait for an already-dispatched close-step task to reach a terminal state. + + The wall-clock deadline path enqueues the report and dispatches it + before CLOSE is entered, so the sequencer can find its own step already + under way. Handing that row to ``run_task`` asks the registry for + ``running -> running``, which it refuses, taking the close step down + with it — and the session that ran out of time is the session whose + report is worth the most. + + How long to wait is a question about the work, not about the budget: + the closing reserve says how much of the session to hold back for + CLOSE, which for a short session is a few seconds — less than any + report takes to write, so bounding the wait by it is the same as not + waiting. :func:`_close_step_wait_sec` bounds it by what the step's own + action typically takes instead, so a task that never lands costs CLOSE + that bound and no more. + + Args: + task: The close-step task found in ``running``. + step: Close-step label, for logging. + + Returns: + The state the task ended in, or ``running`` when the bound elapsed + first — which the caller records the same way it records a failure. + """ + bound_sec = self._close_step_wait_sec(task) + poll_sec = float(getattr(self, "_dispatcher_poll_sec", _DEFAULT_TASK_POLL_SEC)) + deadline = time.monotonic() + bound_sec + log.info( + "CLOSE step %s: task_id=%s is already running; waiting up to %.0fs for it", + step, + task.task_id, + bound_sec, + ) + state = _TASK_STATE_RUNNING + while True: + try: + state = str(getattr(await self.tasks.get(task.task_id), "state", "") or "") + except Exception: # noqa: BLE001 — TaskNotFound + friends + log.warning( + "CLOSE step %s: task_id=%s vanished while the sequencer waited for it", + step, + task.task_id, + ) + return state + if state != _TASK_STATE_RUNNING: + log.info( + "CLOSE step %s: task_id=%s finished as %s while the sequencer waited", + step, + task.task_id, + state, + ) + return state + remaining = deadline - time.monotonic() + if remaining <= 0.0: + log.warning( + "CLOSE step %s: task_id=%s still running after %.0fs; recording the step as failed", + step, + task.task_id, + bound_sec, + ) + return state + await asyncio.sleep(min(poll_sec, remaining)) + + async def _run_close_task(self, task: Task, *, step: str) -> str | None: + """Run one close-step task and return the state it ended in. + + ``run_task`` transitions ``queued -> running``, which the registry + refuses for a row that is already terminal or already running — and + refuses correctly: the rejection is the double-spawn guard. So such a + row is reported or waited on here instead of run, which keeps one row + the sequencer did not create from taking down the step that was + supposed to salvage the session. + + Args: + task: The task to run. + step: Close-step label, for logging. + + Returns: + The state the task ended in. + """ + state = str(getattr(task, "state", "") or "") + if state == _TASK_STATE_DONE: log.info( - "internal-session_breakdown task reused (idempotent: task_id=%s, state=%s)", + "CLOSE step %s: task_id=%s already succeeded; keeping its artifact", + step, task.task_id, - task.state, ) - return task + return state + if state in _DEAD_TASK_STATES: + log.warning( + "CLOSE step %s: task_id=%s is %s and cannot be run; recording the step as failed", + step, + task.task_id, + state, + ) + return state + if state == _TASK_STATE_RUNNING: + return await self._await_running_close_task(task, step=step) + result = await self.sub.run_task(task) + return result.state async def _record_close_step( self, @@ -454,20 +669,6 @@ async def _record_close_step( task_id: Optional task id associated with the step. detail: Optional free-text detail recorded on the row. """ - history = self.shared_state.phase_history or [] - if not history: - return - row = history[-1] - if not isinstance(row, dict): - return - evidence = row.get("evidence") - if not isinstance(evidence, dict): - evidence = {} - row["evidence"] = evidence - steps = evidence.get("close_steps") - if not isinstance(steps, list): - steps = [] - evidence["close_steps"] = steps entry: dict[str, Any] = { "step": step, "status": status, @@ -477,7 +678,12 @@ async def _record_close_step( entry["task_id"] = task_id if detail: entry["detail"] = detail - steps.append(entry) + if not _phase_state.append_phase_evidence_row( + self.shared_state.phase_history, + key="close_steps", + row=entry, + ): + return try: self.shared_state.save(self.session_dir) except Exception: # noqa: BLE001 diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 5ea0a2b61f..3c055e2989 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -12,6 +12,7 @@ import subprocess import time import tempfile +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any @@ -4369,7 +4370,9 @@ async def _maybe_route_build_outcomes(self) -> None: A succeeded build no longer synthesises status='kept' directly. Instead it enqueues an integrate_patch launch probe so the runtime must actually - boot the model before KEEP is declared. + boot the model before KEEP is declared. Each row is read once, except that + a build whose probe was cancelled before it ran is read again: nothing + launched the runtime, so nothing has decided anything about that build. """ try: all_tasks = [] @@ -4382,45 +4385,23 @@ async def _maybe_route_build_outcomes(self) -> None: # Pick the most recent terminal row (by updated_at). task = sorted(all_tasks, key=lambda t: str(getattr(t, "updated_at", "") or ""))[-1] task_id = str(getattr(task, "task_id", "") or "") - # Skip rows already accounted for (tracked by enablement_build_manifest). + # Skip rows already accounted for (tracked by enablement_build_manifest), + # unless what they were routed to was cancelled before it ran, which + # leaves the build no more launched than an unrouted one. state = self.shared_state - manifest = list(state.enablement.build_manifest or []) - seen_ids = {str(m.get("task_id") or "") for m in manifest if isinstance(m, dict)} - if task_id in seen_ids: + routed = self._build_routing_record(task_id) + if routed is not None and not await self._build_probe_was_cancelled(routed): return - # Mark as seen immediately so we don't process the same row twice. - manifest.append({"task_id": task_id, "routed": True}) - state.enablement.build_manifest = manifest - fc = "" if task.state == "succeeded": - # Load the BuildResult from result.json for the rich runtime. - attempt_root = str((getattr(task, "params", {}) or {}).get("attempt_root") or "") - # The build's attempt_root is resolved at pump time and is NOT - # written back into the task params (they keep the enqueue-time - # default ""). Fall back to the deterministic build path keyed by - # task_id so a *successful* build is not wrongly rejected as - # ``artifact_unreadable`` (mirrors BuildLifecycle._attempt_root). - if not attempt_root and task_id: - attempt_root = str(self.session_dir / "enablement" / "builds" / task_id) - br = None - if attempt_root: - from ..framework.targeted_build import _load_result_json - - br = _load_result_json(attempt_root) - - # If the runtime can't be read, it can't be launched → reverted. - if br is None or not br.ok or not br.runtime.to_runtime_override(): - res: dict = {"enablement": True, "status": "reverted", "reason": "artifact_unreadable"} - log.info("ENABLEMENT: targeted_build artifact-unreadable task=%s", task_id) - self._maybe_rearm_enablement(res) - return - - # Enqueue a launch probe; KEEP is declared by the probe result. - log.info("ENABLEMENT: targeted_build artifact-verified → enqueue launch probe task=%s", task_id) - await self._enqueue_build_launch_probe(task_id, br) + await self._route_succeeded_build(task, routed) return + # A failed build is accounted for the moment it is read: the rearm + # below is the whole outcome, so re-reading the row would charge the + # stall streak twice for one build. + self._note_build_routed(task_id) + fc = "" # failed row — read failure_class from history or last_build_failure history = getattr(task, "history", None) or [] if isinstance(history, (list, tuple)) and history: @@ -4476,7 +4457,128 @@ async def _maybe_route_build_outcomes(self) -> None: except Exception: # noqa: BLE001 — never wedge the tick log.debug("enablement: route_build_outcomes failed", exc_info=True) - async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None: + async def _route_succeeded_build(self, task: "Task", routed: dict[str, Any] | None) -> None: + """Turn a succeeded targeted build into a launch probe, or a no-progress round. + + KEEP is declared by the probe, never here: an artifact that builds is not + a runtime that boots. So this either opens a probe -- and remembers which + one, because a probe cancelled before it ran leaves the build unlaunched + and worth another -- or, when the built runtime cannot even be read, ends + the round as a revert. + + The build is recorded as accounted for only once there is something to + account for. A probe the budget refused leaves it unrouted on purpose: it + is still a verified build nothing has launched, and the manifest saying + otherwise is what would strand it for the rest of the session. + + Args: + task: The succeeded ``targeted_build`` row. + routed: What this build was routed to before, when it is being routed + again after its probe was cancelled; ``None`` on the first pass. + """ + task_id = str(getattr(task, "task_id", "") or "") + attempt_root = str((getattr(task, "params", {}) or {}).get("attempt_root") or "") + # The build's attempt_root is resolved at pump time and is NOT written + # back into the task params (they keep the enqueue-time default ""). Fall + # back to the deterministic build path keyed by task_id so a *successful* + # build is not wrongly rejected as ``artifact_unreadable`` (mirrors + # BuildLifecycle._attempt_root). + if not attempt_root and task_id: + attempt_root = str(self.session_dir / "enablement" / "builds" / task_id) + br = None + if attempt_root: + from ..framework.targeted_build import _load_result_json + + br = _load_result_json(attempt_root) + + # If the runtime can't be read, it can't be launched → reverted. + if br is None or not br.ok or not br.runtime.to_runtime_override(): + self._note_build_routed(task_id) + log.info("ENABLEMENT: targeted_build artifact-unreadable task=%s", task_id) + self._maybe_rearm_enablement( + {"enablement": True, "status": "reverted", "reason": "artifact_unreadable"} + ) + return + + log.info("ENABLEMENT: targeted_build artifact-verified → enqueue launch probe task=%s", task_id) + probe_tid, generation = await self._enqueue_build_launch_probe( + task_id, + br, + generation=int((routed or {}).get("probe_generation") or 0), + ) + if not probe_tid: + return + self._note_build_routed(task_id, probe_task_id=probe_tid, probe_generation=generation) + + def _build_routing_record(self, build_task_id: str) -> dict[str, Any] | None: + """The record of what a build's outcome was already routed to, if any. + + Only the routing sentinels carry a ``task_id``; the build attempts the + lifecycle appends to the same manifest carry ``ok`` instead. + + Args: + build_task_id: The ``targeted_build`` row to look for. + + Returns: + The sentinel dict, or ``None`` when this build has not been routed. + """ + for entry in reversed(list(self.shared_state.enablement.build_manifest or [])): + if isinstance(entry, dict) and str(entry.get("task_id") or "") == build_task_id: + return entry + return None + + def _note_build_routed(self, build_task_id: str, **fields: Any) -> None: + """Record that a build's outcome has been routed, and to what. + + Args: + build_task_id: The ``targeted_build`` row that was routed. + fields: What it was routed to, for a build whose outcome is a probe. + """ + manifest = list(self.shared_state.enablement.build_manifest or []) + for idx, entry in enumerate(manifest): + if isinstance(entry, dict) and str(entry.get("task_id") or "") == build_task_id: + manifest[idx] = {**entry, **fields} + break + else: + manifest.append({"task_id": build_task_id, "routed": True, **fields}) + self.shared_state.enablement.build_manifest = manifest + + async def _build_probe_was_cancelled(self, routed: dict[str, Any]) -> bool: + """Whether the probe a build was routed to was stopped before it ran. + + A cancelled probe is no evidence about the build: the queue scan drops a + queued row the wall-clock budget can no longer fit, and a phase boundary + drops one the new phase does not allow. Either way the built runtime was + never launched, so the build is still owed a probe -- and without noticing + that, the manifest entry written when the first one was opened keeps this + build accounted for permanently, across resumes included. + + Args: + routed: The build's routing record. + + Returns: + ``True`` when the recorded probe exists and was cancelled. + """ + from ..state.task_registry import TaskNotFound + + probe_tid = str(routed.get("probe_task_id") or "").strip() + if not probe_tid: + return False + try: + probe = await self.tasks.get(probe_tid) + except TaskNotFound: + # Pruned rather than cancelled; re-probing on a row that is gone + # would re-probe on every later tick too. + return False + return str(getattr(probe, "state", "") or "") == "cancelled" + + async def _enqueue_build_launch_probe( + self, + build_task_id: str, + br: Any, + *, + generation: int = 0, + ) -> tuple[str, int]: """Enqueue an integrate_patch launch probe for a verified build. Runs the built runtime through the enablement runnable gate without @@ -4485,9 +4587,34 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None _maybe_rearm_authored_lane → _maybe_rearm_enablement, producing a genuine KEEP/advanced/reverted outcome. The whole-machine GPU pool is acquired via _framework_gpu_params. + + The probe is what declares KEEP for a build, so it must not be opened + into a session that cannot run it: the queue scan drops a queued row the + wall-clock budget can no longer fit, and a probe cancelled that way + leaves the build verified but never launched. So the same gate the scan + asks is asked here first, and a denial enqueues nothing -- the build + stays unrouted, and the tick or resume that can afford a probe opens one. + + Args: + build_task_id: The verified build this probe launches. + br: Its ``BuildResult``, read for the runtime override. + generation: The probe generation to try first, from what this build + was routed to before. + + Returns: + The probe ``task_id`` and the generation it sits on; the id is empty + when nothing was enqueued. """ from hyperloom.agents.framework.enablement import classify_failure + denied = self._time_budget_denial_for_action("integrate_patch") + if denied is not None: + log.info( + "ENABLEMENT: build launch probe held for build=%s, not enqueued -- %s", + build_task_id, + denied, + ) + return "", generation state = self.shared_state runtime_override = br.runtime.to_runtime_override() launch_log = str(state.enablement.launch_log or "") @@ -4509,22 +4636,27 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None ) if cfg: params["config_path"] = cfg - idem = f"build_launch_probe:{build_task_id}" lanes, ttl = self._framework_authoring_lanes_ttl(params, base_ttl_sec=3600) - probe_task, existing = await self.tasks.create_or_return_existing( + probe_task, generation = await self._open_row_past_spent_generations( kind="integrate_patch", params=params, - idempotency_key=idem, + key_for=lambda gen: f"build_launch_probe:{build_task_id}:gen{gen}", + generation=generation, + label="build launch probe", requires_lanes=lanes, lease_ttl_sec=ttl, ) + if probe_task is None: + return "", generation probe_tid = str(getattr(probe_task, "task_id", "") or "") log.info( - "ENABLEMENT: build launch probe %s task=%s (existing=%s)", - "re-used" if existing else "enqueued", + "ENABLEMENT: build launch probe task=%s gen%d state=%s (build=%s)", probe_tid, - existing, + generation, + getattr(probe_task, "state", ""), + build_task_id, ) + return probe_tid, generation async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: """Enqueue one genuine baseline to revalidate a KEEP'd eval-origin patch. @@ -4549,6 +4681,17 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: return tracked_tid except Exception: # noqa: BLE001 — defensive pass + # Do not open a row the dispatcher would cancel on sight. A revalidation + # is a full baseline, and the queue scan drops a queued one the session + # budget can no longer fit -- which leaves a cancelled row owning this + # window's idempotency key, and a row cancelled at dispatch never + # produces a result to route, so nothing would advance the generation + # past it. Holding the window shut for now costs nothing: it stays open, + # and the resume that has budget again enqueues it. + denied = self._time_budget_denial_for_action("baseline") + if denied is not None: + log.info("ENABLEMENT revalidation: window held open, not enqueued -- %s", denied) + return "" params: dict[str, Any] = { "source": "coordinator_internal", "reason": ENABLEMENT_REVALIDATION_REASON, @@ -4574,14 +4717,9 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: rt_override = rt_obj.to_runtime_override() if rt_override: params["runtime_override"] = rt_override - # Use generation in the idempotency key so each revalidation window gets - # a fresh row even when a prior window's row is in a terminal state. - gen = int(state.enablement.revalidation_generation or 0) - task, _existing = await self.tasks.create_or_return_existing( - kind="baseline", - params=params, - idempotency_key=f"enablement_revalidation:gen{gen}", - ) + task = await self._open_revalidation_row(params) + if task is None: + return "" task_id = str(getattr(task, "task_id", "") or "") # Persist the task_id so _promote_baseline can verify identity. if task_id and task_id != str(state.enablement.revalidation_task_id or ""): @@ -4592,6 +4730,87 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: log.debug("enablement revalidation: save of task_id failed", exc_info=True) return task_id + async def _open_revalidation_row(self, params: dict[str, Any]) -> "Task | None": + """Resolve this revalidation window's task row, on a generation it can use. + + Args: + params: The baseline params for the revalidation row. + + Returns: + The row to track, or ``None`` when this tick found only spent + generations -- the window stays open and the next tick tries again. + """ + state = self.shared_state + task, generation = await self._open_row_past_spent_generations( + kind="baseline", + params=params, + key_for=lambda gen: f"enablement_revalidation:gen{gen}", + generation=int(state.enablement.revalidation_generation or 0), + label="revalidation", + ) + state.enablement.revalidation_generation = generation + return task + + async def _open_row_past_spent_generations( + self, + *, + kind: str, + params: dict[str, Any], + key_for: Callable[[int], str], + generation: int, + label: str, + attempts: int = 2, + **create_kwargs: Any, + ) -> tuple["Task | None", int]: + """Create or re-use a task row, skipping generations already spent. + + A generation in the idempotency key is what lets one piece of work get a + fresh row after an earlier attempt at it went terminal. That only holds if + a key resolving to a terminal row is recognised as a spent generation + rather than an enqueue: a row cancelled at dispatch -- which is what the + queue scan does to work the wall-clock budget can no longer fit -- never + produces a result to route, so nothing downstream advances the generation + past it, and every later attempt resolves to a row that measured nothing. + + Args: + kind: The task kind to create. + params: The task params. + key_for: Builds the idempotency key for a generation number. + generation: The generation to try first. + label: How this work is named in the log when a generation is spent. + attempts: How many generations to try before giving up this pass. + **create_kwargs: Passed through to + :meth:`TaskRegistry.create_or_return_existing` (lanes, TTL). + + Returns: + The row to use and the generation it sits on, or ``None`` with the + generation to try next when every attempt this pass found a spent + one. The caller persists the generation, since only the caller knows + where it lives. + """ + from ..state.task_registry import TERMINAL_STATES + + for _attempt in range(max(1, attempts)): + task, _existing = await self.tasks.create_or_return_existing( + kind=kind, + params=params, + idempotency_key=key_for(generation), + **create_kwargs, + ) + if str(getattr(task, "state", "") or "") not in TERMINAL_STATES: + return task, generation + log.warning( + "ENABLEMENT %s: gen%d resolves to terminal task %s (%s); opening " + "generation %d so the work is not stuck on it", + label, + generation, + getattr(task, "task_id", ""), + getattr(task, "state", ""), + generation + 1, + ) + generation += 1 + return None, generation + async def _pump_enablement_safely(self, *, caller: str) -> None: """Phase-independent enablement pump — runs every tick. diff --git a/src/hyperloom/orchestrator/phases/kernel.py b/src/hyperloom/orchestrator/phases/kernel.py index d21985a28d..775f06efd4 100644 --- a/src/hyperloom/orchestrator/phases/kernel.py +++ b/src/hyperloom/orchestrator/phases/kernel.py @@ -33,7 +33,6 @@ _resolve_roofline_watermark_ratio, _resolve_serving_fidelity, _split_env_and_flags, - effective_closing_grace_sec, ) from .base import PhaseHandler @@ -597,10 +596,7 @@ def _geak_timeouts(self) -> tuple[int, int, bool]: if deadline is None: return env_default_timeout, env_default_timeout + 600, False remaining = deadline - time.monotonic() - grace = effective_closing_grace_sec( - float(getattr(self.shared_state, "max_minutes", 0) or 0), - None, - ) + grace = self.shared_state.closing_reserve_sec() margin = float(os.environ.get("GEAK_BUDGET_MARGIN_S", "300")) # Reserve the closing window: kill the subprocess with at least ``grace`` left. kill_budget = remaining - grace diff --git a/src/hyperloom/orchestrator/phases/machine.py b/src/hyperloom/orchestrator/phases/machine.py index 0922c91e23..7a7661bb96 100644 --- a/src/hyperloom/orchestrator/phases/machine.py +++ b/src/hyperloom/orchestrator/phases/machine.py @@ -35,6 +35,9 @@ def _ensure_phase_initialised(self) -> None: if not state.phase_budget_pct: state.phase_budget_pct = dict(self._phase_budget_pct) current = (state.phase or "").strip().upper() + if current == _phase_state.PHASE_CLOSE: + self._reopen_a_session_that_was_left_closed() + current = _phase_state.PHASE_PRELUDE if current in _phase_state.PHASE_NAMES: # Already initialised; keep the CLI-side budget override authoritative. state.phase_budget_pct = dict(self._phase_budget_pct) @@ -54,6 +57,47 @@ def _ensure_phase_initialised(self) -> None: except Exception: # noqa: BLE001 — defensive log.exception("Coordinator: save after phase init failed") + def _reopen_a_session_that_was_left_closed(self) -> None: + """Put a session persisted in CLOSE back at the phase machine's entrance. + + CLOSE is terminal -- the machine has no transition out of it -- so a + resumed session that loads it stays there for the whole leg. The run loop + does not stop on CLOSE either, so what such a leg actually does is tick + in a phase that admits only ``report``, ``session_breakdown`` and + ``recover``: it spends its new clock on none of the work it was resumed + for. + + Reopened at PRELUDE rather than at the phase CLOSE was entered from, + because PRELUDE is the phase that works out where a session belongs. It + exits on its first evaluation when the anchor the run needs is already + measured, and it measures one when it is not. A session stopped for a + cold anchor is the second case, and re-measuring is the whole reason its + stop was worth resuming from. + + A session that still cannot fund the work is not kept open by this: the + PRELUDE exits price the new clock and route it back to CLOSE, this time + against the budget it actually has. + + Reached only from the constructor, so a session cannot reopen itself + mid-run -- within one leg, CLOSE is entered long after this has run. + """ + state = self.shared_state + log.info( + "Coordinator: session resumed in CLOSE, a phase with no way out; " + "reopening at PRELUDE so the new budget can be spent on the work " + "the earlier leg stopped short of." + ) + state.record_phase_transition( + to_phase=_phase_state.PHASE_PRELUDE, + reason="phase_entered", + evidence={"trigger": "resumed_from_close"}, + ) + # Locked True by the CLOSE sequencer and read by the end-of-run safety + # nets as "the sequencer already wrote the breakdown". Carried into a leg + # that then never reaches CLOSE, it suppresses the write that would have + # stood in for it, and the leg produces no breakdown at all. + state.close_sequence_done = False + def _ensure_recipe_kb_t0_anchored(self) -> None: """Defensive T0 anchor for SDK callers constructed without cli plumbing. Skips when recipe_kb is None or recipe_kb_session_id set.""" client = self.recipe_kb @@ -224,6 +268,15 @@ async def _advance_phase_if_needed(self) -> None: # Consume escalate hint after a hint-driven transition. if isinstance(evidence, dict) and (evidence.get("evidence") == "llm_escalation" or "hint" in evidence): state.consume_pending_escalate_hint() + elif ( + str(prior or "").strip().upper() == _phase_state.PHASE_SWEEP + and str(getattr(state, "pending_escalate_hint", "") or "").strip() + == _phase_state.ESCALATE_HINT_SKIP_TO_CLOSE + ): + # SWEEP already had an honest closeout, so skip_to_close was + # suppressed. Drop it here or the next phase inherits it and + # becomes robustness_escalated. + state.consume_pending_escalate_hint() # Terminal transition (target=CLOSE): mirror the stop_reason onto state. if ( target == _phase_state.PHASE_CLOSE diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 1fba9944db..39a511e6f6 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -235,6 +235,7 @@ def render_phase_proposable_bullets( "recipe_kb_drain_failed", "recipe_kb_commit_failed", "prelude_baseline_failed", + "prelude_cold_anchor_low_budget", # PRELUDE → CLOSE; only a cold anchor, nothing comparable to it affordable "prelude_policy_loop", "policy_loop", "crash_threshold_exceeded", @@ -267,6 +268,7 @@ def render_phase_proposable_bullets( "robustness_escalated", "user_stop_requested", "prelude_baseline_failed", + "prelude_cold_anchor_low_budget", "prelude_policy_loop", "time_exhausted_during_prelude", "recipe_kb_t0_failed", @@ -352,6 +354,34 @@ def is_valid_phase_exit_reason(value: str) -> bool: PHASE_CLOSE: 0.02, } +# Share of the session held back for the phases that actually produce a result. +# +# PRELUDE's ``DEFAULT_PHASE_BUDGET_PCT`` entry is 3%, but nothing enforced it: +# :func:`exit_normal_prelude` tests only whether a baseline landed, so the phase +# ran to whatever its contents cost. Two sessions on unrelated models (different +# quantization, different PRELUDE composition -- one baseline-dominated, one +# split with a TraceLens roofline) spent 73.8% and 72.8% of a three-hour budget +# in it, and both reached FRAMEWORK_AGENT with ~47 minutes left against its +# 108-minute entry threshold. Both budgets were honoured; both sessions produced +# nothing. Stopping the run on time is necessary and not sufficient -- the +# phases that spend the budget have to be the ones that produce the result. +# +# This bounds the preparation rather than the session, and it is deliberately +# looser than 3%: a baseline that legitimately takes an hour should still run, +# it just cannot also buy an 80-minute roofline out of the optimization phases' +# time. +# +# A second bound used to sit beside it -- a ceiling on PRELUDE's own banked +# spend -- and it was removed because the two clocks it straddled can disagree. +# Banked phase spend accumulates against the phase ledger; the reserve is read +# off the session clock. A session resumed with a reanchored budget restarts the +# session clock and carries the ledger over, so preparation was born over its +# ceiling with hours genuinely left, and every resumed session was refused the +# measured half of its baseline. The reserve alone answers the question that +# decides the matter: after this work, is enough left for the phases that +# produce the result? +OPTIMIZATION_RESERVE_PCT: float = 0.50 + # Wall-clock ceiling for an unbounded run (``max_minutes`` == 0): the container # lifetime. Used both as the global deadline and as the basis for the absolute # per-phase cap so an unbounded run still forces phase rotation. @@ -372,6 +402,9 @@ def is_valid_phase_exit_reason(value: str) -> bool: # EXPLORE hard force-exit thresholds (IR-6 HARD time gate; overrides plateau). # Fires when remaining wall-clock < HOURS_REMAINING OR EXPLORE budget fraction < BUDGET_PCT. +# HOURS_REMAINING is a leave-behind for later phases on long runs. When it is +# not strictly smaller than the session, the hours gate is ignored so a 3h +# smoke does not skip EXPLORE the moment the phase starts. DEFAULT_EXPLORE_FORCE_EXIT_HOURS_REMAINING: float = 3.0 DEFAULT_EXPLORE_FORCE_EXIT_BUDGET_PCT: float = 0.20 @@ -1262,6 +1295,29 @@ def phase_cap_exceeded( # EXPLORE hard force-exit (HARD time gate) +def _explore_hours_leavebehind_applies(state: Any, *, threshold_hours: float) -> bool: + """Whether IR-6's hours-remaining gate can fire for this session. + + Remaining starts at ``max_hours``, so a leave-behind that is not strictly + smaller than the session fires the moment EXPLORE starts. The 3h default + is for long runs; a 3h session (the CI smoke, the 3h example) must not + inherit it. + + Args: + state (Any): Frozen SharedState view. + threshold_hours (float): Configured hours-remaining leave-behind. + + Returns: + bool: ``True`` when the hours gate may fire. + """ + if float(threshold_hours) <= 0.0: + return False + session_hours = _max_minutes(state) / 60.0 + if session_hours <= 0.0: + return False + return float(threshold_hours) < session_hours + + def session_remaining_seconds( state: Any, *, @@ -1314,12 +1370,14 @@ def should_force_exit_explore( """Return ``(True, evidence)`` when HARD EXPLORE force-exit fires (IR-6). Fires when session remaining ≤ hours_threshold*3600 OR phase remaining - pct ≤ budget_pct_threshold; ``evidence`` records which fired. + pct ≤ budget_pct_threshold; ``evidence`` records which fired. The hours + gate is ignored when the leave-behind is not strictly smaller than the + session (it would otherwise fire the moment EXPLORE starts). Args: state (Any): Frozen SharedState view. hours_remaining_threshold (float): Session-hours-remaining gate; - non-positive disables it. + non-positive disables it. Also ignored when it covers the session. budget_pct_threshold (float): Phase-budget-fraction gate; non-positive disables it. budget_pct (dict[str, float] | None): Phase-budget overrides; defaults @@ -1338,8 +1396,16 @@ def should_force_exit_explore( fired_reasons: list[str] = [] # Non-positive threshold = disabled; both disabled turns force-exit off. - hours_threshold_enabled = float(hours_remaining_threshold) > 0.0 + # A leave-behind that covers the whole session is also disabled: remaining + # starts at max_hours, so it would fire the moment EXPLORE starts. + hours_threshold_enabled = _explore_hours_leavebehind_applies( + state, threshold_hours=hours_remaining_threshold + ) pct_threshold_enabled = float(budget_pct_threshold) > 0.0 + if float(hours_remaining_threshold) > 0.0 and not hours_threshold_enabled: + session_hours = _max_minutes(state) / 60.0 + if session_hours > 0.0: + evidence["hours_remaining_gate"] = "disabled_leavebehind_covers_session" session_remaining = session_remaining_seconds(state, now_unix=now_unix) if session_remaining is not None and hours_threshold_enabled: @@ -1595,11 +1661,36 @@ def compute_plateau_kernel( } +# Statuses on last_sweep / last_conc_sweep that exit_normal_sweep already +# treats as SWEEP closeout. skip_to_close must not override those: the LLM +# emits it when conc_sweep was refused, and mapping that to +# robustness_escalated turns a successful run into a CI failure. +_SWEEP_DONE_STATUSES: frozenset[str] = frozenset({"succeeded", "partial", "completed"}) +_CONC_SWEEP_CLOSEOUT_STATUSES: frozenset[str] = frozenset( + {"succeeded", "partial", "completed", "skipped", "failed"} +) + + +def _sweep_has_recorded_closeout(state: Any) -> bool: + """Whether SWEEP already recorded a result the phase machine can close on.""" + last_sweep = getattr(state, "last_sweep", None) or {} + if isinstance(last_sweep, dict): + if str(last_sweep.get("status") or "").lower() in _SWEEP_DONE_STATUSES: + return True + last_conc = getattr(state, "last_conc_sweep", None) or {} + if isinstance(last_conc, dict): + if str(last_conc.get("status") or "").lower() in _CONC_SWEEP_CLOSEOUT_STATUSES: + return True + return False + + # terminal / abort (global) def _global_terminal(state: Any) -> tuple[str, dict[str, Any]] | None: """Return ``(stop_reason, evidence)`` for a phase-orthogonal stop. - Priority: 1. ``skip_to_close`` → ``robustness_escalated``; 2. Coordinator ``stop_reason``. + Priority: 1. ``skip_to_close`` → ``robustness_escalated``, except in SWEEP + when a sweep/conc_sweep closeout is already recorded (the honest SWEEP + exit wins); 2. Coordinator ``stop_reason``. Args: state (Any): Frozen SharedState view exposing ``stop_reason`` and any @@ -1611,6 +1702,9 @@ def _global_terminal(state: Any) -> tuple[str, dict[str, Any]] | None: """ hint = _pending_escalate_hint(state) if hint == ESCALATE_HINT_SKIP_TO_CLOSE: + current = (getattr(state, "phase", "") or "").strip().upper() + if current == PHASE_SWEEP and _sweep_has_recorded_closeout(state): + return None return "robustness_escalated", { "evidence": "llm_escalation", "hint": hint, @@ -1946,6 +2040,14 @@ def enablement_engaged(state: Any) -> bool: def exit_normal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """``baseline_tput > 0`` and warm-replay settled → ``prelude_done`` (else ``None``). + A figure whose hot pass was dropped for budget does not finish preparation, + even though it is a figure. It is depressed by the boot, the compile and the + graph capture it could not discard, so declaring PRELUDE done on it would + hand the optimization phases a denominator that was never the baseline. The + phase stays open instead, and what happens next is decided by the budget: + :func:`exit_cold_anchor_prelude` closes a session that still cannot afford a + comparable baseline, and one resumed with a fresh clock measures another. + Args: state (Any): Frozen SharedState view exposing ``baseline_tput`` and the warm-replay outcome. @@ -1956,15 +2058,419 @@ def exit_normal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """ if warm_replay_in_flight(state): return None + if bool(getattr(state, "baseline_measure_round_dropped", False)): + return None try: tput = float(getattr(state, "baseline_tput", 0.0) or 0.0) except (TypeError, ValueError): return None if tput > 0.0: - return "prelude_done", {"baseline_tput": tput} + return "prelude_done", {"baseline_tput": tput, **prelude_exit_viability(state)} return None +def measured_seconds(state: Any, field: str) -> float | None: + """Read a duration an earlier round measured, or ``None`` when none did. + + Every budget decision that prices work off a measurement has to tell "no + round has run" apart from "a round ran and took no time", because the first + means the decision cannot be made and the second would make everything look + free. + + Args: + state (Any): Frozen SharedState view. + field (str): The ``SharedState`` attribute holding the duration. + + Returns: + float | None: The duration when one was measured, else ``None``. + """ + try: + value = float(getattr(state, field, 0.0) or 0.0) + except (TypeError, ValueError): + return None + return value if value > 0.0 else None + + +def boot_cost_sec(state: Any) -> float | None: + """What bringing this workload's server up costs, or ``None`` when unmeasured. + + The baseline's cold round is the one round that pays this in the open, so it + is where the figure comes from: its wall-clock less the part of it that ran + after the server reported ready. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Seconds spent before the server was ready, or ``None`` when + no round has reported the split (no baseline yet, or a scriptable + workload, which runs no server and so has no boot to separate). + """ + total_sec = measured_seconds(state, "baseline_runtime_sec") + post_ready_sec = measured_seconds(state, "baseline_post_ready_runtime_sec") + if total_sec is None or post_ready_sec is None: + return None + return max(0.0, total_sec - post_ready_sec) + + +def benchmark_cost_sec(state: Any) -> float | None: + """What one benchmark pass costs on a server already up, or ``None``. + + Two figures can answer this and they are not equally good, so the better one + wins when it exists: + + * The measured hot pass (``baseline_warm_runtime_sec``) is the answer, being + exactly a benchmark against a server someone else booted. + * The cold round's post-ready segment is the fallback, and it over-predicts: + that segment also pays the first request's kernel compile, which a pass on + a now-populated JIT cache does not. It is what a session has before its hot + pass runs, and over-predicting a benchmark is a smaller error than pricing + one at a whole cold round. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Seconds one benchmark pass costs, or ``None`` when neither + figure has been measured. + """ + hot_sec = measured_seconds(state, "baseline_warm_runtime_sec") + if hot_sec is not None: + return hot_sec + return measured_seconds(state, "baseline_post_ready_runtime_sec") + + +def baseline_round_cost_sec(state: Any, *, double_run: bool) -> float | None: + """What a baseline round costs, or ``None`` when unmeasured. + + A round is one or two passes and they are priced apart, because they buy + different things. The first brings a server up and benchmarks it cold, paying + the first request's kernel compile on the way; the session has measured that + whole pass directly, so it is read rather than reconstructed. The second + re-attaches to the server the first left running, so it costs a benchmark and + no second boot. + + Reconstructing the first pass as boot-plus-hot-benchmark would drop the + compile and under-price the round, which matters because this figure guards + ignition while the post-warmup gate that follows prices the same work from + the pass it just watched. A round admitted here and then certainly refused + there costs a whole cold pass to learn nothing. + + Reading the measured total also gives a price to rounds with no boot/benchmark + split to reconstruct from -- multi-node and scriptable workloads, which never + report one -- so those are gated rather than waved through. + + Args: + state (Any): Frozen SharedState view. + double_run (bool): Whether the round runs a warmup pass and a measured + one, or a single pass. + + Returns: + float | None: Seconds the round costs, or ``None`` when the session has + measured nothing to price it from. + """ + first_pass_sec = measured_seconds(state, "baseline_runtime_sec") + if first_pass_sec is None or not double_run: + return first_pass_sec + second_pass_sec = benchmark_cost_sec(state) + return first_pass_sec if second_pass_sec is None else first_pass_sec + second_pass_sec + + +def one_more_measurement_sec(state: Any) -> float | None: + """What the next measured variant will cost, or ``None`` when unmeasured. + + A variant is not a benchmark; it is a boot and then a benchmark. Its config + differs from the baseline's in the very knobs that decide how a server comes + up -- parallelism, quantization, kernel backends -- so it cannot re-attach to + a server already running and has to bring up its own. + + This is the unit every "is there time left to use a result" question is asked + in, because a result nothing gets measured against is a result the session + could not have used. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Seconds one further measured variant costs, or ``None`` + when either half of it is unmeasured. + """ + boot_sec = boot_cost_sec(state) + benchmark_sec = benchmark_cost_sec(state) + if boot_sec is None or benchmark_sec is None: + return None + return boot_sec + benchmark_sec + + +def prelude_exit_viability(state: Any) -> dict[str, Any]: + """Report whether the budget PRELUDE leaves behind can still fund one optimization round. + + A session can honour its wall clock and still be over: both field sessions + left FRAMEWORK_AGENT ~47 minutes against a 108-minute threshold, and every + later phase then declined in turn, each for its own local reason. Nothing + said the plain thing — preparation had spent the run. + + Stated here, on the exit that caused it, because this is the last moment + the answer is actionable and the first moment it is knowable: the baseline + is measured, so one benchmark round has a price rather than an estimate. + + Priced as what an optimization round actually is -- one boot and one + benchmark (:func:`one_more_measurement_sec`) -- rather than as the baseline's + whole cold wall-clock, which also carries the first request's compile and so + over-reports a prepared session as a spent one. The cold figure remains the + fallback for a session whose baseline reported no split, with ``priced_by`` + naming which ruler answered so two runs' evidence cannot be compared as + though they used the same one. + + Args: + state (Any): Frozen SharedState view. + + Returns: + dict[str, Any]: Evidence for the phase record; empty when the budget is + unbounded or no round has been measured. + """ + usable = session_usable_seconds(state) + round_sec = one_more_measurement_sec(state) + priced_by = "boot_plus_benchmark" + if round_sec is None: + round_sec = measured_seconds(state, "baseline_runtime_sec") + priced_by = "cold_round" + if usable is None or round_sec is None: + return {} + return { + "session_usable_sec": round(usable, 1), + "measured_round_sec": round(round_sec, 1), + "priced_by": priced_by, + "affordable_rounds": round(usable / round_sec, 2), + "fits_one_optimization_round": usable >= round_sec, + } + + +def append_phase_evidence_row(history: Any, *, key: str, row: dict[str, Any]) -> bool: + """Append ``row`` to the current phase's ``evidence[key]`` list. + + Phases record what happened inside them on the newest ``phase_history`` + row, which the session breakdown exports verbatim. Creates the ``evidence`` + dict and the list under ``key`` when absent, and replaces a non-list value + rather than raising — a malformed row must not take a phase down. + + Args: + history (Any): The ``phase_history`` list. + key (str): Evidence key holding the list of rows. + row (dict[str, Any]): The row to append. + + Returns: + bool: ``True`` when the row landed; ``False`` when there was no usable + history row to attach it to. + """ + if not isinstance(history, list) or not history: + return False + current = history[-1] + if not isinstance(current, dict): + return False + evidence = current.get("evidence") + if not isinstance(evidence, dict): + evidence = {} + current["evidence"] = evidence + rows = evidence.get(key) + if not isinstance(rows, list): + rows = [] + evidence[key] = rows + rows.append(row) + return True + + +def session_usable_seconds(state: Any) -> float | None: + """Seconds a unit of work may still claim, from the session's own accounting. + + Prefers ``SharedState.session_budget_usable_sec`` — the single number + admission control and the grid deadline both read — so this policy cannot + disagree with them about how much budget is left. Falls back to the raw + remaining time for the frozen views and test doubles that expose attributes + only. Called rather than imported because ``shared_state`` imports this + module. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Usable seconds, or ``None`` when the budget is unbounded. + """ + getter = getattr(state, "session_budget_usable_sec", None) + if callable(getter): + try: + return getter() + except Exception: # noqa: BLE001 — fall back to the attribute path + pass + return session_remaining_seconds(state) + + +def prelude_affordable_seconds(state: Any) -> tuple[float | None, dict[str, Any]]: + """Seconds PRELUDE may still spend, and the numbers the figure is built from. + + What is left on the session clock once the optimization phases' reserve + (:data:`OPTIMIZATION_RESERVE_PCT`) is held back, measured against the same + usable remainder every other budget decision reads, so the figure survives + a resume that reanchors the budget. Work that does not fit is not work the + session needed — it is work the session could not have used the result of. + + Read directly by callers that have to *size* a unit of work rather than + judge one they can already price, which is the position a first baseline is + in: it has no measured runtime to judge against, but the share it may spend + is known before anything runs. + + Args: + state (Any): Frozen SharedState view. + + Returns: + tuple[float | None, dict[str, Any]]: The affordable seconds — which may + be negative once the reserve is eaten into — or ``None`` on an + unbounded budget, plus the evidence behind it. + """ + max_sec = _max_minutes(state) * 60.0 + usable = session_usable_seconds(state) + if max_sec <= 0.0 or usable is None: + return None, {"reason": "unbounded_budget"} + reserve_sec = max_sec * OPTIMIZATION_RESERVE_PCT + affordable_sec = usable - reserve_sec + return affordable_sec, { + "optimization_reserve_sec": round(reserve_sec, 1), + "session_usable_sec": round(usable, 1), + "affordable_sec": round(affordable_sec, 1), + "bound": "optimization_reserve", + } + + +def prelude_can_afford( + state: Any, + *, + expected_cost_sec: float, +) -> tuple[bool, dict[str, Any]]: + """Decide whether PRELUDE can still buy an optional arm costing ``expected_cost_sec``. + + The share itself is :func:`prelude_affordable_seconds`; this judges one cost + against it. + + Args: + state (Any): Frozen SharedState view. + expected_cost_sec (float): What the arm is expected to cost. Callers + should anchor this on something this session measured; the static + per-action estimates are calibrated on small models and understate + a large one by an order of magnitude. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. Evidence + carries the numbers behind the decision so a skip is legible in the log + and the phase record. + """ + cost = max(0.0, float(expected_cost_sec or 0.0)) + affordable_sec, evidence = prelude_affordable_seconds(state) + priced = {"expected_cost_sec": round(cost, 1), **evidence} + if affordable_sec is None: + return True, priced + return affordable_sec >= cost, priced + + +def exit_time_exhausted_prelude( + state: Any, + *, + now_unix: float | None = None, +) -> tuple[str, dict[str, Any]] | None: + """Route to CLOSE when the session clock runs out before PRELUDE lands a baseline. + + ``time_exhausted_during_prelude`` was in the terminal-reason vocabulary and + in the report's reason glossary, but no code ever assigned it: the state + machine had a word for this failure and no way to reach it. A session that + burns its whole budget preparing then read as an ordinary exit. + + Only fires while PRELUDE is still incomplete. Once a baseline exists the + later phases have their own force-exits, and reporting the run as "never + began optimizing" would be false. + + Args: + state (Any): Frozen SharedState view. + now_unix (float | None): Override for the current time. + + Returns: + tuple[str, dict[str, Any]] | None: ``("time_exhausted_during_prelude", + evidence)`` when the budget is gone, else ``None``. + """ + usable = session_usable_seconds(state) + if usable is None or usable > 0.0: + return None + return "time_exhausted_during_prelude", { + "session_usable_sec": round(usable, 1), + "prelude_spent_sec": round( + phase_cumulative_seconds(state, phase=PHASE_PRELUDE, now_unix=now_unix), + 1, + ), + } + + +def exit_cold_anchor_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: + """Route to CLOSE when PRELUDE could only produce a cold anchor. + + A baseline's hot pass is dropped when the budget cannot cover it together with + one variant to read against it. What survives is the cold pass's figure, and + it is depressed: it carries the server boot, the first request's kernel + compile and the graph capture in its throughput denominator. + + Continuing on it is worse than stopping. Every variant measured against a + depressed denominator reads as an improvement over a baseline that was never + the baseline, so the session would spend the rest of its clock producing + findings that a later run cannot reproduce. Stopping keeps the number and the + marker that says what it is. + + Fires only on the dropped-pass marker, not on a cold figure as such: a session + configured for a single-round baseline reports a cold figure by design, and + its comparisons are consistent because everything downstream is measured the + same way. + + And only while the budget still cannot buy a comparable baseline. A session + resumed with a fresh clock carries the earlier leg's marker but not its + shortfall, and it can now do the thing it was stopped for: measure a hot + baseline. Firing on the marker alone would send it straight back to CLOSE on + the strength of a constraint that no longer holds, and no later baseline could + ever clear the marker because none would run. + + Args: + state (Any): Frozen SharedState view. + + Returns: + tuple[str, dict[str, Any]] | None: ``("prelude_cold_anchor_low_budget", + evidence)`` when the hot pass was dropped and still cannot be afforded, + else ``None``. + """ + if not bool(getattr(state, "baseline_measure_round_dropped", False)): + return None + usable = session_usable_seconds(state) + if usable is None: + return None + # Without the boot/benchmark split -- a scriptable workload runs no server, so + # it has no ready boundary to split on -- the cold round's whole wall-clock + # stands in for each half, the same upper bound the round's own gate falls + # back to. Undecidable rather than assumed only when nothing was measured at + # all, which cannot happen with the marker set but is not worth closing on. + cold_sec = measured_seconds(state, "baseline_runtime_sec") + round_sec = ( + baseline_round_cost_sec( + state, + double_run=bool(getattr(state, "baseline_double_run", False)), + ) + or cold_sec + ) + use_sec = one_more_measurement_sec(state) or cold_sec + if round_sec is None or use_sec is None: + return None + if usable >= round_sec + use_sec: + return None + return "prelude_cold_anchor_low_budget", { + "baseline_anchor": "cold", + "retry_round_sec": round(round_sec, 1), + **prelude_exit_viability(state), + } + + def exit_terminal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """Decide the PRELUDE terminal exit on repeated baseline failures. @@ -2592,7 +3098,18 @@ def compute_next_phase( term = exit_terminal_prelude(state) if term is not None: return PHASE_CLOSE, term[0], {"terminal": True, **term[1]} + # Asked before the normal exit, which sees only that a figure exists: a + # cold anchor is a figure the later phases cannot honestly compare to. + cold = exit_cold_anchor_prelude(state) + if cold is not None: + return PHASE_CLOSE, cold[0], {"terminal": True, **cold[1]} norm = exit_normal_prelude(state) + if norm is None: + # No baseline and no clock left: name the failure instead of + # letting the run read as an ordinary exit. + exhausted = exit_time_exhausted_prelude(state, now_unix=now_unix) + if exhausted is not None: + return PHASE_CLOSE, exhausted[0], {"terminal": True, **exhausted[1]} if norm is not None: if framework_agent_phase_enabled: return PHASE_FRAMEWORK_AGENT, norm[0], norm[1] @@ -3102,6 +3619,7 @@ def record_lifecycle_event( "DEFAULT_EXPLORE_FORCE_EXIT_BUDGET_PCT", "DEFAULT_EXPLORE_FORCE_EXIT_HOURS_REMAINING", "DEFAULT_PHASE_BUDGET_PCT", + "OPTIMIZATION_RESERVE_PCT", "DEFAULT_PLATEAU_EXPLORE_EMPTY_STREAK", "DEFAULT_PLATEAU_EXPLORE_KEEP_GAIN_PCT", "DEFAULT_PLATEAU_EXPLORE_LOOKBACK", @@ -3158,9 +3676,21 @@ def record_lifecycle_event( "exit_normal_explore", "exit_normal_framework_agent", "exit_normal_kernel", + "exit_cold_anchor_prelude", "exit_normal_prelude", "exit_normal_sweep", "exit_terminal_prelude", + "exit_time_exhausted_prelude", + "append_phase_evidence_row", + "baseline_round_cost_sec", + "benchmark_cost_sec", + "boot_cost_sec", + "measured_seconds", + "one_more_measurement_sec", + "prelude_affordable_seconds", + "prelude_can_afford", + "prelude_exit_viability", + "session_usable_seconds", "is_action_allowed_in_phase", "is_action_llm_proposable_in_phase", "llm_proposable_actions_for", diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index 51ec98637d..a11d620f7e 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -24,6 +24,10 @@ from ..loop.coordinator import ( _DEFAULT_WARM_REPLAY_MIN_CONFIDENCE, ) +from ..loop.coordinator_helpers import ( + expected_action_cost_minutes, + measured_baseline_runtime_sec, +) from .base import PhaseHandler log = _logging.getLogger(__name__) @@ -152,6 +156,51 @@ def _internal_analysis_kind(self) -> str: else "profile" ) + def _measured_analysis_cost_sec(self) -> float: + """Expected cost of the initial roofline/profile arm, in seconds. + + The analysis arm boots its own server and runs the same benchmark under + a profiler, so one measured baseline round is a floor on its cost rather + than a guess at it; :func:`expected_action_cost_minutes` applies that + floor and falls back to the catalog for the first analysis of a session + that has no measurement yet. + + Returns: + float: Expected cost in seconds; ``0.0`` when nothing is on record, + which :func:`machine_state.prelude_can_afford` reads as free. + """ + registry = getattr(self, "action_registry", None) + meta = registry.get(self._internal_analysis_kind()) if registry is not None else None + return ( + expected_action_cost_minutes( + meta, + measured_baseline_sec=measured_baseline_runtime_sec(self.shared_state), + ) + * 60.0 + ) + + def _record_prelude_arm_dropped(self, arm: str, evidence: dict[str, Any]) -> None: + """Record a PRELUDE arm dropped for budget on the current phase record. + + The phase record is what the session breakdown exports, so a dropped + arm reads as a decision with numbers behind it rather than as an arm + that silently never ran. + + Args: + arm: The arm that was dropped. + evidence: The affordability numbers behind the decision. + """ + if not _phase_state.append_phase_evidence_row( + getattr(self.shared_state, "phase_history", None), + key="budget_dropped_arms", + row={"arm": arm, **evidence}, + ): + return + try: + self.shared_state.save(self.session_dir) + except Exception: # noqa: BLE001 — best-effort record + log.exception("PRELUDE: failed to persist the dropped-arm record for %r", arm) + def _warm_recipe_proven_items(self) -> list[dict[str, str]]: """Summarise warm-start ``what_worked`` items the scout can skip ({name, source}); fail-soft. @@ -2303,6 +2352,22 @@ async def _maybe_enqueue_prelude_initial_analysis_after_baseline( return if (state.auto_roofline_pending_task_id or "").strip(): return + affordable, evidence = _phase_state.prelude_can_afford( + state, + expected_cost_sec=self._measured_analysis_cost_sec(), + ) + if not affordable: + log.warning( + "PRELUDE: skipping the initial %s — %.0fs of preparation budget " + "left (bound=%s) against an expected %.0fs. The optimization " + "phases keep the time instead.", + self._internal_analysis_kind(), + evidence.get("affordable_sec", 0.0), + evidence.get("bound", ""), + evidence.get("expected_cost_sec", 0.0), + ) + self._record_prelude_arm_dropped("initial_analysis", evidence) + return try: rl_task = await self._enqueue_internal_analysis_task( reason="prelude_initial", diff --git a/src/hyperloom/orchestrator/phases/sweep.py b/src/hyperloom/orchestrator/phases/sweep.py index a4872d2958..cffe471790 100644 --- a/src/hyperloom/orchestrator/phases/sweep.py +++ b/src/hyperloom/orchestrator/phases/sweep.py @@ -59,6 +59,16 @@ async def _on_enter_sweep(self, *, from_phase: str) -> None: auto_conc_sweep_skipped_validated_gain=cur_validated, ) return + denied = self._time_budget_denial_for_action("conc_sweep") + if denied is not None: + log.info( + "SWEEP entry (from=%s): conc_sweep cannot fit the session budget " + "(%s); recording terminal skip.", + from_phase or "", + denied, + ) + self._record_session_budget_conc_sweep_skip(denied=denied) + return try: task = await self._enqueue_internal_conc_sweep_task( reason="phase_entry", @@ -167,6 +177,21 @@ async def _enqueue_internal_conc_sweep_task( self._record_phase_entry_evidence(auto_conc_sweep_task_id=task.task_id) return task + def _record_session_budget_conc_sweep_skip(self, *, denied: object) -> None: + """Stamp last_conc_sweep skipped when the session clock refused conc_sweep. + + No-op when a conc_sweep result is already on the session: a later + over-budget cancel must not erase a measurement the phase can close on. + """ + last = getattr(self.shared_state, "last_conc_sweep", None) or {} + if str(last.get("status") or "").strip(): + return + self._record_terminal_conc_sweep_skip( + skip_reason="session_time_budget", + auto_conc_sweep_skipped="session_time_budget", + auto_conc_sweep_denied=str(denied), + ) + def _record_terminal_conc_sweep_skip( self, *, diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index b751ee3567..db505c9250 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -556,6 +556,11 @@ def _source_file_candidates(value: str) -> tuple[str, ...]: # leg's CLOSE transition back the right to speak for this one. "resumed_ts", "max_minutes", + # Sizes the closing reserve, so it decides how much of ``max_minutes`` + # is still usable: locking the budget without locking this one leaves + # the same forgery one field over -- a large value spends the session + # outright, a zero one erases the window the CLOSE report needs. + "closing_grace_sec", # fact-layer KEEP ledger; Coordinator is the sole writer. "optimization_stack", "gain_per_stack_entry", @@ -1538,7 +1543,19 @@ def _validate_baseline_singleton( self, payload: dict[str, Any], ) -> None: - """Deny a baseline proposal when an enablement round is in flight or the anchor is established.""" + """Deny a baseline proposal when an enablement round is in flight or the anchor is established. + + "Established" is the whole rule: a repeat baseline is refused because it + re-measures a reference the run already has. A cold anchor is the case + where it does not. The session kept a warmup's figure because the clock + could not fund the hot pass that would have made it comparable, and + marked it as such; PRELUDE will not finish while the mark is set, because + every variant read against a depressed denominator reads as an + improvement over a baseline that never existed. Refusing the round that + would clear the mark leaves the phase with no way forward and no way out, + which is the state a session resumed on a fresh clock arrives in -- + exactly the one the mark exists to make recoverable. + """ ss = getattr(self, "shared_state", None) if ss is None: return @@ -1553,6 +1570,12 @@ def _validate_baseline_singleton( "before re-running baseline." ), ) + # Checked after the authoring round, which is a reason to wait whatever + # the anchor says: a specialist rewriting the framework underneath a + # baseline would have this round measuring a stack that changes as it + # runs. + if bool(getattr(ss, "baseline_measure_round_dropped", False)): + return anchor = getattr(ss, "baseline_tput", 0.0) if not isinstance(anchor, (int, float)) or anchor <= 0: return diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 0bb1006f32..d13d4fd39f 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -259,6 +259,7 @@ def render_model_arch_compact(arch: dict | None) -> str: # How many hot / skipped kernels ``record_trace_analyze`` keeps in the trace # summary (matches the ``*_top15`` field names). _TRACE_HOT_KERNEL_TOP_N = 15 + # Session-level kernel-roofline report the analyzer writes for a non-close run; # read back when the trace_analyze envelope arrives without its payload keys. _DEFAULT_ROOFLINE_REPORT_NAME = "kernel_roofline_current.json" @@ -339,6 +340,33 @@ def render_model_arch_compact(arch: dict | None) -> str: _FRAMEWORK_VARIANT_PREFIX_V5: str = "framework:" +def effective_closing_grace_sec( + max_minutes: float | None, + closing_grace_sec: float | None, +) -> float: + """Resolve the closing-phase grace window after the wall-clock deadline. + + Explicit ``closing_grace_sec`` (including ``0`` to disable the closing + phase) wins; otherwise default to ``min(120, max_minutes * 60 * 0.02)``. + + Lives beside the budget accessors rather than next to its Coordinator + caller because the window is also the reserve those accessors hold back, + and this module is the leaf both sides can import. + + Args: + max_minutes (float | None): The wall-clock budget in minutes, used for + the default. + closing_grace_sec (float | None): Explicit grace window in seconds; + when not ``None`` it is used verbatim. + + Returns: + float: The closing-phase grace window in seconds. + """ + if closing_grace_sec is not None: + return float(closing_grace_sec) + return min(120.0, (max_minutes or 0.0) * 60.0 * 0.02) + + def _cap_tested_ledger(tested: dict[str, Any]) -> dict[str, Any]: """Bound the explore_search negative ledger for multi-day runs. @@ -567,6 +595,18 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # Baseline WARM measure-round wall-clock (client-only, no boot); anchors the # explore overtime kill apples-to-apples. Zero => fall back to the cold anchor. baseline_warm_runtime_sec: float = 0.0 + # Whether the baseline's hot pass was dropped because the budget could not + # cover it plus one variant to read against it, leaving the cold pass's + # depressed figure as the anchor. Routes PRELUDE to CLOSE rather than letting + # later phases compare against a denominator that was never the baseline, and + # keeps a resumed session from treating preparation as finished. + baseline_measure_round_dropped: bool = False + # The benchmark's own share of the COLD round above, from the server-ready + # marker onward. Kept beside that total rather than replacing it because the + # difference between them is what booting this workload costs, and every + # later variant boots again: the pair prices one variant, while this figure + # alone prices a pass that re-attaches. Zero => never measured. + baseline_post_ready_runtime_sec: float = 0.0 current_best: dict[str, Any] = field(default_factory=dict) # Reference launch recipe from the operator's --reference-script: lowest-priority # base server args/envs seeding every baseline. Persisted. @@ -646,6 +686,8 @@ class SharedState(_RenderMixin, _ExploreStateMixin): pruned_families: list[str] = field(default_factory=list) start_ts: str = field(default_factory=_now_iso) max_minutes: int = 0 + # Operator's ``--closing-grace-sec``; ``None`` derives it from max_minutes. + closing_grace_sec: float | None = None last_profile_trace: str = "" # ``succeeded``/``failed`` for most recent profile; failed allows re-run even when last_profile_trace is non-empty. last_profile_status: str = "" @@ -3602,7 +3644,54 @@ def remaining_minutes(self, *, now: datetime | None = None) -> float | None: return None return max(0.0, float(self.max_minutes) - self.elapsed_minutes(now=now)) - def grid_session_deadline_sec(self, *, reserve_sec: float = 120.0) -> float | None: + def closing_reserve_sec(self) -> float: + """Seconds held back from every unit of work so CLOSE can still report. + + Resolved through :func:`effective_closing_grace_sec`, the same function + the Coordinator uses to size the closing phase itself, so the budget + reserved for that phase and the budget it actually gets are one number. + A hardcoded reserve told the truth only for sessions of at least 100 + minutes, and charged 120 seconds even to an operator who had disabled + the closing phase outright. + + Returns: + float: The closing reserve in seconds; ``0.0`` when the operator + disabled the closing phase. + """ + return max(0.0, effective_closing_grace_sec(float(self.max_minutes or 0), self.closing_grace_sec)) + + def session_budget_usable_sec( + self, + *, + reserve_sec: float | None = None, + ) -> float | None: + """Seconds of wall-clock budget left once the closing reserve is held back. + + The single source for "how much time may a unit of work still claim". + Admission control (which action may start) and the grid deadline (how + long a variant may run) both read it, so they cannot disagree about how + much budget exists. + + Args: + reserve_sec (float | None): Seconds held back for the CLOSE phase + and its report; ``None`` takes :meth:`closing_reserve_sec`. An + explicit ``0`` is honoured as "reserve nothing". + + Returns: + float | None: Usable seconds (clamped at 0.0), or ``None`` when + ``max_minutes`` is unset (unbounded budget). + """ + remaining = self.remaining_minutes() + if remaining is None: + return None + reserve = self.closing_reserve_sec() if reserve_sec is None else float(reserve_sec) + return max(0.0, remaining * 60.0 - reserve) + + def grid_session_deadline_sec( + self, + *, + reserve_sec: float | None = None, + ) -> float | None: """``time.monotonic()`` deadline for grid variant loops, or ``None`` when the budget is unbounded. Reserves ``reserve_sec`` so the CLOSE phase and report still have room @@ -3611,18 +3700,16 @@ def grid_session_deadline_sec(self, *, reserve_sec: float = 120.0) -> float | No the whole grid. Args: - reserve_sec (float): Seconds held back from the raw remaining budget. + reserve_sec (float | None): Seconds held back from the raw remaining + budget; ``None`` takes :meth:`closing_reserve_sec`. Returns: - float | None: A monotonic-clock deadline, or ``None`` when unbounded - or already past the reserve. + float | None: A monotonic-clock deadline, or ``None`` when unbounded; + ``time.monotonic()`` (i.e. already due) once the reserve is gone. """ - remaining = self.remaining_minutes() - if remaining is None: + usable = self.session_budget_usable_sec(reserve_sec=reserve_sec) + if usable is None: return None - usable = remaining * 60.0 - reserve_sec - if usable <= 0.0: - return time.monotonic() return time.monotonic() + usable def optimization_stack_has_unvalidated_keeps(self) -> bool: