From ae9636de2d213aa4f48a22b074ff2550d03f5181 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 03:37:50 +0000 Subject: [PATCH 01/65] fix(loop): refuse an action the session budget cannot fit An action was admitted on its own merits and only met the wall clock once it was already running, so a 60-minute explore could start with 20 minutes left and be reaped half-done -- spending the budget and yielding no measurement. Worse, the reaped attempt landed in the failure ledgers, teaching the KB that the action fails when all that happened was the run ran out of time. Gate it before a task row exists, so nothing records a refusal as a failure. Admission judges on expected cost (p50) rather than the p75 tail: judging on the tail abandons minutes that would have been used more than half the time, and the overruns are what the later defences are for. The closing actions stay startable on an empty budget -- refusing those would strand the session with nothing to show. ``SharedState.session_budget_usable_sec`` becomes the one number admission and the grid deadline both read, so the two cannot disagree about how much budget is left. A queued task re-runs the gate immediately before dispatch, since a task can wait for a busy lane long enough for the budget to drain underneath it. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 293 ++++++++++++++++++ .../orchestrator/loop/coordinator.py | 2 + .../orchestrator/loop/coordinator_helpers.py | 43 +++ src/hyperloom/orchestrator/loop/dispatcher.py | 133 +++++++- .../orchestrator/loop/intent_router.py | 4 +- .../orchestrator/state/shared_state.py | 47 ++- 6 files changed, 510 insertions(+), 12 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_session_time_budget.py diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py new file mode 100644 index 0000000000..a66ab7c233 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Admission control for the session wall-clock budget. + +The first of the time-budget defences: an action whose expected cost cannot fit +the budget that is left never starts. Covers the pure fit decision, the +SharedState accessor both it and the grid deadline read, the dispatcher gate, the +three intent paths that share it, and the pre-dispatch backstop for a task that +sat queued until its budget drained. +""" + +from __future__ import annotations + +import pytest + +from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType +from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.loop.coordinator_helpers import ( + TIME_BUDGET_EXEMPT_ACTIONS, + action_fits_time_budget, +) +from hyperloom.orchestrator.policy.gate import PolicyDenied +from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan +from hyperloom.orchestrator.state.shared_state import CLOSING_RESERVE_SEC, SharedState + +# An action costing an hour at p50, so a short budget cannot fit it. +_EXPENSIVE_ACTION = "kernel_opt" +_EXPENSIVE_COST_MIN = 60.0 +# Cheap enough to fit anything but a nearly-spent budget. +_CHEAP_ACTION = "profile" + + +def _heartbeat() -> Intent: + return Intent(type=IntentType.SEND_MESSAGE, payload={"topic": "heartbeat", "body_md": "ok"}) + + +def _backends() -> dict[str, Backend]: + silent = ScriptedPlan(turns=[], default_intent=_heartbeat()) + return {name: MockBackend(silent, name=name) for name in ("orchestration", "critic", "robustness")} + + +@pytest.fixture +def coord(session_dir) -> Coordinator: + c = Coordinator(session_dir, backends=_backends()) + # Past the baseline prerequisite so the sequence gate stays out of the way. + c.shared_state.baseline_tput = 800.0 + return c + + +def _set_budget(coord: Coordinator, *, minutes: float, elapsed_min: float = 0.0) -> None: + """Give the session a finite budget with ``elapsed_min`` already spent.""" + coord.shared_state.max_minutes = int(minutes) + coord.shared_state.elapsed_minutes = lambda **_kw: elapsed_min # type: ignore[method-assign] + + +class TestFitDecision: + """The pure fit rule, independent of any Coordinator.""" + + def test_an_unbounded_budget_fits_everything(self): + assert action_fits_time_budget(usable_sec=None, expected_cost_minutes=600.0) + + def test_an_action_with_no_cost_on_record_is_admitted(self): + assert action_fits_time_budget(usable_sec=60.0, expected_cost_minutes=0.0) + assert action_fits_time_budget(usable_sec=60.0, expected_cost_minutes=-1.0) + + def test_an_action_that_fits_is_admitted(self): + assert action_fits_time_budget(usable_sec=30 * 60.0, expected_cost_minutes=30.0) + + def test_an_action_that_does_not_fit_is_refused(self): + assert not action_fits_time_budget(usable_sec=30 * 60.0 - 1, expected_cost_minutes=30.0) + + def test_the_expected_cost_is_the_anchor_not_the_p75_backstop(self): + """A 90-minute budget admits a 60/120 action: the tail is not the bar. + + Judging fit on p75 would refuse work that finishes in the budget half the + time, abandoning usable minutes. The session reaper handles the overruns. + """ + assert action_fits_time_budget(usable_sec=90 * 60.0, expected_cost_minutes=60.0) + assert not action_fits_time_budget(usable_sec=90 * 60.0, expected_cost_minutes=120.0) + + +class TestUsableBudgetAccessor: + """``session_budget_usable_sec`` is the one number admission and the grid share.""" + + def test_an_unset_budget_reads_as_unbounded(self): + assert SharedState(session_id="s").session_budget_usable_sec() is None + + def test_the_closing_reserve_is_held_back(self): + state = SharedState(session_id="s", max_minutes=60) + state.elapsed_minutes = lambda **_kw: 0.0 # type: ignore[method-assign] + assert state.session_budget_usable_sec() == pytest.approx(3600.0 - CLOSING_RESERVE_SEC) + + def test_a_budget_inside_the_reserve_reads_as_spent(self): + state = SharedState(session_id="s", max_minutes=60) + state.elapsed_minutes = lambda **_kw: 59.9 # type: ignore[method-assign] + assert state.session_budget_usable_sec() == 0.0 + + def test_the_grid_deadline_is_derived_from_the_same_number(self, monkeypatch): + """Both wall-clock layers must agree on how much budget is left.""" + import time as _time + + state = SharedState(session_id="s", max_minutes=60) + state.elapsed_minutes = lambda **_kw: 10.0 # type: ignore[method-assign] + monkeypatch.setattr(_time, "monotonic", lambda: 1000.0) + usable = state.session_budget_usable_sec() + assert state.grid_session_deadline_sec() == pytest.approx(1000.0 + usable) + + +class TestTimeBudgetGate: + """The dispatcher gate that turns a fit failure into a refusal.""" + + def test_an_action_too_big_for_the_budget_is_denied(self, coord: Coordinator): + _set_budget(coord, minutes=20) + denied = coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) + assert isinstance(denied, PolicyDenied) + assert denied.rule == "time_budget" + assert "60 min" in str(denied) + assert "report" in str(getattr(denied, "hint", "")) + + def test_an_action_that_fits_is_admitted(self, coord: Coordinator): + _set_budget(coord, minutes=20) + assert coord._time_budget_denial_for_action(_CHEAP_ACTION) is None + + def test_an_unbounded_budget_admits_the_most_expensive_action(self, coord: Coordinator): + coord.shared_state.max_minutes = 0 + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + + def test_an_action_with_no_registry_entry_is_admitted(self, coord: Coordinator): + _set_budget(coord, minutes=1) + assert coord._time_budget_denial_for_action("frobnicate") is None + + def test_the_closing_actions_stay_startable_on_an_empty_budget(self, coord: Coordinator): + """Refusing these would strand the session with nothing to show.""" + _set_budget(coord, minutes=60, elapsed_min=60.0) + assert coord.shared_state.session_budget_usable_sec() == 0.0 + for action in TIME_BUDGET_EXEMPT_ACTIONS: + assert coord._time_budget_denial_for_action(action) is None, action + + def test_a_stopping_session_leaves_the_gate_to_the_stop_path(self, coord: Coordinator): + _set_budget(coord, minutes=1) + coord.shared_state.stop_reason = "time_exhausted" + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + + def test_the_budget_shrinks_the_gate_as_the_session_runs(self, coord: Coordinator): + _set_budget(coord, minutes=120, elapsed_min=0.0) + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + _set_budget(coord, minutes=120, elapsed_min=70.0) + assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is not None + + +class TestAdmissionGateOrder: + """``_admission_denial_for_action`` chains the gates; the first one wins.""" + + def test_the_baseline_prerequisite_is_reported_before_the_budget(self, coord: Coordinator): + coord.shared_state.baseline_tput = 0.0 + _set_budget(coord, minutes=1) + denied = coord._admission_denial_for_action("explore") + assert denied is not None and denied.rule == "execution_order" + + def test_the_budget_gate_runs_once_the_sequence_gate_passes(self, coord: Coordinator): + _set_budget(coord, minutes=20) + denied = coord._admission_denial_for_action(_EXPENSIVE_ACTION) + assert denied is not None and denied.rule == "time_budget" + + def test_an_action_clearing_both_gates_is_admitted(self, coord: Coordinator): + _set_budget(coord, minutes=600) + assert coord._admission_denial_for_action(_EXPENSIVE_ACTION) is None + + +def _delegate(action_name: str, key: str) -> Intent: + return Intent( + type=IntentType.DELEGATE, + payload={"action_name": action_name, "params": {}, "idempotency_key": key}, + ) + + +class TestIntentPathsAreGated: + """A refusal must land before a task row exists, so no ledger sees it.""" + + @pytest.mark.asyncio + async def test_delegating_an_over_budget_action_queues_nothing( + self, + coord: Coordinator, + monkeypatch, + ): + _set_budget(coord, minutes=20) + recorded: list[PolicyDenied] = [] + + async def _rec(source, intent, denied, action_name=None): + recorded.append(denied) + + monkeypatch.setattr(coord.writeback, "_record_policy_denied", _rec) + await coord._handle_delegate("orchestration", _delegate(_EXPENSIVE_ACTION, "d-budget")) + assert [d.rule for d in recorded] == ["time_budget"] + assert [t for t in await coord.tasks.queued() if t.kind == _EXPENSIVE_ACTION] == [] + + @pytest.mark.asyncio + async def test_delegating_an_affordable_action_still_queues( + self, + coord: Coordinator, + monkeypatch, + ): + _set_budget(coord, minutes=600) + monkeypatch.setattr(coord.shared_state, "is_pruned", lambda a: False) + await coord._handle_delegate("orchestration", _delegate(_EXPENSIVE_ACTION, "d-ok")) + assert [t for t in await coord.tasks.queued() if t.kind == _EXPENSIVE_ACTION] + + @pytest.mark.asyncio + async def test_proposing_an_over_budget_action_never_reaches_the_critic( + self, + coord: Coordinator, + monkeypatch, + ): + _set_budget(coord, minutes=20) + recorded: list[PolicyDenied] = [] + + async def _rec(source, intent, denied, action_name=None): + recorded.append(denied) + + monkeypatch.setattr(coord.writeback, "_record_policy_denied", _rec) + intent = Intent( + type=IntentType.PROPOSE_ACTION, + payload={"action_name": _EXPENSIVE_ACTION, "predicted_gain_pct": 5.0}, + ) + await coord._handle_propose_action("orchestration", intent) + assert [d.rule for d in recorded] == ["time_budget"] + assert not coord.state.pending_proposals + + @pytest.mark.asyncio + async def test_the_inline_runner_reports_the_refusal(self, coord: Coordinator, monkeypatch): + _set_budget(coord, minutes=20) + + async def _rec(source, intent, denied, action_name=None): + return None + + monkeypatch.setattr(coord.writeback, "_record_policy_denied", _rec) + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + out = await coord._run_action_now(_EXPENSIVE_ACTION, {}) + assert "denied" in out + assert [t for t in await coord.tasks.queued() if t.kind == _EXPENSIVE_ACTION] == [] + + +class TestPreDispatchBackstop: + """A task can wait for a lane long enough for its budget to drain.""" + + @pytest.mark.asyncio + async def test_a_queued_task_the_budget_outlived_is_dropped_before_dispatch( + self, + coord: Coordinator, + ): + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind=_EXPENSIVE_ACTION, + params={}, + idempotency_key="q-drained", + ) + # The budget drains while the task waits in the queue. + _set_budget(coord, minutes=600, elapsed_min=590.0) + + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert [t.task_id for t, _, _ in spawned] == [] + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + + @pytest.mark.asyncio + async def test_the_drop_is_not_recorded_as_an_action_failure(self, coord: Coordinator): + """A task that never ran is not evidence about the action.""" + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind=_EXPENSIVE_ACTION, + params={}, + idempotency_key="q-no-failure", + ) + _set_budget(coord, minutes=600, elapsed_min=590.0) + + await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + failures = list(getattr(coord.shared_state, "last_action_failures", []) or []) + assert [f for f in failures if str(f.get("action") or "") == _EXPENSIVE_ACTION] == [] + + @pytest.mark.asyncio + async def test_a_queued_task_that_still_fits_is_left_alone(self, coord: Coordinator): + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind=_EXPENSIVE_ACTION, + params={}, + idempotency_key="q-fits", + ) + assert await coord.dispatcher._cancel_queued_task_over_budget(task) is False + assert (await coord.tasks.get(task.task_id)).state == "queued" + diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index b78f962bbc..cbde600cea 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1154,6 +1154,8 @@ def router(self) -> IntentRouter: "_account_dead_holder_failures": "dispatcher", "_lanes_fit": "dispatcher", "_sequence_denial_for_action": "dispatcher", + "_time_budget_denial_for_action": "dispatcher", + "_admission_denial_for_action": "dispatcher", "_sequence_denial_for_request": "dispatcher", "_skip_gemm_tuning": "dispatcher", "_gemm_tuning_required_before_kernel_opt": "dispatcher", diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index e26ac35bc8..16092e5be0 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -238,6 +238,49 @@ def effective_closing_grace_sec( return min(120.0, (max_minutes or 0.0) * 60.0 * 0.02) +# Actions that must stay startable no matter how little budget is left: they are +# how a session ends cleanly (report/breakdown) or unsticks itself (recover), so +# a time gate that refused them would strand the run with nothing to show. +TIME_BUDGET_EXEMPT_ACTIONS: frozenset[str] = frozenset( + { + "report", + "session_breakdown", + "recover", + } +) + + +def action_fits_time_budget( + *, + usable_sec: float | None, + expected_cost_minutes: float, +) -> bool: + """Decide whether an action's expected cost still fits the remaining budget. + + The anchor is the action's *expected* cost (``cost_minutes_p50``), not its + p75 backstop. Judging fit on the pessimistic tail would abandon usable + budget — with 90 minutes left we would refuse an action that finishes in 60 + minutes half the time — and the session already has a wall-clock reaper for + the overruns, so the optimistic anchor is the one that keeps the tail of a + session productive. This mirrors how the grid admits variants. + + Args: + usable_sec: Budget left after the closing reserve, from + ``SharedState.session_budget_usable_sec``; ``None`` means unbounded. + expected_cost_minutes: The action's expected cost in minutes; values at + or below zero mean "no estimate on record". + + Returns: + ``True`` when the action may start: the budget is unbounded, no estimate + is on record, or the expected cost fits what is left. + """ + if usable_sec is None: + return True + if expected_cost_minutes <= 0.0: + return True + return usable_sec >= expected_cost_minutes * 60.0 + + def _parse_iso_unix(ts: str) -> float: """Parse an ISO 8601 UTC timestamp into unix seconds; ``0.0`` on failure. diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 37d053246a..5790eafec4 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -31,7 +31,11 @@ ) from .sub_agent_runner import SubAgentResult from ..state.task_registry import Task -from .coordinator_helpers import coerce_needs_gpu +from .coordinator_helpers import ( + TIME_BUDGET_EXEMPT_ACTIONS, + action_fits_time_budget, + coerce_needs_gpu, +) from .coordinator import ( _format_inbox_event, @@ -358,6 +362,8 @@ async def _spawn_fitting_queued( # Off-loop builds run in their own process group and are pumped/ # reaped by BuildLifecycleCollaborator; never drain them here. continue + if await self._cancel_queued_task_over_budget(task): + continue lanes_needed = list(task.requires_lanes or []) if lanes_needed: # SQLite lane gate. Under single-node Ray execution the @@ -1157,6 +1163,129 @@ def _sequence_denial_for_action( ) return None + def _time_budget_denial_for_action( + self, + action_name: str, + ) -> PolicyDenied | None: + """Refuse an action whose expected cost cannot fit the remaining session budget. + + The first of the wall-clock defences: cheaper to never start a 60-minute + action with 20 minutes left than to reap it half-done, because a reaped + action spends the budget and yields no measurement. Denying here also + keeps the refusal out of the failure ledgers — no task row is created, so + nothing teaches the KB that the action failed. + + Args: + action_name: The proposed/delegated/inline action name. + + Returns: + A :class:`PolicyDenied` when the budget cannot fit the action, else + ``None`` (unbounded budget, exempt action, or no cost on record). + """ + action = str(action_name or "").strip() + if not action or action in TIME_BUDGET_EXEMPT_ACTIONS: + return None + if self.shared_state.stop_reason: + return None + reg = getattr(self, "action_registry", None) + meta = reg.get(action) if reg is not None else None + if meta is None: + return None + expected_min = float(getattr(meta, "cost_minutes_p50", 0.0) or 0.0) + usable_sec = self.shared_state.session_budget_usable_sec() + if action_fits_time_budget( + usable_sec=usable_sec, + expected_cost_minutes=expected_min, + ): + return None + remaining_min = (usable_sec or 0.0) / 60.0 + return PolicyDenied( + f"action={action!r} denied: needs ~{expected_min:.0f} min but only " + f"{remaining_min:.0f} min of the session budget is left", + rule="time_budget", + hint=( + "the wall-clock budget cannot fit this action; delegate `report` " + "to close the session, or pick an action that fits the time left" + ), + ) + + def _admission_denial_for_action( + self, + action_name: str, + ) -> PolicyDenied | None: + """Run every pre-dispatch gate for an action name, first denial wins. + + The single entry point the intent handlers and the inline runner share, + so a new gate reaches all three paths at once. + + Args: + action_name: The proposed/delegated/inline action name. + + Returns: + The first :class:`PolicyDenied` that fires, else ``None``. + """ + denied = self._sequence_denial_for_action(action_name) + if denied is not None: + return denied + return self._time_budget_denial_for_action(action_name) + + async def _cancel_queued_task_over_budget(self, task: Task) -> bool: + """Drop a queued task the budget can no longer fit, before it takes a lane. + + The admission gate runs when the action is proposed, but a task can wait + for a busy lane long enough for the budget to drain underneath it. This + is the same gate re-applied at the last moment it is still free to say + no. The row is cancelled rather than left queued so the pump does not + re-examine it every tick, and it stays out of the failure ledgers: a task + that never ran is not evidence about the action. + + Args: + task: The queued task about to be considered for dispatch. + + Returns: + ``True`` when the task was cancelled and must be skipped this pass. + """ + denied = self._time_budget_denial_for_action(task.kind) + if denied is None: + return False + try: + await self.tasks.transition( + task.task_id, + "cancelled", + evidence={"reason": "time_budget", "error": str(denied)}, + ) + except Exception: # noqa: BLE001 — a lost row must not abort the pump + log.exception( + "dispatcher: could not cancel over-budget task=%s kind=%s", + task.task_id, + task.kind, + ) + return True + log.warning( + "dispatcher: dropped queued task=%s kind=%s before dispatch: %s", + task.task_id, + task.kind, + denied, + ) + try: + await self._record_observation( + "coordinator", + "observation", + { + "kind": "dispatch_denied_time_budget", + "task_id": task.task_id, + "action": task.kind, + "error": str(denied), + "hint": getattr(denied, "hint", ""), + }, + ) + except Exception: # noqa: BLE001 — observability must not block dispatch + log.exception( + "dispatcher: could not record time-budget denial for task=%s", + task.task_id, + ) + return True + def _sequence_denial_for_request( self, target_agent: str, @@ -1382,7 +1511,7 @@ async def _run_action_now( f"{getattr(denied, 'rule', '')!s} — " f"{str(getattr(denied, 'hint', denied))[:200]})" ) - seq_denied = self._sequence_denial_for_action(action_name) + seq_denied = self._admission_denial_for_action(action_name) if seq_denied is not None: await self._record_policy_denied( "orchestration", diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 7ba9685686..dd6ef0cd2b 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -168,7 +168,7 @@ async def _handle_propose_action(self, source: str, intent: Intent) -> None: ), }, ) - denied = self._sequence_denial_for_action(action_name) + denied = self._admission_denial_for_action(action_name) if denied is not None: await self._record_policy_denied(source, intent, denied) return @@ -506,7 +506,7 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: ), }, ) - denied = self._sequence_denial_for_action(action_name) + denied = self._admission_denial_for_action(action_name) if denied is not None: await self._record_policy_denied( source, diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 0bb1006f32..aab3c491e6 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -259,6 +259,11 @@ def render_model_arch_compact(arch: dict | None) -> str: # How many hot / skipped kernels ``record_trace_analyze`` keeps in the trace # summary (matches the ``*_top15`` field names). _TRACE_HOT_KERNEL_TOP_N = 15 + +# Wall-clock budget held back from every unit of work so the CLOSE phase can +# still write its report. Matches the default closing grace window. +CLOSING_RESERVE_SEC = 120.0 + # Session-level kernel-roofline report the analyzer writes for a non-close run; # read back when the trace_analyze envelope arrives without its payload keys. _DEFAULT_ROOFLINE_REPORT_NAME = "kernel_roofline_current.json" @@ -3602,7 +3607,36 @@ def remaining_minutes(self, *, now: datetime | None = None) -> float | None: return None return max(0.0, float(self.max_minutes) - self.elapsed_minutes(now=now)) - def grid_session_deadline_sec(self, *, reserve_sec: float = 120.0) -> float | None: + def session_budget_usable_sec( + self, + *, + reserve_sec: float = CLOSING_RESERVE_SEC, + ) -> float | None: + """Seconds of wall-clock budget left once the closing reserve is held back. + + The single source for "how much time may a unit of work still claim". + Admission control (which action may start) and the grid deadline (how + long a variant may run) both read it, so they cannot disagree about how + much budget exists. + + Args: + reserve_sec (float): Seconds held back for the CLOSE phase and its + report. + + Returns: + float | None: Usable seconds (clamped at 0.0), or ``None`` when + ``max_minutes`` is unset (unbounded budget). + """ + remaining = self.remaining_minutes() + if remaining is None: + return None + return max(0.0, remaining * 60.0 - reserve_sec) + + def grid_session_deadline_sec( + self, + *, + reserve_sec: float = CLOSING_RESERVE_SEC, + ) -> float | None: """``time.monotonic()`` deadline for grid variant loops, or ``None`` when the budget is unbounded. Reserves ``reserve_sec`` so the CLOSE phase and report still have room @@ -3614,15 +3648,12 @@ def grid_session_deadline_sec(self, *, reserve_sec: float = 120.0) -> float | No reserve_sec (float): Seconds held back from the raw remaining budget. Returns: - float | None: A monotonic-clock deadline, or ``None`` when unbounded - or already past the reserve. + float | None: A monotonic-clock deadline, or ``None`` when unbounded; + ``time.monotonic()`` (i.e. already due) once the reserve is gone. """ - remaining = self.remaining_minutes() - if remaining is None: + usable = self.session_budget_usable_sec(reserve_sec=reserve_sec) + if usable is None: return None - usable = remaining * 60.0 - reserve_sec - if usable <= 0.0: - return time.monotonic() return time.monotonic() + usable def optimization_stack_has_unvalidated_keeps(self) -> bool: From 5d7ac118afb0cc24e0fd26d8e5580bf1a6b08b39 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 03:38:09 +0000 Subject: [PATCH 02/65] fix(executors): stop one sentinel returncode from naming two causes ``_run_magpie`` returns ``AGENTX_PREFLIGHT_RETURNCODE`` when the execution boundary fails preflight and, thirty lines on, whatever the serving lease's actor returned -- including ``_ACTOR_TIMEOUT_RC``. Both were -912, and both reach their consumer as a bare returncode carrying no other tag, so a hung Ray actor was recorded as a missing aiperf: an infrastructure timeout filed under a setup mistake, in the ledgers the next run reads. Move the Ray code rather than the AgentX one. Downstream and every record written so far already read -912 as the preflight failure, so moving that side would reinterpret history a second time. The space is shared by two modules that were coordinating through a comment, which is how the overlap survived. Say so where the codes are declared, and name the guard test that now enumerates both modules and fails on reuse. Co-authored-by: Cursor --- .../orchestrator/actions/executors/_ray_serving.py | 9 ++++++--- .../orchestrator/actions/executors/_subprocess_kill.py | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py index 892d291c13..79b4b16f73 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py @@ -16,9 +16,12 @@ log = logging.getLogger(__name__) -# Ray-side sentinel returncodes. -912 is also used by -# ``_subprocess_kill.AGENTX_PREFLIGHT_RETURNCODE``. -_ACTOR_TIMEOUT_RC: int = -912 +# Ray-side sentinel returncodes, allocated out of the same space as +# ``_subprocess_kill``'s -- read the note there before claiming a new one. Both +# of these leave ``_grid_runner._run_magpie`` through the very return channel +# that carries ``AGENTX_PREFLIGHT_RETURNCODE``, so an overlap would have an +# actor timeout recorded as a failed AgentX preflight. +_ACTOR_TIMEOUT_RC: int = -916 _RAY_ACTOR_DIED_RC: int = -913 # Timeout for ray.get probes on specialist actor methods (is_alive/exit_code/stop). diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 90f1b71640..a3b7321ab5 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -182,6 +182,13 @@ def kill_my_spawned_server( pass +# Sentinel ``returncode`` allocation. This module is not the only owner of the +# space -- ``_ray_serving`` hands out Ray-actor codes from it too, and both +# arrive at their consumer as a bare ``returncode`` carrying no other tag. A +# number claimed by a second cause therefore makes attribution a coin flip, so +# allocate an unused one; ``test_every_sentinel_returncode_names_exactly_one_cause`` +# enumerates both modules and fails on reuse. + # Sentinel ``returncode`` when ``run_with_session_kill`` reaps a child for an # elapsed ``soft_deadline_sec`` (vs the ``timeout=`` hard cap, which raises # ``TimeoutExpired``). Chosen not to collide with a real signal-based returncode. From 63ff1abee9ffe5528ee7e1db3a82c8a22c03bb88 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 03:38:30 +0000 Subject: [PATCH 03/65] fix(executors): hold every unit of work to the session clock Two holes let one action outlast the whole run. Action timeouts were derived from the action alone, so explore could be granted four hours inside a three-hour session; and once a subprocess was running, the only clocks watching it were the soft deadline -- which retires at eval start, by design, because it judges whether a variant is abnormally slow -- and the hard cap, which was the unclamped grant. An accuracy eval starting a minute before the session ends therefore ran to completion. Clamp the grants to the budget that is left, and give ``run_with_session_kill`` a session deadline on its own channel. The separate channel is the point: the eval-start boundary is meaningful for "is this variant abnormally slow" and meaningless for "is the run out of time", so the session deadline is checked in every phase and never suspended. It reaps the tree with a returncode of its own rather than reusing the overtime kill, which would have taught the KB that a variant is slow whenever a session happened to end during it. The stack rebench never consulted the budget at all -- the same hole, second instance -- and a budget skip was filed as a failure with no cause attached. Both now carry the session_time_exhausted attribution. Co-authored-by: Cursor --- .../tests/test_critic_verdict_map.py | 2 +- .../tests/test_explore_executor.py | 133 ++++++ .../tests/test_framework_agent_executor.py | 154 ++++++- .../tests/test_grid_runner.py | 398 +++++++++++++++++- .../test_integrate_patch_coverage_unit.py | 218 ++++++++++ .../tests/test_kill_spawned_server.py | 110 +++++ .../actions/executors/_grid_runner.py | 232 +++++++++- .../actions/executors/_stack_rebench.py | 23 +- .../actions/executors/_subprocess_kill.py | 77 +++- .../orchestrator/actions/executors/explore.py | 65 ++- .../actions/executors/framework_agent.py | 21 +- .../actions/executors/integrate_patch.py | 46 ++ .../orchestrator/actions/executors/sweep.py | 12 +- 13 files changed, 1449 insertions(+), 42 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index 5e3d58c696..6946d13193 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -1911,7 +1911,7 @@ def reset_policy_denial_streak(self, _action_name: str) -> None: c.bus = _StubBus() c._record_observation = AsyncMock() # type: ignore[method-assign] c._record_policy_denied = AsyncMock() # type: ignore[method-assign] - c._sequence_denial_for_action = lambda *a, **k: None # type: ignore[method-assign] + c._admission_denial_for_action = lambda *a, **k: None # type: ignore[method-assign] c._registry_lanes_ttl = lambda _name: (set(), 0) # type: ignore[method-assign] c.policy = None return c diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 8db389445d..156be866ac 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -25,6 +25,7 @@ canonical_fingerprint, ) from hyperloom.orchestrator.actions.executors._grid_runner import ( + _SESSION_KILL_GRACE_SEC, apply_compatibility_filter, ) from hyperloom.orchestrator.actions.executors._accuracy_gate import ( @@ -1605,6 +1606,138 @@ def _fake_kill(cmd, *args, **kwargs): assert out["status"] == "succeeded" +@pytest.mark.asyncio +async def test_explore_variant_cap_is_clamped_to_the_session_budget( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """A granted cap never exceeds what is left of the session. + + explore derives the cap from the measured baseline (up to 4h) and never + consulted the budget, so a 3h session could hand a single variant more time + than the whole run was given. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 3.0 # 180s budget - 120s close reserve => ~60s usable + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + granted: list[int] = [] + + def _fake_run(cmd, *args, **kwargs): + # Only benchmark rounds carry --output-dir; the interpreter probe does not, + # and it is module-memoized, so counting it would make this order-dependent. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + granted.append(int(kwargs["timeout"])) + slot = Path(cmd[cmd.index("--output-dir") + 1]) + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-clamp"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_fits", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-clamp", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert res.result["status"] == "succeeded" + assert granted, "the variant should have been admitted (20s expected, ~60s left)" + # The hard cap is allowed to sit a grace window past the deadline so the + # in-process session watchdog reaps the tree first and the kill is attributed + # to the budget rather than to a slow variant. + assert all(t <= 60 + _SESSION_KILL_GRACE_SEC for t in granted), ( + f"caps must be clamped to the ~60s budget, got {granted}" + ) + assert all(t < 3600 for t in granted), f"the declared 3600s cap must not survive the budget, got {granted}" + + +@pytest.mark.asyncio +async def test_explore_skips_a_variant_the_budget_cannot_fit( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """Admission is judged on the expected runtime, and refused when it does not fit.""" + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 3.0 # ~60s usable + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + granted: list[int] = [] + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + granted.append(int(kwargs["timeout"])) + slot = Path(cmd[cmd.index("--output-dir") + 1]) + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-nofit"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_too_long", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 600.0, + }, + idempotency_key="ex-budget-nofit", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert granted == [], "a variant needing 600s must not start with ~60s left" + # Measuring nothing because the budget ran out is not the same as variants + # failing, and it must not be reported as a bare, unattributed failure. + assert res.result["error_class"] == "session_time_exhausted" + assert res.result["session_budget_untested"] == 1 + # Untested variants stay out of the ledger so a resume can retry them. + assert res.result["losers"] == [] + assert res.result["explore_search_update"]["tested"] == {} + + @pytest.mark.asyncio async def test_explore_executor_empty_grid_returns_failed(sub_agent_runner, tmp_path): sub, tr, _ = sub_agent_runner diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py index 387ab81830..fb96cd66a6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py @@ -76,7 +76,7 @@ def _make_candidate( } -def _make_ctx(task_id: str, params: dict[str, Any]) -> RunnerContext: +def _make_ctx(task_id: str, params: dict[str, Any], extra: dict[str, Any] | None = None) -> RunnerContext: task = Task( task_id=task_id, kind="framework_agent", @@ -85,7 +85,7 @@ def _make_ctx(task_id: str, params: dict[str, Any]) -> RunnerContext: idempotency_key=task_id, requires_lanes=tuple(), ) - return RunnerContext(task=task, lease=None, extra={}) + return RunnerContext(task=task, lease=None, extra=extra if extra is not None else {}) def test_candidate_slug_prefers_repo_and_pr_number(): @@ -333,7 +333,7 @@ async def test_executor_keep_when_delta_above_threshold(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -360,6 +360,128 @@ async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 assert (repo / "src.py").read_text().endswith("return 2\n") +@pytest.mark.asyncio +async def test_bench_is_bounded_by_the_session_budget(tmp_path: Path): + """The candidate bench is handed the session budget, as the other arms are. + + Its declared cap answers "how long before this counts as hung", not "how much + budget is left", so without the session deadline a candidate benched near the + end of a run outlives the run itself. + """ + from unittest.mock import MagicMock + + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + captured: dict[str, Any] = {} + + async def fake_bench(self, *, params, output_root, slug, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return ( + {"status": "succeeded", "output_throughput": 1100.0}, + {"accuracy_pass": None}, + ) + + shared_state = MagicMock() + shared_state.grid_session_deadline_sec.return_value = 4242.0 + shared_state.baseline_runtime_sec = 600.0 + + ctx = _make_ctx( + "t-fp-budget", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + extra={"shared_state": shared_state}, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench): + result = await executor(ctx) + + assert result["status"] == "kept" + assert captured["session_deadline_sec"] == 4242.0 + # The expected runtime, not the backstop cap: admitting on the backstop + # abandons the tail of the budget. + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_bench_budget_is_unbounded_without_a_session(tmp_path: Path): + """No session context means no deadline, not a deadline of zero.""" + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + captured: dict[str, Any] = {} + + async def fake_bench(self, *, params, output_root, slug, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return ( + {"status": "succeeded", "output_throughput": 1100.0}, + {"accuracy_pass": None}, + ) + + ctx = _make_ctx( + "t-fp-nobudget", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench): + result = await executor(ctx) + + assert result["status"] == "kept" + assert captured["session_deadline_sec"] is None + assert captured["variant_expected_sec"] is None + + +@pytest.mark.asyncio +async def test_bench_candidate_forwards_the_session_budget_to_the_grid(tmp_path: Path): + session_dir = tmp_path / "session" + session_dir.mkdir() + config_path = tmp_path / "baseline.yaml" + config_path.write_text("benchmark: {}\n", encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + captured: dict[str, Any] = {} + + async def fake_run_grid(*args, **kwargs): # noqa: ARG001 + captured.update(kwargs) + return [_mk_variant_result(tput=1100.0, status="succeeded")] + + from hyperloom.orchestrator.actions.executors import framework_agent as fp_mod + + with ( + patch.object(fp_mod, "run_grid", new=fake_run_grid), + patch.object(fp_mod, "materialize_config_with_envs", return_value=config_path), + ): + await executor._bench_candidate( + params={"config_path": str(config_path)}, + output_root=tmp_path / "out", + slug="budget", + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + @pytest.mark.asyncio async def test_executor_reverts_when_live_anchor_exceeds_queued_baseline(tmp_path: Path): session_dir = tmp_path / "session" @@ -372,7 +494,7 @@ async def test_executor_reverts_when_live_anchor_exceeds_queued_baseline(tmp_pat state.baseline_tput = 1000.0 state.current_best = {"tput": 1150.0} - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None} ctx = _make_ctx( @@ -412,7 +534,7 @@ async def test_executor_keep_writes_kb_lessons(tmp_path: Path, monkeypatch): cand = _make_candidate() cand["pr_url"] = "https://github.com/sgl-project/sglang/pull/1234" - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -458,7 +580,7 @@ async def test_executor_revert_writes_kb_lessons(tmp_path: Path, monkeypatch): cand = _make_candidate() cand["pr_url"] = "https://github.com/sgl-project/sglang/pull/1234" - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 980.0}, {"accuracy_pass": None}, @@ -500,7 +622,7 @@ async def test_executor_revert_when_delta_below_threshold(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 980.0}, {"accuracy_pass": None}, @@ -539,7 +661,7 @@ async def test_executor_revert_on_accuracy_regression(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": False}, @@ -576,7 +698,7 @@ async def test_executor_bench_exception_triggers_revert(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def boom(self, *, params, output_root, slug): # noqa: ARG001 + async def boom(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 raise RuntimeError("simulated bench crash") ctx = _make_ctx( @@ -623,13 +745,13 @@ async def test_reject_after_keep_preserves_kept_changes(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) - async def keep_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def keep_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, ) - async def reject_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def reject_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 980.0}, {"accuracy_pass": None}, @@ -684,7 +806,7 @@ async def test_apply_failure_after_keep_preserves_kept_changes(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) - async def keep_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def keep_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -810,7 +932,7 @@ async def test_executor_checkout_head_mode_applies_and_keeps(tmp_path: Path, mon "apply_mode": "checkout_head", } - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -877,7 +999,7 @@ async def test_executor_keep_adds_new_file_pr(tmp_path: Path): executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -1078,7 +1200,7 @@ async def test_executor_require_accuracy_blocks_keep_when_unevaluated(tmp_path: executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, @@ -1117,7 +1239,7 @@ async def test_executor_require_accuracy_degrades_without_baseline(tmp_path: Pat executor = FrameworkAgentExecutor(session_dir=session_dir) cand = _make_candidate() - async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 return ( {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None}, diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index 4b73b955d5..e80cfdad13 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -11,7 +11,8 @@ import subprocess import time from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest import yaml @@ -26,6 +27,7 @@ _build_variant_yaml, _parse_skip_spec, _run_magpie, + _SESSION_KILL_GRACE_SEC, apply_runtime_benchmark_overrides, apply_user_skip_list, coerce_extra_envs, @@ -1391,6 +1393,400 @@ def fake_run(cmd, *args, **kwargs): assert [r.status for r in results] == ["succeeded", "succeeded"] +def _capture_timeouts(recorded: list[tuple[str, int]]): + """A ``run_with_session_kill`` double that records each round's granted timeout. + + Only benchmark rounds are recorded: the interpreter probe carries no + ``--output-dir`` and is module-memoized, so counting it would make these + assertions depend on which test ran first. + + Args: + recorded: Appended to as ``(slot_name, timeout)`` per launched round. + + Returns: + A callable usable as ``side_effect``. + """ + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + recorded.append((slot.name, int(kwargs["timeout"]))) + _fake_workspace(slot) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + return fake_run + + +class TestSessionGridBounds: + """One definition of the two numbers every benching arm needs. + + A deadline derived one way in one executor and another way in the next + produces arms that abandon different amounts of the tail budget. + """ + + def test_no_session_means_no_bounds(self): + assert _grid_runner.session_grid_bounds(None) == (None, None) + + def test_reads_the_deadline_and_the_measured_baseline(self): + state = MagicMock() + state.grid_session_deadline_sec.return_value = 4242.0 + state.baseline_runtime_sec = 600.0 + assert _grid_runner.session_grid_bounds(state) == (4242.0, 600.0) + + def test_unmeasured_baseline_yields_no_estimate(self): + """Zero is "not measured yet", which must not read as "needs 0 seconds".""" + state = MagicMock() + state.grid_session_deadline_sec.return_value = 4242.0 + state.baseline_runtime_sec = 0.0 + assert _grid_runner.session_grid_bounds(state) == (4242.0, None) + + def test_unparseable_baseline_yields_no_estimate(self): + state = MagicMock() + state.grid_session_deadline_sec.return_value = None + state.baseline_runtime_sec = "not-a-number" + assert _grid_runner.session_grid_bounds(state) == (None, None) + + def test_state_without_the_deadline_accessor_is_tolerated(self): + state = SimpleNamespace(baseline_runtime_sec=600.0) + assert _grid_runner.session_grid_bounds(state) == (None, 600.0) + + +class TestSessionBudgetAdmission: + """A variant is admitted on what it is expected to need, not on its backstop. + + ``variant_timeout_sec`` is the catastrophic-hang cap (~baseline x 2 for + explore). Gating admission on it abandons the tail of the budget: with a + 20-minute baseline the grid refuses to start a round with 30 minutes left. + """ + + @pytest.mark.asyncio + async def test_variant_runs_when_budget_fits_expected_but_not_the_backstop(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=30.0, + ) + + assert [r.status for r in results] == ["succeeded"] + assert len(recorded) == 1 + + @pytest.mark.asyncio + async def test_variant_skipped_when_budget_cannot_fit_the_expected_runtime(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 10.0, + variant_expected_sec=300.0, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + + @pytest.mark.asyncio + async def test_without_an_estimate_the_stricter_backstop_check_is_kept(self, tmp_path): + """Callers that cannot estimate keep the pre-existing, stricter gate.""" + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=None, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + + +class TestSessionBudgetTimeoutClamp: + """A granted cap never exceeds what the session can still pay for. + + explore derives caps from the measured baseline (up to 4h) and never + consulted the budget, so a 3h session could hand a single variant more time + than the whole run was given. + """ + + @pytest.mark.asyncio + async def test_granted_cap_is_clamped_to_the_remaining_budget(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=30.0, + ) + + assert len(recorded) == 1 + granted = recorded[0][1] + # The cap is allowed a small grace past the deadline so the in-process + # session watchdog trips first and attributes the kill correctly. + assert 60 <= granted <= 120 + _SESSION_KILL_GRACE_SEC, ( + f"expected a cap clamped to the ~120s budget, got {granted}" + ) + + @pytest.mark.asyncio + async def test_declared_cap_is_kept_when_the_budget_is_larger(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 36000.0, + variant_expected_sec=30.0, + ) + + assert [t for _, t in recorded] == [600] + + @pytest.mark.asyncio + async def test_no_deadline_leaves_the_declared_cap_untouched(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=None, + variant_expected_sec=30.0, + ) + + assert [t for _, t in recorded] == [600] + + +class TestSessionKillAttribution: + """A round reaped for the session budget is not a verdict about the variant.""" + + @pytest.mark.asyncio + async def test_mid_round_budget_kill_is_recorded_as_skipped_not_failed(self, tmp_path): + from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + SESSION_TIME_EXHAUSTED_RETURNCODE, + ) + + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + + def fake_run(cmd, *args, **kwargs): + return subprocess.CompletedProcess(cmd, SESSION_TIME_EXHAUSTED_RETURNCODE, "", "") + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 120.0, + variant_expected_sec=30.0, + ) + + assert [r.status for r in results] == ["skipped"] + assert results[0].error_class == "session_time_exhausted" + # Never the overtime label, which asserts the variant is abnormally slow. + assert not getattr(results[0], "killed_overtime", False) + assert results[0].output_throughput is None + + @pytest.mark.asyncio + async def test_the_hard_cap_leaves_room_for_the_session_watchdog_to_win(self, tmp_path): + """Both fire at the same instant, and the sentinel must get there first. + + The hard cap raises ``TimeoutExpired``, which the ledger reads as a variant + timeout, so it is granted a small grace past the session deadline. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 60.0, + variant_expected_sec=30.0, + ) + + assert len(recorded) == 1 + granted = recorded[0][1] + assert granted > 60, f"hard cap {granted}s must sit past the ~60s deadline, not on it" + assert granted <= 90, f"the grace must stay small, got {granted}s" + + @pytest.mark.asyncio + async def test_the_session_deadline_reaches_the_subprocess_layer(self, tmp_path): + """Regression: the clamped cap alone bounds the round but mislabels the kill.""" + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + deadline = time.monotonic() + 120.0 + seen: list[float | None] = [] + + def fake_run(cmd, *args, **kwargs): + # The module-memoized interpreter probe is not a benchmark round and + # correctly carries no session deadline; only rounds are of interest. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + seen.append(kwargs.get("session_deadline_sec")) + _fake_workspace(Path(cmd[cmd.index("--output-dir") + 1])) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=deadline, + variant_expected_sec=30.0, + ) + + assert seen == [deadline] + + +class TestSessionBudgetWarmupRounds: + """A warmup round costs a full pass, and the measured round is paid first.""" + + @pytest.mark.asyncio + async def test_admission_accounts_for_the_warmup_pass(self, tmp_path): + """Budget for one round is not budget for a warmup plus a measure round. + + Admitting on a single round's estimate would let a variant in and then + clamp its measured round to nothing, turning a budget shortfall into a + ledger full of spurious timeouts. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 40.0, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + + assert recorded == [] + assert [r.status for r in results] == ["skipped"] + + @pytest.mark.asyncio + async def test_warmup_cap_reserves_budget_for_the_measured_round(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[tuple[str, int]] = [] + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_timeouts(recorded), + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 300.0, + variant_expected_sec=60.0, + warmup_before_measure=True, + ) + + by_round = dict(recorded) + warmup = next(t for slot, t in recorded if "warmup" in slot) + measure = next(t for slot, t in recorded if "warmup" not in slot) + assert warmup <= 240 + _SESSION_KILL_GRACE_SEC, ( + f"warmup cap must hold back the measured round's 60s, got {warmup}" + ) + assert warmup < measure, "the warmup must be granted less than the round it reserves budget for" + assert len(by_round) == 2, f"expected a warmup and a measured round, got {list(by_round)}" + + class TestCompactJsonServerArgs: """JSON-valued flags must be space-free to survive Magpie's unquoted ``$EXTRA_VLLM_ARGS`` splice (otherwise spec-decode / compilation-config diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py index 2f7c00ae9c..10d1851abb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py @@ -238,6 +238,159 @@ def get_specialist_patch_verdict(self, tid): assert (repo / "src.py").read_text().endswith("return 1\n") +def _capture_bench(captured: dict, result: dict, gate: dict): + """A ``_bench_patch`` stub that records the kwargs it was handed.""" + + async def _b(self, **kwargs): + captured.update(kwargs) + return result, gate + + return _b + + +@pytest.mark.asyncio +async def test_bench_is_bounded_by_the_session_budget(tmp_path, monkeypatch): + """The patch bench is handed the session budget, as the other arms are. + + Its declared cap answers "how long before this counts as hung", not "how much + budget is left", so without the session deadline a patch benched near the end + of a run outlives the run itself. + """ + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + class _SS: + baseline_runtime_sec = 600.0 + + def get_specialist_patch_verdict(self, tid): + return "approve" + + def grid_session_deadline_sec(self, **_kwargs): + return 4242.0 + + captured: dict[str, Any] = {} + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr( + IntegratePatchExecutor, + "_bench_patch", + _capture_bench(captured, {"output_throughput": 200.0, "status": "succeeded"}, {"accuracy_pass": None}), + ) + res = await ex( + _make_ctx( + "t", + { + "specialist_task_id": "spec", + "framework_source_root": str(repo), + "base_tput": 100.0, + "enable_stack_rebench": False, + }, + extra={"shared_state": _SS()}, + ) + ) + + assert res["status"] == "kept" + assert captured["session_deadline_sec"] == 4242.0 + # The expected runtime, not the backstop cap: admitting on the backstop + # abandons the tail of the budget. + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_bench_budget_is_unbounded_without_a_session(tmp_path, monkeypatch): + """No session context means no deadline, not a deadline of zero.""" + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + captured: dict[str, Any] = {} + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr( + IntegratePatchExecutor, + "_bench_patch", + _capture_bench(captured, {"output_throughput": 200.0, "status": "succeeded"}, {"accuracy_pass": None}), + ) + res = await ex( + _make_ctx( + "t", + { + "specialist_task_id": "spec", + "framework_source_root": str(repo), + "base_tput": 100.0, + "enable_stack_rebench": False, + }, + ) + ) + + assert res["status"] == "kept" + assert captured["session_deadline_sec"] is None + assert captured["variant_expected_sec"] is None + + +@pytest.mark.asyncio +async def test_bench_patch_forwards_the_session_budget_to_the_grid(tmp_path, monkeypatch): + session = tmp_path / "s" + session.mkdir() + config_path = tmp_path / "baseline.yaml" + config_path.write_text("benchmark: {}\n", encoding="utf-8") + + from hyperloom.orchestrator.actions.executors import integrate_patch as ip_mod + + captured: dict[str, Any] = {} + + async def fake_run_grid(**kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr(ip_mod, "run_grid", fake_run_grid) + monkeypatch.setattr(ip_mod, "materialize_config_with_envs", lambda *a, **k: config_path) + + await IntegratePatchExecutor(session_dir=session)._bench_patch( + params={"config_path": str(config_path)}, + output_root=tmp_path / "out", + extra_server_args_applied="", + extra_envs_applied={}, + specialist_task_id="spec", + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_switch_off_parity_leg_is_bounded_by_the_session_budget(tmp_path, monkeypatch): + """The parity leg is a second full bench, so it needs the same bound.""" + session = tmp_path / "s" + session.mkdir() + captured: dict[str, Any] = {} + + async def _capture(**kwargs): + captured.update(kwargs) + return ({"output_throughput": 100.0, "status": "succeeded"}, {"accuracy_pass": None}) + + ex = IntegratePatchExecutor(session_dir=session) + monkeypatch.setattr(ex, "_bench_patch", _capture) + + await ex._switch_off_parity( + params={}, + output_root=tmp_path, + specialist_task_id="spec", + switch_manifest=[{"switch": "HYPERLOOM_REWRITE_X"}], + base_tput=100.0, + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + @pytest.mark.asyncio async def test_keep_path(tmp_path, monkeypatch): session = tmp_path / "s" @@ -308,6 +461,71 @@ async def _measure(**kwargs): assert captured["stable_threshold_pct"] == pytest.approx(0.2) +@pytest.mark.asyncio +async def test_stack_rebench_is_bounded_by_the_session_budget(tmp_path, monkeypatch): + """The confirmation round must not outlive the run it is confirming for.""" + config_path = tmp_path / "baseline.yaml" + config_path.write_text("benchmark: {}\n", encoding="utf-8") + captured: dict[str, Any] = {} + + async def _measure(**kwargs): + captured.update(kwargs) + return StackRebenchResult(tput=None, workspace=None) + + monkeypatch.setattr(ip, "materialize_config_with_envs", lambda *a, **k: config_path) + monkeypatch.setattr(ip, "measure_stack_rebench", _measure) + + await IntegratePatchExecutor(session_dir=tmp_path)._confirm_stack_rebench( + params={"config_path": str(config_path)}, + output_root=tmp_path / "output", + extra_server_args_applied="", + extra_envs_applied={}, + specialist_task_id="spec", + base_tput=100.0, + session_deadline_sec=4242.0, + variant_expected_sec=600.0, + ) + + assert captured["session_deadline_sec"] == 4242.0 + assert captured["variant_expected_sec"] == 600.0 + + +@pytest.mark.asyncio +async def test_rebench_dropped_for_budget_is_not_reported_as_a_failed_measurement(tmp_path): + """Not measuring a variant is not evidence that the variant is unstable.""" + from unittest.mock import patch + + from hyperloom.orchestrator.actions.executors._grid_runner import GridVariant, VariantResult + from hyperloom.orchestrator.actions.executors import _stack_rebench as sr + + skipped = VariantResult( + name="v", + extra_server_args="", + extra_envs={}, + status="skipped", + error="session wall-clock budget exhausted before this variant ran", + error_class="session_time_exhausted", + ) + + async def _fake_run_grid(**_kwargs): + return [skipped] + + with patch.object(sr, "run_grid", new=_fake_run_grid): + result = await sr.measure_stack_rebench( + config_path=tmp_path / "base.yaml", + base_extra_args="", + variant=GridVariant("v"), + base_tput=100.0, + stable_threshold_pct=0.5, + output_slot=tmp_path / "slot", + variant_timeout_sec=600, + ) + + assert result.skipped_for_session_budget + assert result.warnings == ["stack_rebench_skipped:session_time_exhausted"] + assert not any("stack_rebench_failed" in w for w in result.warnings) + + @pytest.mark.asyncio async def test_keep_confirmed_by_rebench(tmp_path, monkeypatch): session = tmp_path / "s" diff --git a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py index 9a62476410..cc5d44e5d5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py +++ b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py @@ -23,6 +23,7 @@ DETOKENIZER_STALL_RETURNCODE, OVERTIME_KILL_RETURNCODE, SERVER_DEAD_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, _scan_logs_increment, _scan_server_log_increment, _server_log_shows_death, @@ -298,6 +299,115 @@ def test_run_with_session_kill_soft_deadline_still_fires_without_eval_marker(tmp assert elapsed < 10.0, f"soft-deadline path took {elapsed:.2f}s" +class TestSessionDeadline: + """The session budget is a separate channel from the soft deadline. + + The soft deadline answers "is this variant abnormally slow", which is why it + retires when the accuracy eval starts. The session budget answers "is the run + out of time", which no phase boundary changes. + """ + + def test_expired_session_budget_reaps_the_tree_with_its_own_sentinel(self): + start = time.monotonic() + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + session_deadline_sec=time.monotonic() - 1.0, + ) + elapsed = time.monotonic() - start + assert cp.returncode == SESSION_TIME_EXHAUSTED_RETURNCODE + assert cp.returncode != OVERTIME_KILL_RETURNCODE, ( + "a budget kill must not share the overtime code, which asserts the variant is slow" + ) + assert elapsed < 10.0, f"session-deadline path took {elapsed:.2f}s" + + def test_eval_start_does_not_retire_the_session_budget(self, tmp_path): + """The marker that retires the soft deadline must not retire this one. + + An accuracy eval that starts one minute before the run is out of time + still has to stop; this is the whole reason the two are separate channels. + """ + log_path = tmp_path / "server.log" + log_path.write_text("Application startup complete\nHYPERLOOM_EVAL_START\n") + start = time.monotonic() + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + soft_deadline_sec=1.0, + server_log_path=str(log_path), + session_deadline_sec=time.monotonic() + 1.5, + ) + elapsed = time.monotonic() - start + assert cp.returncode == SESSION_TIME_EXHAUSTED_RETURNCODE + assert elapsed < 10.0, f"session budget was not enforced during eval ({elapsed:.2f}s)" + + def test_a_budget_with_room_left_leaves_the_child_alone(self): + start = time.monotonic() + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(2)"], + timeout=60, + session_deadline_sec=time.monotonic() + 3600.0, + ) + elapsed = time.monotonic() - start + assert cp.returncode == 0 + assert elapsed >= 1.5, f"child was cut short at {elapsed:.2f}s" + + def test_no_session_deadline_keeps_the_previous_behaviour(self): + cp = run_with_session_kill( + [sys.executable, "-c", "print('done')"], + timeout=30, + session_deadline_sec=None, + ) + assert cp.returncode == 0 + assert "done" in (cp.stdout or "") + + +def _sentinel_returncodes() -> dict[int, set[str]]: + """Map every sentinel returncode to the qualified names that claim it.""" + from hyperloom.orchestrator.actions.executors import _ray_serving, _subprocess_kill + + assigned: dict[int, set[str]] = {} + for module in (_subprocess_kill, _ray_serving): + short = module.__name__.rsplit(".", 1)[-1] + for name, value in vars(module).items(): + if not isinstance(value, int) or isinstance(value, bool): + continue + if not (name.endswith("_RETURNCODE") or name.endswith("_RC")): + continue + assigned.setdefault(value, set()).add(f"{short}.{name}") + return assigned + + +def test_every_sentinel_returncode_names_exactly_one_cause(): + """A sentinel shared by two causes makes attribution a coin flip. + + The codes are handed out in more than one module and all arrive at their + consumer as a plain ``returncode``, so a new one can quietly reuse a number + already taken. That is how the session-budget code first landed on the Ray + actor-died number, which would have had every actor death read as a spent + budget and taught the ledger the wrong thing about both. + """ + assigned = _sentinel_returncodes() + + collisions = {code: sorted(names) for code, names in assigned.items() if len(names) > 1} + assert not collisions, f"sentinel return codes collide: {collisions}" + assert SESSION_TIME_EXHAUSTED_RETURNCODE in assigned + + +def test_an_actor_timeout_is_not_recorded_as_a_failed_agentx_preflight(): + """The two causes that share ``_run_magpie``'s return channel stay apart. + + ``_run_magpie`` returns ``AGENTX_PREFLIGHT_RETURNCODE`` when the execution + boundary fails preflight and, a few lines on, whatever the serving lease's + actor returned -- including ``_ACTOR_TIMEOUT_RC``. Callers see one + ``returncode`` either way, so the two sharing a number (which they did) is + enough to have a hung actor blamed on a missing aiperf. + """ + from hyperloom.orchestrator.actions.executors import _ray_serving, _subprocess_kill + + assert _ray_serving._ACTOR_TIMEOUT_RC != _subprocess_kill.AGENTX_PREFLIGHT_RETURNCODE + + def test_run_with_session_kill_streams_child_output_to_parent(capsys): """Captured child output is also mirrored immediately to parent streams.""" code = "import sys\nprint('child-out', flush=True)\nprint('child-err', file=sys.stderr, flush=True)\n" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 2de6d32297..2766b8f7a7 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -41,6 +41,7 @@ EVAL_PROBE_UNPATCHABLE_RETURNCODE, OVERTIME_KILL_RETURNCODE, SERVER_DEAD_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, run_with_session_kill, server_log_death_excerpt, ) @@ -938,6 +939,7 @@ def _run_magpie( server_already_ready: bool = False, serving_lease: Any = None, on_output: Callable[[], None] | None = None, + session_deadline_sec: float | None = None, ) -> tuple[int, str, str]: """Blocking subprocess wrapper. Returns (rc, stdout, stderr). @@ -971,6 +973,10 @@ def _run_magpie( line the benchmark emits, so the caller's heartbeat can keep reporting across a run that blocks for hours. Ignored on the ``serving_lease`` path — see the note there. + session_deadline_sec (float | None): Absolute ``time.monotonic()`` instant + at which the session budget expires. Reaps the tree and returns + ``SESSION_TIME_EXHAUSTED_RETURNCODE``. Enforced in every phase, + including the accuracy eval that retires ``soft_deadline_sec``. Returns: tuple[int, str, str]: ``(returncode, stdout, stderr)``. @@ -1097,6 +1103,7 @@ def _run_magpie( server_log_path=str(output_dir / "server.log"), server_already_ready=server_already_ready, on_output=on_output, + session_deadline_sec=session_deadline_sec, ) return proc.returncode, proc.stdout or "", proc.stderr or "" @@ -1224,6 +1231,53 @@ def _variant_progress_note( } +# Seconds by which a round's hard cap is allowed to sit past the session deadline, +# so the in-process session watchdog (which attributes the kill correctly) trips +# before the hard cap (which the ledger reads as a variant timeout). Small enough +# that it cannot meaningfully eat the close-window reserve. +_SESSION_KILL_GRACE_SEC: int = 15 + +# Labels a round that never ran because the session wall-clock budget was spent. +# Distinct from any measurement failure: a round that was not run is not evidence +# about the variant, and the ledger must not read it as one. +SESSION_TIME_EXHAUSTED_CLASS = "session_time_exhausted" + + +def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: + """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call. + + Every arm that benches on the GPU needs the same two numbers, and they have + to agree: a deadline derived one way in one executor and another way in the + next produces arms that abandon different amounts of the tail budget. Both + are read here so there is one definition. + + ``variant_expected_sec`` is the measured baseline runtime -- what a + normally-behaving variant needs -- and is deliberately not the declared + timeout, which is a catastrophic-hang backstop roughly twice as large. + Admitting on the backstop would refuse to start a 20-minute round with 30 + minutes left. + + Args: + shared_state: The session ``SharedState``, or ``None`` when the caller + has no session context (direct executor invocation, tests). + + Returns: + tuple[float | None, float | None]: The monotonic-clock session deadline + and the expected per-variant runtime. Either is ``None`` when + unknown, which leaves the corresponding check disabled rather than + guessing a bound. + """ + if shared_state is None: + return (None, None) + deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) + deadline = deadline_fn() if callable(deadline_fn) else None + try: + baseline_sec = float(getattr(shared_state, "baseline_runtime_sec", 0.0) or 0.0) + except (TypeError, ValueError): + baseline_sec = 0.0 + return (deadline, baseline_sec if baseline_sec > 0 else None) + + async def run_grid( *, base_yaml_path: Path, @@ -1245,6 +1299,7 @@ async def run_grid( server_already_ready: bool = False, serving_lease: Any = None, session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> list[VariantResult]: """Execute each grid variant and return all per-variant results. @@ -1257,9 +1312,22 @@ async def run_grid( decision). ``session_deadline_sec`` is a ``time.monotonic()`` deadline for the whole - session budget; a variant is skipped once the remaining budget cannot fit - another ``variant_timeout_sec`` worst-case run, so a wall-clock timeout stops - the grid mid-way and the last variant never overruns the close window. + session budget, already excluding the close-window reserve. It does two + things, deliberately split: + + * **Whether to start.** A variant is skipped, and every variant after it, + once the remaining budget cannot fit ``variant_expected_sec`` -- what a + normally-behaving variant needs. Judging by ``variant_timeout_sec`` + instead means judging by the catastrophic backstop (``baseline x 2`` for + explore), which abandons the tail of the budget to variants that would + have finished comfortably. ``None`` falls back to ``variant_timeout_sec``, + preserving the stricter behaviour for callers that cannot estimate. + * **How long it may run.** The cap handed to each round is clamped to what + the session can still pay for, recomputed per round because a warmup + consumes budget too. Without this a variant admitted near the end can + outlive the whole session: explore derives caps up to 4h from the measured + baseline and never consulted the budget, so a 3h run could grant one + variant more time than the run was given. Every pass runs with ``output_root`` as its working directory, the way the baseline arm anchors Magpie to its own output dir. That is what marks the @@ -1388,19 +1456,96 @@ async def _pulse_after_variant(idx: int) -> None: except Exception as exc: # noqa: BLE001 log.debug("robustness pulse swallowed: %r", exc) + def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: float = 0.0) -> int: + """``variant_timeout_sec`` capped at what the session can still pay for. + + Recomputed per round rather than once per variant: a warmup round spends + budget the measure round would otherwise still count on. + + ``reserve_sec`` holds back what the variant's remaining rounds still + need, so a slow warmup cannot consume the budget belonging to the + measured round. The measured round is the only one that yields a usable + data point, so it is the round the budget is kept for; a discarded + warmup that overruns is the cheaper thing to cut short. + + ``session_deadline_sec`` already excludes the close-window reserve, so + no further margin is taken for the session itself. Never returns less + than 1 -- a non-positive cap would read as "no timeout" to the + subprocess layer, which is the opposite of what an exhausted budget + means. Deciding whether the round should start at all belongs to the + caller's fit check. + + A small grace is added past the session deadline so the in-process + session watchdog trips first: both fire at the same instant, and the hard + cap raises ``TimeoutExpired``, which the ledger records as a variant + timeout -- a claim about the variant. The watchdog's sentinel says the run + ran out of time, which is what actually happened. + + Args: + idx (int): Zero-based variant index, for the log line. + name (str): Variant name, for the log line. + round_label (str): Which round is being capped (warmup / measure). + reserve_sec (float): Seconds held back for this variant's later + rounds. Zero for the last round. + + Returns: + int: The hard timeout to grant this round, in seconds. + """ + cap = int(variant_timeout_sec) + if session_deadline_sec is None: + return cap + usable = int(session_deadline_sec - time.monotonic() - max(0.0, reserve_sec)) + _SESSION_KILL_GRACE_SEC + if usable >= cap: + return cap + clamped = max(1, usable) + log.info( + "grid_runner: variant %d/%d name=%s %s cap clamped %ds -> %ds by the session budget", + idx + 1, + len(grid), + name, + round_label, + cap, + clamped, + ) + return clamped + + # How many full benchmark passes one variant costs. Both warmups run the same + # workload as the measured pass -- neither is a reduced one -- so a variant + # that warms up costs twice what its measured round does. Admitting on a + # single round's estimate would systematically let in variants that then get + # their measured round clamped to nothing, turning a budget shortfall into a + # ledger full of spurious timeouts. Both flags are run-level, so this is known + # before the loop; the per-variant ``auto_warmup`` is only ever narrower. + _mn_warmup_rounds = 0 + if variant_expected_sec is not None: + from ._multi_node_env import ( + is_multi_node as _mn_is_multi_node, + mn_bench_warmup_enabled as _mn_bench_warmup_enabled, + ) + + _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 + variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds + for i, variant in enumerate(grid): # Session-budget stop: skip the remaining variants once the wall-clock - # deadline is reached or the remaining budget cannot fit another - # variant's worst-case runtime, so a timeout halts the grid instead of - # draining it (and the last variant cannot overrun the close window). + # deadline is reached or the remaining budget cannot fit another variant, + # so a timeout halts the grid instead of draining it (and the last variant + # cannot overrun the close window). if session_deadline_sec is not None: remaining_sec = session_deadline_sec - time.monotonic() - if remaining_sec < float(variant_timeout_sec): + # Falls back to a single ``variant_timeout_sec`` when no estimate was + # given, which is what callers that cannot estimate already got. + required_sec = ( + float(variant_expected_sec) * variant_rounds + if variant_expected_sec is not None + else float(variant_timeout_sec) + ) + if remaining_sec < required_sec: log.warning( - "grid_runner: session budget exhausted (%.0fs left < variant cap %ds); " + "grid_runner: session budget exhausted (%.0fs left < %.0fs needed); " "skipping %d remaining variant(s)", max(0.0, remaining_sec), - variant_timeout_sec, + required_sec, len(grid) - i, ) for skipped_variant in grid[i:]: @@ -1591,12 +1736,18 @@ async def _pulse_after_variant(idx: int) -> None: magpie_python=magpie_python, config_path=warmup_cfg_path, output_dir=warmup_slot, - timeout_sec=variant_timeout_sec, + timeout_sec=_round_timeout_sec( + i, + variant.name, + round_label="warmup", + reserve_sec=float(variant_expected_sec or 0.0) * (1 + _mn_warmup_rounds), + ), cwd=cwd, result_dir=result_dir, soft_deadline_sec=None, preclean=True, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, ) except subprocess.TimeoutExpired as exc: from ._server_lifecycle import teardown_lifecycle_server @@ -1812,12 +1963,18 @@ async def _pulse_after_variant(idx: int) -> None: magpie_python=magpie_python, config_path=cfg_path, output_dir=_mn_warm_slot, - timeout_sec=variant_timeout_sec, + timeout_sec=_round_timeout_sec( + i, + variant.name, + round_label="mn_warmup", + reserve_sec=float(variant_expected_sec or 0.0), + ), cwd=cwd, result_dir=None, soft_deadline_sec=None, preclean=False, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, ) log.info( "grid_runner: MN warmup pass done (discarded) %d/%d name=%s", @@ -1843,13 +2000,14 @@ async def _pulse_after_variant(idx: int) -> None: magpie_python=magpie_python, config_path=cfg_path, output_dir=slot, - timeout_sec=variant_timeout_sec, + timeout_sec=_round_timeout_sec(i, variant.name, round_label="measure"), cwd=cwd, result_dir=result_dir, soft_deadline_sec=soft_deadline_sec, preclean=(False if auto_warmup else preclean_before_run), server_already_ready=(server_already_ready or auto_warmup), serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, ) except subprocess.TimeoutExpired as exc: # Harvest pre-timeout leaks. @@ -2048,6 +2206,52 @@ async def _pulse_after_variant(idx: int) -> None: break continue + # Session budget ran out mid-round and the tree was reaped. Recorded as + # ``skipped``, exactly like a variant the budget refused to start: in both + # cases nothing was measured, so there is no verdict to record about the + # variant. Grading it as a failure -- or worse as ``killed_overtime``, + # which asserts the variant is abnormally slow -- would put a conclusion + # the run never reached into the ledger and the KB. + # + # The remaining variants are left to the fit check at the top of the loop, + # which now sees a deadline in the past and skips them all under the same + # label rather than duplicating that decision here. + if rc == SESSION_TIME_EXHAUSTED_RETURNCODE: + variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) + log.warning( + "grid_runner: variant %d/%d name=%s reaped after %.1fs: the session " + "wall-clock budget ran out mid-round; recorded as skipped, not failed", + i + 1, + len(grid), + variant.name, + variant_runtime_sec, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=SESSION_TIME_EXHAUSTED_CLASS, + error_summary="session wall-clock budget exhausted mid-round; tree reaped", + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + returncode=rc, + runtime_sec=variant_runtime_sec, + error="session wall-clock budget exhausted while this variant was running", + error_class=SESSION_TIME_EXHAUSTED_CLASS, + server_log_path=_existing_log_path(server_log), + note=variant.note, + ) + ) + await _pulse_after_variant(i) + if not keep_going_on_failure: + break + continue + # Soft overtime gate fired: record a ``killed_overtime=True`` result with # no tput and still harvest leaks for post-mortem. if rc == OVERTIME_KILL_RETURNCODE: @@ -2275,7 +2479,7 @@ def _session_deadline_skip_result(variant: GridVariant) -> VariantResult: extra_envs=dict(variant.extra_envs), status="skipped", error="session wall-clock budget exhausted before this variant ran", - error_class="session_time_exhausted", + error_class=SESSION_TIME_EXHAUSTED_CLASS, note=variant.note, ) @@ -2412,6 +2616,7 @@ def _write_variant_abort_marker_impl( "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", "GridVariant", "MULTI_NODE_DEFAULT_KEEP_THRESHOLD_PCT", + "SESSION_TIME_EXHAUSTED_CLASS", "SGLANG_WATCHDOG_TIMEOUT_ENV", "SINGLE_NODE_DEFAULT_KEEP_THRESHOLD_PCT", "VariantResult", @@ -2429,6 +2634,7 @@ def _write_variant_abort_marker_impl( "sanitize_result_dir", "sanitize_script_name", "server_args_env_name", + "session_grid_bounds", # Re-exported from the sibling modules to keep the namespace intact. "coerce_extra_envs", "compact_json_server_args", diff --git a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py index dbe72fe402..7444321d9a 100644 --- a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py +++ b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -from ._grid_runner import GridVariant, run_grid +from ._grid_runner import SESSION_TIME_EXHAUSTED_CLASS, GridVariant, run_grid # Single source of truth for the post-KEEP confirmation floor shared by every @@ -43,6 +43,15 @@ def stable(self) -> bool: """True when the measured throughput cleared the stability floor.""" return self.tput is not None and self.tput >= self.stable_floor + @property + def skipped_for_session_budget(self) -> bool: + """True when the round never ran because the session budget was spent. + + Lets a caller tell "the confirmation did not happen" apart from "the + confirmation failed", which are different facts about the variant. + """ + return any(w.startswith(f"stack_rebench_skipped:{SESSION_TIME_EXHAUSTED_CLASS}") for w in self.warnings) + async def measure_stack_rebench( *, @@ -64,6 +73,8 @@ async def measure_stack_rebench( soft_deadline_sec: float | None = None, server_already_ready: bool = False, serving_lease: Any = None, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> StackRebenchResult: """Run ``variant`` once on the stack and grade it against the floor. @@ -76,6 +87,12 @@ async def measure_stack_rebench( execution, §12 T1) routes the round through the caller's held Ray lease so the rebench shares the same lease as the warmup/decision rounds it reuses the hot server from; ``None`` keeps the local path. + + ``session_deadline_sec`` bounds the rebench by the session wall-clock, so a + confirmation round cannot outlive the run it is confirming for. + A rebench dropped for lack of budget is reported as its own warning rather + than as a failed measurement: not measuring a variant is not evidence that + the variant is unstable, and the caller grades on the distinction. """ output_slot.mkdir(parents=True, exist_ok=True) results = await run_grid( @@ -95,6 +112,8 @@ async def measure_stack_rebench( soft_deadline_sec=soft_deadline_sec, server_already_ready=server_already_ready, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) rb = results[0] if results else None tput: float | None = None @@ -104,6 +123,8 @@ async def measure_stack_rebench( tput = rb.output_throughput workspace = rb.workspace warnings = list(rb.nonfatal_warnings) + elif rb is not None and getattr(rb, "error_class", "") == SESSION_TIME_EXHAUSTED_CLASS: + warnings.append(f"stack_rebench_skipped:{SESSION_TIME_EXHAUSTED_CLASS}") elif rb is not None: warnings.append(f"stack_rebench_failed:{(rb.error or '')[-120:]}") else: diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index a3b7321ab5..7310c626eb 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -260,6 +260,14 @@ def kill_my_spawned_server( # already fails with, so a bounds gap reads identically on both arms. EVAL_PROBE_UNPATCHABLE_RETURNCODE: int = -914 +# Sentinel ``returncode`` when the session wall-clock budget ran out mid-round and +# the tree was reaped. Deliberately distinct from ``OVERTIME_KILL_RETURNCODE``: +# that one says "this variant ran far longer than the baseline", which is a +# judgement about the variant, while this one says "the run was out of time", +# which says nothing about the variant at all. Sharing a code would teach the KB +# that a variant is slow whenever a session happened to end during it. +SESSION_TIME_EXHAUSTED_RETURNCODE: int = -915 + # Server-ready markers: their appearance in ``server.log`` means the server has # finished startup and is accepting traffic. Only after one is observed does the # detokenizer-stall clock start. Covers the uvicorn frontend (vLLM + sglang) and @@ -686,13 +694,25 @@ def run_with_session_kill( detok_stall_grace_sec: float | None = None, server_already_ready: bool = False, on_output: Callable[[], None] | None = None, + session_deadline_sec: float | None = None, ) -> subprocess.CompletedProcess: """Run a subprocess in its own session and reap descendants on every exit path. + ``session_deadline_sec`` is an absolute ``time.monotonic()`` instant, unlike + ``soft_deadline_sec`` which is a relative duration. It is the session's own + wall-clock budget and is enforced in every phase, including the accuracy + eval -- the phase that retires ``soft_deadline_sec``. That distinction is the + reason it is a separate channel rather than a reuse of the soft deadline: the + eval-start boundary is meaningful for "is this variant abnormally slow" and + meaningless for "is the run out of time". + Args: on_output: Called from a reader thread each time the child produces output, so a caller can report a long step alive on the child's own activity rather than on a timer. + session_deadline_sec: Absolute ``time.monotonic()`` instant at which the + session budget expires. Reaps the tree and returns + ``SESSION_TIME_EXHAUSTED_RETURNCODE``. """ if server_dead_grace_sec is None: try: @@ -738,6 +758,7 @@ def run_with_session_kill( detok_stall_grace_sec=detok_stall_grace_sec, capture=capture, server_already_ready=server_already_ready, + session_deadline_sec=session_deadline_sec, ) except subprocess.TimeoutExpired: kill_my_spawned_server(proc) @@ -783,6 +804,24 @@ def run_with_session_kill( stdout=stdout if stdout is not None else ("" if text else b""), stderr=stderr if stderr is not None else ("" if text else b""), ) + except _SessionDeadlineExceeded as exc: + kill_my_spawned_server(proc) + stdout, stderr = ( + capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") + ) + log.warning( + "_subprocess_kill: session wall-clock budget exhausted %.1fs ago " + "(round elapsed=%.1fs); reaped tree with sentinel returncode=%d.", + exc.overrun_sec, + exc.elapsed_sec, + SESSION_TIME_EXHAUSTED_RETURNCODE, + ) + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout=stdout if stdout is not None else ("" if text else b""), + stderr=stderr if stderr is not None else ("" if text else b""), + ) except _SoftDeadlineExceeded as exc: kill_my_spawned_server(proc) stdout, stderr = ( @@ -811,6 +850,25 @@ def run_with_session_kill( kill_my_spawned_server(proc) +class _SessionDeadlineExceeded(Exception): + """Internal sentinel: the session wall-clock budget ran out mid-round. + + Never bubbles past :func:`run_with_session_kill` (converted to a + ``CompletedProcess`` carrying ``SESSION_TIME_EXHAUSTED_RETURNCODE``). + """ + + def __init__(self, *, overrun_sec: float, elapsed_sec: float) -> None: + """Record how far past the session deadline the round got. + + Args: + overrun_sec (float): Seconds past the session deadline at trip time. + elapsed_sec (float): Wall-clock elapsed for this round at trip time. + """ + super().__init__(f"session budget exhausted {overrun_sec:.1f}s ago (round elapsed={elapsed_sec:.1f}s)") + self.overrun_sec = float(overrun_sec) + self.elapsed_sec = float(elapsed_sec) + + class _SoftDeadlineExceeded(Exception): """Internal sentinel for an elapsed soft deadline. Never bubbles past :func:`run_with_session_kill` (converted to a ``CompletedProcess``). @@ -897,6 +955,7 @@ def _communicate_with_soft_deadline( detok_stall_grace_sec: float | None = None, capture: _StreamCapture | None = None, server_already_ready: bool = False, + session_deadline_sec: float | None = None, ) -> tuple[str | bytes, str | bytes]: """Communicate with a child while enforcing soft and server-log watchdogs.""" watchdog_active = bool(server_log_path) and ( @@ -904,9 +963,10 @@ def _communicate_with_soft_deadline( ) stall_active = bool(server_log_path) and (detok_stall_grace_sec is not None and float(detok_stall_grace_sec) > 0.0) soft_active = soft_deadline_sec is not None and float(soft_deadline_sec) > 0.0 - if capture is None and not soft_active and not watchdog_active and not stall_active: + session_active = session_deadline_sec is not None + if capture is None and not soft_active and not watchdog_active and not stall_active and not session_active: return proc.communicate(timeout=hard_timeout) - if capture is not None and not soft_active and not watchdog_active and not stall_active: + if capture is not None and not soft_active and not watchdog_active and not stall_active and not session_active: proc.wait(timeout=hard_timeout) return capture.finish() @@ -950,6 +1010,16 @@ def _communicate_with_soft_deadline( while True: now = time.monotonic() elapsed = now - start + # Session budget. Checked before every other gate and never suspended: + # unlike the soft deadline it makes no claim about the variant, so the + # eval-start boundary that retires the soft deadline does not apply. An + # accuracy eval that starts one minute before the run is out of time still + # has to stop. + if session_active and session_deadline_sec is not None and now >= session_deadline_sec: + raise _SessionDeadlineExceeded( + overrun_sec=now - float(session_deadline_sec), + elapsed_sec=elapsed, + ) # Advance the log scan, latching the server-ready, last-activity # and eval-start signals. if scan_active: @@ -1023,6 +1093,8 @@ def _communicate_with_soft_deadline( # Slice bounded by every active remaining window so the right gate # fires first; the child can still finish inside any slice. slice_sec = poll_interval + if session_active and session_deadline_sec is not None: + slice_sec = min(slice_sec, float(session_deadline_sec) - now) if soft_active and deadline_sec is not None and not soft_deadline_suspended: if soft_from_ready: if server_ready_since is not None: @@ -1050,6 +1122,7 @@ def _communicate_with_soft_deadline( "EVAL_PROBE_UNPATCHABLE_RETURNCODE", "OVERTIME_KILL_RETURNCODE", "SERVER_DEAD_RETURNCODE", + "SESSION_TIME_EXHAUSTED_RETURNCODE", "kill_my_spawned_server", "new_session_kwargs", "run_with_session_kill", diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index c5826da120..5a67692c69 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -63,9 +63,11 @@ apply_aiter_moe_pin_filter, apply_multi_node_invalid_variants, reorder_grid_for_multi_node, + SESSION_TIME_EXHAUSTED_CLASS, run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from ._grid_server_args import compose_server_args, server_args_env_name from ._ray_serving import maybe_serving_lease @@ -1112,13 +1114,47 @@ async def __call__(self, ctx) -> dict[str, Any]: round_serving_lease = maybe_serving_lease(num_gpus=_num_gpus_for_config(config_path)) if runnable else None # Stop testing further variants once the session wall-clock budget runs # out; untested variants stay out of the ledger so a resume can retry them. - _ss = extra.get("shared_state") or extra.get("state") - session_deadline_sec = _ss.grid_session_deadline_sec() if _ss is not None else None + # + # What a normally-behaving round needs, as opposed to ``timeout_sec``, + # which is the catastrophic backstop (``baseline x (kill_ratio + margin)`` + # ~= baseline x 2). Gating on the backstop abandons the tail of the budget + # to variants that would have finished comfortably: with a 20-min baseline + # it refuses to start with 30 min left, for a round that needs ~20. The + # params values win over the session's because an operator may override + # them per task; ``None`` when neither is known, which leaves the stricter + # backstop check in place rather than guessing. + # + # The two rounds are estimated separately because they cost different + # amounts: the warmup pass pays a cold server boot and is discarded, while + # the decision round is client-only against the hot server -- which is + # exactly the split ``decision_anchor_sec`` already draws for the overtime + # kill. + session_deadline_sec, session_expected_sec = session_grid_bounds( + extra.get("shared_state") or extra.get("state") + ) + warmup_expected_sec = (baseline_runtime_sec if baseline_runtime_sec > 0 else None) or session_expected_sec + decision_expected_sec = (decision_anchor_sec if decision_anchor_sec > 0 else None) or session_expected_sec + # Set when the loop stops early for lack of budget, so the round can say + # so instead of reporting a bare, unattributed failure: a variant that + # never ran is not a variant that failed. + session_budget_untested = 0 try: for idx, gv in enumerate(runnable): - if session_deadline_sec is not None and (session_deadline_sec - time.monotonic()) < float(timeout_sec): + # A warm-decision variant pays for both rounds, so admitting it on + # the decision round alone would let it in and then strand it + # mid-variant with a discarded warmup and no measurement. + if decision_expected_sec is not None: + fit_required_sec = float(decision_expected_sec) + ( + float(warmup_expected_sec or 0.0) if use_warm_decision else 0.0 + ) + else: + fit_required_sec = float(timeout_sec) + if session_deadline_sec is not None and (session_deadline_sec - time.monotonic()) < fit_required_sec: + session_budget_untested = len(runnable) - idx log.warning( - "explore: session budget cannot fit another variant; stopping after %d/%d variant(s)", + "explore: session budget cannot fit another variant " + "(needs %.0fs); stopping after %d/%d variant(s)", + fit_required_sec, idx, len(runnable), ) @@ -1205,6 +1241,8 @@ async def __call__(self, ctx) -> dict[str, Any]: server_lifecycle=round1_lifecycle, base_args_mode=stack_base_args_mode, serving_lease=variant_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=warmup_expected_sec, ) w = warmup_results[0] if warmup_results else None if w is None or getattr(w, "status", "") != "succeeded": @@ -1296,6 +1334,8 @@ async def __call__(self, ctx) -> dict[str, Any]: preclean_before_run=not use_warm_decision, server_already_ready=use_warm_decision, serving_lease=variant_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=decision_expected_sec, ) if not results: # run_grid returns one result per grid entry. @@ -1606,6 +1646,8 @@ async def __call__(self, ctx) -> dict[str, Any]: soft_deadline_sec=decision_deadline_sec, server_already_ready=lifecycle_eligible, serving_lease=variant_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=decision_expected_sec, ) stack_rebench_tput = rebench.tput stack_rebench_workspace = rebench.workspace @@ -1914,9 +1956,24 @@ async def __call__(self, ctx) -> dict[str, Any]: if t.get("round_id") == round_id ) status = "succeeded" if produced_measurement or winners else "failed" + # A round that measured nothing because the budget ran out is not the same + # as one whose variants failed, and it used to be reported as a bare + # ``failed`` with no error_class at all -- nothing downstream could tell + # the two apart, so the KB could learn that these variants are bad. + budget_error: dict[str, Any] = {} + if status == "failed" and session_budget_untested > 0: + budget_error = { + "error_class": SESSION_TIME_EXHAUSTED_CLASS, + "error": ( + f"session wall-clock budget exhausted; {session_budget_untested} variant(s) never ran " + "and stay out of the ledger so a resume can retry them" + ), + } return { "status": status, + **budget_error, + "session_budget_untested": session_budget_untested, "base_tput": base_tput, "running_base_tput": running_base_tput, "output_throughput": output_throughput, diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 0d41d5361e..29babb7d54 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -28,6 +28,7 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from ._workload_envs import ( FrameworkScriptMismatchError, @@ -796,12 +797,21 @@ async def __call__(self, ctx) -> dict[str, Any]: }, ) - # Bench via run_grid (size=1). + # Bench via run_grid (size=1). Bound it by the session wall-clock, as the + # sweep and explore arms already are: without it the declared cap is the + # only limit, and that cap answers "how long before this counts as hung", + # not "how much budget is left" -- so a candidate benched near the end of + # a run could outlive the run itself. + session_deadline_sec, variant_expected_sec = session_grid_bounds( + extra.get("shared_state") or extra.get("state") + ) try: bench_result, gate_evidence = await self._bench_candidate( params=params, output_root=output_root, slug=slug, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) except FrameworkScriptMismatchError as exc: reverted = self._revert_patches( @@ -1127,6 +1137,8 @@ async def _bench_candidate( params: dict[str, Any], output_root: Path, slug: str, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Run a 1-variant Magpie bench under the patched server + accuracy gate. Mirrors :meth:`IntegratePatchExecutor._bench_patch`. @@ -1135,6 +1147,11 @@ async def _bench_candidate( params: The task params (config / model / bench knobs). output_root: The per-task workspace root for the bench. slug: The candidate slug used to name the variant. + session_deadline_sec: Monotonic-clock session budget deadline, or + ``None`` when unbounded. Resolved by the caller, which owns the + session context. + variant_expected_sec: Expected bench runtime used to decide whether + the remaining budget can fit this bench at all. Returns: A ``(bench, gate_evidence)`` tuple: the bench result dict and a @@ -1205,6 +1222,8 @@ async def _bench_candidate( benchmark_script=override_script, result_dir=override_result_dir, serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) finally: if serving_lease is not None: diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 58089ca0f9..4c280db011 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -55,6 +55,7 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from . import _framework_switch_manifest as _switch_manifest from ._grid_server_args import compose_server_args @@ -2505,6 +2506,13 @@ async def _stage_gate( params = dict(params) params["runtime_override"] = provision_result.runtime.to_runtime_override() + # Bound both bench legs by the session wall-clock, as the sweep and explore + # arms already are: the declared cap answers "how long before this counts + # as hung", not "how much budget is left", so without this a patch benched + # near the end of a run could outlive the run itself. Resolved once here + # and reused by the parity leg -- the deadline is an absolute monotonic + # timestamp, so it stays correct as the gate progresses. + session_deadline_sec, variant_expected_sec = session_grid_bounds(shared_state) try: bench_result, gate_evidence = await self._bench_patch( params=params, @@ -2512,6 +2520,8 @@ async def _stage_gate( extra_server_args_applied=extra_server_args_applied, extra_envs_applied=extra_envs_applied, specialist_task_id=specialist_task_id, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) except FrameworkScriptMismatchError as exc: artifacts_reverted = self._revert_artifacts(applied_artifacts) @@ -2988,6 +2998,13 @@ async def _gate_perf( # is genuinely unchanged with every switch unset, which is exactly what this # leg measures. An unswitched patch has no "off" state to fall back to, so # it still reverts without spending the leg. + # Both the parity leg and the stack rebench below are additional full + # benches, so they need the same session bound the first bench got. + # Resolved here rather than threaded from the caller because the deadline + # is an absolute monotonic timestamp: the budget the first bench spent is + # already reflected in it. + session_deadline_sec, variant_expected_sec = session_grid_bounds(shared_state) + parity: dict[str, Any] = {"ran": False, "ok": True, "reason": ""} if switch_manifest: parity = await self._switch_off_parity( @@ -2996,6 +3013,8 @@ async def _gate_perf( specialist_task_id=specialist_task_id, switch_manifest=switch_manifest, base_tput=base_tput, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) if not parity.get("ok"): # An unmeasurable parity leg reverts under its own verdict: the patch @@ -3149,6 +3168,8 @@ async def _gate_perf( extra_envs_applied=extra_envs_applied, specialist_task_id=specialist_task_id, base_tput=base_tput, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) rb_acc_block, rb_acc_reason, _rb_degraded = accuracy_keep_block( confirm["accuracy_pass"], @@ -3340,6 +3361,8 @@ async def _switch_off_parity( specialist_task_id: str, switch_manifest: list[dict[str, Any]], base_tput: float, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> dict[str, Any]: """Verify the patch is genuinely inert with every rewrite switch unset. @@ -3363,6 +3386,10 @@ async def _switch_off_parity( specialist_task_id: The originating specialist. switch_manifest: Parsed switch manifest. base_tput: Pre-patch throughput to compare against. + session_deadline_sec: Monotonic-clock session budget deadline for the + parity bench, or ``None`` when unbounded. + variant_expected_sec: Expected bench runtime, used to decide whether + the remaining budget can fit the parity leg at all. Returns: ``{"ran", "ok", "tput", "delta_pct", "band_pct", "accuracy_pass", @@ -3389,6 +3416,8 @@ async def _switch_off_parity( specialist_task_id=specialist_task_id, unset_envs=switch_names, variant_suffix="-parity", + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) except Exception as exc: # noqa: BLE001 — a failed probe must not read as a pass return { @@ -3865,6 +3894,8 @@ async def _bench_patch( specialist_task_id: str, unset_envs: "list[str] | None" = None, variant_suffix: str = "", + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: """Run a 1-variant Magpie bench under the patched server + accuracy gate. @@ -3881,6 +3912,11 @@ async def _bench_patch( an earlier KEEP put them into the base configuration. variant_suffix: Appended to the variant name so a second leg does not collide with the first one's grid slot. + session_deadline_sec: Monotonic-clock session budget deadline, or + ``None`` when unbounded. Resolved by the caller, which owns the + session context. + variant_expected_sec: Expected bench runtime used to decide whether + the remaining budget can fit this bench at all. Returns: A ``(bench_result_dict, gate_evidence)`` tuple where @@ -3969,6 +4005,8 @@ async def _bench_patch( result_dir=override_result_dir, base_args_mode=str(params.get("base_args_mode") or "append"), serving_lease=serving_lease, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) finally: if serving_lease is not None: @@ -4141,12 +4179,18 @@ async def _confirm_stack_rebench( extra_envs_applied: dict[str, str], specialist_task_id: str, base_tput: float, + session_deadline_sec: float | None = None, + variant_expected_sec: float | None = None, ) -> dict[str, Any]: """Re-bench the patched stack once more and re-grade throughput + accuracy. Mirrors the explore ledger's post-KEEP confirmation: a patch only KEEPs if a second full-stack run still clears the stability floor and the accuracy gate. Returns ``stable`` / ``tput`` / ``accuracy_pass`` / etc. + + ``session_deadline_sec`` / ``variant_expected_sec`` bound the + confirmation round by the session wall-clock, so it cannot outlive the + run it is confirming for. """ config_path = Path(params.get("config_path") or self.default_config_path or default_baseline_config()) resolved_model = str(params.get("model_path") or "").strip() or os.environ.get("MODEL_PATH", "").strip() @@ -4206,6 +4250,8 @@ async def _confirm_stack_rebench( result_dir=override_result_dir, magpie_python=params.get("magpie_python") or None, base_args_mode=str(params.get("base_args_mode") or "append"), + session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) # See ``_bench_patch``: lm-eval writes to the grid slot (the parent of # ``rebench.workspace``), so grade from there, honoring ``result_dir``. diff --git a/src/hyperloom/orchestrator/actions/executors/sweep.py b/src/hyperloom/orchestrator/actions/executors/sweep.py index 0a12642e3b..311f20f58c 100644 --- a/src/hyperloom/orchestrator/actions/executors/sweep.py +++ b/src/hyperloom/orchestrator/actions/executors/sweep.py @@ -43,6 +43,7 @@ run_grid, sanitize_result_dir, sanitize_script_name, + session_grid_bounds, ) from ._ray_serving import maybe_serving_lease from ._workload_envs import ( @@ -310,9 +311,13 @@ async def __call__(self, ctx) -> dict[str, Any]: # outlives its GPU lease. ``None`` on the local path (multi-node / # RAY_EXEC off / tests) keeps the legacy behaviour. sweep_lease = maybe_serving_lease(num_gpus=_num_gpus_for_config(config_path)) - # Stop the sweep grid mid-way when the session wall-clock budget runs out. - _ss = extra.get("shared_state") or extra.get("state") - session_deadline_sec = _ss.grid_session_deadline_sec() if _ss is not None else None + # Stop the sweep grid mid-way when the session wall-clock budget runs out, + # and cap each variant at what the budget can still pay for. Admission is + # judged on the expected runtime rather than ``timeout_sec``, which is the + # catastrophic-hang backstop and would abandon the tail of the budget. + session_deadline_sec, variant_expected_sec = session_grid_bounds( + extra.get("shared_state") or extra.get("state") + ) try: # Pass resolved_model / resolved_gpu so variant servers inherit TP/precision. results = await run_grid( @@ -327,6 +332,7 @@ async def __call__(self, ctx) -> dict[str, Any]: result_dir=override_result_dir, serving_lease=sweep_lease, session_deadline_sec=session_deadline_sec, + variant_expected_sec=variant_expected_sec, ) finally: if sweep_lease is not None: From bf80774a56b16e4ea61895878d0692ddde7a942b Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 03:38:51 +0000 Subject: [PATCH 04/65] fix(loop): cancel the in-flight actions the session cannot wait for The earlier defences all act before or around a running action: admission refuses what cannot fit, the clamp bounds the grant, and the subprocess reaper stops the child trees that were handed a session deadline. Nothing reached an action already under way whose executor takes no deadline, or that is not spending its time in a subprocess at all -- and nothing reached any of them on SIGTERM, which only set the stop event. Since the pump does not return until everything it dispatched has finished, shutdown latency was the remaining runtime of the longest action, and teardown then closed the database out from under work still using it. The handles were the obstacle: they lived only in the pump's frame, so the sole caller able to cancel an action was the frame already blocked awaiting it. Keep them on the dispatcher instead, retiring each as its action ends, and the budget guard, the stop event and ``Coordinator.stop`` can all reach them. Two details worth naming. A spent budget spares the closing actions, because the reserve it trips on is held back precisely so they can run, and it does not stop the queue scan -- that scan is what cancels the rows it can no longer fit. And the runner now lands a cancelled task on a terminal state before re-raising: ``CancelledError`` is not an ``Exception``, so it skipped the handler that records failures, and the row would sit at ``running`` holding its lanes and reading as live work to every phase gate until the TTL sweep. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 261 +++++++++++++++++- .../orchestrator/loop/coordinator.py | 23 +- src/hyperloom/orchestrator/loop/dispatcher.py | 191 ++++++++++--- .../orchestrator/loop/sub_agent_runner.py | 17 ++ 4 files changed, 432 insertions(+), 60 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index a66ab7c233..1e44677dd3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -1,17 +1,30 @@ # SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Admission control for the session wall-clock budget. - -The first of the time-budget defences: an action whose expected cost cannot fit -the budget that is left never starts. Covers the pure fit decision, the -SharedState accessor both it and the grid deadline read, the dispatcher gate, the -three intent paths that share it, and the pre-dispatch backstop for a task that -sat queued until its budget drained. +"""The session wall-clock budget defences that live in the orchestrator loop. + +Two of the four layers are here, the ones outside the executors: + +* Admission -- an action whose expected cost cannot fit the budget that is left + never starts. Covers the pure fit decision, the SharedState accessor both it + and the grid deadline read, the dispatcher gate, the three intent paths that + share it, and the pre-dispatch backstop for a task that sat queued until its + budget drained. +* In-flight cancellation -- the backstop for work already running when the + budget goes or the process is asked to stop. Covers the handles the dispatcher + keeps, the cancellation itself, the closing-action carve-out, the task row + landing terminal instead of stranding at ``running``, and the pump and + ``Coordinator.stop`` paths that trigger it. + +The remaining two layers are enforced inside the executors and tested next to +them: the timeout clamp in ``test_explore_executor``, and the subprocess session +reaper in ``test_kill_spawned_server``. """ from __future__ import annotations +import asyncio + import pytest from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType @@ -23,6 +36,7 @@ from hyperloom.orchestrator.policy.gate import PolicyDenied from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan from hyperloom.orchestrator.state.shared_state import CLOSING_RESERVE_SEC, SharedState +from hyperloom.orchestrator.state.task_registry import Task # An action costing an hour at p50, so a short budget cannot fit it. _EXPENSIVE_ACTION = "kernel_opt" @@ -291,3 +305,236 @@ async def test_a_queued_task_that_still_fits_is_left_alone(self, coord: Coordina assert await coord.dispatcher._cancel_queued_task_over_budget(task) is False assert (await coord.tasks.get(task.task_id)).state == "queued" + +# One of the closing actions, exempt from the budget because the closing reserve +# is held back so it can run. +_CLOSING_ACTION = "report" +# The lane ``_CHEAP_ACTION`` holds while it runs, so a leak is observable. +_CHEAP_ACTION_LANE = "profile_lane" + + +def _never_finishes(started: asyncio.Event): + """Build an executor that only ever ends by being cancelled.""" + + async def _run(_ctx) -> dict: + started.set() + await asyncio.sleep(3600.0) + return {} + + return _run + + +async def _queue_action(coord: Coordinator, *, kind: str, key: str) -> tuple[Task, asyncio.Event]: + """Queue an action that only ends by being cancelled, with its real lanes.""" + started = asyncio.Event() + coord.sub.register_executor(kind, _never_finishes(started)) + lanes, ttl_sec = coord.dispatcher._registry_lanes_ttl(kind) + task, _ = await coord.tasks.create_or_return_existing( + kind=kind, + params={}, + idempotency_key=key, + requires_lanes=lanes, + lease_ttl_sec=ttl_sec, + ) + return task, started + + +async def _start_action(coord: Coordinator, *, kind: str, key: str) -> tuple[Task, asyncio.Task]: + """Dispatch the action with no pump running, for the pieces under it.""" + task, started = await _queue_action(coord, kind=kind, key=key) + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + assert [t.task_id for t, _, _ in spawned] == [task.task_id] + await asyncio.wait_for(started.wait(), timeout=5.0) + return task, spawned[0][1] + + +async def _start_action_under_pump( + coord: Coordinator, + *, + kind: str, + key: str, +) -> tuple[Task, asyncio.Task, asyncio.Task]: + """Let a running pump dispatch the action, the way a tick does. + + Returns ``(task, action task, pump task)``. The pump owns what it spawned, + so the triggers can only be tested against a pump that spawned the work. + """ + task, started = await _queue_action(coord, kind=kind, key=key) + pump = asyncio.create_task(coord._pump_dispatcher_once()) + await asyncio.wait_for(started.wait(), timeout=5.0) + return task, coord.dispatcher._inflight_actions[task.task_id][1], pump + + +async def _settle(atask: asyncio.Task) -> None: + """Wait for an action to finish unwinding, however it ended.""" + await asyncio.wait_for(asyncio.gather(atask, return_exceptions=True), timeout=5.0) + + +class TestInflightHandles: + """Something other than the pump has to be able to reach a running action.""" + + @pytest.mark.asyncio + async def test_a_running_action_is_reachable_by_task_id(self, coord: Coordinator): + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="h-live") + try: + assert coord.dispatcher._inflight_actions[task.task_id] == (_CHEAP_ACTION, atask) + finally: + atask.cancel() + await _settle(atask) + + @pytest.mark.asyncio + async def test_the_handle_retires_itself_when_the_action_ends(self, coord: Coordinator): + """Self-removal is what keeps the set from outliving the work.""" + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="h-retire") + atask.cancel() + await _settle(atask) + assert task.task_id not in coord.dispatcher._inflight_actions + + @pytest.mark.asyncio + async def test_an_action_that_finishes_normally_leaves_no_handle(self, coord: Coordinator): + coord.sub.register_executor(_CHEAP_ACTION, lambda _ctx: _done({"ok": True})) + task, _ = await coord.tasks.create_or_return_existing( + kind=_CHEAP_ACTION, + params={}, + idempotency_key="h-quick", + ) + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + await _settle(spawned[0][1]) + assert task.task_id not in coord.dispatcher._inflight_actions + + +async def _done(payload: dict) -> dict: + return payload + + +class TestCancellingInflightActions: + """The cancellation itself, and who it spares.""" + + @pytest.mark.asyncio + async def test_it_stops_the_action_and_names_what_it_stopped(self, coord: Coordinator): + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="c-stop") + cancelled = await coord.dispatcher.cancel_inflight_actions(reason="test") + assert cancelled == [task.task_id] + assert atask.cancelled() + + @pytest.mark.asyncio + async def test_the_closing_actions_can_be_spared(self, coord: Coordinator): + """Cancelling the report to save time would leave nothing to show for the run.""" + _, atask = await _start_action(coord, kind=_CLOSING_ACTION, key="c-exempt") + try: + assert ( + await coord.dispatcher.cancel_inflight_actions( + reason="test", + exempt=TIME_BUDGET_EXEMPT_ACTIONS, + ) + == [] + ) + assert not atask.done() + finally: + atask.cancel() + await _settle(atask) + + @pytest.mark.asyncio + async def test_cancelling_with_nothing_running_is_a_no_op(self, coord: Coordinator): + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [] + + @pytest.mark.asyncio + async def test_the_lane_is_free_again_afterwards(self, coord: Coordinator): + """A cancelled action that kept its lane would wedge every later one.""" + await _start_action(coord, kind=_CHEAP_ACTION, key="c-lane") + assert (await coord.locks.lane_holders()).get(_CHEAP_ACTION_LANE, 0) == 1 + await coord.dispatcher.cancel_inflight_actions(reason="test") + assert (await coord.locks.lane_holders()).get(_CHEAP_ACTION_LANE, 0) == 0 + + +class TestTheRunnerRecordsACancellation: + """``CancelledError`` is not an ``Exception``, so the runner must name it.""" + + @pytest.mark.asyncio + async def test_a_cancelled_action_does_not_stay_running(self, coord: Coordinator): + """A row stuck at ``running`` reads as live work to every phase gate.""" + task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="r-terminal") + atask.cancel() + await _settle(atask) + row = await coord.tasks.get(task.task_id) + assert row.state == "cancelled" + assert "cancelled_in_flight" in str(row.history) + + @pytest.mark.asyncio + async def test_the_cancellation_still_reaches_the_caller(self, coord: Coordinator): + """Recording it must not turn a cancellation into a normal return.""" + _task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="r-propagates") + atask.cancel() + await _settle(atask) + assert atask.cancelled() + + +class TestThePumpStopsWorkItCannotWaitFor: + """The trigger side: a spent budget, and a shutdown request.""" + + @staticmethod + def _quick_poll(coord: Coordinator) -> None: + coord._dispatcher_poll_sec = 0.05 + + @pytest.mark.asyncio + async def test_a_budget_that_runs_out_stops_the_action(self, coord: Coordinator): + self._quick_poll(coord) + _set_budget(coord, minutes=600) + task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-budget") + _set_budget(coord, minutes=600, elapsed_min=600.0) + + await asyncio.wait_for(pump, timeout=10.0) + + assert atask.cancelled() + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + + @pytest.mark.asyncio + async def test_the_closing_actions_keep_their_reserve(self, coord: Coordinator): + """The budget hits zero with the closing window still to spend.""" + self._quick_poll(coord) + _set_budget(coord, minutes=600, elapsed_min=600.0) + _task, atask, pump = await _start_action_under_pump(coord, kind=_CLOSING_ACTION, key="p-closing") + await asyncio.sleep(0.3) + + assert not atask.done() + + pump.cancel() + await _settle(pump) + + @pytest.mark.asyncio + async def test_a_shutdown_request_stops_the_action(self, coord: Coordinator): + """SIGTERM sets the stop event; before this it only stopped the tick.""" + self._quick_poll(coord) + _set_budget(coord, minutes=600) + _task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-signal") + coord._stop.set() + + await asyncio.wait_for(pump, timeout=10.0) + + assert atask.cancelled() + + @pytest.mark.asyncio + async def test_a_cancelled_pump_does_not_orphan_its_actions(self, coord: Coordinator): + """The handles live in the pump's frame; leaving must not drop them.""" + self._quick_poll(coord) + _set_budget(coord, minutes=600) + _task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-orphan") + + pump.cancel() + await _settle(pump) + + assert atask.cancelled() + assert coord.dispatcher._inflight_actions == {} + + +class TestCoordinatorStop: + """Teardown closes the database, so it cannot leave actions using it.""" + + @pytest.mark.asyncio + async def test_stop_cancels_the_actions_still_running(self, coord: Coordinator): + _task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="s-stop") + + await coord.stop() + + assert atask.cancelled() + assert coord.dispatcher._inflight_actions == {} diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index cbde600cea..6a7470642e 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1398,15 +1398,24 @@ def _reap_orphaned_servers_best_effort(self) -> None: # Lifecycle async def stop(self) -> None: - """Signal shutdown, cancel reactor tasks, finalize, and close the DB. - - Sets the stop event, cancels and awaits every running reactor task, - runs the Recipe KB T4 safety-net finalize hook when CLOSE never reached - a terminal publication status or its earlier attempt failed, then closes - the SQLite connection. Exceptions raised by reactor tasks during - teardown are logged, not propagated. + """Signal shutdown, cancel in-flight work, finalize, and close the DB. + + Sets the stop event, cancels and awaits the dispatched actions still + running plus every running reactor task, runs the Recipe KB T4 + safety-net finalize hook when CLOSE never reached a terminal + publication status or its earlier attempt failed, then closes the + SQLite connection. Exceptions raised by reactor tasks during teardown + are logged, not propagated. + + Dispatched actions are cancelled first and awaited: the stop event alone + only asks the loop to stop between ticks, so a teardown that skipped + them would close the database out from under work still using it. """ self._stop.set() + try: + await self.dispatcher.cancel_inflight_actions(reason="coordinator_stop") + except Exception: # noqa: BLE001 — teardown proceeds even if cancellation misbehaves + log.exception("Coordinator.stop: cancelling in-flight actions raised") for t in self._tasks_running: if not t.done(): t.cancel() diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 5790eafec4..ec10067c44 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -53,6 +53,13 @@ def __init__(self, coordinator) -> None: # Task ids already charged a failure by the dead-holder reclaim path, so # a late normal result for the same task cannot double-count it. self._dead_holder_accounted: set[str] = set() + # Handles on the actions currently running, ``task_id -> (kind, task)``. + # Kept on the collaborator and not only in the pump's frame: an action + # whose handle lives in a frame can only be stopped by the frame that is + # already blocked awaiting it, which is precisely the situation shutdown + # and an exhausted wall-clock budget have to break. Entries remove + # themselves in :meth:`_run_dispatched_with_gpu_release`. + self._inflight_actions: dict[str, tuple[str, asyncio.Task[SubAgentResult]]] = {} def __getattr__(self, name: str): return getattr(object.__getattribute__(self, "_coord"), name) @@ -120,6 +127,77 @@ def _dispatch_paused_for_phase_budget(self) -> bool: return False return remaining is not None and remaining <= 0.0 + async def cancel_inflight_actions( + self, + *, + reason: str, + exempt: frozenset[str] = frozenset(), + ) -> list[str]: + """Cancel the running dispatched actions and wait for them to unwind. + + The last of the wall-clock defences, and the only one that reaches work + already under way: admission refuses what cannot fit, the timeout clamp + bounds what does, and the subprocess reaper stops the child trees that + were handed a session deadline -- an executor that never takes that + argument, or that is not spending its time in a subprocess at all, has + nothing stopping it before it finishes. + + Cancellation is cooperative: every cancel is delivered before the first + await, so a caller that is itself being cancelled still leaves no action + running unattended. + + Args: + reason: Short cause, logged and used as evidence. + exempt: Action kinds to leave running -- the closing actions, when + the trigger is a budget that already reserved time for them. + + Returns: + list[str]: Task ids that were cancelled (empty when nothing ran). + """ + victims = [ + (task_id, kind, atask) + for task_id, (kind, atask) in self._inflight_actions.items() + if kind not in exempt and not atask.done() + ] + if not victims: + return [] + log.warning( + "dispatcher: cancelling %d in-flight action(s) [%s]: %s", + len(victims), + reason, + ", ".join(f"{kind}/{task_id[:12]}" for task_id, kind, _ in victims), + ) + for _task_id, _kind, atask in victims: + atask.cancel() + await asyncio.gather(*(atask for _tid, _kind, atask in victims), return_exceptions=True) + return [task_id for task_id, _kind, _atask in victims] + + async def _cancel_inflight_that_outlived_the_session(self) -> bool: + """Stop in-flight actions the session can no longer wait for. + + Two causes, both of which mean no result is coming: the process was + asked to shut down, or the wall-clock budget is spent. The budget case + spares :data:`TIME_BUDGET_EXEMPT_ACTIONS` because the closing reserve + this gate trips on exists precisely so those actions can run. + + Returns: + bool: ``True`` when the pump must not start anything new. Only a + shutdown says that; a spent budget does not, because the queue scan + is what cancels the rows it can no longer fit, and the closing + actions it exempts still have their reserve to run in. + """ + stop_event = getattr(self, "_stop", None) + if stop_event is not None and stop_event.is_set(): + await self.cancel_inflight_actions(reason="shutdown_requested") + return True + usable_sec = self.shared_state.session_budget_usable_sec() + if usable_sec is not None and usable_sec <= 0.0: + await self.cancel_inflight_actions( + reason="session_time_exhausted", + exempt=TIME_BUDGET_EXEMPT_ACTIONS, + ) + return False + async def _pump_dispatcher_once(self) -> None: """Dispatch queued tasks respecting per-lane capacity, re-scanning for newly-fittable tasks while in-flight tasks run. @@ -179,38 +257,50 @@ async def _pump_dispatcher_once(self) -> None: # fast task reaped before its queued->running transition is visible is # not re-dispatched. A task is dispatched at most once per pump. dispatched_ids: set[str] = set() - while True: - # Budget guard: stop launching NEW phase-scoped variants once the - # phase's cyclic budget is spent; drain in-flight then return. - if not self._dispatch_paused_for_phase_budget(): - spawned = await self._spawn_fitting_queued(exclude_ids=dispatched_ids) - dispatched_ids.update(t.task_id for t, _, _ in spawned) - inflight.extend(spawned) - if not inflight: - return - done, _pending = await asyncio.wait( - [atask for _, atask, _ in inflight], - timeout=self._dispatcher_poll_sec, - return_when=asyncio.FIRST_COMPLETED, - ) - if not done: - # Poll elapsed with no completion; re-scan in case a lane freed. - continue - remaining: list[tuple[Task, asyncio.Task[SubAgentResult], Any]] = [] - completed: list[tuple[Task, Any, Any]] = [] - for entry in inflight: - task, atask, gpu_lease = entry - if atask in done: - try: - maybe_result: Any = atask.result() - except (Exception, asyncio.CancelledError) as exc: # noqa: BLE001 — mirror gather(return_exceptions=True); capture task error + cancellation, never KeyboardInterrupt/SystemExit - maybe_result = exc - completed.append((task, maybe_result, gpu_lease)) - else: - remaining.append(entry) - inflight = remaining - for task, maybe_result, gpu_lease in completed: - await self._reap_dispatched_task(task, maybe_result, gpu_lease) + try: + while True: + # Wall-clock guard: a spent session budget (or a shutdown + # request) stops the actions already running, because waiting + # for them is what the budget no longer allows. + shutting_down = await self._cancel_inflight_that_outlived_the_session() + # Budget guard: stop launching NEW phase-scoped variants once the + # phase's cyclic budget is spent; drain in-flight then return. + if not shutting_down and not self._dispatch_paused_for_phase_budget(): + spawned = await self._spawn_fitting_queued(exclude_ids=dispatched_ids) + dispatched_ids.update(t.task_id for t, _, _ in spawned) + inflight.extend(spawned) + if not inflight: + return + done, _pending = await asyncio.wait( + [atask for _, atask, _ in inflight], + timeout=self._dispatcher_poll_sec, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + # Poll elapsed with no completion; re-scan in case a lane freed. + continue + remaining: list[tuple[Task, asyncio.Task[SubAgentResult], Any]] = [] + completed: list[tuple[Task, Any, Any]] = [] + for entry in inflight: + task, atask, gpu_lease = entry + if atask in done: + try: + maybe_result: Any = atask.result() + except (Exception, asyncio.CancelledError) as exc: # noqa: BLE001 — mirror gather(return_exceptions=True); capture task error + cancellation, never KeyboardInterrupt/SystemExit + maybe_result = exc + completed.append((task, maybe_result, gpu_lease)) + else: + remaining.append(entry) + inflight = remaining + for task, maybe_result, gpu_lease in completed: + await self._reap_dispatched_task(task, maybe_result, gpu_lease) + finally: + # The pump is the only owner of the actions it spawned. Leaving by + # any door other than the drained one -- cancelled at shutdown, or a + # raise from the bookkeeping -- would otherwise leave them running + # with every handle on them gone. A drained pump has nothing left to + # cancel, so the normal exit pays nothing for this. + await self.cancel_inflight_actions(reason="dispatcher_pump_exit") async def _reconcile_cancelled_policy_denied_integrate_tasks(self) -> list[str]: """Re-queue integrate_patch rows cancelled at dispatch when policy now passes. @@ -572,21 +662,17 @@ async def _spawn_fitting_queued( ) except Exception: # noqa: BLE001 - audit must never affect dispatch pass - spawned.append( - ( + atask = asyncio.create_task( + self._run_dispatched_with_gpu_release( task, - asyncio.create_task( - self._run_dispatched_with_gpu_release( - task, - prebound_lease=lease, - extra_context=extra_context, - gpu_lease=gpu_lease, - gpu_specialist_lease=gpu_specialist_lease, - ), - ), - gpu_lease, - ) + prebound_lease=lease, + extra_context=extra_context, + gpu_lease=gpu_lease, + gpu_specialist_lease=gpu_specialist_lease, + ), ) + self._inflight_actions[task.task_id] = (task.kind, atask) + spawned.append((task, atask, gpu_lease)) return spawned async def _run_dispatched_with_gpu_release( @@ -606,7 +692,9 @@ async def _run_dispatched_with_gpu_release( ``release`` is idempotent, so the release in :meth:`_reap_dispatched_task` remains harmless. When a Ray ``GpuSpecialistLease`` was acquired it is closed here too so - the ``num_gpus`` lease is released on every exit path. + the ``num_gpus`` lease is released on every exit path. The same finally + retires this task's ``_inflight_actions`` handle, so the set cannot + outlive the work it points at. Args: task: The dispatched task. @@ -628,6 +716,7 @@ async def _run_dispatched_with_gpu_release( extra_context=extra_context, ) finally: + self._inflight_actions.pop(task.task_id, None) if gpu_lease is not None: try: await self.gpu_specialist_pool.release(gpu_lease) @@ -864,6 +953,16 @@ async def _reap_dispatched_task( "dispatcher: failed to release GPU specialist lease for task=%s", task.task_id, ) + if isinstance(maybe_result, asyncio.CancelledError): + # Asked for, not gone wrong: the wall-clock defences stop + # in-flight actions on purpose. Logged as the deliberate act it + # is so a shutdown does not read as a crash. + log.warning( + "dispatcher: in-flight action task=%s kind=%s was cancelled", + task.task_id, + task.kind, + ) + continue if isinstance(maybe_result, BaseException): log.exception( "dispatcher: spawned task %s raised: %r", diff --git a/src/hyperloom/orchestrator/loop/sub_agent_runner.py b/src/hyperloom/orchestrator/loop/sub_agent_runner.py index de1bef9ebc..4fe85fd4be 100644 --- a/src/hyperloom/orchestrator/loop/sub_agent_runner.py +++ b/src/hyperloom/orchestrator/loop/sub_agent_runner.py @@ -11,6 +11,7 @@ from __future__ import annotations +import asyncio from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, Awaitable, Callable @@ -293,6 +294,22 @@ async def run_task( try: with progress_scope(self._progress_reporter(task.task_id)): result_payload = await runner(ctx) + except asyncio.CancelledError: + # Stopped from outside -- shutdown, or a wall-clock budget that + # ran out while this was running. ``CancelledError`` is not an + # ``Exception``, so it skips the handler below and nothing else + # would move the row off ``running``: it would hold its lanes + # and read as live work to every phase gate until the TTL sweep + # noticed. Recorded as ``cancelled`` rather than ``failed`` + # because the action was never given the chance to fail. + await self._transition_resilient( + task.task_id, + "cancelled", + evidence={"reason": "cancelled_in_flight"}, + context="executor_cancelled", + allow_terminal=True, + ) + raise except Exception as exc: # noqa: BLE001 — surface to task.history await self._transition_resilient( task.task_id, From ea95a73eab596af1042bae3f58178edc0ea4f284 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 03:54:16 +0000 Subject: [PATCH 05/65] fix(loop): let the wall-clock defences reach an inline action too ``run_action_now`` hands the coroutine to the coordinator loop, waits out the inline timeout, and then abandons the future -- the action keeps running by design, so the agent's turn is not held hostage by it. But abandoning the future abandoned the only handle on it, so nothing could stop it afterwards: not the spent budget, not shutdown, and not the teardown that closes the database it is still using. The blast radius was bounded (the inline whitelist admits only lane-light actions, and admission still gates them) but the task row would sit at ``running`` until the TTL sweep, and the run kept spending wall clock that was already accounted for elsewhere. Publish the coroutine's own task while the action runs, the same way the dispatched path does, and the existing cancellation reaches it unchanged. The sync bridge now names cancellation before its generic handler. It was already caught there -- but reported as ``errored``, which reads as a fault in the action rather than a deliberate stop. Both cancellation classes are caught because whether ``concurrent.futures.CancelledError`` and ``asyncio.CancelledError`` are the same class varies by Python version, and on the versions where they are, it is a ``BaseException`` that would escape the bridge and take the agent's turn down with it. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 78 +++++++++++++++++++ src/hyperloom/orchestrator/loop/dispatcher.py | 27 ++++++- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 1e44677dd3..17180596a4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -24,6 +24,7 @@ from __future__ import annotations import asyncio +import threading import pytest @@ -527,6 +528,83 @@ async def test_a_cancelled_pump_does_not_orphan_its_actions(self, coord: Coordin assert coord.dispatcher._inflight_actions == {} +class TestInlineActionsAreReachableToo: + """The inline path abandons its future, so it needs the same handle.""" + + @staticmethod + def _allow_inline(coord: Coordinator, monkeypatch) -> asyncio.Event: + """Register a never-finishing executor and clear the gates around it.""" + started = asyncio.Event() + coord.sub.register_executor(_CHEAP_ACTION, _never_finishes(started)) + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + _set_budget(coord, minutes=600) + return started + + @pytest.mark.asyncio + async def test_an_inline_action_that_outlived_its_caller_can_be_stopped( + self, + coord: Coordinator, + monkeypatch, + ): + """Before this, the only thing that ended it was the action itself.""" + started = self._allow_inline(coord, monkeypatch) + inline = asyncio.create_task(coord.dispatcher._run_action_now(_CHEAP_ACTION, {})) + await asyncio.wait_for(started.wait(), timeout=5.0) + task_id = next(iter(coord.dispatcher._inflight_actions)) + + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task_id] + + await _settle(inline) + assert inline.cancelled() + assert (await coord.tasks.get(task_id)).state == "cancelled" + assert coord.dispatcher._inflight_actions == {} + + @pytest.mark.asyncio + async def test_an_inline_action_that_finishes_leaves_no_handle( + self, + coord: Coordinator, + monkeypatch, + ): + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + _set_budget(coord, minutes=600) + coord.sub.register_executor(_CHEAP_ACTION, lambda _ctx: _done({"ok": True})) + + await coord.dispatcher._run_action_now(_CHEAP_ACTION, {}) + + assert coord.dispatcher._inflight_actions == {} + + @pytest.mark.asyncio + async def test_the_sync_bridge_reports_the_cancellation_instead_of_raising( + self, + coord: Coordinator, + monkeypatch, + ): + """It runs on an agent's turn thread, which a ``CancelledError`` would end.""" + started = self._allow_inline(coord, monkeypatch) + monkeypatch.setattr( + coord.dispatcher, + "_inline_action_whitelist", + lambda: frozenset({_CHEAP_ACTION}), + ) + coord._inline_fast_actions_enabled = True + coord._coordinator_loop = asyncio.get_running_loop() + + outcome: list[str] = [] + caller = threading.Thread( + target=lambda: outcome.append(coord.dispatcher._run_action_now_sync(_CHEAP_ACTION, {})), + daemon=True, + ) + caller.start() + try: + await asyncio.wait_for(started.wait(), timeout=5.0) + await coord.dispatcher.cancel_inflight_actions(reason="test") + await asyncio.to_thread(caller.join, 5.0) + finally: + caller.join(5.0) + + assert outcome and "was cancelled" in outcome[0] + + class TestCoordinatorStop: """Teardown closes the database, so it cannot leave actions using it.""" diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index ec10067c44..d6747f09b6 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -8,6 +8,7 @@ import hashlib import json import os +from concurrent.futures import CancelledError as FuturesCancelledError from concurrent.futures import TimeoutError as FuturesTimeoutError from typing import Any from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType @@ -59,7 +60,7 @@ def __init__(self, coordinator) -> None: # already blocked awaiting it, which is precisely the situation shutdown # and an exhausted wall-clock budget have to break. Entries remove # themselves in :meth:`_run_dispatched_with_gpu_release`. - self._inflight_actions: dict[str, tuple[str, asyncio.Task[SubAgentResult]]] = {} + self._inflight_actions: dict[str, tuple[str, asyncio.Task[Any]]] = {} def __getattr__(self, name: str): return getattr(object.__getattribute__(self, "_coord"), name) @@ -1576,6 +1577,16 @@ def _run_action_now_sync( "get_recent_outcomes or the next-tick inbox for its " "delegated_result)" ) + except (FuturesCancelledError, asyncio.CancelledError): + # The wall-clock defences stopped it on purpose. Named before the + # generic handler, which would file a deliberate stop as ``errored`` + # and read as a fault in the action. Both classes are caught because + # whether the two are the same one varies by Python version, and on + # the versions where they are, it is a ``BaseException`` that would + # escape this bridge entirely and take the agent's turn down with it. + return ( + f"(run_action_now: {name!r} was cancelled — the session is shutting down or out of wall-clock budget)" + ) except Exception as exc: # noqa: BLE001 — never crash the turn log.exception("run_action_now: inline run of %r failed", name) return f"(run_action_now: {name!r} errored: {exc!r})" @@ -1642,7 +1653,19 @@ async def _run_action_now( f"(run_action_now: an identical {action_name!r} task is " f"already {task.state!r}; wait for its delegated_result)" ) - result = await self.sub.run_task(task) + # Publish a handle for as long as the action runs. This path abandons its + # future once the caller's inline wait elapses -- the action keeps going + # by design, so without a handle nothing could stop it, including a + # teardown that is about to close the database underneath it. The handle + # is this coroutine's own task: cancelling it drops the audit publication + # below, which the runner's terminal transition already accounts for. + inline_handle = asyncio.current_task() + if inline_handle is not None: + self._inflight_actions[task.task_id] = (task.kind, inline_handle) + try: + result = await self.sub.run_task(task) + finally: + self._inflight_actions.pop(task.task_id, None) result_payload = { "task_id": task.task_id, "kind": task.kind, From 180ad5ce234221afe1a1d037f80682c7a51e301f Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 04:00:14 +0000 Subject: [PATCH 06/65] fix(loop): declare the time-budget helpers in the module interface CodeQL read ``TIME_BUDGET_EXEMPT_ACTIONS`` as unused. It is not -- the dispatcher imports it for both the admission gate and the in-flight carve-out -- but the module publishes an ``__all__`` that did not list it, so the interface the module declares disagreed with the one it actually has, and a reader (human or tool) that trusts the declaration concludes nobody uses it. List both new names. The test constant ``_EXPENSIVE_COST_MIN`` was genuinely unused: the assertion it was written for spelled the cost out as a literal instead. Use it, so the documented cost and the asserted message cannot drift apart. Co-authored-by: Cursor --- .../inference_optimizer/tests/test_session_time_budget.py | 2 +- src/hyperloom/orchestrator/loop/coordinator_helpers.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 17180596a4..3915e153a3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -130,7 +130,7 @@ def test_an_action_too_big_for_the_budget_is_denied(self, coord: Coordinator): denied = coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) assert isinstance(denied, PolicyDenied) assert denied.rule == "time_budget" - assert "60 min" in str(denied) + assert f"{_EXPENSIVE_COST_MIN:.0f} min" in str(denied) assert "report" in str(getattr(denied, "hint", "")) def test_an_action_that_fits_is_admitted(self, coord: Coordinator): diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index 16092e5be0..0e4b0ecb4c 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -31,8 +31,10 @@ # Constants below are read from other modules; listed here to mark them as # intentionally exported. __all__ = [ + "TIME_BUDGET_EXEMPT_ACTIONS", "_GEAK_MEASUREMENT_DIVERGENCE_WARN_PCT", "_MIN_KERNEL_ENGAGED_GAIN_PCT", + "action_fits_time_budget", "coerce_needs_gpu", ] From ec558e4d3e8b87e730e38a24d95ccce5e98a3e4d Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 15:08:14 +0800 Subject: [PATCH 07/65] fix(close): give the sequencer a report task it can still run CLOSE reached for a report task by asking "was one enqueued?" when it needed "is one runnable?". Both reuse paths -- the ``closing_report_task_id`` the wall-clock deadline path writes, and ``create_or_return_existing`` -- read ``task.state`` only to log it, so a cancelled row came straight back and ``run_task``'s ``queued -> running`` failed the registry's terminal check: IllegalTransition: cannot transition ... from 'cancelled' to 'running' The session that hits this is the one whose report matters most, and it lost ``final.md`` entirely (#1167). This lands ahead of the enforcement layers on purpose. Cancelling in-flight work at the deadline is the point of those layers, and it widens the set of ways a report task can be sitting in ``cancelled`` when CLOSE arrives -- so without the guard the crash gets easier to hit, not harder. A dead row now falls through to a fresh enqueue under a suffixed idempotency key, shared by the report and session_breakdown steps rather than written twice, and ``_run_close_task`` reports a terminal row instead of running it so no path can hand one to ``run_task`` again. Also adds the test that would have caught the AttributeError a field run hit on ``_record_explore_variant_failures``: every ``_DELEGATED`` entry must resolve on its collaborator. It found two stale entries (``_resolve_issue_canonical``, ``_wait_for_task_terminal``), both naming methods that no longer exist anywhere; removed. Co-authored-by: Cursor --- .../tests/test_close_phase_sequencer.py | 84 ++++++++- .../test_coordinator_async_batch2_unit.py | 16 ++ .../orchestrator/loop/coordinator.py | 4 +- src/hyperloom/orchestrator/phases/close.py | 159 +++++++++++++++--- 4 files changed, 233 insertions(+), 30 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py index 4772c6b4a4..56cd5a1b9e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py +++ b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py @@ -83,7 +83,8 @@ async def create_or_return_existing( row = _StubTaskRow( task_id=tid, kind=kind, - state="succeeded", + # Matches the registry's INSERT: a new row is always ``queued``. + state="queued", params=dict(params), idempotency_key=idempotency_key, ) @@ -249,6 +250,87 @@ async def test_enqueue_internal_report_task_reuses_existing(coord): assert "internal-report-close_phase_entry" not in coord.tasks._by_key +@pytest.mark.asyncio +async def test_enqueue_internal_report_task_replaces_a_cancelled_one(coord): + """A report the deadline path enqueued and then cancelled cannot be run; the sequencer needs a live one. + + Regression for the ``cannot transition from 'cancelled' to 'running'`` + crash: the helper used to answer "was one enqueued?" when the caller needs + "is one runnable?". + """ + dead = _StubTaskRow( + task_id="wallclock-report", + kind="report", + state="cancelled", + params={}, + idempotency_key="closing-report-1234", + ) + coord.tasks._by_id["wallclock-report"] = dead + coord.shared_state.closing_report_task_id = "wallclock-report" + + task = await coord._enqueue_internal_report_task(reason="close_phase_entry") + + assert task is not dead + assert task.state == "queued" + assert coord.shared_state.closing_report_task_id == task.task_id + + +@pytest.mark.asyncio +async def test_enqueue_internal_report_task_retries_past_a_dead_idempotent_row(coord): + """The idempotency key itself can resolve to a corpse; the retry key mints a runnable row.""" + coord.tasks._by_key["internal-report-close_phase_entry"] = _StubTaskRow( + task_id="dead-idempotent", + kind="report", + state="cancelled", + params={}, + idempotency_key="internal-report-close_phase_entry", + ) + + task = await coord._enqueue_internal_report_task(reason="close_phase_entry") + + assert task.task_id != "dead-idempotent" + assert task.idempotency_key == "internal-report-close_phase_entry-retry" + assert task.state == "queued" + + +@pytest.mark.asyncio +async def test_close_sequencer_still_reports_when_the_first_report_task_was_cancelled(coord): + """End to end: a session that hits its deadline is the one whose report matters most.""" + coord.shared_state.phase_history = [_close_phase_history_row()] + coord.tasks._by_id["wallclock-report"] = _StubTaskRow( + task_id="wallclock-report", + kind="report", + state="cancelled", + params={}, + idempotency_key="closing-report-1234", + ) + coord.shared_state.closing_report_task_id = "wallclock-report" + + await coord._on_enter_close(from_phase="SWEEP") + + rows = coord.shared_state.phase_history[-1]["evidence"]["close_steps"] + by_step = {r["step"]: r for r in rows} + assert by_step["report"]["status"] == "done" + assert [t.task_id for t in coord.sub.run_calls] != ["wallclock-report"] + + +@pytest.mark.asyncio +async def test_a_terminal_task_is_reported_not_run(coord): + """Backstop: nothing hands a terminal row to ``run_task``, whose ``queued -> running`` would raise.""" + done = _StubTaskRow( + task_id="already-done", + kind="report", + state="succeeded", + params={}, + idempotency_key="internal-report-close_phase_entry", + ) + + state = await coord._run_close_task(done, step="1 (report)") + + assert state == "succeeded" + assert coord.sub.run_calls == [] + + @pytest.mark.asyncio async def test_enqueue_internal_session_breakdown_task(coord): task = await coord._enqueue_internal_session_breakdown_task( diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 0c818c0cc1..407b497fcc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -189,6 +189,22 @@ def coord(session_dir) -> Coordinator: return Coordinator(session_dir, backends=_build_backends()) +def test_every_delegated_name_resolves_on_its_collaborator(coord: Coordinator) -> None: + """A map entry naming a method its collaborator never defined is a crash at first call, not at import. + + A field run lost every EXPLORE variant-failure record to exactly that: the + entry was there, the method was not, and ``__getattr__`` raised only once + the reap loop reached for it. + """ + unresolved = [] + for name in Coordinator._DELEGATED: + try: + getattr(coord, name) + except AttributeError as exc: + unresolved.append(f"{name}: {exc}") + assert unresolved == [] + + # -- _context_inbox_reader -------------------------------------------------- def test_context_inbox_reader_empty(coord: Coordinator) -> None: out = coord._context_inbox_reader() diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 6a7470642e..57b818efb8 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -943,8 +943,10 @@ def router(self) -> IntentRouter: "_session_integrated_kernel_patch": "phase_close", "_maybe_run_close_post_opt_roofline": "phase_close", "_on_enter_close": "phase_close", + "_enqueue_runnable_internal_task": "phase_close", "_enqueue_internal_report_task": "phase_close", "_enqueue_internal_session_breakdown_task": "phase_close", + "_run_close_task": "phase_close", "_record_close_step": "phase_close", "_enter_closing_phase": "phase_close", "_closing_report_terminal": "phase_close", @@ -1128,7 +1130,6 @@ def router(self) -> IntentRouter: "_current_primary_gap": "conversation", "_recent_proposed_variants": "conversation", "_priors_match_advisory_block": "conversation", - "_resolve_issue_canonical": "proposals", "_workload_canonical_id": "proposals", "_read_local_recipe_row": "proposals", "_extract_kept_best_config": "proposals", @@ -1140,7 +1141,6 @@ def router(self) -> IntentRouter: "_record_proposal_task_map": "proposals", "_registry_lanes_ttl": "dispatcher", "_cycle_idem_suffix": "dispatcher", - "_wait_for_task_terminal": "dispatcher", "_cursor_advance_to_latest": "dispatcher", "_dispatch_paused_for_phase_budget": "dispatcher", "_pump_dispatcher_once": "dispatcher", diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index 9f8917f357..3a5b344ff3 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -18,6 +18,34 @@ log = _logging.getLogger(__name__) +# Terminal task states, split by what the CLOSE sequencer can still do with a +# task in one. Both are dead ends for ``run_task``: ``enter_running`` refuses +# any terminal row, so handing one over takes the close step down with it. +# ``succeeded`` means the step's artifact is already on disk (skip it); +# ``cancelled``/``failed`` mean the work never happened and the sequencer needs +# a fresh row (re-enqueue under a distinct idempotency key). +_TASK_STATE_DONE: str = "succeeded" +_DEAD_TASK_STATES: frozenset[str] = frozenset({"cancelled", "failed"}) +# Appended to a close step's idempotency key when its first row is dead, so +# ``create_or_return_existing`` mints a new task instead of returning the +# corpse. +_RETRY_KEY_SUFFIX: str = "retry" + + +def _task_is_dead(task: Task | None) -> bool: + """True when ``task`` reached a terminal state without producing its artifact. + + Args: + task: The task to inspect; ``None`` reads as not dead (there is nothing + to reuse, which the caller handles as a fresh enqueue anyway). + + Returns: + ``True`` when the task is ``cancelled`` or ``failed``. + """ + if task is None: + return False + return str(getattr(task, "state", "") or "") in _DEAD_TASK_STATES + class ClosePhase(PhaseHandler): """Extracted phase handler; delegates unknown attrs to its Coordinator.""" @@ -189,8 +217,7 @@ async def _on_enter_close(self, *, from_phase: str) -> None: report_task = await self._enqueue_internal_report_task( reason="close_phase_entry", ) - report_result = await self.sub.run_task(report_task) - terminal_state = report_result.state + terminal_state = await self._run_close_task(report_task, step="1 (report)") if terminal_state in {"succeeded", None}: await self._record_close_step( "report", @@ -243,8 +270,7 @@ async def _on_enter_close(self, *, from_phase: str) -> None: bd_task = await self._enqueue_internal_session_breakdown_task( reason="close_phase_entry", ) - bd_result = await self.sub.run_task(bd_task) - terminal_state = bd_result.state + terminal_state = await self._run_close_task(bd_task, step="2 (session_breakdown)") if terminal_state in {"succeeded", None}: await self._record_close_step( "session_breakdown", @@ -340,6 +366,61 @@ async def _on_enter_close(self, *, from_phase: str) -> None: await self._record_close_step("done", status="done") log.info("CLOSE 7-step sequencer complete") + async def _enqueue_runnable_internal_task( + self, + *, + kind: str, + params: dict[str, Any], + idempotency_key: str, + ) -> Task: + """Enqueue a Coordinator-internal close-step task the sequencer can still run. + + Idempotency is what lets the wall-clock deadline path and the CLOSE + sequencer reach for the same task instead of writing the artifact + twice. Its cost is that the key can resolve to a row that is already + terminal — most often ``cancelled``, because the deadline path that + enqueued the task is also the path that cancels in-flight work. Such a + row cannot be run, so one retry under a suffixed key mints a fresh one. + + Args: + kind: Task kind (``report`` / ``session_breakdown``). + params: Task parameters, identical across attempts. + idempotency_key: The step's key; the retry appends a suffix. + + Returns: + The created or reused :class:`Task`. Still terminal only when the + retry also resolved to a dead row, which + :meth:`_run_close_task` reports rather than runs. + """ + task: Task | None = None + for key in (idempotency_key, f"{idempotency_key}-{_RETRY_KEY_SUFFIX}"): + task, was_existing = await self.tasks.create_or_return_existing( + kind=kind, + params=params, + idempotency_key=key, + requires_lanes=[], + allowed_tools=["Read"], + side_effects=["writes_results"], + lease_ttl_sec=120, + ) + if not was_existing: + return task + if not _task_is_dead(task): + log.info( + "internal-%s task reused (idempotent: task_id=%s, state=%s)", + kind, + task.task_id, + task.state, + ) + return task + log.warning( + "internal-%s task %s is %s and cannot be run; re-enqueueing under a fresh key", + kind, + task.task_id, + task.state, + ) + return task # type: ignore[return-value] # loop body always binds it + async def _enqueue_internal_report_task( self, *, @@ -357,8 +438,12 @@ async def _enqueue_internal_report_task( """ existing_id = (self.shared_state.closing_report_task_id or "").strip() if existing_id: + task = None try: task = await self.tasks.get(existing_id) + except Exception: # noqa: BLE001 — TaskNotFound + friends + pass # Stale id; fall through to fresh enqueue. + if task is not None and not _task_is_dead(task): log.info( "internal-report task already enqueued by wall-clock " "deadline path (task_id=%s, state=%s); sequencer will " @@ -367,9 +452,15 @@ async def _enqueue_internal_report_task( task.state, ) return task - except Exception: # noqa: BLE001 — TaskNotFound + friends - # Stale id; fall through to fresh enqueue. - pass + # Dead or vanished: the id names a report that will never be + # written, so drop it before the fresh enqueue mirrors its own. + if task is not None: + log.warning( + "internal-report task %s recorded on closing_report_task_id is %s; re-enqueueing", + task.task_id, + task.state, + ) + self.shared_state.closing_report_task_id = "" params: dict[str, Any] = { "source": "coordinator_internal", @@ -377,14 +468,10 @@ async def _enqueue_internal_report_task( "session_dir": str(self.session_dir), "max_highlights": 50, } - task, was_existing = await self.tasks.create_or_return_existing( + task = await self._enqueue_runnable_internal_task( kind="report", params=params, idempotency_key=f"internal-report-{reason}", - requires_lanes=[], - allowed_tools=["Read"], - side_effects=["writes_results"], - lease_ttl_sec=120, ) # Mirror onto closing_report_task_id. if not self.shared_state.closing_report_task_id: @@ -393,12 +480,6 @@ async def _enqueue_internal_report_task( self.shared_state.save(self.session_dir) except Exception: # noqa: BLE001 log.exception("internal-report: closing_report_task_id save failed") - if was_existing: - log.info( - "internal-report task reused (idempotent: task_id=%s, state=%s)", - task.task_id, - task.state, - ) return task async def _enqueue_internal_session_breakdown_task( @@ -420,22 +501,46 @@ async def _enqueue_internal_session_breakdown_task( "reason": str(reason), "session_dir": str(self.session_dir), } - task, was_existing = await self.tasks.create_or_return_existing( + return await self._enqueue_runnable_internal_task( kind="session_breakdown", params=params, idempotency_key=f"internal-session_breakdown-{reason}", - requires_lanes=[], - allowed_tools=["Read"], - side_effects=["writes_results"], - lease_ttl_sec=120, ) - if was_existing: + + async def _run_close_task(self, task: Task, *, step: str) -> str | None: + """Run one close-step task and return its terminal state. + + ``run_task`` transitions ``queued -> running``, which the registry + refuses for a row that is already terminal — and refuses correctly: + the rejection is the double-spawn guard. So a terminal row is reported + here instead of run, which keeps one dead task from taking down the + step that was supposed to salvage the session. + + Args: + task: The task to run. + step: Close-step label, for logging. + + Returns: + The task's terminal state. + """ + state = str(getattr(task, "state", "") or "") + if state == _TASK_STATE_DONE: log.info( - "internal-session_breakdown task reused (idempotent: task_id=%s, state=%s)", + "CLOSE step %s: task_id=%s already succeeded; keeping its artifact", + step, task.task_id, - task.state, ) - return task + return state + if state in _DEAD_TASK_STATES: + log.warning( + "CLOSE step %s: task_id=%s is %s and cannot be run; recording the step as failed", + step, + task.task_id, + state, + ) + return state + result = await self.sub.run_task(task) + return result.state async def _record_close_step( self, From 8291c0c5161732cb5d4aa64ef0ab4f13a4307269 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 15:32:57 +0800 Subject: [PATCH 08/65] fix(prelude): stop preparation spending the optimization phases' budget PRELUDE's budget share is 3%, and nothing enforced it: exit_normal_prelude tests whether a baseline landed and nothing else, so the phase ran to whatever its contents happened to cost. Two sessions on unrelated models -- different quantization, different PRELUDE composition, one baseline-dominated and one split with a TraceLens roofline -- spent 73.8% and 72.8% of a three-hour budget in it, and both reached FRAMEWORK_AGENT with ~47 minutes against its 108-minute entry threshold. Landing on time would not have saved either run: they were unrecoverable the moment PRELUDE ended. So PRELUDE now decides before it spends. ``prelude_can_afford`` bounds an optional arm by the tighter of its own share and what the optimization phases have reserved, and two arms consult it: - the initial roofline/profile, which in one run took 81 minutes -- 45% of the session -- and was already known-degraded at second zero because the vLLM version patch had failed; - the baseline's measured round, which cost the other run 59 minutes to move the anchor 0.2%, and whose absence leaves the single-round baseline the codebase already supports rather than a new half-measured state. Both estimates come from what this session measured, not from the action catalog: the catalog's p50s (baseline 5 min, roofline 8 min) are calibrated on small models, and a guard reading them would have admitted both arms. ``time_exhausted_during_prelude`` finally gets a producer. It was in the terminal-reason vocabulary and in the report's glossary, but nothing ever assigned it -- the state machine had a word for this failure and no way to reach it. And PRELUDE's normal exit now states whether the budget it leaves behind still funds one benchmark round, which is the plain sentence neither field session ever got. Extracts ``append_phase_evidence_row`` so the dropped-arm record and the CLOSE step record share one definition of "append to the current phase record". Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 64 ++++++ .../tests/test_phase_state_machine.py | 87 ++++++++ .../tests/test_warm_replay.py | 45 ++++ .../actions/executors/baseline.py | 63 ++++++ .../orchestrator/loop/coordinator.py | 2 + src/hyperloom/orchestrator/phases/close.py | 21 +- .../orchestrator/phases/machine_state.py | 211 +++++++++++++++++- src/hyperloom/orchestrator/phases/prelude.py | 69 ++++++ 8 files changed, 546 insertions(+), 16 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 6215789206..bd0cd78ead 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -329,6 +329,70 @@ def test_a_failing_warmup_round_still_reported_that_it_started(tmp_path): assert [(n["label"], n["status"]) for n in notes] == [("warmup", "started")] +def _prelude_shared_state(*, spent_sec: float, usable_sec: float) -> SimpleNamespace: + """A PRELUDE session state with an explicit clock, as the budget policy reads it.""" + return SimpleNamespace( + baseline_double_run=True, + phase="PRELUDE", + max_minutes=180, + phase_elapsed_totals={"PRELUDE": spent_sec}, + phase_started_unix=0.0, + session_budget_usable_sec=lambda: usable_sec, + ) + + +def _run_double_run_baseline(tmp_path, shared_state) -> dict: + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + output_dir = tmp_path / "ws" + fake_run, state = _cold_then_hot_fake_run() + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=shared_state, + ) + ctx = _make_ctx({"output_dir": str(output_dir), "timeout_sec": 10, "gpu_type": "mi300x"}) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + result["_rounds_run"] = state["calls"] + return result + + +def test_measured_round_is_dropped_when_preparation_has_spent_the_budget(tmp_path): + """The MiniMax-M2 shape: a 66-minute warmup, then a second round the session cannot use. + + The warmup already carries accuracy and a throughput figure, so dropping + the measured round leaves the single-round baseline the codebase supports + rather than a half-measured state. + """ + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(spent_sec=10_000.0, usable_sec=500.0), + ) + + assert result["status"] == "succeeded" + assert result["_rounds_run"] == 1 + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert "baseline_measure_round_dropped_low_budget" in result["nonfatal_warnings"] + assert result["measure_round_dropped"]["bound"] == "prelude_ceiling" + + +def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): + """The guard must not turn every double-run into a single one.""" + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(spent_sec=300.0, usable_sec=10_000.0), + ) + + assert result["_rounds_run"] == 2 + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + assert "measure_round_dropped" not in result + + def test_deferred_accuracy_skips_eval_when_hot_throughput_regresses( tmp_path, ): diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py index 5b833e28c8..d59a60f3e4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py @@ -220,6 +220,93 @@ def test_exit_normal_prelude_blocked_while_warm_replay_in_flight(): assert out is not None and out[0] == "prelude_done" +def _prelude_state( + *, + max_minutes: int = 180, + spent_sec: float = 0.0, + usable_sec: float | None = None, + baseline_tput: float = 0.0, + baseline_runtime_sec: float = 0.0, +) -> SimpleNamespace: + """A PRELUDE-phase state with an explicit clock, as the budget policy reads it.""" + return SimpleNamespace( + phase="PRELUDE", + max_minutes=max_minutes, + phase_elapsed_totals={"PRELUDE": spent_sec}, + phase_started_unix=0.0, + baseline_tput=baseline_tput, + baseline_runtime_sec=baseline_runtime_sec, + session_budget_usable_sec=lambda: usable_sec, + ) + + +def test_prelude_can_afford_an_arm_the_budget_still_covers(): + """The normal case must be untouched: a cheap arm early in a session runs.""" + state = _prelude_state(spent_sec=600.0, usable_sec=10_000.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=300.0) + assert affordable is True + assert evidence["prelude_spent_sec"] == 600.0 + + +def test_prelude_refuses_an_arm_past_its_own_ceiling(): + """The Qwen3.5-397B shape: 51 minutes of baseline, then a roofline that costs another 45+.""" + state = _prelude_state(spent_sec=3090.0, usable_sec=7700.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=2706.0) + assert affordable is False + assert evidence["bound"] == "prelude_ceiling" + # 40% of 180 minutes is 4320s; 3090 spent leaves 1230s, well under the arm. + assert evidence["affordable_sec"] == pytest.approx(1230.0) + + +def test_prelude_refuses_an_arm_that_would_eat_the_optimization_reserve(): + """Under the phase ceiling but over the session reserve: the tighter bound wins.""" + state = _prelude_state(spent_sec=60.0, usable_sec=6000.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=3000.0) + assert affordable is False + assert evidence["bound"] == "optimization_reserve" + + +def test_prelude_budget_policy_is_inert_without_a_clock(): + """An unbounded run has no budget to protect, so nothing is refused.""" + state = _prelude_state(max_minutes=0, usable_sec=None) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=99_999.0) + assert affordable is True + assert evidence["reason"] == "unbounded_budget" + + +def test_time_exhausted_during_prelude_finally_has_a_producer(): + """The reason was in the vocabulary and in the report glossary with no code path to it.""" + state = _prelude_state(spent_sec=10_800.0, usable_sec=0.0) + out = phase_state.compute_next_phase(state, kernel_enabled=True) + assert out is not None + next_phase, reason, evidence = out + assert (next_phase, reason) == ("CLOSE", "time_exhausted_during_prelude") + assert evidence["terminal"] is True + assert phase_state.is_valid_stop_reason(reason) + + +def test_a_landed_baseline_outranks_the_exhausted_clock(): + """With a baseline in hand the run has something to optimize; the later phases judge for themselves.""" + state = _prelude_state(spent_sec=10_800.0, usable_sec=0.0, baseline_tput=1074.7) + out = phase_state.compute_next_phase(state, kernel_enabled=True) + assert out is not None + assert out[1] == "prelude_done" + + +def test_prelude_exit_states_whether_one_optimization_round_still_fits(): + """The plain statement neither field session ever got: preparation spent the run.""" + state = _prelude_state(baseline_tput=1074.7, baseline_runtime_sec=2705.7, usable_sec=2796.0) + out = phase_state.exit_normal_prelude(state) + assert out is not None + evidence = out[1] + assert evidence["fits_one_optimization_round"] is True + assert evidence["affordable_rounds"] == pytest.approx(1.03, abs=0.01) + + state.session_budget_usable_sec = lambda: 1200.0 + evidence = phase_state.exit_normal_prelude(state)[1] + assert evidence["fits_one_optimization_round"] is False + + def test_exit_terminal_prelude_after_three_baseline_failures(): state = SimpleNamespace(baseline_failure_streak=2) assert phase_state.exit_terminal_prelude(state) is None diff --git a/src/hyperloom/inference_optimizer/tests/test_warm_replay.py b/src/hyperloom/inference_optimizer/tests/test_warm_replay.py index b1054c1852..8dd1fe525d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_warm_replay.py +++ b/src/hyperloom/inference_optimizer/tests/test_warm_replay.py @@ -1436,6 +1436,51 @@ async def test_prelude_initial_analysis_enqueued_after_warm_replay_finishes( assert coord.shared_state.auto_roofline_pending_task_id +@pytest.mark.asyncio +async def test_prelude_initial_analysis_dropped_when_it_would_cost_the_optimization_phases( + tmp_path, +): + """A roofline is worth an hour only if the session can still use what it finds. + + The Qwen3.5-397B shape: 51 minutes of baseline, then an 81-minute TraceLens + arm that left FRAMEWORK_AGENT 46 minutes against its 108-minute threshold. + """ + coord = _make_coord(tmp_path) + state = coord.shared_state + state.baseline_tput = 600.0 + state.max_minutes = 180 + state.baseline_runtime_sec = 2705.7 + state.phase_elapsed_totals = {"PRELUDE": 3090.0} + state.phase_history = [{"to_phase": "PRELUDE", "evidence": {}}] + state.session_budget_usable_sec = lambda: 7700.0 + + await coord._maybe_enqueue_prelude_initial_analysis_after_baseline() + + assert coord.tasks.calls == [] + assert not coord.shared_state.auto_roofline_pending_task_id + dropped = state.phase_history[-1]["evidence"]["budget_dropped_arms"] + assert dropped[0]["arm"] == "initial_analysis" + assert dropped[0]["expected_cost_sec"] == pytest.approx(2705.7) + + +@pytest.mark.asyncio +async def test_prelude_initial_analysis_runs_when_the_budget_covers_it(tmp_path): + """Same wiring, ordinary session: the arm is not dropped just because the guard exists.""" + coord = _make_coord(tmp_path) + state = coord.shared_state + state.baseline_tput = 600.0 + state.max_minutes = 180 + state.baseline_runtime_sec = 300.0 + state.phase_elapsed_totals = {"PRELUDE": 320.0} + state.phase_history = [{"to_phase": "PRELUDE", "evidence": {}}] + state.session_budget_usable_sec = lambda: 10_300.0 + + await coord._maybe_enqueue_prelude_initial_analysis_after_baseline() + + assert len(coord.tasks.calls) == 1 + assert coord.shared_state.auto_roofline_pending_task_id + + def test_prelude_bootstrap_runs_on_positive_baseline(tmp_path): coord = _make_coord(tmp_path) assert coord._should_run_prelude_bootstrap(600.0) is True diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index a48de07d13..9fb0b7958e 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -36,6 +36,7 @@ from ...framework.paths import resolve_session_framework_root from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import heartbeat_while_output_flows, report_progress +from ...phases import machine_state as _phase_state from . import _server_lifecycle as _lifecycle from ._file_lock import best_effort_file_lock from ._aiter_jit import ( @@ -3035,6 +3036,28 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: runtime_sec=warmup_runtime, ) + if not defer_accuracy_until_after_measure: + affordable, budget_evidence = self._measure_round_affordable( + warmup_runtime_sec=warmup_runtime, + ctx_extra=extra, + ) + if not affordable: + log.warning( + "baseline_executor: skipping the measured round — the warmup " + "took %.0fs and only %.0fs of preparation budget is left " + "(bound=%s). Keeping the warmup as the baseline; it is the " + "cold anchor a single-round baseline would have produced.", + float(warmup_runtime or 0.0), + budget_evidence.get("affordable_sec", 0.0), + budget_evidence.get("bound", ""), + ) + warmup_result.setdefault("nonfatal_warnings", []) + warmup_result["nonfatal_warnings"].append( + "baseline_measure_round_dropped_low_budget", + ) + warmup_result["measure_round_dropped"] = budget_evidence + return warmup_result + # Round 2 (measured): re-attach to the hot server (client only). # Warm re-attach is intentional — all comparison points (baseline, # explore decision, stack_rebench, and their grading anchor) are @@ -3201,6 +3224,46 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() + def _measure_round_affordable( + self, + *, + warmup_runtime_sec: Any, + ctx_extra: dict[str, Any] | None = None, + ) -> tuple[bool, dict[str, Any]]: + """Whether PRELUDE's remaining budget still covers the measured round. + + The double-run exists to keep the baseline off cold-start numbers, and + on a normal model it costs minutes. On a large one it does not: two + field sessions spent 51 and 125 minutes on the pair, and in both the + cold and hot figures landed within 1% of each other — a correction the + session then had no time left to use. Dropping the second round when it + no longer fits leaves the single-round baseline the codebase already + supports and treats as valid, rather than a new half-measured state. + + The warmup's own runtime is the estimate: the measured round re-attaches + to the hot server, so it is an upper bound rather than a guess. Only + PRELUDE is guarded — a re-baseline in a later phase answers to that + phase's budget. + + Args: + warmup_runtime_sec: Wall-clock the warmup round took. + ctx_extra: The runner context extras carrying ``shared_state``. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. + """ + state = (ctx_extra or {}).get("shared_state") or self.shared_state + if state is None: + return True, {"reason": "no_session_state"} + phase = str(getattr(state, "phase", "") or "").strip().upper() + if phase != _phase_state.PHASE_PRELUDE: + return True, {"reason": "not_prelude", "phase": phase} + try: + cost = float(warmup_runtime_sec or 0.0) + except (TypeError, ValueError): + cost = 0.0 + return _phase_state.prelude_can_afford(state, expected_cost_sec=cost) + def _double_run_enabled( self, *, diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 57b818efb8..0a6fcd8e58 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -928,6 +928,8 @@ def router(self) -> IntentRouter: "_reseed_orch_prompt_for_phase": "phase_machine", "_record_phase_entry_evidence": "phase_machine", "_internal_analysis_kind": "phase_prelude", + "_measured_analysis_cost_sec": "phase_prelude", + "_record_prelude_arm_dropped": "phase_prelude", "_warm_recipe_proven_items": "phase_prelude", "_inject_warm_recipe_history_into_ledger": "phase_prelude", "_filter_warm_patches_with_kg": "phase_prelude", diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index 3a5b344ff3..7e20510efe 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -559,20 +559,6 @@ async def _record_close_step( task_id: Optional task id associated with the step. detail: Optional free-text detail recorded on the row. """ - history = self.shared_state.phase_history or [] - if not history: - return - row = history[-1] - if not isinstance(row, dict): - return - evidence = row.get("evidence") - if not isinstance(evidence, dict): - evidence = {} - row["evidence"] = evidence - steps = evidence.get("close_steps") - if not isinstance(steps, list): - steps = [] - evidence["close_steps"] = steps entry: dict[str, Any] = { "step": step, "status": status, @@ -582,7 +568,12 @@ async def _record_close_step( entry["task_id"] = task_id if detail: entry["detail"] = detail - steps.append(entry) + if not _phase_state.append_phase_evidence_row( + self.shared_state.phase_history, + key="close_steps", + row=entry, + ): + return try: self.shared_state.save(self.session_dir) except Exception: # noqa: BLE001 diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 1fba9944db..002662c296 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -352,6 +352,26 @@ def is_valid_phase_exit_reason(value: str) -> bool: PHASE_CLOSE: 0.02, } +# Share of the session PRELUDE may spend before its optional arms are dropped, +# and the share held for the phases that actually produce a result. +# +# PRELUDE's ``DEFAULT_PHASE_BUDGET_PCT`` entry is 3%, but nothing enforced it: +# :func:`exit_normal_prelude` tests only whether a baseline landed, so the phase +# ran to whatever its contents cost. Two sessions on unrelated models (different +# quantization, different PRELUDE composition -- one baseline-dominated, one +# split with a TraceLens roofline) spent 73.8% and 72.8% of a three-hour budget +# in it, and both reached FRAMEWORK_AGENT with ~47 minutes left against its +# 108-minute entry threshold. Both budgets were honoured; both sessions produced +# nothing. Stopping the run on time is necessary and not sufficient -- the +# phases that spend the budget have to be the ones that produce the result. +# +# These bound the preparation rather than the session. They are deliberately +# looser than 3%: a baseline that legitimately takes an hour should still run, +# it just cannot also buy an 80-minute roofline out of the optimization phases' +# time. +PRELUDE_SPEND_CEILING_PCT: float = 0.40 +OPTIMIZATION_RESERVE_PCT: float = 0.50 + # Wall-clock ceiling for an unbounded run (``max_minutes`` == 0): the container # lifetime. Used both as the global deadline and as the basis for the absolute # per-phase cap so an unbounded run still forces phase rotation. @@ -1961,10 +1981,187 @@ def exit_normal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: except (TypeError, ValueError): return None if tput > 0.0: - return "prelude_done", {"baseline_tput": tput} + return "prelude_done", {"baseline_tput": tput, **prelude_exit_viability(state)} return None +def prelude_exit_viability(state: Any) -> dict[str, Any]: + """Report whether the budget PRELUDE leaves behind can still fund one optimization round. + + A session can honour its wall clock and still be over: both field sessions + left FRAMEWORK_AGENT ~47 minutes against a 108-minute threshold, and every + later phase then declined in turn, each for its own local reason. Nothing + said the plain thing — preparation had spent the run. + + Stated here, on the exit that caused it, because this is the last moment + the answer is actionable and the first moment it is knowable: the baseline + is measured, so one benchmark round has a price rather than an estimate. + + Args: + state (Any): Frozen SharedState view. + + Returns: + dict[str, Any]: Evidence for the phase record; empty when the budget is + unbounded or no round has been measured. + """ + usable = _session_usable_seconds(state) + try: + round_sec = float(getattr(state, "baseline_runtime_sec", 0.0) or 0.0) + except (TypeError, ValueError): + round_sec = 0.0 + if usable is None or round_sec <= 0.0: + return {} + return { + "session_usable_sec": round(usable, 1), + "measured_round_sec": round(round_sec, 1), + "affordable_rounds": round(usable / round_sec, 2), + "fits_one_optimization_round": usable >= round_sec, + } + + +def append_phase_evidence_row(history: Any, *, key: str, row: dict[str, Any]) -> bool: + """Append ``row`` to the current phase's ``evidence[key]`` list. + + Phases record what happened inside them on the newest ``phase_history`` + row, which the session breakdown exports verbatim. Creates the ``evidence`` + dict and the list under ``key`` when absent, and replaces a non-list value + rather than raising — a malformed row must not take a phase down. + + Args: + history (Any): The ``phase_history`` list. + key (str): Evidence key holding the list of rows. + row (dict[str, Any]): The row to append. + + Returns: + bool: ``True`` when the row landed; ``False`` when there was no usable + history row to attach it to. + """ + if not isinstance(history, list) or not history: + return False + current = history[-1] + if not isinstance(current, dict): + return False + evidence = current.get("evidence") + if not isinstance(evidence, dict): + evidence = {} + current["evidence"] = evidence + rows = evidence.get(key) + if not isinstance(rows, list): + rows = [] + evidence[key] = rows + rows.append(row) + return True + + +def _session_usable_seconds(state: Any) -> float | None: + """Seconds a unit of work may still claim, from the session's own accounting. + + Prefers ``SharedState.session_budget_usable_sec`` — the single number + admission control and the grid deadline both read — so this policy cannot + disagree with them about how much budget is left. Falls back to the raw + remaining time for the frozen views and test doubles that expose attributes + only. Called rather than imported because ``shared_state`` imports this + module. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Usable seconds, or ``None`` when the budget is unbounded. + """ + getter = getattr(state, "session_budget_usable_sec", None) + if callable(getter): + try: + return getter() + except Exception: # noqa: BLE001 — fall back to the attribute path + pass + return session_remaining_seconds(state) + + +def prelude_can_afford( + state: Any, + *, + expected_cost_sec: float, + now_unix: float | None = None, +) -> tuple[bool, dict[str, Any]]: + """Decide whether PRELUDE can still buy an optional arm costing ``expected_cost_sec``. + + Two bounds apply and the tighter wins: what is left of PRELUDE's own share + (:data:`PRELUDE_SPEND_CEILING_PCT`), and what is left of the session once + the optimization phases' reserve (:data:`OPTIMIZATION_RESERVE_PCT`) is held + back. An arm that fits neither is not refused work the session needed — it + is refused work the session could not have used the result of. + + Args: + state (Any): Frozen SharedState view. + expected_cost_sec (float): What the arm is expected to cost. Callers + should anchor this on something this session measured; the static + per-action estimates are calibrated on small models and understate + a large one by an order of magnitude. + now_unix (float | None): Override for the current time. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. Evidence + carries the numbers behind the decision so a skip is legible in the log + and the phase record. + """ + cost = max(0.0, float(expected_cost_sec or 0.0)) + max_sec = _max_minutes(state) * 60.0 + usable = _session_usable_seconds(state) + if max_sec <= 0.0 or usable is None: + return True, {"reason": "unbounded_budget", "expected_cost_sec": round(cost, 1)} + spent = phase_cumulative_seconds(state, phase=PHASE_PRELUDE, now_unix=now_unix) + phase_headroom = max_sec * PRELUDE_SPEND_CEILING_PCT - spent + session_headroom = usable - max_sec * OPTIMIZATION_RESERVE_PCT + affordable_sec = min(phase_headroom, session_headroom) + evidence: dict[str, Any] = { + "expected_cost_sec": round(cost, 1), + "prelude_spent_sec": round(spent, 1), + "prelude_ceiling_sec": round(max_sec * PRELUDE_SPEND_CEILING_PCT, 1), + "optimization_reserve_sec": round(max_sec * OPTIMIZATION_RESERVE_PCT, 1), + "session_usable_sec": round(usable, 1), + "affordable_sec": round(affordable_sec, 1), + "bound": "prelude_ceiling" if phase_headroom <= session_headroom else "optimization_reserve", + } + return affordable_sec >= cost, evidence + + +def exit_time_exhausted_prelude( + state: Any, + *, + now_unix: float | None = None, +) -> tuple[str, dict[str, Any]] | None: + """Route to CLOSE when the session clock runs out before PRELUDE lands a baseline. + + ``time_exhausted_during_prelude`` was in the terminal-reason vocabulary and + in the report's reason glossary, but no code ever assigned it: the state + machine had a word for this failure and no way to reach it. A session that + burns its whole budget preparing then read as an ordinary exit. + + Only fires while PRELUDE is still incomplete. Once a baseline exists the + later phases have their own force-exits, and reporting the run as "never + began optimizing" would be false. + + Args: + state (Any): Frozen SharedState view. + now_unix (float | None): Override for the current time. + + Returns: + tuple[str, dict[str, Any]] | None: ``("time_exhausted_during_prelude", + evidence)`` when the budget is gone, else ``None``. + """ + usable = _session_usable_seconds(state) + if usable is None or usable > 0.0: + return None + return "time_exhausted_during_prelude", { + "session_usable_sec": round(usable, 1), + "prelude_spent_sec": round( + phase_cumulative_seconds(state, phase=PHASE_PRELUDE, now_unix=now_unix), + 1, + ), + } + + def exit_terminal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """Decide the PRELUDE terminal exit on repeated baseline failures. @@ -2593,6 +2790,12 @@ def compute_next_phase( if term is not None: return PHASE_CLOSE, term[0], {"terminal": True, **term[1]} norm = exit_normal_prelude(state) + if norm is None: + # No baseline and no clock left: name the failure instead of + # letting the run read as an ordinary exit. + exhausted = exit_time_exhausted_prelude(state, now_unix=now_unix) + if exhausted is not None: + return PHASE_CLOSE, exhausted[0], {"terminal": True, **exhausted[1]} if norm is not None: if framework_agent_phase_enabled: return PHASE_FRAMEWORK_AGENT, norm[0], norm[1] @@ -3102,6 +3305,8 @@ def record_lifecycle_event( "DEFAULT_EXPLORE_FORCE_EXIT_BUDGET_PCT", "DEFAULT_EXPLORE_FORCE_EXIT_HOURS_REMAINING", "DEFAULT_PHASE_BUDGET_PCT", + "OPTIMIZATION_RESERVE_PCT", + "PRELUDE_SPEND_CEILING_PCT", "DEFAULT_PLATEAU_EXPLORE_EMPTY_STREAK", "DEFAULT_PLATEAU_EXPLORE_KEEP_GAIN_PCT", "DEFAULT_PLATEAU_EXPLORE_LOOKBACK", @@ -3161,6 +3366,10 @@ def record_lifecycle_event( "exit_normal_prelude", "exit_normal_sweep", "exit_terminal_prelude", + "exit_time_exhausted_prelude", + "append_phase_evidence_row", + "prelude_can_afford", + "prelude_exit_viability", "is_action_allowed_in_phase", "is_action_llm_proposable_in_phase", "llm_proposable_actions_for", diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index 51ec98637d..93423cc040 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -152,6 +152,59 @@ def _internal_analysis_kind(self) -> str: else "profile" ) + def _measured_analysis_cost_sec(self) -> float: + """Expected cost of the initial roofline/profile arm, in seconds. + + Anchored on this session's own baseline round rather than the action + catalog. The catalog's estimates (``baseline`` 5 min, ``roofline`` + 8 min) are calibrated on small models: the two sessions that motivated + this guard measured 51 and 125 minutes of baseline and an 81-minute + roofline, so a catalog-anchored guard admits an arm it cannot pay for. + + The analysis arm boots its own server and runs the same benchmark under + a profiler, so one measured baseline round is a floor on its cost + rather than a guess at it. The catalog is the fallback for the first + analysis of a session that has no measurement yet. + + Returns: + float: Expected cost in seconds; ``0.0`` when nothing is on record, + which :func:`machine_state.prelude_can_afford` reads as free. + """ + try: + measured = float(getattr(self.shared_state, "baseline_runtime_sec", 0.0) or 0.0) + except (TypeError, ValueError): + measured = 0.0 + if measured > 0.0: + return measured + registry = getattr(self, "action_registry", None) + meta = registry.get(self._internal_analysis_kind()) if registry is not None else None + try: + return float(getattr(meta, "cost_minutes_p50", 0.0) or 0.0) * 60.0 + except (TypeError, ValueError): + return 0.0 + + def _record_prelude_arm_dropped(self, arm: str, evidence: dict[str, Any]) -> None: + """Record a PRELUDE arm dropped for budget on the current phase record. + + The phase record is what the session breakdown exports, so a dropped + arm reads as a decision with numbers behind it rather than as an arm + that silently never ran. + + Args: + arm: The arm that was dropped. + evidence: The affordability numbers behind the decision. + """ + if not _phase_state.append_phase_evidence_row( + getattr(self.shared_state, "phase_history", None), + key="budget_dropped_arms", + row={"arm": arm, **evidence}, + ): + return + try: + self.shared_state.save(self.session_dir) + except Exception: # noqa: BLE001 — best-effort record + log.exception("PRELUDE: failed to persist the dropped-arm record for %r", arm) + def _warm_recipe_proven_items(self) -> list[dict[str, str]]: """Summarise warm-start ``what_worked`` items the scout can skip ({name, source}); fail-soft. @@ -2303,6 +2356,22 @@ async def _maybe_enqueue_prelude_initial_analysis_after_baseline( return if (state.auto_roofline_pending_task_id or "").strip(): return + affordable, evidence = _phase_state.prelude_can_afford( + state, + expected_cost_sec=self._measured_analysis_cost_sec(), + ) + if not affordable: + log.warning( + "PRELUDE: skipping the initial %s — %.0fs of preparation budget " + "left (bound=%s) against an expected %.0fs. The optimization " + "phases keep the time instead.", + self._internal_analysis_kind(), + evidence.get("affordable_sec", 0.0), + evidence.get("bound", ""), + evidence.get("expected_cost_sec", 0.0), + ) + self._record_prelude_arm_dropped("initial_analysis", evidence) + return try: rl_task = await self._enqueue_internal_analysis_task( reason="prelude_initial", From 7b4cb28fd9130974a1282836950510ab4d2e482e Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 10:40:59 +0000 Subject: [PATCH 09/65] fix(executors): carry the session clock into the Ray worker On a single node the Ray execution backend is the production default, and the only reason every test on this branch exercised the local subprocess path is the pytest guard inside ``_should_use_ray_backend``. So the session reaper -- the defence that makes a run out of time say so -- was absent exactly where the work actually runs. Handing the reaper's ``session_deadline_sec`` straight to the actor would have been worse than omitting it: it is a ``time.monotonic()`` instant, and the actor is another process whose clock counts from its own unrelated origin, so the number names an arbitrary moment there -- an immediate kill or one that never fires, with nothing to distinguish the two. The deadline therefore crosses as the seconds it has left and is re-anchored on the far side, with a name that cannot be mistaken for the absolute form. Without it the hard timeout is the last line, and it is deliberately set slightly past the session deadline so the watchdog trips first; whichever one fires writes the ledger's account of why the run stopped. On the Ray path the cap won that race and the ledger blamed the variant for the session running out of time, which is the attribution this branch exists to correct. The accuracy eval made it worse: ``soft_deadline_sec`` retires at eval start by design, so an eval starting near the end had nothing bounding it but that same misattributing cap. baseline.py is left alone. It has neither defence -- no clamp on its timeout and no deadline on its subprocess -- and it also has no branch for the session sentinel, so plumbing one in would only turn running out of time into an unclassified crash. That needs its own design, not a keyword argument. Co-authored-by: Cursor --- .../tests/test_kill_spawned_server.py | 35 ++++++ .../tests/test_ray_backend_unit.py | 114 +++++++++++++++++- .../actions/executors/_grid_runner.py | 8 +- .../actions/executors/_ray_backend.py | 12 +- .../actions/executors/_ray_serving.py | 28 ++++- .../actions/executors/_subprocess_kill.py | 44 +++++++ 6 files changed, 237 insertions(+), 4 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py index cc5d44e5d5..6c216c85ff 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py +++ b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py @@ -31,6 +31,8 @@ new_session_kwargs, run_with_session_kill, server_log_death_excerpt, + session_deadline_to_remaining_sec, + session_remaining_to_deadline_sec, ) @@ -362,6 +364,39 @@ def test_no_session_deadline_keeps_the_previous_behaviour(self): assert "done" in (cp.stdout or "") +class TestSessionDeadlineCrossesAProcessBoundary: + """A ``time.monotonic()`` instant is only meaningful in the process that read it. + + Handing the absolute deadline to a Ray worker would name an instant on the + worker's own clock, whose origin is unrelated -- an immediate kill or one + that never fires, both silently. Only a duration survives the trip. + """ + + def test_an_unbounded_budget_stays_unbounded_in_both_directions(self): + assert session_deadline_to_remaining_sec(None) is None + assert session_remaining_to_deadline_sec(None) is None + + def test_a_deadline_becomes_the_seconds_it_has_left(self, monkeypatch): + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + assert session_deadline_to_remaining_sec(1300.0) == pytest.approx(300.0) + + def test_a_spent_budget_crosses_as_a_non_positive_duration(self, monkeypatch): + """Not floored at zero: the receiver must reap, not read it as "no budget given".""" + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + assert session_deadline_to_remaining_sec(940.0) == pytest.approx(-60.0) + + def test_the_duration_re_anchors_onto_the_reading_clock(self, monkeypatch): + """The same duration names a different instant on each side; that is the point.""" + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + remaining = session_deadline_to_remaining_sec(1300.0) + monkeypatch.setattr(time, "monotonic", lambda: 5_000_000.0) + assert session_remaining_to_deadline_sec(remaining) == pytest.approx(5_000_300.0) + + def test_the_round_trip_is_the_identity_within_one_clock(self, monkeypatch): + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + assert session_remaining_to_deadline_sec(session_deadline_to_remaining_sec(1234.5)) == pytest.approx(1234.5) + + def _sentinel_returncodes() -> dict[int, set[str]]: """Map every sentinel returncode to the qualified names that claim it.""" from hyperloom.orchestrator.actions.executors import _ray_serving, _subprocess_kill diff --git a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py index 573bb99b16..76b592a995 100644 --- a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py @@ -28,6 +28,9 @@ ServingLease, maybe_serving_lease, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + SESSION_TIME_EXHAUSTED_RETURNCODE, +) # ── flag gate ──────────────────────────────────────────────────────────────── @@ -397,8 +400,10 @@ def test_strip_visible_devices_noop_when_absent(tmp_path: Path): class _FakeMethod: def __init__(self, ret): self._ret = ret + self.calls: list[dict] = [] - def remote(self, *_a, **_k): + def remote(self, *_a, **kw): + self.calls.append(dict(kw)) return self._ret @@ -599,6 +604,113 @@ def _fake_build(*, python_exe, config_path, output_dir): assert lease.calls[0]["timeout"] == 10 +# ── the session budget across the Ray process boundary ─────────────────────── +class TestTheSessionBudgetReachesTheRayWorker: + """Production takes the Ray path on a single node; the local path is the test default. + + So the session reaper has to be carried across the boundary explicitly, and + as a duration: the absolute deadline is a ``time.monotonic()`` instant, and + the worker is another process whose clock starts somewhere else. Without it + the hard timeout is the only thing left, and a run that ran out of time gets + recorded as a variant that timed out. + """ + + def test_run_magpie_converts_the_deadline_before_handing_it_over( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + from hyperloom.orchestrator.actions.executors import _grid_runner as gr + + cfg = tmp_path / "config.yaml" + cfg.write_text("benchmark:\n framework: sglang\n envs:\n TP: 1\n", encoding="utf-8") + out_dir = tmp_path / "out" + out_dir.mkdir() + monkeypatch.setattr(gr, "build_benchmark_command", lambda **_kw: ["magpie"]) + monkeypatch.setattr(time, "monotonic", lambda: 1000.0) + + lease = _RecordingLease() + gr._run_magpie( + magpie_python="python3", + config_path=cfg, + output_dir=out_dir, + timeout_sec=10, + cwd=str(tmp_path), + serving_lease=lease, + session_deadline_sec=1250.0, + ) + + assert lease.calls[0]["session_remaining_sec"] == pytest.approx(250.0) + assert "session_deadline_sec" not in lease.calls[0], ( + "an absolute monotonic instant is meaningless in the actor's process" + ) + + def test_an_unbounded_budget_reaches_the_lease_as_none( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + from hyperloom.orchestrator.actions.executors import _grid_runner as gr + + cfg = tmp_path / "config.yaml" + cfg.write_text("benchmark:\n framework: sglang\n envs:\n TP: 1\n", encoding="utf-8") + out_dir = tmp_path / "out" + out_dir.mkdir() + monkeypatch.setattr(gr, "build_benchmark_command", lambda **_kw: ["magpie"]) + + lease = _RecordingLease() + gr._run_magpie( + magpie_python="python3", + config_path=cfg, + output_dir=out_dir, + timeout_sec=10, + cwd=str(tmp_path), + serving_lease=lease, + ) + + assert lease.calls[0]["session_remaining_sec"] is None + + def test_the_lease_forwards_the_budget_to_its_actor(self, monkeypatch: pytest.MonkeyPatch): + """The lease is the last in-process hop; dropping it here loses the reaper.""" + monkeypatch.setitem(sys.modules, "ray", _LeaseFakeRay()) + lease = ServingLease(num_gpus=1) + lease._actor = _FakeActor((0, "ok", "")) # pre-set so ensure() is a no-op + + lease.run_session_kill(["echo", "hi"], timeout=5, session_remaining_sec=42.0) + + assert lease._actor.run_blocking.calls[0]["session_remaining_sec"] == pytest.approx(42.0) + + def test_the_worker_reaps_a_child_whose_budget_is_already_spent(self): + """End of the chain: the duration becomes a deadline on the worker's own clock.""" + start = time.monotonic() + rc, _out, _err = rb._run_subprocess_worker( + cmd=[sys.executable, "-c", "import time; time.sleep(30)"], + env=None, + cwd=None, + timeout_s=60, + soft_deadline_sec=None, + server_log_path=None, + server_already_ready=False, + session_remaining_sec=-1.0, + ) + assert rc == SESSION_TIME_EXHAUSTED_RETURNCODE + assert time.monotonic() - start < 10.0 + + def test_the_worker_leaves_a_child_with_budget_left_alone(self): + rc, out, _err = rb._run_subprocess_worker( + cmd=["echo", "still-running"], + env=None, + cwd=None, + timeout_s=60, + soft_deadline_sec=None, + server_log_path=None, + server_already_ready=False, + session_remaining_sec=3600.0, + ) + assert rc == 0 + assert "still-running" in out + + # ── P2: ManagedServerProcess.exit_code (real subprocess) ───────────────────── def test_managed_process_exit_code_none_then_latched(): """exit_code is None before start / while alive, then the real return code.""" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 2766b8f7a7..b9cc44183e 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -44,6 +44,7 @@ SESSION_TIME_EXHAUSTED_RETURNCODE, run_with_session_kill, server_log_death_excerpt, + session_deadline_to_remaining_sec, ) from .benchmark_result import ( estimate_killed_variant_throughput, @@ -976,7 +977,9 @@ def _run_magpie( session_deadline_sec (float | None): Absolute ``time.monotonic()`` instant at which the session budget expires. Reaps the tree and returns ``SESSION_TIME_EXHAUSTED_RETURNCODE``. Enforced in every phase, - including the accuracy eval that retires ``soft_deadline_sec``. + including the accuracy eval that retires ``soft_deadline_sec``. The + lease path sends it across as a remaining-seconds duration, the only + form that means the same thing in the actor's process. Returns: tuple[int, str, str]: ``(returncode, stdout, stderr)``. @@ -1085,6 +1088,9 @@ def _run_magpie( soft_deadline_sec=soft_deadline_sec, server_log_path=str(output_dir / "server.log"), server_already_ready=server_already_ready, + # Converted here, at the last moment before the process boundary: + # the actor cannot read this process's monotonic clock. + session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), ) cmd = build_benchmark_command( diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_backend.py b/src/hyperloom/orchestrator/actions/executors/_ray_backend.py index 991be587e4..2b61d767b0 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_backend.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_backend.py @@ -184,12 +184,15 @@ def _run_subprocess_worker( soft_deadline_sec: float | None, server_log_path: str | None, server_already_ready: bool, + session_remaining_sec: float | None = None, ) -> tuple[int, str, str]: """Ray worker body: run the subprocess under session-kill semantics. Executes on a Ray worker where ``*_VISIBLE_DEVICES`` are already set by Ray. Reuses :func:`run_with_session_kill` so kill/soft-deadline behaviour matches - the local path exactly. + the local path exactly -- including the session reaper, which is the only + defence that attributes running out of time to the run rather than to the + variant that happened to be in flight. Args: cmd: The command to execute. @@ -199,12 +202,18 @@ def _run_subprocess_worker( soft_deadline_sec: Overtime soft deadline. server_log_path: Path to the server log for watchdog markers. server_already_ready: Start the soft clock from spawn (warm reuse). + session_remaining_sec: Seconds left on the session budget when the + submitter made the call. A duration rather than the in-process + absolute deadline because this body runs in a Ray worker, whose + ``time.monotonic()`` origin is its own; it is re-anchored here onto + this process's clock. Returns: ``(returncode, stdout, stderr)``. """ from hyperloom.orchestrator.actions.executors._subprocess_kill import ( run_with_session_kill, + session_remaining_to_deadline_sec, ) worker_env = _merge_worker_env(env) @@ -216,6 +225,7 @@ def _run_subprocess_worker( soft_deadline_sec=soft_deadline_sec, server_log_path=server_log_path, server_already_ready=server_already_ready, + session_deadline_sec=session_remaining_to_deadline_sec(session_remaining_sec), ) return proc.returncode, proc.stdout or "", proc.stderr or "" diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py index 79b4b16f73..0cf5387f3c 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py @@ -277,8 +277,14 @@ def run_blocking( soft_deadline_sec=None, server_log_path=None, server_already_ready=False, + session_remaining_sec=None, ): - """Run one benchmark round to completion; return ``(rc, stdout, stderr)``.""" + """Run one benchmark round to completion; return ``(rc, stdout, stderr)``. + + ``session_remaining_sec`` is a duration, not the submitter's absolute + session deadline: this actor is a separate process with its own + ``time.monotonic()`` origin, so only a duration survives the trip. + """ import subprocess as _sp # noqa: PLC0415 from ._ray_backend import _run_subprocess_worker # noqa: PLC0415 @@ -292,6 +298,7 @@ def run_blocking( soft_deadline_sec=soft_deadline_sec, server_log_path=server_log_path, server_already_ready=server_already_ready, + session_remaining_sec=session_remaining_sec, ) except _sp.TimeoutExpired as exc: return _ACTOR_TIMEOUT_RC, "", f"TimeoutExpired: {exc}" @@ -392,12 +399,30 @@ def run_session_kill( soft_deadline_sec: float | None = None, server_log_path: str | None = None, server_already_ready: bool = False, + session_remaining_sec: float | None = None, ) -> tuple[int, str, str]: """Run one benchmark round inside the lease's actor; return ``(rc, stdout, stderr)``. Drop-in for ``run_with_session_kill``; re-raises ``subprocess.TimeoutExpired`` on hard timeout. Cluster-ensure failures and Ray worker errors degrade to a benchmark failure (rc=1) rather than crashing the session. + + Args: + cmd: The benchmark command to run inside the actor. + env: Caller env for the subprocess. + cwd: Working directory for the subprocess. + timeout: Hard timeout in seconds. + soft_deadline_sec: Overtime soft deadline. + server_log_path: Path to the server log for the watchdogs. + server_already_ready: Warm reuse round (soft clock from spawn). + session_remaining_sec: Seconds left on the session budget, as + produced by ``session_deadline_to_remaining_sec``. The absolute + deadline the local path uses cannot cross into the actor: it is + a ``time.monotonic()`` instant, and the actor's clock has its own + origin. The actor re-anchors this duration onto its own clock. + + Returns: + ``(returncode, stdout, stderr)`` from the round. """ import subprocess as _sp # noqa: PLC0415 @@ -416,6 +441,7 @@ def run_session_kill( soft_deadline_sec=soft_deadline_sec, server_log_path=server_log_path, server_already_ready=server_already_ready, + session_remaining_sec=session_remaining_sec, ) # Resolve Ray's exception classes defensively. Real ray always exposes # both, but this is a failure hot-path: a partial test double or a future diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 7310c626eb..c1a3df563a 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -681,6 +681,48 @@ def _scan_server_log_increment(path: str, from_offset: int) -> tuple[int, bool, return size, saw_ready, saw_progress, saw_eval_start +def session_deadline_to_remaining_sec(session_deadline_sec: float | None) -> float | None: + """Convert an in-process session deadline into seconds still left on it. + + The pair of this and :func:`session_remaining_to_deadline_sec` is how a + session deadline crosses a process boundary. ``time.monotonic()`` has an + unspecified, per-process origin, so the absolute instant means nothing to a + reader in another process; a duration means the same thing everywhere. + + Args: + session_deadline_sec: Absolute ``time.monotonic()`` instant at which the + session budget expires, or ``None`` when the budget is unbounded. + + Returns: + Seconds left on the budget, or ``None`` when unbounded. Non-positive + when the deadline has already passed, which the receiving side is meant + to act on immediately rather than treat as "no budget given". + """ + if session_deadline_sec is None: + return None + return float(session_deadline_sec) - time.monotonic() + + +def session_remaining_to_deadline_sec(session_remaining_sec: float | None) -> float | None: + """Re-anchor a remaining session budget onto this process's monotonic clock. + + The inverse of :func:`session_deadline_to_remaining_sec`. Whatever the + budget spent in transit is charged to the receiver, since no clock is shared + across the boundary to measure it with. + + Args: + session_remaining_sec: Seconds left on the session budget as measured by + the sender, or ``None`` when the budget is unbounded. + + Returns: + An absolute ``time.monotonic()`` deadline usable in this process, or + ``None`` when unbounded. + """ + if session_remaining_sec is None: + return None + return time.monotonic() + float(session_remaining_sec) + + def run_with_session_kill( cmd: list[str], *, @@ -1127,4 +1169,6 @@ def _communicate_with_soft_deadline( "new_session_kwargs", "run_with_session_kill", "server_log_death_excerpt", + "session_deadline_to_remaining_sec", + "session_remaining_to_deadline_sec", ] From 4d7ecd8c518a6616ade1cdd3cceb54184ee3f057 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 10:50:04 +0000 Subject: [PATCH 10/65] fix(dispatcher): stop the pump cancelling work it does not own ``_inflight_actions`` has two writers. The pump registers what it spawns, and the inline path registers the caller's own task -- an action explicitly designed to keep running after the inline wait gives up on it, which is why it publishes a handle at all. The pump's exit sweep ran over the whole registry on every exit, including the ordinary early return taken by a tick that found nothing queued. So the cheapest possible tick killed a live inline action, silently undoing the one guarantee that path makes. The comment above it argued that a drained pump has nothing left to cancel; true of a pump-local registry, and this one is not. The sweep now covers the entries still in the pump's own in-flight list, which is exactly what it spawned. Leaving by any door other than the drained one still orphans nothing of its own. The triggers that must reach everything -- shutdown, a spent budget, ``Coordinator.stop`` -- keep the whole registry. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 115 ++++++++++++++---- src/hyperloom/orchestrator/loop/dispatcher.py | 24 ++-- 2 files changed, 109 insertions(+), 30 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 3915e153a3..3f4dcca84f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -13,8 +13,9 @@ * In-flight cancellation -- the backstop for work already running when the budget goes or the process is asked to stop. Covers the handles the dispatcher keeps, the cancellation itself, the closing-action carve-out, the task row - landing terminal instead of stranding at ``running``, and the pump and - ``Coordinator.stop`` paths that trigger it. + landing terminal instead of stranding at ``running``, which of those handles + the pump owns on its way out, and the pump and ``Coordinator.stop`` paths that + trigger it. The remaining two layers are enforced inside the executors and tested next to them: the timeout clamp in ``test_explore_executor``, and the subprocess session @@ -470,16 +471,17 @@ async def test_the_cancellation_still_reaches_the_caller(self, coord: Coordinato assert atask.cancelled() +def _quick_poll(coord: Coordinator) -> None: + """Shorten the pump's re-scan interval so a pump test is not a wall-clock test.""" + coord._dispatcher_poll_sec = 0.05 + + class TestThePumpStopsWorkItCannotWaitFor: """The trigger side: a spent budget, and a shutdown request.""" - @staticmethod - def _quick_poll(coord: Coordinator) -> None: - coord._dispatcher_poll_sec = 0.05 - @pytest.mark.asyncio async def test_a_budget_that_runs_out_stops_the_action(self, coord: Coordinator): - self._quick_poll(coord) + _quick_poll(coord) _set_budget(coord, minutes=600) task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-budget") _set_budget(coord, minutes=600, elapsed_min=600.0) @@ -492,7 +494,7 @@ async def test_a_budget_that_runs_out_stops_the_action(self, coord: Coordinator) @pytest.mark.asyncio async def test_the_closing_actions_keep_their_reserve(self, coord: Coordinator): """The budget hits zero with the closing window still to spend.""" - self._quick_poll(coord) + _quick_poll(coord) _set_budget(coord, minutes=600, elapsed_min=600.0) _task, atask, pump = await _start_action_under_pump(coord, kind=_CLOSING_ACTION, key="p-closing") await asyncio.sleep(0.3) @@ -505,7 +507,7 @@ async def test_the_closing_actions_keep_their_reserve(self, coord: Coordinator): @pytest.mark.asyncio async def test_a_shutdown_request_stops_the_action(self, coord: Coordinator): """SIGTERM sets the stop event; before this it only stopped the tick.""" - self._quick_poll(coord) + _quick_poll(coord) _set_budget(coord, minutes=600) _task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-signal") coord._stop.set() @@ -517,7 +519,7 @@ async def test_a_shutdown_request_stops_the_action(self, coord: Coordinator): @pytest.mark.asyncio async def test_a_cancelled_pump_does_not_orphan_its_actions(self, coord: Coordinator): """The handles live in the pump's frame; leaving must not drop them.""" - self._quick_poll(coord) + _quick_poll(coord) _set_budget(coord, minutes=600) _task, atask, pump = await _start_action_under_pump(coord, kind=_CHEAP_ACTION, key="p-orphan") @@ -528,18 +530,26 @@ async def test_a_cancelled_pump_does_not_orphan_its_actions(self, coord: Coordin assert coord.dispatcher._inflight_actions == {} +def _allow_inline(coord: Coordinator, monkeypatch) -> asyncio.Event: + """Register a never-finishing executor and clear the gates around it.""" + started = asyncio.Event() + coord.sub.register_executor(_CHEAP_ACTION, _never_finishes(started)) + monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) + _set_budget(coord, minutes=600) + return started + + +async def _start_inline_action(coord: Coordinator, monkeypatch) -> asyncio.Task: + """Run an inline action and wait until it is registered and under way.""" + started = _allow_inline(coord, monkeypatch) + inline = asyncio.create_task(coord.dispatcher._run_action_now(_CHEAP_ACTION, {})) + await asyncio.wait_for(started.wait(), timeout=5.0) + return inline + + class TestInlineActionsAreReachableToo: """The inline path abandons its future, so it needs the same handle.""" - @staticmethod - def _allow_inline(coord: Coordinator, monkeypatch) -> asyncio.Event: - """Register a never-finishing executor and clear the gates around it.""" - started = asyncio.Event() - coord.sub.register_executor(_CHEAP_ACTION, _never_finishes(started)) - monkeypatch.setattr(coord.policy, "validate_intent", lambda *a, **k: None) - _set_budget(coord, minutes=600) - return started - @pytest.mark.asyncio async def test_an_inline_action_that_outlived_its_caller_can_be_stopped( self, @@ -547,9 +557,7 @@ async def test_an_inline_action_that_outlived_its_caller_can_be_stopped( monkeypatch, ): """Before this, the only thing that ended it was the action itself.""" - started = self._allow_inline(coord, monkeypatch) - inline = asyncio.create_task(coord.dispatcher._run_action_now(_CHEAP_ACTION, {})) - await asyncio.wait_for(started.wait(), timeout=5.0) + inline = await _start_inline_action(coord, monkeypatch) task_id = next(iter(coord.dispatcher._inflight_actions)) assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task_id] @@ -580,7 +588,7 @@ async def test_the_sync_bridge_reports_the_cancellation_instead_of_raising( monkeypatch, ): """It runs on an agent's turn thread, which a ``CancelledError`` would end.""" - started = self._allow_inline(coord, monkeypatch) + started = _allow_inline(coord, monkeypatch) monkeypatch.setattr( coord.dispatcher, "_inline_action_whitelist", @@ -605,6 +613,67 @@ async def test_the_sync_bridge_reports_the_cancellation_instead_of_raising( assert outcome and "was cancelled" in outcome[0] +class TestThePumpOnlyCancelsWhatItSpawned: + """The registry is dispatcher-wide; the pump's exit sweep is not. + + An inline action is registered by whoever ran it, not by the pump, and is + designed to keep going after that caller stops waiting. A tick with nothing + queued returns immediately, so an exit sweep over the whole registry would + make the emptiest possible pump the thing that kills it. + """ + + @pytest.mark.asyncio + async def test_a_tick_with_nothing_queued_leaves_an_inline_action_running( + self, + coord: Coordinator, + monkeypatch, + ): + inline = await _start_inline_action(coord, monkeypatch) + try: + await asyncio.wait_for(coord._pump_dispatcher_once(), timeout=10.0) + + assert not inline.done() + assert coord.dispatcher._inflight_actions + finally: + inline.cancel() + await _settle(inline) + + @pytest.mark.asyncio + async def test_a_cancelled_pump_takes_its_own_and_only_its_own( + self, + coord: Coordinator, + monkeypatch, + ): + """Narrowing the sweep must not cost the pump the actions it does own.""" + _quick_poll(coord) + inline = await _start_inline_action(coord, monkeypatch) + _task, spawned, pump = await _start_action_under_pump(coord, kind=_CLOSING_ACTION, key="own-spawn") + try: + pump.cancel() + await _settle(pump) + + assert spawned.cancelled() + assert not inline.done() + finally: + inline.cancel() + await _settle(inline) + + @pytest.mark.asyncio + async def test_a_shutdown_still_reaches_an_inline_action( + self, + coord: Coordinator, + monkeypatch, + ): + """The narrower sweep must not blunt the trigger that has to reach everything.""" + inline = await _start_inline_action(coord, monkeypatch) + coord._stop.set() + + await asyncio.wait_for(coord._pump_dispatcher_once(), timeout=10.0) + + await _settle(inline) + assert inline.cancelled() + + class TestCoordinatorStop: """Teardown closes the database, so it cannot leave actions using it.""" diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index d6747f09b6..633e04770e 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -8,6 +8,7 @@ import hashlib import json import os +from collections.abc import Collection from concurrent.futures import CancelledError as FuturesCancelledError from concurrent.futures import TimeoutError as FuturesTimeoutError from typing import Any @@ -133,6 +134,7 @@ async def cancel_inflight_actions( *, reason: str, exempt: frozenset[str] = frozenset(), + only_task_ids: Collection[str] | None = None, ) -> list[str]: """Cancel the running dispatched actions and wait for them to unwind. @@ -151,6 +153,10 @@ async def cancel_inflight_actions( reason: Short cause, logged and used as evidence. exempt: Action kinds to leave running -- the closing actions, when the trigger is a budget that already reserved time for them. + only_task_ids: Restrict the cancel to these ids, for a caller that + owns part of the registry rather than all of it. ``None`` reaches + every action that is not exempt, which is what a shutdown or a + spent budget needs. Returns: list[str]: Task ids that were cancelled (empty when nothing ran). @@ -158,7 +164,7 @@ async def cancel_inflight_actions( victims = [ (task_id, kind, atask) for task_id, (kind, atask) in self._inflight_actions.items() - if kind not in exempt and not atask.done() + if kind not in exempt and not atask.done() and (only_task_ids is None or task_id in only_task_ids) ] if not victims: return [] @@ -296,12 +302,16 @@ async def _pump_dispatcher_once(self) -> None: for task, maybe_result, gpu_lease in completed: await self._reap_dispatched_task(task, maybe_result, gpu_lease) finally: - # The pump is the only owner of the actions it spawned. Leaving by - # any door other than the drained one -- cancelled at shutdown, or a - # raise from the bookkeeping -- would otherwise leave them running - # with every handle on them gone. A drained pump has nothing left to - # cancel, so the normal exit pays nothing for this. - await self.cancel_inflight_actions(reason="dispatcher_pump_exit") + # ``_inflight_actions`` is dispatcher-wide: the inline path registers + # a handle there too, and that action is meant to outlive the caller + # that started it. The pump owns exactly the entries still in its own + # ``inflight``, so leaving by any door other than the drained one -- + # cancelled at shutdown, or a raise from the bookkeeping -- takes + # those and nothing else. A drained pump has nothing left to cancel. + await self.cancel_inflight_actions( + reason="dispatcher_pump_exit", + only_task_ids={task.task_id for task, _atask, _gpu_lease in inflight}, + ) async def _reconcile_cancelled_policy_denied_integrate_tasks(self) -> list[str]: """Re-queue integrate_patch rows cancelled at dispatch when policy now passes. From 5b784b28a01d7ba238d4ea83c5716a6b17c93e1f Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 10:58:49 +0000 Subject: [PATCH 11/65] fix(close): wait for a report already running instead of re-running it The wall-clock deadline path enqueues the report AND dispatches it before CLOSE is entered, so the sequencer routinely meets its own step in ``running``. The registry allows ``running`` only into a terminal state, so handing that row to ``run_task`` raised ``IllegalTransition`` and took the close step down -- costing the report of the session that had the least left to show for itself, which is the one the report is worth the most for. The intent was already written down: the enqueue helper logs that the sequencer will wait for such a row. It just never did. It does now, by polling the registry at the dispatcher's own interval, bounded by the closing grace window -- the same budget the deadline path set aside for this work, so a task that never lands costs CLOSE that window and nothing more. Whatever state the row is in when the wait ends is reported the way every other branch reports one. Co-authored-by: Cursor --- .../tests/test_close_phase_sequencer.py | 91 +++++++++++++++++++ src/hyperloom/orchestrator/phases/close.py | 88 ++++++++++++++++-- 2 files changed, 173 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py index 56cd5a1b9e..23d76b1c73 100644 --- a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py +++ b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py @@ -36,6 +36,7 @@ class _BareState: recipe_finalize_attempts: int = 0 recipe_finalize_outcome: dict[str, Any] = field(default_factory=dict) phase_history: list[dict[str, Any]] = field(default_factory=list) + max_minutes: int = 0 save_count: int = 0 def save(self, _session_dir: Path | None) -> None: @@ -331,6 +332,96 @@ async def test_a_terminal_task_is_reported_not_run(coord): assert coord.sub.run_calls == [] +class _FinishesWhileWaiting(_StubTaskRegistry): + """Registry whose running row lands terminal once the sequencer looks again.""" + + def __init__(self, terminal_state: str): + super().__init__() + self._terminal_state = terminal_state + self.gets = 0 + + async def get(self, task_id): + row = await super().get(task_id) + self.gets += 1 + if self.gets >= 2: + row.state = self._terminal_state + return row + + +def _running_report_row(coord) -> _StubTaskRow: + """Register a report the wall-clock deadline path already enqueued AND dispatched.""" + row = _StubTaskRow( + task_id="wallclock-report", + kind="report", + state="running", + params={}, + idempotency_key="closing-report-1234", + ) + coord.tasks._by_id[row.task_id] = row + return row + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal_state", ["succeeded", "failed"]) +async def test_a_running_task_is_waited_for_not_re_run(coord, terminal_state: str): + """``running -> running`` is not a transition the registry has; asking for it kills the step. + + The deadline path dispatches the report before CLOSE is entered, so the + sequencer routinely meets its own step already under way. It was documented + as "the sequencer will wait for it" and implemented as a second dispatch. + """ + coord.tasks = _FinishesWhileWaiting(terminal_state) + coord._dispatcher_poll_sec = 0.01 + coord.shared_state.max_minutes = 60 + + state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") + + assert state == terminal_state + assert coord.sub.run_calls == [] + + +@pytest.mark.asyncio +async def test_a_running_task_that_never_lands_is_reported_not_waited_on_forever(coord): + """The closing grace window is the budget for this work, so it is also the bound.""" + coord._dispatcher_poll_sec = 0.01 + # Resolves to a 1.2s grace window, the same arithmetic a real session uses. + coord.shared_state.max_minutes = 1 + + state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") + + assert state == "running" + assert coord.sub.run_calls == [] + + +@pytest.mark.asyncio +async def test_a_zero_grace_window_buys_no_wait_at_all(coord): + """The row is still looked at once: no window is not the same as no answer.""" + coord.shared_state.max_minutes = 0 # no budget -> no grace window + + state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") + + assert state == "running" + assert coord.sub.run_calls == [] + + +@pytest.mark.asyncio +async def test_the_sequencer_records_the_state_a_running_report_ended_in(coord): + """End to end: the waited-for report is reported like any other outcome.""" + coord.tasks = _FinishesWhileWaiting("succeeded") + coord._dispatcher_poll_sec = 0.01 + coord.shared_state.max_minutes = 60 + coord.shared_state.phase_history = [_close_phase_history_row()] + _running_report_row(coord) + coord.shared_state.closing_report_task_id = "wallclock-report" + + await coord._on_enter_close(from_phase="SWEEP") + + rows = coord.shared_state.phase_history[-1]["evidence"]["close_steps"] + report = next(r for r in rows if r["step"] == "report") + assert report["status"] == "done" + assert "wallclock-report" not in [t.task_id for t in coord.sub.run_calls] + + @pytest.mark.asyncio async def test_enqueue_internal_session_breakdown_task(coord): task = await coord._enqueue_internal_session_breakdown_task( diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index 7e20510efe..bd6b25c07b 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -13,6 +13,7 @@ import logging as _logging from . import machine_state as _phase_state from ..bus.message_bus import Message +from ..loop.coordinator_helpers import effective_closing_grace_sec from ..state.task_registry import Task from .base import PhaseHandler @@ -26,10 +27,15 @@ # a fresh row (re-enqueue under a distinct idempotency key). _TASK_STATE_DONE: str = "succeeded" _DEAD_TASK_STATES: frozenset[str] = frozenset({"cancelled", "failed"}) +# A row the wall-clock deadline path already dispatched. Not terminal, and not +# runnable either: the registry allows ``running`` only into a terminal state. +_TASK_STATE_RUNNING: str = "running" # Appended to a close step's idempotency key when its first row is dead, so # ``create_or_return_existing`` mints a new task instead of returning the # corpse. _RETRY_KEY_SUFFIX: str = "retry" +# Fallback registry poll interval, for a caller with no dispatcher poll set. +_DEFAULT_TASK_POLL_SEC: float = 10.0 def _task_is_dead(task: Task | None) -> bool: @@ -507,21 +513,89 @@ async def _enqueue_internal_session_breakdown_task( idempotency_key=f"internal-session_breakdown-{reason}", ) + async def _await_running_close_task(self, task: Task, *, step: str) -> str: + """Wait for an already-dispatched close-step task to reach a terminal state. + + The wall-clock deadline path enqueues the report and dispatches it + before CLOSE is entered, so the sequencer can find its own step already + under way. Handing that row to ``run_task`` asks the registry for + ``running -> running``, which it refuses, taking the close step down + with it — and the session that ran out of time is the session whose + report is worth the most. + + The wait is bounded by the closing grace window, which is the budget + reserved for exactly this work, so a task that never lands costs CLOSE + that window and no more. + + Args: + task: The close-step task found in ``running``. + step: Close-step label, for logging. + + Returns: + The state the task ended in, or ``running`` when the bound elapsed + first — which the caller records the same way it records a failure. + """ + bound_sec = max( + 0.0, + effective_closing_grace_sec( + float(getattr(self.shared_state, "max_minutes", 0) or 0), + None, + ), + ) + poll_sec = float(getattr(self, "_dispatcher_poll_sec", _DEFAULT_TASK_POLL_SEC)) + deadline = time.monotonic() + bound_sec + log.info( + "CLOSE step %s: task_id=%s is already running; waiting up to %.0fs for it", + step, + task.task_id, + bound_sec, + ) + state = _TASK_STATE_RUNNING + while True: + try: + state = str(getattr(await self.tasks.get(task.task_id), "state", "") or "") + except Exception: # noqa: BLE001 — TaskNotFound + friends + log.warning( + "CLOSE step %s: task_id=%s vanished while the sequencer waited for it", + step, + task.task_id, + ) + return state + if state != _TASK_STATE_RUNNING: + log.info( + "CLOSE step %s: task_id=%s finished as %s while the sequencer waited", + step, + task.task_id, + state, + ) + return state + remaining = deadline - time.monotonic() + if remaining <= 0.0: + log.warning( + "CLOSE step %s: task_id=%s still running after %.0fs; recording the step as failed", + step, + task.task_id, + bound_sec, + ) + return state + await asyncio.sleep(min(poll_sec, remaining)) + async def _run_close_task(self, task: Task, *, step: str) -> str | None: - """Run one close-step task and return its terminal state. + """Run one close-step task and return the state it ended in. ``run_task`` transitions ``queued -> running``, which the registry - refuses for a row that is already terminal — and refuses correctly: - the rejection is the double-spawn guard. So a terminal row is reported - here instead of run, which keeps one dead task from taking down the - step that was supposed to salvage the session. + refuses for a row that is already terminal or already running — and + refuses correctly: the rejection is the double-spawn guard. So such a + row is reported or waited on here instead of run, which keeps one row + the sequencer did not create from taking down the step that was + supposed to salvage the session. Args: task: The task to run. step: Close-step label, for logging. Returns: - The task's terminal state. + The state the task ended in. """ state = str(getattr(task, "state", "") or "") if state == _TASK_STATE_DONE: @@ -539,6 +613,8 @@ async def _run_close_task(self, task: Task, *, step: str) -> str | None: state, ) return state + if state == _TASK_STATE_RUNNING: + return await self._await_running_close_task(task, step=step) result = await self.sub.run_task(task) return result.state From 7051ebf663e1efceefc9b151082c5b6c40cf95e7 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 11:21:22 +0000 Subject: [PATCH 12/65] fix(state): reserve the closing grace window, not a fixed two minutes The budget held back from every unit of work was hardcoded at 120s and commented "matches the default closing grace window", but that window is ``min(120, 2% of the session)`` unless the operator names one. The comment was true only for sessions of at least 100 minutes: a 1-hour session reserved 120s against a 72s grace, and ``--closing-grace-sec 0``, documented as disabling the closing phase, still cost 120s of budget for work that never runs. This PR made that constant load-bearing beyond the grid: session_budget_usable_sec is now the admission criterion, so the same wrong number decides whether an action may start at all. Resolve the reserve through effective_closing_grace_sec, so the budget kept for the closing phase and the budget that phase actually gets are one number. The operator's value has to reach SharedState for that, since it arrived only at the Coordinator; the helper moves next to the budget accessors, which is the leaf both sides can import. The CLOSE wait bound and the GEAK subprocess cap read the same accessor instead of re-deriving the window with the operator's choice dropped. Co-authored-by: Cursor --- .../tests/test_close_phase_sequencer.py | 5 ++ .../tests/test_coordinator_helpers_unit.py | 15 ---- .../tests/test_session_time_budget.py | 77 +++++++++++++++++-- .../tests/test_shared_state_units.py | 6 +- .../orchestrator/loop/coordinator.py | 9 ++- .../orchestrator/loop/coordinator_helpers.py | 22 ------ src/hyperloom/orchestrator/phases/close.py | 9 +-- src/hyperloom/orchestrator/phases/kernel.py | 6 +- .../orchestrator/state/shared_state.py | 64 ++++++++++++--- 9 files changed, 139 insertions(+), 74 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py index 23d76b1c73..4ef6b60368 100644 --- a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py +++ b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py @@ -21,6 +21,7 @@ ) from hyperloom.orchestrator.loop.coordinator import Coordinator from hyperloom.orchestrator.policy.gate import CORE_STATE_FIELDS +from hyperloom.orchestrator.state.shared_state import effective_closing_grace_sec @dataclass @@ -37,6 +38,7 @@ class _BareState: recipe_finalize_outcome: dict[str, Any] = field(default_factory=dict) phase_history: list[dict[str, Any]] = field(default_factory=list) max_minutes: int = 0 + closing_grace_sec: float | None = None save_count: int = 0 def save(self, _session_dir: Path | None) -> None: @@ -45,6 +47,9 @@ def save(self, _session_dir: Path | None) -> None: def set_stop_reason(self, reason: str) -> None: self.stop_reason = reason + def closing_reserve_sec(self) -> float: + return effective_closing_grace_sec(self.max_minutes, self.closing_grace_sec) + @dataclass class _StubTaskRow: diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py index ad3eb327d7..992225cfbb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_helpers_unit.py @@ -46,21 +46,6 @@ def test_infer_model_class_ignores_bool_experts(tmp_path): assert ch._infer_model_class_from_config(str(tmp_path)) == "dense" -# ---- effective_closing_grace_sec ---- - - -def test_closing_grace_explicit(): - assert ch.effective_closing_grace_sec(100, 0) == 0.0 - assert ch.effective_closing_grace_sec(100, 5) == 5.0 - - -def test_closing_grace_default(): - # min(120, max_minutes*60*0.02). - assert ch.effective_closing_grace_sec(200, None) == 120.0 - assert ch.effective_closing_grace_sec(10, None) == 12.0 - assert ch.effective_closing_grace_sec(None, None) == 0.0 - - # ---- _parse_iso_unix ---- diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 3f4dcca84f..47762ea5ea 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -37,7 +37,7 @@ ) from hyperloom.orchestrator.policy.gate import PolicyDenied from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan -from hyperloom.orchestrator.state.shared_state import CLOSING_RESERVE_SEC, SharedState +from hyperloom.orchestrator.state.shared_state import SharedState, effective_closing_grace_sec from hyperloom.orchestrator.state.task_registry import Task # An action costing an hour at p50, so a short budget cannot fit it. @@ -64,6 +64,18 @@ def coord(session_dir) -> Coordinator: return c +def _budgeted_state( + *, + minutes: float, + elapsed_min: float = 0.0, + closing_grace_sec: float | None = None, +) -> SharedState: + """A standalone state with a finite budget and ``elapsed_min`` already spent.""" + state = SharedState(session_id="s", max_minutes=int(minutes), closing_grace_sec=closing_grace_sec) + state.elapsed_minutes = lambda **_kw: elapsed_min # type: ignore[method-assign] + return state + + def _set_budget(coord: Coordinator, *, minutes: float, elapsed_min: float = 0.0) -> None: """Give the session a finite budget with ``elapsed_min`` already spent.""" coord.shared_state.max_minutes = int(minutes) @@ -103,25 +115,74 @@ def test_an_unset_budget_reads_as_unbounded(self): assert SharedState(session_id="s").session_budget_usable_sec() is None def test_the_closing_reserve_is_held_back(self): - state = SharedState(session_id="s", max_minutes=60) - state.elapsed_minutes = lambda **_kw: 0.0 # type: ignore[method-assign] - assert state.session_budget_usable_sec() == pytest.approx(3600.0 - CLOSING_RESERVE_SEC) + state = _budgeted_state(minutes=60) + assert state.session_budget_usable_sec() == pytest.approx(3600.0 - 72.0) def test_a_budget_inside_the_reserve_reads_as_spent(self): - state = SharedState(session_id="s", max_minutes=60) - state.elapsed_minutes = lambda **_kw: 59.9 # type: ignore[method-assign] + state = _budgeted_state(minutes=60, elapsed_min=59.9) assert state.session_budget_usable_sec() == 0.0 def test_the_grid_deadline_is_derived_from_the_same_number(self, monkeypatch): """Both wall-clock layers must agree on how much budget is left.""" import time as _time - state = SharedState(session_id="s", max_minutes=60) - state.elapsed_minutes = lambda **_kw: 10.0 # type: ignore[method-assign] + state = _budgeted_state(minutes=60, elapsed_min=10.0) + monkeypatch.setattr(_time, "monotonic", lambda: 1000.0) + usable = state.session_budget_usable_sec() + assert state.grid_session_deadline_sec() == pytest.approx(1000.0 + usable) + + +class TestTheReserveIsTheClosingGraceWindow: + """The budget held back must be the budget the CLOSE phase actually gets. + + A fixed 120s reserve was only ever right for sessions of at least 100 + minutes: a shorter one was charged more than its closing phase can spend, + and an operator who passed ``--closing-grace-sec 0`` to disable that phase + paid 120 seconds for work that never runs. + """ + + @pytest.mark.parametrize( + ("minutes", "closing_grace_sec", "expected"), + [ + (120, None, 120.0), # the default session: unchanged by this fix + (60, None, 72.0), # min(120, 2% of the budget) + (60, 0.0, 0.0), # closing phase disabled: reserve nothing + (60, 600.0, 600.0), # an explicit window wins verbatim + (0, None, 0.0), # unbounded budget: nothing to reserve from + ], + ) + def test_the_reserve_tracks_the_resolved_grace_window(self, minutes, closing_grace_sec, expected): + state = SharedState(session_id="s", max_minutes=minutes, closing_grace_sec=closing_grace_sec) + assert state.closing_reserve_sec() == pytest.approx(expected) + assert state.closing_reserve_sec() == pytest.approx(effective_closing_grace_sec(minutes, closing_grace_sec)) + + @pytest.mark.parametrize("closing_grace_sec", [None, 0.0, 600.0]) + def test_admission_and_the_grid_deadline_agree_on_every_reserve(self, closing_grace_sec, monkeypatch): + import time as _time + + state = _budgeted_state(minutes=60, elapsed_min=20.0, closing_grace_sec=closing_grace_sec) monkeypatch.setattr(_time, "monotonic", lambda: 1000.0) usable = state.session_budget_usable_sec() + assert usable == pytest.approx(max(0.0, 2400.0 - state.closing_reserve_sec())) assert state.grid_session_deadline_sec() == pytest.approx(1000.0 + usable) + def test_a_disabled_closing_phase_leaves_the_last_minutes_spendable(self): + """The 120s a disabled phase used to cost is the difference here.""" + spent = _budgeted_state(minutes=60, elapsed_min=59.0) + kept = _budgeted_state(minutes=60, elapsed_min=59.0, closing_grace_sec=0.0) + assert spent.session_budget_usable_sec() == 0.0 + assert kept.session_budget_usable_sec() == pytest.approx(60.0) + + @pytest.mark.asyncio + async def test_the_coordinator_hands_the_operators_window_to_the_state(self, coord: Coordinator): + """The reserve lives on SharedState, but the flag arrives at the Coordinator.""" + try: + await coord.run(max_ticks=1, max_minutes=60, closing_grace_sec=0.0) + finally: + await coord.stop() + assert coord.shared_state.closing_grace_sec == 0.0 + assert coord.shared_state.closing_reserve_sec() == 0.0 + class TestTimeBudgetGate: """The dispatcher gate that turns a fit failure into a refusal.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py index f6f702046c..2c3d163fe2 100644 --- a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py @@ -122,8 +122,8 @@ def test_future_deadline_when_ample_budget(self, monkeypatch): s.max_minutes = 60.0 monkeypatch.setattr(type(s), "remaining_minutes", lambda self, **_: 10.0) monkeypatch.setattr(ss_mod.time, "monotonic", lambda: 1000.0) - # 10 min remaining, 120s reserve -> now + (600 - 120). - assert s.grid_session_deadline_sec() == pytest.approx(1000.0 + 480.0) + # 10 min remaining, 72s closing reserve -> now + (600 - 72). + assert s.grid_session_deadline_sec() == pytest.approx(1000.0 + 528.0) def test_deadline_is_now_when_under_reserve(self, monkeypatch): import hyperloom.orchestrator.state.shared_state as ss_mod @@ -132,7 +132,7 @@ def test_deadline_is_now_when_under_reserve(self, monkeypatch): s.max_minutes = 60.0 monkeypatch.setattr(type(s), "remaining_minutes", lambda self, **_: 1.0) monkeypatch.setattr(ss_mod.time, "monotonic", lambda: 500.0) - # 60s remaining < 120s reserve -> deadline == now (already exhausted). + # 60s remaining < 72s closing reserve -> deadline == now (already exhausted). assert s.grid_session_deadline_sec() == pytest.approx(500.0) diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 0a6fcd8e58..dfb5bd8cc4 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -77,7 +77,7 @@ ResourceLockManager, SqliteLeaseBackend, ) -from ..state.shared_state import SharedState +from ..state.shared_state import SharedState, effective_closing_grace_sec from .intent_router import IntentRouter from .sub_agent_runner import SubAgentRunner from ..state.task_registry import TaskRegistry @@ -89,7 +89,6 @@ ) from .coordinator_helpers import ( _infer_model_class_from_config, - effective_closing_grace_sec, format_exc_brief, serialize_verdict_advisory, ) @@ -1607,6 +1606,10 @@ async def run( self._coordinator_loop = asyncio.get_running_loop() except RuntimeError: self._coordinator_loop = None + # The reserve every unit of work holds back IS this window, so the + # admission gate has to see the operator's choice too, not only the + # closing phase that spends it. + self.shared_state.closing_grace_sec = closing_grace_sec max_minutes_value = max_minutes if max_minutes is not None else 0 # Persist budget so prompts and Resume can see it. if max_minutes is not None: @@ -2093,7 +2096,7 @@ async def _track_backend_error_streak( "CoordinatorState", "PendingProposal", "SharedState", - # Re-exported from coordinator_helpers for callers/tests. + # Re-exported from coordinator_helpers / state.shared_state for callers/tests. "_infer_model_class_from_config", "effective_closing_grace_sec", # Re-exported from policy.gate; referenced via ``coordinator.`` in tests. diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index 0e4b0ecb4c..d191990ead 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -218,28 +218,6 @@ def _positive_int(*keys: str) -> bool: _MAX_ROOFLINE_FAILURE_RETRIES: int = 3 -def effective_closing_grace_sec( - max_minutes: float | None, - closing_grace_sec: float | None, -) -> float: - """Resolve the closing-phase grace window after the wall-clock deadline. - - Explicit ``closing_grace_sec`` (including ``0`` to disable) wins; - otherwise default to ``min(120, max_minutes * 60 * 0.02)``. - - Args: - max_minutes: The wall-clock budget in minutes (used for the default). - closing_grace_sec: Explicit grace window in seconds; when not - ``None`` it is used verbatim. - - Returns: - The closing-phase grace window in seconds. - """ - if closing_grace_sec is not None: - return float(closing_grace_sec) - return min(120.0, (max_minutes or 0.0) * 60.0 * 0.02) - - # Actions that must stay startable no matter how little budget is left: they are # how a session ends cleanly (report/breakdown) or unsticks itself (recover), so # a time gate that refused them would strand the run with nothing to show. diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index bd6b25c07b..f2e8c77dab 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -13,7 +13,6 @@ import logging as _logging from . import machine_state as _phase_state from ..bus.message_bus import Message -from ..loop.coordinator_helpers import effective_closing_grace_sec from ..state.task_registry import Task from .base import PhaseHandler @@ -535,13 +534,7 @@ async def _await_running_close_task(self, task: Task, *, step: str) -> str: The state the task ended in, or ``running`` when the bound elapsed first — which the caller records the same way it records a failure. """ - bound_sec = max( - 0.0, - effective_closing_grace_sec( - float(getattr(self.shared_state, "max_minutes", 0) or 0), - None, - ), - ) + bound_sec = self.shared_state.closing_reserve_sec() poll_sec = float(getattr(self, "_dispatcher_poll_sec", _DEFAULT_TASK_POLL_SEC)) deadline = time.monotonic() + bound_sec log.info( diff --git a/src/hyperloom/orchestrator/phases/kernel.py b/src/hyperloom/orchestrator/phases/kernel.py index d21985a28d..775f06efd4 100644 --- a/src/hyperloom/orchestrator/phases/kernel.py +++ b/src/hyperloom/orchestrator/phases/kernel.py @@ -33,7 +33,6 @@ _resolve_roofline_watermark_ratio, _resolve_serving_fidelity, _split_env_and_flags, - effective_closing_grace_sec, ) from .base import PhaseHandler @@ -597,10 +596,7 @@ def _geak_timeouts(self) -> tuple[int, int, bool]: if deadline is None: return env_default_timeout, env_default_timeout + 600, False remaining = deadline - time.monotonic() - grace = effective_closing_grace_sec( - float(getattr(self.shared_state, "max_minutes", 0) or 0), - None, - ) + grace = self.shared_state.closing_reserve_sec() margin = float(os.environ.get("GEAK_BUDGET_MARGIN_S", "300")) # Reserve the closing window: kill the subprocess with at least ``grace`` left. kill_budget = remaining - grace diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index aab3c491e6..6e7cc7b68a 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -260,10 +260,6 @@ def render_model_arch_compact(arch: dict | None) -> str: # summary (matches the ``*_top15`` field names). _TRACE_HOT_KERNEL_TOP_N = 15 -# Wall-clock budget held back from every unit of work so the CLOSE phase can -# still write its report. Matches the default closing grace window. -CLOSING_RESERVE_SEC = 120.0 - # Session-level kernel-roofline report the analyzer writes for a non-close run; # read back when the trace_analyze envelope arrives without its payload keys. _DEFAULT_ROOFLINE_REPORT_NAME = "kernel_roofline_current.json" @@ -344,6 +340,33 @@ def render_model_arch_compact(arch: dict | None) -> str: _FRAMEWORK_VARIANT_PREFIX_V5: str = "framework:" +def effective_closing_grace_sec( + max_minutes: float | None, + closing_grace_sec: float | None, +) -> float: + """Resolve the closing-phase grace window after the wall-clock deadline. + + Explicit ``closing_grace_sec`` (including ``0`` to disable the closing + phase) wins; otherwise default to ``min(120, max_minutes * 60 * 0.02)``. + + Lives beside the budget accessors rather than next to its Coordinator + caller because the window is also the reserve those accessors hold back, + and this module is the leaf both sides can import. + + Args: + max_minutes (float | None): The wall-clock budget in minutes, used for + the default. + closing_grace_sec (float | None): Explicit grace window in seconds; + when not ``None`` it is used verbatim. + + Returns: + float: The closing-phase grace window in seconds. + """ + if closing_grace_sec is not None: + return float(closing_grace_sec) + return min(120.0, (max_minutes or 0.0) * 60.0 * 0.02) + + def _cap_tested_ledger(tested: dict[str, Any]) -> dict[str, Any]: """Bound the explore_search negative ledger for multi-day runs. @@ -651,6 +674,8 @@ class SharedState(_RenderMixin, _ExploreStateMixin): pruned_families: list[str] = field(default_factory=list) start_ts: str = field(default_factory=_now_iso) max_minutes: int = 0 + # Operator's ``--closing-grace-sec``; ``None`` derives it from max_minutes. + closing_grace_sec: float | None = None last_profile_trace: str = "" # ``succeeded``/``failed`` for most recent profile; failed allows re-run even when last_profile_trace is non-empty. last_profile_status: str = "" @@ -3607,10 +3632,26 @@ def remaining_minutes(self, *, now: datetime | None = None) -> float | None: return None return max(0.0, float(self.max_minutes) - self.elapsed_minutes(now=now)) + def closing_reserve_sec(self) -> float: + """Seconds held back from every unit of work so CLOSE can still report. + + Resolved through :func:`effective_closing_grace_sec`, the same function + the Coordinator uses to size the closing phase itself, so the budget + reserved for that phase and the budget it actually gets are one number. + A hardcoded reserve told the truth only for sessions of at least 100 + minutes, and charged 120 seconds even to an operator who had disabled + the closing phase outright. + + Returns: + float: The closing reserve in seconds; ``0.0`` when the operator + disabled the closing phase. + """ + return max(0.0, effective_closing_grace_sec(float(self.max_minutes or 0), self.closing_grace_sec)) + def session_budget_usable_sec( self, *, - reserve_sec: float = CLOSING_RESERVE_SEC, + reserve_sec: float | None = None, ) -> float | None: """Seconds of wall-clock budget left once the closing reserve is held back. @@ -3620,8 +3661,9 @@ def session_budget_usable_sec( much budget exists. Args: - reserve_sec (float): Seconds held back for the CLOSE phase and its - report. + reserve_sec (float | None): Seconds held back for the CLOSE phase + and its report; ``None`` takes :meth:`closing_reserve_sec`. An + explicit ``0`` is honoured as "reserve nothing". Returns: float | None: Usable seconds (clamped at 0.0), or ``None`` when @@ -3630,12 +3672,13 @@ def session_budget_usable_sec( remaining = self.remaining_minutes() if remaining is None: return None - return max(0.0, remaining * 60.0 - reserve_sec) + reserve = self.closing_reserve_sec() if reserve_sec is None else float(reserve_sec) + return max(0.0, remaining * 60.0 - reserve) def grid_session_deadline_sec( self, *, - reserve_sec: float = CLOSING_RESERVE_SEC, + reserve_sec: float | None = None, ) -> float | None: """``time.monotonic()`` deadline for grid variant loops, or ``None`` when the budget is unbounded. @@ -3645,7 +3688,8 @@ def grid_session_deadline_sec( the whole grid. Args: - reserve_sec (float): Seconds held back from the raw remaining budget. + reserve_sec (float | None): Seconds held back from the raw remaining + budget; ``None`` takes :meth:`closing_reserve_sec`. Returns: float | None: A monotonic-clock deadline, or ``None`` when unbounded; From 117d74cefc7683c99f61b77542d2b8c7e264be14 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 11:31:08 +0000 Subject: [PATCH 13/65] fix(loop): read the action cost the catalogue actually carries Admission judged every action free. It read ``cost_minutes_p50`` off the catalogue entry, a field that stopped existing when main inlined the action YAML into ACTION_CATALOGUE and dropped the cost columns as unread -- this branch had added the reader in parallel, so the merge left a getattr default of 0.0 behind, which action_fits_time_budget treats as "no estimate on record" and admits. Layer 1 of the wall-clock defence was inert in production, and the same default made the PRELUDE affordability guard rate its analysis arm free. Read ``typical_runtime_min``, which carries the same numbers the p50 column did, through one helper both guards share, so the field is named in one place. The regression that hid this is that a wrong field name looks exactly like an uncatalogued action, so the guard test asserts no catalogued action reads as free. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 19 ++++++++++++++ .../orchestrator/loop/coordinator_helpers.py | 26 +++++++++++++++++-- src/hyperloom/orchestrator/loop/dispatcher.py | 3 ++- src/hyperloom/orchestrator/phases/prelude.py | 6 ++--- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 47762ea5ea..9f33c9cc3e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -29,11 +29,13 @@ import pytest +from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType from hyperloom.orchestrator.loop.coordinator import Coordinator from hyperloom.orchestrator.loop.coordinator_helpers import ( TIME_BUDGET_EXEMPT_ACTIONS, action_fits_time_budget, + expected_action_cost_minutes, ) from hyperloom.orchestrator.policy.gate import PolicyDenied from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan @@ -82,6 +84,23 @@ def _set_budget(coord: Coordinator, *, minutes: float, elapsed_min: float = 0.0) coord.shared_state.elapsed_minutes = lambda **_kw: elapsed_min # type: ignore[method-assign] +class TestTheCostTheGateJudgesOn: + """Where the expected cost comes from: the action catalogue, or nowhere.""" + + def test_a_catalogued_action_reads_its_expected_runtime(self): + assert expected_action_cost_minutes(ACTION_CATALOGUE[_EXPENSIVE_ACTION]) == pytest.approx(_EXPENSIVE_COST_MIN) + + def test_an_action_the_catalogue_does_not_carry_has_no_estimate(self): + assert expected_action_cost_minutes(None) == 0.0 + + def test_no_catalogued_action_reads_as_free(self): + """A zero cost admits an action on any budget, so a whole catalogue of + them is an admission gate that is not there — which is what reading a + renamed field through a ``getattr`` default silently produced.""" + free = sorted(name for name, meta in ACTION_CATALOGUE.items() if expected_action_cost_minutes(meta) <= 0.0) + assert free == [] + + class TestFitDecision: """The pure fit rule, independent of any Coordinator.""" diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index d191990ead..e4bfb8209f 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -230,6 +230,28 @@ def _positive_int(*keys: str) -> bool: ) +def expected_action_cost_minutes(meta: Any | None) -> float: + """Read an action's expected cost out of its catalogue metadata. + + Both budget guards go through here so the field is named once. Reading it + inline with a ``getattr`` default turned the catalogue's move off YAML — + which renamed the field — into a gate that admitted everything without a + word, because "no estimate on record" and "the field moved" look the same + from a default. + + Args: + meta (Any | None): The action's catalogue metadata, or ``None`` for an + action the catalogue does not carry. + + Returns: + float: The expected cost in minutes; ``0.0`` when nothing is on record. + """ + try: + return float(getattr(meta, "typical_runtime_min", 0.0) or 0.0) + except (TypeError, ValueError): + return 0.0 + + def action_fits_time_budget( *, usable_sec: float | None, @@ -237,8 +259,8 @@ def action_fits_time_budget( ) -> bool: """Decide whether an action's expected cost still fits the remaining budget. - The anchor is the action's *expected* cost (``cost_minutes_p50``), not its - p75 backstop. Judging fit on the pessimistic tail would abandon usable + The anchor is the action's *expected* cost (its typical runtime), not its + pessimistic tail. Judging fit on the pessimistic tail would abandon usable budget — with 90 minutes left we would refuse an action that finishes in 60 minutes half the time — and the session already has a wall-clock reaper for the overruns, so the optimistic anchor is the one that keeps the tail of a diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 633e04770e..31cc4e2f4e 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -37,6 +37,7 @@ TIME_BUDGET_EXEMPT_ACTIONS, action_fits_time_budget, coerce_needs_gpu, + expected_action_cost_minutes, ) from .coordinator import ( @@ -1301,7 +1302,7 @@ def _time_budget_denial_for_action( meta = reg.get(action) if reg is not None else None if meta is None: return None - expected_min = float(getattr(meta, "cost_minutes_p50", 0.0) or 0.0) + expected_min = expected_action_cost_minutes(meta) usable_sec = self.shared_state.session_budget_usable_sec() if action_fits_time_budget( usable_sec=usable_sec, diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index 93423cc040..7cac792a77 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -24,6 +24,7 @@ from ..loop.coordinator import ( _DEFAULT_WARM_REPLAY_MIN_CONFIDENCE, ) +from ..loop.coordinator_helpers import expected_action_cost_minutes from .base import PhaseHandler log = _logging.getLogger(__name__) @@ -178,10 +179,7 @@ def _measured_analysis_cost_sec(self) -> float: return measured registry = getattr(self, "action_registry", None) meta = registry.get(self._internal_analysis_kind()) if registry is not None else None - try: - return float(getattr(meta, "cost_minutes_p50", 0.0) or 0.0) * 60.0 - except (TypeError, ValueError): - return 0.0 + return expected_action_cost_minutes(meta) * 60.0 def _record_prelude_arm_dropped(self, arm: str, evidence: dict[str, Any]) -> None: """Record a PRELUDE arm dropped for budget on the current phase record. From 402a7770bdff76cc571b8e8486b3a2a6913618fa Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 11:46:48 +0000 Subject: [PATCH 14/65] test: stop two doubles asserting the behaviour these fixes removed Both encoded the state of the world before this branch. The CLOSE lifecycle tests handed the sequencer a report row already in ``running`` -- a state a freshly enqueued row is never in -- and expected it to be dispatched anyway, which is precisely the illegal transition the close fix exists to stop; they now hand over the ``queued`` row the enqueue helper really returns. The explore clamp test spelled the closing reserve as a literal 120 seconds against a 3-minute session, so it asserted a bound the reserve no longer has; it reads the budget accessor the clamp reads, before the run, when the budget is at its largest. Co-authored-by: Cursor --- .../tests/test_explore_executor.py | 11 +++++++---- .../tests/test_lifecycle_wiring.py | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 156be866ac..e5128eada3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -1622,8 +1622,11 @@ async def test_explore_variant_cap_is_clamped_to_the_session_budget( sub, tr, _ = sub_agent_runner state = SharedState() state.baseline_tput = 800.0 - state.max_minutes = 3.0 # 180s budget - 120s close reserve => ~60s usable + state.max_minutes = 3.0 sub.shared_state = state + # Read before the run: the budget only shrinks from here, so a cap granted + # later can only be smaller than what this allows. + usable_sec = state.session_budget_usable_sec() base = tmp_path / "base.yaml" _write_baseline_yaml(base) @@ -1666,12 +1669,12 @@ def _fake_run(cmd, *args, **kwargs): res = await sub.run_task(task) assert res.result["status"] == "succeeded" - assert granted, "the variant should have been admitted (20s expected, ~60s left)" + assert granted, f"the variant should have been admitted (20s expected, ~{usable_sec:.0f}s left)" # The hard cap is allowed to sit a grace window past the deadline so the # in-process session watchdog reaps the tree first and the kill is attributed # to the budget rather than to a slow variant. - assert all(t <= 60 + _SESSION_KILL_GRACE_SEC for t in granted), ( - f"caps must be clamped to the ~60s budget, got {granted}" + assert all(t <= usable_sec + _SESSION_KILL_GRACE_SEC for t in granted), ( + f"caps must be clamped to the ~{usable_sec:.0f}s budget, got {granted}" ) assert all(t < 3600 for t in granted), f"the declared 3600s cap must not survive the budget, got {granted}" diff --git a/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py b/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py index 294e79740e..80f2830185 100644 --- a/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py +++ b/src/hyperloom/inference_optimizer/tests/test_lifecycle_wiring.py @@ -460,7 +460,7 @@ async def test_on_enter_close_emits_report_end(session_dir, monkeypatch): report_task = Task( task_id="rpt-1", kind="report", - state="running", + state="queued", params={}, idempotency_key="internal-report-close", requires_lanes=[], @@ -468,7 +468,7 @@ async def test_on_enter_close_emits_report_end(session_dir, monkeypatch): bd_task = Task( task_id="bd-1", kind="session_breakdown", - state="running", + state="queued", params={}, idempotency_key="internal-breakdown-close", requires_lanes=[], @@ -532,7 +532,7 @@ async def test_on_enter_close_emits_report_error_for_failed_task( report_task = Task( task_id="rpt-1", kind="report", - state="running", + state="queued", params={}, idempotency_key="internal-report-close", requires_lanes=[], @@ -540,7 +540,7 @@ async def test_on_enter_close_emits_report_error_for_failed_task( bd_task = Task( task_id="bd-1", kind="session_breakdown", - state="running", + state="queued", params={}, idempotency_key="internal-breakdown-close", requires_lanes=[], @@ -597,7 +597,7 @@ async def test_on_enter_close_emits_report_error_for_exception( report_task = Task( task_id="rpt-1", kind="report", - state="running", + state="queued", params={}, idempotency_key="internal-report-close", requires_lanes=[], @@ -605,7 +605,7 @@ async def test_on_enter_close_emits_report_error_for_exception( bd_task = Task( task_id="bd-1", kind="session_breakdown", - state="running", + state="queued", params={}, idempotency_key="internal-breakdown-close", requires_lanes=[], From 39895690496897996d36a502f88f418fce31d0cb Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 12:52:20 +0000 Subject: [PATCH 15/65] fix(loop): stop the work, not just the coroutine awaiting it Layer 4 cancelled the action's task and returned. Every benchmark executor spends its time inside ``asyncio.to_thread``, and a thread that has started cannot be cancelled: the ``await`` raised a clean ``CancelledError`` at the canceller while ``run_with_session_kill`` ran on to its own ``timeout=``. So ``Coordinator.stop`` gathered a clean return and closed SQLite, and the executor's ``finally`` blocks released the GPU specialist lease and the lanes, with the benchmark still holding the card. The only place work actually stopped was ``run_grid``'s local path, and only because layer 3 fires inside the thread -- layer 3 doing layer 4's job, which is also why the layers were not as independent as claimed. The suite missed it because the executor it cancels is a bare ``asyncio.sleep``, which nothing real resembles. The test added first blocks in a thread the way every benchmark does and asserts the work is over before the cancel returns; it failed on the assertion, with the abandoned thread keeping the suite alive for the full 30s sleep. Give the thread something it can check. A ``CancelScope`` carries a ``threading.Event`` on a ``ContextVar`` published when the dispatcher starts the action; ``asyncio.to_thread`` copies the context, so the subprocess poll loop -- which already checks the session deadline every 0.5s -- sees it many frames down without a parameter on every executor. Tripping it reaps the process group and raises, reported as its own sentinel returncode: a cancel is not a timeout, not a slow variant, and not the same fact as a spent budget, which is still the attribution recorded when both are true. Cancelling then goes out on the scopes first, before any await, so work that can hear it stops even if the canceller is itself cancelled mid-wait. Work that is listening is given a bounded grace to unwind through its own ``finally`` blocks, which is what makes the lease release follow the benchmark rather than precede it. Anything not listening is cancelled as before, so a shutdown can never block on work that cannot hear the channel. The scope is in-process: a round inside a Ray actor still stops on the session deadline it was handed, and nothing else. Adding a fifth sentinel meant a fifth copy of the same reap-log-return block, so the causes now carry their own returncode and log level and share one handler. Co-authored-by: Cursor --- .../tests/test_kill_spawned_server.py | 88 ++++++ .../tests/test_session_time_budget.py | 171 +++++++++- .../orchestrator/actions/cancel_channel.py | 166 ++++++++++ .../actions/executors/_grid_runner.py | 102 ++++-- .../actions/executors/_subprocess_kill.py | 291 ++++++++++-------- src/hyperloom/orchestrator/loop/dispatcher.py | 139 +++++++-- 6 files changed, 774 insertions(+), 183 deletions(-) create mode 100644 src/hyperloom/orchestrator/actions/cancel_channel.py diff --git a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py index 6c216c85ff..023f53dfda 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py +++ b/src/hyperloom/inference_optimizer/tests/test_kill_spawned_server.py @@ -19,8 +19,10 @@ import pytest +from hyperloom.orchestrator.actions.cancel_channel import CancelScope, use_cancel_scope from hyperloom.orchestrator.actions.executors._subprocess_kill import ( DETOKENIZER_STALL_RETURNCODE, + ORCHESTRATOR_CANCELLED_RETURNCODE, OVERTIME_KILL_RETURNCODE, SERVER_DEAD_RETURNCODE, SESSION_TIME_EXHAUSTED_RETURNCODE, @@ -364,6 +366,92 @@ def test_no_session_deadline_keeps_the_previous_behaviour(self): assert "done" in (cp.stdout or "") +class TestAnOrchestratorCancelReachesTheChild: + """The last defence has to stop the child, not just the coroutine above it. + + The executor blocks in a worker thread, so cancelling its task frees the + lanes and the GPU lease while the benchmark is still running. The cancel + scope is the channel the thread checks, at the poll it already runs. + """ + + def test_a_cancel_raised_before_the_call_reaps_the_tree(self): + scope = CancelScope() + scope.cancel(reason="shutdown_requested") + start = time.monotonic() + with use_cancel_scope(scope): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + ) + elapsed = time.monotonic() - start + assert cp.returncode == ORCHESTRATOR_CANCELLED_RETURNCODE + assert elapsed < 10.0, f"cancel path took {elapsed:.2f}s" + + def test_a_cancel_that_arrives_mid_run_still_reaches_it(self): + """The interesting case: nothing is wrong when the child is launched.""" + scope = CancelScope() + timer = threading.Timer(0.5, lambda: scope.cancel(reason="session_time_exhausted")) + timer.start() + start = time.monotonic() + try: + with use_cancel_scope(scope): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + ) + finally: + timer.cancel() + elapsed = time.monotonic() - start + assert cp.returncode == ORCHESTRATOR_CANCELLED_RETURNCODE + assert elapsed < 10.0, f"mid-run cancel took {elapsed:.2f}s" + + def test_a_scope_nobody_cancelled_leaves_the_child_alone(self): + """The channel must cost nothing on the path every healthy round takes.""" + with use_cancel_scope(CancelScope()): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(1); print('done')"], + timeout=60, + ) + assert cp.returncode == 0 + assert "done" in (cp.stdout or "") + + def test_a_spent_budget_keeps_its_own_attribution(self): + """Both are true at once whenever the budget is what triggered the cancel. + + The budget is a fact about the run and the cancel is only the dispatcher + acting on it, so the ledger gets the cause, not the mechanism. + """ + scope = CancelScope() + scope.cancel(reason="session_time_exhausted") + with use_cancel_scope(scope): + cp = run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + session_deadline_sec=time.monotonic() - 1.0, + ) + assert cp.returncode == SESSION_TIME_EXHAUSTED_RETURNCODE + + def test_the_call_registers_as_a_listener_while_the_child_lives(self): + """The canceller waits only for work that can hear it, so this is load-bearing.""" + scope = CancelScope() + seen: list[bool] = [] + timer = threading.Timer( + 0.5, + lambda: (seen.append(scope.has_listeners), scope.cancel(reason="test")), + ) + timer.start() + try: + with use_cancel_scope(scope): + run_with_session_kill( + [sys.executable, "-c", "import time; time.sleep(30)"], + timeout=60, + ) + finally: + timer.cancel() + assert seen == [True] + assert not scope.has_listeners + + class TestSessionDeadlineCrossesAProcessBoundary: """A ``time.monotonic()`` instant is only meaningful in the process that read it. diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 9f33c9cc3e..2c632a64fc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -26,12 +26,25 @@ import asyncio import threading +import time +from collections.abc import Callable +from typing import Any import pytest from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType +from hyperloom.orchestrator.actions.cancel_channel import ( + CancelScope, + current_cancel_scope, + use_cancel_scope, +) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + run_with_session_kill, +) from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.loop.dispatcher import _COOPERATIVE_CANCEL_GRACE_SEC from hyperloom.orchestrator.loop.coordinator_helpers import ( TIME_BUDGET_EXEMPT_ACTIONS, action_fits_time_budget, @@ -406,10 +419,16 @@ async def _run(_ctx) -> dict: return _run -async def _queue_action(coord: Coordinator, *, kind: str, key: str) -> tuple[Task, asyncio.Event]: - """Queue an action that only ends by being cancelled, with its real lanes.""" +async def _queue_action( + coord: Coordinator, + *, + kind: str, + key: str, + make_executor: Callable[[asyncio.Event], Any] = _never_finishes, +) -> tuple[Task, asyncio.Event]: + """Queue an action with its real lanes; ``make_executor`` shapes what it does.""" started = asyncio.Event() - coord.sub.register_executor(kind, _never_finishes(started)) + coord.sub.register_executor(kind, make_executor(started)) lanes, ttl_sec = coord.dispatcher._registry_lanes_ttl(kind) task, _ = await coord.tasks.create_or_return_existing( kind=kind, @@ -421,9 +440,15 @@ async def _queue_action(coord: Coordinator, *, kind: str, key: str) -> tuple[Tas return task, started -async def _start_action(coord: Coordinator, *, kind: str, key: str) -> tuple[Task, asyncio.Task]: +async def _start_action( + coord: Coordinator, + *, + kind: str, + key: str, + make_executor: Callable[[asyncio.Event], Any] = _never_finishes, +) -> tuple[Task, asyncio.Task]: """Dispatch the action with no pump running, for the pieces under it.""" - task, started = await _queue_action(coord, kind=kind, key=key) + task, started = await _queue_action(coord, kind=kind, key=key, make_executor=make_executor) spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) assert [t.task_id for t, _, _ in spawned] == [task.task_id] await asyncio.wait_for(started.wait(), timeout=5.0) @@ -459,7 +484,9 @@ class TestInflightHandles: async def test_a_running_action_is_reachable_by_task_id(self, coord: Coordinator): task, atask = await _start_action(coord, kind=_CHEAP_ACTION, key="h-live") try: - assert coord.dispatcher._inflight_actions[task.task_id] == (_CHEAP_ACTION, atask) + entry = coord.dispatcher._inflight_actions[task.task_id] + assert (entry.kind, entry.atask) == (_CHEAP_ACTION, atask) + assert not entry.scope.cancelled finally: atask.cancel() await _settle(atask) @@ -529,6 +556,138 @@ async def test_the_lane_is_free_again_afterwards(self, coord: Coordinator): assert (await coord.locks.lane_holders()).get(_CHEAP_ACTION_LANE, 0) == 0 +# Long enough that a round which ran to completion is unmistakable in the +# elapsed time, short enough that an abandoned thread cannot outlive the suite. +_BLOCKING_SEC = 30 + + +def _blocks_in_a_thread(started: asyncio.Event, *, outcome: dict[str, Any]): + """Build an executor shaped like every benchmark one: a subprocess in a thread. + + ``asyncio.to_thread`` is where all of them spend their time, and a thread + that has started cannot be cancelled, so this is the shape the last defence + actually has to stop. ``outcome`` is written after the thread returns, which + is what makes "the work is over" observable rather than inferred. + """ + + async def _run(_ctx) -> dict: + started.set() + proc = await asyncio.to_thread( + run_with_session_kill, + ["sleep", str(_BLOCKING_SEC)], + timeout=_BLOCKING_SEC * 4, + ) + outcome["returncode"] = proc.returncode + return {"returncode": proc.returncode} + + return _run + + +def _sleeps_in_a_thread(started: asyncio.Event, *, seconds: float = 2.0): + """Build an executor whose thread has no way to hear a cancel.""" + + async def _run(_ctx) -> dict: + started.set() + await asyncio.to_thread(time.sleep, seconds) + return {} + + return _run + + +class TestTheCancelChannel: + """The channel itself: what it carries, and how far it reaches.""" + + @pytest.mark.asyncio + async def test_a_worker_thread_sees_the_scope_of_the_task_that_started_it(self): + """The whole idiom rests on ``to_thread`` copying the context.""" + scope = CancelScope() + with use_cancel_scope(scope): + seen = await asyncio.to_thread(current_cancel_scope) + assert seen is scope + + @pytest.mark.asyncio + async def test_code_outside_an_action_finds_no_scope(self): + """A Ray worker and a bare call are the same case: nothing to check.""" + assert await asyncio.to_thread(current_cancel_scope) is None + + def test_the_first_reason_is_the_one_kept(self): + """A blanket cancel arriving second must not overwrite the specific cause.""" + scope = CancelScope() + scope.cancel(reason="session_time_exhausted") + scope.cancel(reason="dispatcher_pump_exit") + assert scope.cancelled + assert scope.reason == "session_time_exhausted" + + def test_a_scope_reports_whether_anything_is_watching_it(self): + scope = CancelScope() + assert not scope.has_listeners + with scope.listening(): + assert scope.has_listeners + assert not scope.has_listeners + + +class TestCancellingWorkThatBlocksInAThread: + """Cancelling the coroutine does not stop the thread it is waiting on. + + The canceller gets a clean ``CancelledError`` off the ``await`` while the + subprocess runs on to its own hard timeout, so the lanes and the GPU lease + are released, and the database closed, with the benchmark still holding the + card. Stopping it takes a channel the thread itself checks. + """ + + @pytest.mark.asyncio + async def test_the_work_is_over_before_the_cancel_returns(self, coord: Coordinator): + outcome: dict[str, Any] = {} + task, atask = await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-thread", + make_executor=lambda started: _blocks_in_a_thread(started, outcome=outcome), + ) + began = time.monotonic() + + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task.task_id] + + assert outcome, "the cancel returned while the thread was still running" + assert time.monotonic() - began < _BLOCKING_SEC + await _settle(atask) + + @pytest.mark.asyncio + async def test_the_stop_is_attributed_to_the_orchestrator(self, coord: Coordinator): + """A cancel is not a timeout and not a slow variant; the ledger reads returncodes.""" + outcome: dict[str, Any] = {} + await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-thread-rc", + make_executor=lambda started: _blocks_in_a_thread(started, outcome=outcome), + ) + + await coord.dispatcher.cancel_inflight_actions(reason="test_reason") + + assert outcome["returncode"] == ORCHESTRATOR_CANCELLED_RETURNCODE + + @pytest.mark.asyncio + async def test_a_thread_with_nothing_listening_is_still_not_waited_for(self, coord: Coordinator): + """The channel is cooperative, so work that cannot hear it is left behind. + + Waiting on it anyway would trade a leaked thread for a shutdown that + hangs on one, which is the worse of the two. + """ + _task, atask = await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-deaf", + make_executor=_sleeps_in_a_thread, + ) + began = time.monotonic() + + await coord.dispatcher.cancel_inflight_actions(reason="test") + + assert time.monotonic() - began < _COOPERATIVE_CANCEL_GRACE_SEC + assert atask.cancelled() + + class TestTheRunnerRecordsACancellation: """``CancelledError`` is not an ``Exception``, so the runner must name it.""" diff --git a/src/hyperloom/orchestrator/actions/cancel_channel.py b/src/hyperloom/orchestrator/actions/cancel_channel.py new file mode 100644 index 0000000000..4a243dbcac --- /dev/null +++ b/src/hyperloom/orchestrator/actions/cancel_channel.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Cooperative cancellation channel between the dispatcher and blocking work. + +Every benchmark executor spends its time inside ``asyncio.to_thread``, and a +thread that has already started cannot be cancelled. Cancelling the coroutine +returns a clean ``CancelledError`` to the canceller while the subprocess it was +waiting on keeps running -- long enough for the lanes and the GPU lease to be +released, and the database closed, under a benchmark that still owns the card. + +So the thread needs something it can check. A :class:`CancelScope` carries a +:class:`threading.Event` on a :class:`contextvars.ContextVar`: the dispatcher +publishes one per action, and ``asyncio.to_thread`` copies the context into the +worker thread, so code many frames down finds it without every executor +signature growing a parameter. Blocking code checks the scope at whatever +interval it already polls at and stops itself. + +Cooperative means exactly that: work that never looks at the scope cannot be +stopped through it, which is why a scope also counts its listeners -- the +canceller waits only for work that can hear it. + +The channel is in-process. A round running inside a Ray actor is in another +process where this ContextVar is unset, and stops on the session deadline it +was handed instead. +""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +__all__ = [ + "CancelScope", + "cancel_scope_listener", + "current_cancel_scope", + "use_cancel_scope", +] + + +class CancelScope: + """One action's cancel channel: the flag, why it was raised, who watches it. + + Every method is safe to call from any thread: the flag is a + :class:`threading.Event` and the listener count is taken under a lock, since + the writer is the event loop and the readers are worker threads. + """ + + def __init__(self) -> None: + """Create an uncancelled scope with no listeners.""" + self._event = threading.Event() + self._lock = threading.Lock() + self._reason = "" + self._listeners = 0 + + def cancel(self, *, reason: str) -> None: + """Ask the work running in this scope to stop. + + Returns as soon as the flag is raised -- whoever is watching it decides + how to stop, and how long that takes. + + Args: + reason (str): Short cause, kept for the log line the stopped work + writes. The first reason wins, so a later blanket cancel cannot + overwrite the specific one that got there first. + """ + with self._lock: + if not self._reason: + self._reason = str(reason) + self._event.set() + + @property + def cancelled(self) -> bool: + """bool: Whether this scope has been cancelled.""" + return self._event.is_set() + + @property + def reason(self) -> str: + """str: Why the scope was cancelled; empty while it is not.""" + with self._lock: + return self._reason + + @property + def has_listeners(self) -> bool: + """bool: Whether any blocking call is currently watching this scope.""" + with self._lock: + return self._listeners > 0 + + @contextmanager + def listening(self) -> Iterator["CancelScope"]: + """Count the caller as a watcher for the duration of the block. + + Yields: + CancelScope: This scope, so the block can check it. + """ + with self._lock: + self._listeners += 1 + try: + yield self + finally: + with self._lock: + self._listeners = max(0, self._listeners - 1) + + +_CURRENT_SCOPE: ContextVar[CancelScope | None] = ContextVar( + "hyperloom_cancel_scope", + default=None, +) + + +def current_cancel_scope() -> CancelScope | None: + """Return the cancel scope of the action running in this context. + + Returns: + CancelScope | None: The published scope, or ``None`` when the caller is + not running under one -- a Ray worker, a unit test, or any code the + dispatcher did not start. + """ + return _CURRENT_SCOPE.get() + + +@contextmanager +def use_cancel_scope(scope: CancelScope | None) -> Iterator[CancelScope | None]: + """Publish ``scope`` for the duration of the block. + + Must be entered inside the task that runs the action: a task copies the + context at creation, so a value set afterwards from outside never reaches + it. + + Args: + scope (CancelScope | None): The scope to publish; ``None`` leaves the + context untouched, for callers with nothing to cancel. + + Yields: + CancelScope | None: The published scope, unchanged. + """ + if scope is None: + yield None + return + token = _CURRENT_SCOPE.set(scope) + try: + yield scope + finally: + _CURRENT_SCOPE.reset(token) + + +@contextmanager +def cancel_scope_listener() -> Iterator[CancelScope | None]: + """Watch the published scope, if there is one, for the duration of the block. + + Registering is what tells the canceller this work can be asked to stop, so + the window has to cover everything the cancel is meant to reach -- from the + moment the child is spawned, not from the first poll. + + Yields: + CancelScope | None: The scope to check, or ``None`` when none is + published, in which case the block is a plain no-op. + """ + scope = _CURRENT_SCOPE.get() + if scope is None: + yield None + return + with scope.listening(): + yield scope diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index b9cc44183e..92a38b17ab 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -19,7 +19,7 @@ import subprocess import time from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, NamedTuple import yaml @@ -39,6 +39,7 @@ AGENTX_PREFLIGHT_RETURNCODE, DETOKENIZER_STALL_RETURNCODE, EVAL_PROBE_UNPATCHABLE_RETURNCODE, + ORCHESTRATOR_CANCELLED_RETURNCODE, OVERTIME_KILL_RETURNCODE, SERVER_DEAD_RETURNCODE, SESSION_TIME_EXHAUSTED_RETURNCODE, @@ -1248,6 +1249,45 @@ def _variant_progress_note( # about the variant, and the ledger must not read it as one. SESSION_TIME_EXHAUSTED_CLASS = "session_time_exhausted" +# Labels a round the orchestrator stopped from outside -- a shutdown, or a +# budget the dispatcher found spent. Kept apart from the class above for the +# same reason their returncodes are: a resume faces the spent budget again and +# does not face the shutdown. +ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" + + +class _StoppedByTheRun(NamedTuple): + """How a returncode that stopped the round from outside is recorded. + + Neither cause is evidence about the variant, so both are recorded as + ``skipped`` rather than failed -- see the branch that consumes this. + """ + + error_class: str + interrupted: str + never_started: str + ends_the_grid: bool + + +# The two causes that stop a round without saying anything about the variant. +# The budget leaves the rest of the grid to the fit check at the top of the loop, +# which sees a deadline in the past and skips them all under the same label; a +# cancel has no such check, and nothing new should start under one anyway. +_STOPPED_BY_THE_RUN: dict[int, _StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_RETURNCODE: _StoppedByTheRun( + error_class=SESSION_TIME_EXHAUSTED_CLASS, + interrupted="session wall-clock budget exhausted while this variant was running", + never_started="session wall-clock budget exhausted before this variant ran", + ends_the_grid=False, + ), + ORCHESTRATOR_CANCELLED_RETURNCODE: _StoppedByTheRun( + error_class=ORCHESTRATOR_CANCELLED_CLASS, + interrupted="the orchestrator cancelled this action while this variant was running", + never_started="the orchestrator cancelled this action before this variant ran", + ends_the_grid=True, + ), +} + def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call. @@ -1555,7 +1595,12 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl len(grid) - i, ) for skipped_variant in grid[i:]: - results.append(_session_deadline_skip_result(skipped_variant)) + results.append( + _not_run_skip_result( + skipped_variant, + _STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_RETURNCODE], + ) + ) break await _unit_started(i, "variant") slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" @@ -2212,31 +2257,29 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl break continue - # Session budget ran out mid-round and the tree was reaped. Recorded as - # ``skipped``, exactly like a variant the budget refused to start: in both - # cases nothing was measured, so there is no verdict to record about the - # variant. Grading it as a failure -- or worse as ``killed_overtime``, - # which asserts the variant is abnormally slow -- would put a conclusion - # the run never reached into the ledger and the KB. - # - # The remaining variants are left to the fit check at the top of the loop, - # which now sees a deadline in the past and skips them all under the same - # label rather than duplicating that decision here. - if rc == SESSION_TIME_EXHAUSTED_RETURNCODE: + # The round was stopped by the run rather than by anything about the + # variant, and the tree was reaped. Recorded as ``skipped``, exactly like + # a variant the budget refused to start: in both cases nothing was + # measured, so there is no verdict to record about the variant. Grading it + # as a failure -- or worse as ``killed_overtime``, which asserts the + # variant is abnormally slow -- would put a conclusion the run never + # reached into the ledger and the KB. + stopped = _STOPPED_BY_THE_RUN.get(rc) + if stopped is not None: variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) log.warning( - "grid_runner: variant %d/%d name=%s reaped after %.1fs: the session " - "wall-clock budget ran out mid-round; recorded as skipped, not failed", + "grid_runner: variant %d/%d name=%s reaped after %.1fs: %s; recorded as skipped, not failed", i + 1, len(grid), variant.name, variant_runtime_sec, + stopped.interrupted, ) _write_variant_abort_marker( slot, variant_name=variant.name, - error_class=SESSION_TIME_EXHAUSTED_CLASS, - error_summary="session wall-clock budget exhausted mid-round; tree reaped", + error_class=stopped.error_class, + error_summary=f"{stopped.interrupted}; tree reaped", extra_args=variant.extra_server_args, ) results.append( @@ -2247,13 +2290,16 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl status="skipped", returncode=rc, runtime_sec=variant_runtime_sec, - error="session wall-clock budget exhausted while this variant was running", - error_class=SESSION_TIME_EXHAUSTED_CLASS, + error=stopped.interrupted, + error_class=stopped.error_class, server_log_path=_existing_log_path(server_log), note=variant.note, ) ) await _pulse_after_variant(i) + if stopped.ends_the_grid: + results.extend(_not_run_skip_result(rest, stopped) for rest in grid[i + 1 :]) + break if not keep_going_on_failure: break continue @@ -2477,15 +2523,24 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl return results -def _session_deadline_skip_result(variant: GridVariant) -> VariantResult: - """Synthetic ``skipped`` result for a variant dropped when the session budget ran out.""" +def _not_run_skip_result(variant: GridVariant, stopped: _StoppedByTheRun) -> VariantResult: + """Build the ``skipped`` result for a variant the run never got to. + + Args: + variant: The variant that was dropped. + stopped: Why the run stopped, which is the whole content of the result: + nothing was measured, so there is nothing else to report. + + Returns: + VariantResult: A synthetic ``skipped`` result carrying that cause. + """ return VariantResult( name=variant.name, extra_server_args=variant.extra_server_args, extra_envs=dict(variant.extra_envs), status="skipped", - error="session wall-clock budget exhausted before this variant ran", - error_class=SESSION_TIME_EXHAUSTED_CLASS, + error=stopped.never_started, + error_class=stopped.error_class, note=variant.note, ) @@ -2622,6 +2677,7 @@ def _write_variant_abort_marker_impl( "DEFAULT_SGLANG_WATCHDOG_TIMEOUT_SEC", "GridVariant", "MULTI_NODE_DEFAULT_KEEP_THRESHOLD_PCT", + "ORCHESTRATOR_CANCELLED_CLASS", "SESSION_TIME_EXHAUSTED_CLASS", "SGLANG_WATCHDOG_TIMEOUT_ENV", "SINGLE_NODE_DEFAULT_KEEP_THRESHOLD_PCT", diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index c1a3df563a..6cf10e7b6a 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -28,6 +28,8 @@ from .bypass_analysis import parse_server_log_throughput +from ..cancel_channel import CancelScope, cancel_scope_listener + log = logging.getLogger(__name__) @@ -268,6 +270,17 @@ def kill_my_spawned_server( # that a variant is slow whenever a session happened to end during it. SESSION_TIME_EXHAUSTED_RETURNCODE: int = -915 +# -916 is ``_ray_serving._ACTOR_TIMEOUT_RC``. + +# Sentinel ``returncode`` when the orchestrator cancelled the action this child +# was launched for -- a shutdown, or a budget that is spent. Distinct from +# ``SESSION_TIME_EXHAUSTED_RETURNCODE`` because the two causes are told apart at +# the source: that one is the round's own deadline elapsing where it runs, this +# one is a decision taken outside it, and only one of them is still true when the +# same session resumes. Distinct from ``OVERTIME_KILL_RETURNCODE`` for the reason +# that one already carries: neither says anything about the variant. +ORCHESTRATOR_CANCELLED_RETURNCODE: int = -917 + # Server-ready markers: their appearance in ``server.log`` means the server has # finished startup and is accepting traffic. Only after one is observed does the # detokenizer-stall clock start. Covers the uvicorn frontend (vLLM + sglang) and @@ -748,6 +761,13 @@ def run_with_session_kill( eval-start boundary is meaningful for "is this variant abnormally slow" and meaningless for "is the run out of time". + The cancel scope published by the dispatcher (:mod:`..cancel_channel`) is + watched for as long as the child lives, so an orchestrator that cancels the + action reaches the tree rather than just the coroutine awaiting this call. + Every cause that reaps the tree is reported as its own sentinel + ``returncode``, which is all a caller of a subprocess gets to tell them apart + by. + Args: on_output: Called from a reader thread each time the child produces output, so a caller can report a long step alive on the child's own @@ -779,126 +799,103 @@ def run_with_session_kill( proc: subprocess.Popen | None = None capture: _StreamCapture | None = None try: - proc = subprocess.Popen( # noqa: S603 — cmd is caller's responsibility - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=text, - env=env, - cwd=cwd, - **new_session_kwargs(), - ) - capture = _StreamCapture(proc, text=text, on_output=on_output) - capture.start() - try: - stdout, stderr = _communicate_with_soft_deadline( - proc, - hard_timeout=timeout, - soft_deadline_sec=soft_deadline_sec, - server_log_path=server_log_path, - server_dead_grace_sec=server_dead_grace_sec, - detok_stall_grace_sec=detok_stall_grace_sec, - capture=capture, - server_already_ready=server_already_ready, - session_deadline_sec=session_deadline_sec, - ) - except subprocess.TimeoutExpired: - kill_my_spawned_server(proc) - if capture is not None: - capture.finish(timeout=2.0) - raise - except _ServerDeadDetected as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.warning( - "_subprocess_kill: server-liveness watchdog reaped tree — " - "engine/worker init died but parent hung (marker=%r, " - "grace=%.1fs, elapsed=%.1fs); returncode=%d.", - exc.marker, - exc.grace_sec, - exc.elapsed_sec, - SERVER_DEAD_RETURNCODE, - ) - return subprocess.CompletedProcess( - args=cmd, - returncode=SERVER_DEAD_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) - except _ServerStalledDetected as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.warning( - "_subprocess_kill: detokenizer-stall watchdog reaped tree — " - "server reported ready but emitted no log output (grace=%.1fs, " - "elapsed=%.1fs); returncode=%d.", - exc.grace_sec, - exc.elapsed_sec, - DETOKENIZER_STALL_RETURNCODE, - ) - return subprocess.CompletedProcess( - args=cmd, - returncode=DETOKENIZER_STALL_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) - except _SessionDeadlineExceeded as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.warning( - "_subprocess_kill: session wall-clock budget exhausted %.1fs ago " - "(round elapsed=%.1fs); reaped tree with sentinel returncode=%d.", - exc.overrun_sec, - exc.elapsed_sec, - SESSION_TIME_EXHAUSTED_RETURNCODE, - ) - return subprocess.CompletedProcess( - args=cmd, - returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) - except _SoftDeadlineExceeded as exc: - kill_my_spawned_server(proc) - stdout, stderr = ( - capture.finish(timeout=2.0) if capture is not None else ("" if text else b"", "" if text else b"") - ) - log.info( - "_subprocess_kill: soft_deadline_sec=%.1fs exceeded " - "(elapsed=%.1fs); reaped tree with sentinel returncode=%d.", - exc.deadline_sec, - exc.elapsed_sec, - OVERTIME_KILL_RETURNCODE, + with cancel_scope_listener() as cancel_scope: + proc = subprocess.Popen( # noqa: S603 — cmd is caller's responsibility + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=text, + env=env, + cwd=cwd, + **new_session_kwargs(), ) + capture = _StreamCapture(proc, text=text, on_output=on_output) + capture.start() + try: + stdout, stderr = _communicate_with_soft_deadline( + proc, + hard_timeout=timeout, + soft_deadline_sec=soft_deadline_sec, + server_log_path=server_log_path, + server_dead_grace_sec=server_dead_grace_sec, + detok_stall_grace_sec=detok_stall_grace_sec, + capture=capture, + server_already_ready=server_already_ready, + session_deadline_sec=session_deadline_sec, + cancel_scope=cancel_scope, + ) + except subprocess.TimeoutExpired: + kill_my_spawned_server(proc) + if capture is not None: + capture.finish(timeout=2.0) + raise + except _ReapedByWatchdog as exc: + kill_my_spawned_server(proc) + stdout, stderr = _finish_capture(capture, text=text) + log.log( + exc.log_level, + "_subprocess_kill: %s; reaped the tree with sentinel returncode=%d.", + exc, + exc.returncode, + ) + return subprocess.CompletedProcess( + args=cmd, + returncode=exc.returncode, + stdout=stdout, + stderr=stderr, + ) + empty: str | bytes = "" if text else b"" return subprocess.CompletedProcess( args=cmd, - returncode=OVERTIME_KILL_RETURNCODE, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), + returncode=proc.returncode, + stdout=stdout if stdout is not None else empty, + stderr=stderr if stderr is not None else empty, ) - return subprocess.CompletedProcess( - args=cmd, - returncode=proc.returncode, - stdout=stdout if stdout is not None else ("" if text else b""), - stderr=stderr if stderr is not None else ("" if text else b""), - ) finally: kill_my_spawned_server(proc) -class _SessionDeadlineExceeded(Exception): - """Internal sentinel: the session wall-clock budget ran out mid-round. +def _finish_capture(capture: _StreamCapture | None, *, text: bool) -> tuple[str | bytes, str | bytes]: + """Drain the capture threads of a reaped child, never returning ``None``. + + Args: + capture: The stream capture to finish, or ``None`` when the child's + output was not captured. + text: Whether the streams are ``str`` (``True``) or ``bytes``, which + decides what an absent stream reads as. + + Returns: + tuple[str | bytes, str | bytes]: The captured ``(stdout, stderr)``. + """ + empty: str | bytes = "" if text else b"" + if capture is None: + return empty, empty + stdout, stderr = capture.finish(timeout=2.0) + return ( + stdout if stdout is not None else empty, + stderr if stderr is not None else empty, + ) + - Never bubbles past :func:`run_with_session_kill` (converted to a - ``CompletedProcess`` carrying ``SESSION_TIME_EXHAUSTED_RETURNCODE``). +class _ReapedByWatchdog(Exception): + """Internal base for a cause that reaps the tree and names itself. + + None of these bubble past :func:`run_with_session_kill`: each is converted + to a ``CompletedProcess`` carrying the subclass's own ``returncode``, since + a returncode is the only thing that survives the trip back to a caller of a + subprocess. Subclasses build their own message and declare how much the + event is worth logging. """ + returncode: int = -1 + log_level: int = logging.WARNING + + +class _SessionDeadlineExceeded(_ReapedByWatchdog): + """Internal sentinel: the session wall-clock budget ran out mid-round.""" + + returncode = SESSION_TIME_EXHAUSTED_RETURNCODE + def __init__(self, *, overrun_sec: float, elapsed_sec: float) -> None: """Record how far past the session deadline the round got. @@ -906,16 +903,45 @@ def __init__(self, *, overrun_sec: float, elapsed_sec: float) -> None: overrun_sec (float): Seconds past the session deadline at trip time. elapsed_sec (float): Wall-clock elapsed for this round at trip time. """ - super().__init__(f"session budget exhausted {overrun_sec:.1f}s ago (round elapsed={elapsed_sec:.1f}s)") + super().__init__( + f"the session wall-clock budget was exhausted {overrun_sec:.1f}s ago " + f"(round elapsed={elapsed_sec:.1f}s)" + ) self.overrun_sec = float(overrun_sec) self.elapsed_sec = float(elapsed_sec) -class _SoftDeadlineExceeded(Exception): - """Internal sentinel for an elapsed soft deadline. Never bubbles - past :func:`run_with_session_kill` (converted to a ``CompletedProcess``). +class _OrchestratorCancelled(_ReapedByWatchdog): + """Internal sentinel: the orchestrator cancelled the action this child serves. + + The cause is a decision taken outside the round -- a shutdown, or a budget + the dispatcher found spent -- which is why it carries the caller's reason + rather than a measurement of its own. """ + returncode = ORCHESTRATOR_CANCELLED_RETURNCODE + + def __init__(self, *, reason: str, elapsed_sec: float) -> None: + """Record who asked for the stop and how far the round had got. + + Args: + reason (str): Short cause from the canceller, e.g. ``shutdown_requested``. + elapsed_sec (float): Wall-clock elapsed for this round at trip time. + """ + super().__init__( + f"the orchestrator cancelled this action ({reason or 'no reason given'}; " + f"round elapsed={elapsed_sec:.1f}s)" + ) + self.reason = str(reason) + self.elapsed_sec = float(elapsed_sec) + + +class _SoftDeadlineExceeded(_ReapedByWatchdog): + """Internal sentinel for an elapsed soft deadline.""" + + returncode = OVERTIME_KILL_RETURNCODE + log_level = logging.INFO + def __init__(self, *, deadline_sec: float, elapsed_sec: float) -> None: """Record the deadline and actual elapsed time on the sentinel. @@ -923,18 +949,18 @@ def __init__(self, *, deadline_sec: float, elapsed_sec: float) -> None: deadline_sec (float): The soft deadline that was exceeded. elapsed_sec (float): The actual wall-clock elapsed at trip time. """ - super().__init__(f"soft deadline {deadline_sec:.1f}s elapsed (actual={elapsed_sec:.1f}s)") + super().__init__(f"soft_deadline_sec={deadline_sec:.1f}s elapsed (actual={elapsed_sec:.1f}s)") self.deadline_sec = float(deadline_sec) self.elapsed_sec = float(elapsed_sec) -class _ServerDeadDetected(Exception): +class _ServerDeadDetected(_ReapedByWatchdog): """Internal sentinel: the server-liveness watchdog saw a terminal engine / - worker init marker that persisted past the grace window. Never bubbles past - :func:`run_with_session_kill` (converted to a ``CompletedProcess`` carrying - ``SERVER_DEAD_RETURNCODE``). + worker init marker that persisted past the grace window. """ + returncode = SERVER_DEAD_RETURNCODE + def __init__( self, *, @@ -950,7 +976,7 @@ def __init__( elapsed_sec: Actual wall-clock elapsed at trip time. """ super().__init__( - f"server init died (marker={marker!r}) and parent hung past " + f"server init died (marker={marker!r}) and the parent hung past " f"grace {grace_sec:.1f}s (elapsed={elapsed_sec:.1f}s)" ) self.marker = marker @@ -958,13 +984,13 @@ def __init__( self.elapsed_sec = float(elapsed_sec) -class _ServerStalledDetected(Exception): +class _ServerStalledDetected(_ReapedByWatchdog): """Internal sentinel: the detokenizer-stall watchdog saw the server report - ready and then produce no generation progress for the grace window. Never - bubbles past :func:`run_with_session_kill` (converted to a - ``CompletedProcess`` carrying ``DETOKENIZER_STALL_RETURNCODE``). + ready and then produce no generation progress for the grace window. """ + returncode = DETOKENIZER_STALL_RETURNCODE + def __init__( self, *, @@ -998,6 +1024,7 @@ def _communicate_with_soft_deadline( capture: _StreamCapture | None = None, server_already_ready: bool = False, session_deadline_sec: float | None = None, + cancel_scope: CancelScope | None = None, ) -> tuple[str | bytes, str | bytes]: """Communicate with a child while enforcing soft and server-log watchdogs.""" watchdog_active = bool(server_log_path) and ( @@ -1006,9 +1033,13 @@ def _communicate_with_soft_deadline( stall_active = bool(server_log_path) and (detok_stall_grace_sec is not None and float(detok_stall_grace_sec) > 0.0) soft_active = soft_deadline_sec is not None and float(soft_deadline_sec) > 0.0 session_active = session_deadline_sec is not None - if capture is None and not soft_active and not watchdog_active and not stall_active and not session_active: + # A cancel scope is polled like any other gate, so its presence rules out the + # single-wait fast paths below: a call that blocks until the child exits + # cannot notice a cancel that arrives while it is blocked. + gated = soft_active or watchdog_active or stall_active or session_active or cancel_scope is not None + if capture is None and not gated: return proc.communicate(timeout=hard_timeout) - if capture is not None and not soft_active and not watchdog_active and not stall_active and not session_active: + if capture is not None and not gated: proc.wait(timeout=hard_timeout) return capture.finish() @@ -1062,6 +1093,15 @@ def _communicate_with_soft_deadline( overrun_sec=now - float(session_deadline_sec), elapsed_sec=elapsed, ) + # Orchestrator cancellation. Checked after the session deadline so a + # round that was already out of time keeps that reason: the budget is a + # fact about the run, while the cancel is only the dispatcher acting on + # it, and the more specific of the two is the one worth recording. + if cancel_scope is not None and cancel_scope.cancelled: + raise _OrchestratorCancelled( + reason=cancel_scope.reason, + elapsed_sec=elapsed, + ) # Advance the log scan, latching the server-ready, last-activity # and eval-start signals. if scan_active: @@ -1162,6 +1202,7 @@ def _communicate_with_soft_deadline( "AGENTX_PREFLIGHT_RETURNCODE", "DETOKENIZER_STALL_RETURNCODE", "EVAL_PROBE_UNPATCHABLE_RETURNCODE", + "ORCHESTRATOR_CANCELLED_RETURNCODE", "OVERTIME_KILL_RETURNCODE", "SERVER_DEAD_RETURNCODE", "SESSION_TIME_EXHAUSTED_RETURNCODE", diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 31cc4e2f4e..ebfe36f072 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -11,11 +11,12 @@ from collections.abc import Collection from concurrent.futures import CancelledError as FuturesCancelledError from concurrent.futures import TimeoutError as FuturesTimeoutError -from typing import Any +from typing import Any, NamedTuple from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType from hyperloom.inference_optimizer.protocol.action_surfaces import ( KERNEL_AGENT_OWNED_ACTIONS, ) +from ..actions.cancel_channel import CancelScope, use_cancel_scope from ..phases import machine_state as _phase_state from ..bus.message_bus import Message from ..kernel.request_handlers import get_handler @@ -47,6 +48,29 @@ log = _logging.getLogger(__name__) +# How long a cancel waits for work that is listening on its cancel scope to stop +# itself. Sized on what stopping actually costs: the poll the blocking side +# checks the scope at, plus the SIGTERM grace before the tree is SIGKILLed, plus +# the drain of the child's pipes. Past that the coroutine is cancelled anyway, +# which is where this started -- the wait buys the guarantee when the work can +# give it, and never turns a shutdown into a hang when it cannot. +_COOPERATIVE_CANCEL_GRACE_SEC: float = 10.0 + +# How long a cancel waits for anything to start listening before deciding +# nothing will. Work that is already blocking is listening before the cancel is +# raised; the window is for the gap between an action starting and reaching the +# call that watches the scope, so it is sized on the same poll interval rather +# than on how long the work takes. +_CANCEL_NOTICE_SEC: float = 0.5 + + +class _InflightAction(NamedTuple): + """A running action's handle: what it is, its task, and how to ask it to stop.""" + + kind: str + atask: asyncio.Task[Any] + scope: CancelScope + class DispatcherCollaborator: """Extracted collaborator; delegates unknown attrs to its Coordinator.""" @@ -56,13 +80,13 @@ def __init__(self, coordinator) -> None: # Task ids already charged a failure by the dead-holder reclaim path, so # a late normal result for the same task cannot double-count it. self._dead_holder_accounted: set[str] = set() - # Handles on the actions currently running, ``task_id -> (kind, task)``. + # Handles on the actions currently running, ``task_id -> _InflightAction``. # Kept on the collaborator and not only in the pump's frame: an action # whose handle lives in a frame can only be stopped by the frame that is # already blocked awaiting it, which is precisely the situation shutdown # and an exhausted wall-clock budget have to break. Entries remove # themselves in :meth:`_run_dispatched_with_gpu_release`. - self._inflight_actions: dict[str, tuple[str, asyncio.Task[Any]]] = {} + self._inflight_actions: dict[str, _InflightAction] = {} def __getattr__(self, name: str): return getattr(object.__getattribute__(self, "_coord"), name) @@ -137,21 +161,33 @@ async def cancel_inflight_actions( exempt: frozenset[str] = frozenset(), only_task_ids: Collection[str] | None = None, ) -> list[str]: - """Cancel the running dispatched actions and wait for them to unwind. + """Stop the running dispatched actions and wait for them to unwind. The last of the wall-clock defences, and the only one that reaches work already under way: admission refuses what cannot fit, the timeout clamp bounds what does, and the subprocess reaper stops the child trees that - were handed a session deadline -- an executor that never takes that - argument, or that is not spending its time in a subprocess at all, has - nothing stopping it before it finishes. - - Cancellation is cooperative: every cancel is delivered before the first - await, so a caller that is itself being cancelled still leaves no action - running unattended. + were handed a session deadline. + + Cancelling the action's task is not by itself enough to stop it. Every + benchmark executor spends its time inside ``asyncio.to_thread``, and a + thread that has started cannot be cancelled: the ``await`` raises here + while the subprocess runs on, so the lanes and the GPU lease would be + released, and the database closed, under a benchmark still holding the + card. So the cancel goes out on the action's :class:`CancelScope` first + -- the channel the blocking side polls -- and work that is listening on + it is given :data:`_COOPERATIVE_CANCEL_GRACE_SEC` to stop itself and + return through its own ``finally`` blocks. Whatever is still running + after that is cancelled the old way, which is no worse than not having + asked. + + Every scope is cancelled before the first await, so a caller that is + itself being cancelled still leaves no action running unattended: the + work that can hear the channel stops on it even if this coroutine never + reaches the wait. Args: - reason: Short cause, logged and used as evidence. + reason: Short cause, logged, used as evidence, and carried to the + blocking side so it can attribute its own stop. exempt: Action kinds to leave running -- the closing actions, when the trigger is a budget that already reserved time for them. only_task_ids: Restrict the cancel to these ids, for a caller that @@ -160,12 +196,14 @@ async def cancel_inflight_actions( spent budget needs. Returns: - list[str]: Task ids that were cancelled (empty when nothing ran). + list[str]: Task ids that were stopped (empty when nothing ran). """ victims = [ - (task_id, kind, atask) - for task_id, (kind, atask) in self._inflight_actions.items() - if kind not in exempt and not atask.done() and (only_task_ids is None or task_id in only_task_ids) + (task_id, entry) + for task_id, entry in self._inflight_actions.items() + if entry.kind not in exempt + and not entry.atask.done() + and (only_task_ids is None or task_id in only_task_ids) ] if not victims: return [] @@ -173,12 +211,46 @@ async def cancel_inflight_actions( "dispatcher: cancelling %d in-flight action(s) [%s]: %s", len(victims), reason, - ", ".join(f"{kind}/{task_id[:12]}" for task_id, kind, _ in victims), + ", ".join(f"{entry.kind}/{task_id[:12]}" for task_id, entry in victims), ) - for _task_id, _kind, atask in victims: - atask.cancel() - await asyncio.gather(*(atask for _tid, _kind, atask in victims), return_exceptions=True) - return [task_id for task_id, _kind, _atask in victims] + for _task_id, entry in victims: + entry.scope.cancel(reason=reason) + try: + await self._wait_for_cooperative_stop(victims) + finally: + for _task_id, entry in victims: + if not entry.atask.done(): + entry.atask.cancel() + await asyncio.gather(*(entry.atask for _task_id, entry in victims), return_exceptions=True) + return [task_id for task_id, _entry in victims] + + async def _wait_for_cooperative_stop(self, victims: list[tuple[str, _InflightAction]]) -> None: + """Give already-cancelled work the chance to stop itself and return. + + Only work watching its scope can be waited for: waiting on the rest + would trade a thread that outlives the cancel for a shutdown that blocks + on one, and the second is the worse failure. So the wait ends at + whichever comes first -- every victim unwound, the grace spent, or the + notice window closing with nothing listening. + + Args: + victims: The ``(task_id, handle)`` pairs whose scopes were just + cancelled. + """ + loop = asyncio.get_running_loop() + notice_deadline = loop.time() + _CANCEL_NOTICE_SEC + grace_deadline = loop.time() + _COOPERATIVE_CANCEL_GRACE_SEC + listening = False + while True: + alive = [entry.atask for _task_id, entry in victims if not entry.atask.done()] + if not alive: + return + listening = listening or any(entry.scope.has_listeners for _task_id, entry in victims) + now = loop.time() + deadline = grace_deadline if listening else notice_deadline + if now >= deadline: + return + await asyncio.wait(alive, timeout=deadline - now, return_when=asyncio.FIRST_COMPLETED) async def _cancel_inflight_that_outlived_the_session(self) -> bool: """Stop in-flight actions the session can no longer wait for. @@ -674,6 +746,7 @@ async def _spawn_fitting_queued( ) except Exception: # noqa: BLE001 - audit must never affect dispatch pass + cancel_scope = CancelScope() atask = asyncio.create_task( self._run_dispatched_with_gpu_release( task, @@ -681,9 +754,10 @@ async def _spawn_fitting_queued( extra_context=extra_context, gpu_lease=gpu_lease, gpu_specialist_lease=gpu_specialist_lease, + cancel_scope=cancel_scope, ), ) - self._inflight_actions[task.task_id] = (task.kind, atask) + self._inflight_actions[task.task_id] = _InflightAction(task.kind, atask, cancel_scope) spawned.append((task, atask, gpu_lease)) return spawned @@ -695,6 +769,7 @@ async def _run_dispatched_with_gpu_release( extra_context: dict[str, Any], gpu_lease: Any, gpu_specialist_lease: Any = None, + cancel_scope: CancelScope | None = None, ) -> "SubAgentResult": """Run a dispatched task, releasing its GPU lease in a structured finally. @@ -717,16 +792,20 @@ async def _run_dispatched_with_gpu_release( gpu_specialist_lease: The Ray ``GpuSpecialistLease`` to close (release the ``num_gpus`` actor lease), or None on the local path. + cancel_scope: The action's cancel channel, published here rather + than at the call site because a task copies its context when it + is created and never sees a value set afterwards. Returns: SubAgentResult: The result from ``sub.run_task``. """ try: - return await self.sub.run_task( - task, - prebound_lease=prebound_lease, - extra_context=extra_context, - ) + with use_cancel_scope(cancel_scope): + return await self.sub.run_task( + task, + prebound_lease=prebound_lease, + extra_context=extra_context, + ) finally: self._inflight_actions.pop(task.task_id, None) if gpu_lease is not None: @@ -1671,10 +1750,12 @@ async def _run_action_now( # is this coroutine's own task: cancelling it drops the audit publication # below, which the runner's terminal transition already accounts for. inline_handle = asyncio.current_task() + cancel_scope = CancelScope() if inline_handle is not None: - self._inflight_actions[task.task_id] = (task.kind, inline_handle) + self._inflight_actions[task.task_id] = _InflightAction(task.kind, inline_handle, cancel_scope) try: - result = await self.sub.run_task(task) + with use_cancel_scope(cancel_scope): + result = await self.sub.run_task(task) finally: self._inflight_actions.pop(task.task_id, None) result_payload = { From 5122e35d6c31b0053fbe0f6d4e58d3585fee4b3a Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 13:11:18 +0000 Subject: [PATCH 16/65] baseline/profile: the session budget reaches the arm that motivated it covered by any of the four layers: admission prices a baseline at five catalogue minutes, `_resolve_timeout` picks a two-hour hang backstop (four cold, and profile's default is four) with no reference to the budget, and `run_with_session_kill` was called without a session deadline at all. The 3941s warmup that motivated the work -- 36% of a 180-minute budget -- ran entirely outside the defence; what the branch added before this commit was an affordability gate on the second round only. The three pieces have to land together. The deadline alone hands the round to the session watchdog, whose sentinel returncode nothing in `baseline.py` knew how to read: a reap leaves exactly what a broken server leaves behind (no workspace, no report, a non-zero returncode), so without the branch a budget kill is filed as `server_init_dead` -- a verdict on the model that the round never reached. And the clamp is what keeps a round from being granted more wall-clock than the session has left in the first place. Both the clamp and the classification are the grid's, promoted rather than copied: `_round_timeout_sec`'s body becomes `session_clamped_timeout_sec` and keeps only the grid's log line, and `_STOPPED_BY_THE_RUN` is read through `stopped_by_the_run`. `_StoppedByTheRun.ends_the_grid` becomes `StoppedByTheRun.ends_the_batch` now that a non-grid caller reads it. Profile is a `BaselineExecutor` subclass that overrides only the config resolver and the trace plumbing, so it inherits all of this; the test covers it as its own arm rather than assuming it. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 143 +++++++++++++++++- .../actions/executors/_grid_runner.py | 125 ++++++++++----- .../actions/executors/baseline.py | 75 +++++++++ 3 files changed, 303 insertions(+), 40 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index bd0cd78ead..757bb8627e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -4,8 +4,10 @@ """Regression tests for the baseline cold-start "warmup artifact". Covers the cold+hot double-run and its server-lifecycle reuse, the pre-start / -teardown cleanup around the reused port, the local InferenceX mirror, and the -subprocess-failure classifier. +teardown cleanup around the reused port, the local InferenceX mirror, the +subprocess-failure classifier, and the session wall-clock budget's reach into +the round (the deadline the reaper is handed, the clamp on the hang backstop, +and how a round the run stopped is told apart from one that failed). """ from __future__ import annotations @@ -26,10 +28,21 @@ from hyperloom.orchestrator.actions.executors.baseline import ( BaselineExecutor, ) +from hyperloom.orchestrator.actions.executors.profile import ( + PROFILE_DEFAULT_TIMEOUT_SEC, + ProfileExecutor, +) from hyperloom.orchestrator.actions.executors._grid_runner import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, GridVariant, + _SESSION_KILL_GRACE_SEC, run_grid, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, +) from hyperloom.orchestrator.state.shared_state import SharedState from hyperloom.orchestrator.trace.task_progress import progress_scope @@ -1821,6 +1834,132 @@ def test_teardown_lifecycle_server_removes_state_files(tmp_path): assert not (pid_dir / "vllm_8888.json").exists() +def _budgeted_state(*, remaining_sec: float | None) -> SimpleNamespace: + """A session state whose only content is how much wall-clock is left.""" + return SimpleNamespace( + baseline_double_run=False, + grid_session_deadline_sec=lambda: (None if remaining_sec is None else time.monotonic() + remaining_sec), + baseline_runtime_sec=0.0, + ) + + +def _capturing_fake_run(returncode: int = 0, *, produces_workspace: bool = True): + """A ``run_with_session_kill`` stand-in that records how it was called.""" + calls: list[dict] = [] + + def fake_run(cmd, *args, **kwargs): + calls.append(dict(kwargs)) + if produces_workspace: + out_idx = cmd.index("--output-dir") + _fake_workspace(Path(cmd[out_idx + 1]), tput=_HOT_TPUT) + return subprocess.CompletedProcess(cmd, returncode, "ok", "") + + return fake_run, calls + + +def _run_baseline_under_budget( + tmp_path, + *, + remaining_sec: float | None, + timeout_sec: int = 7200, + returncode: int = 0, + produces_workspace: bool = True, + executor_cls=BaselineExecutor, +) -> tuple[dict, list[dict]]: + """Run one baseline round against a session with ``remaining_sec`` left.""" + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + fake_run, calls = _capturing_fake_run(returncode, produces_workspace=produces_workspace) + executor = executor_cls( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + ) + ctx = _make_ctx( + { + "output_dir": str(tmp_path / "ws"), + "timeout_sec": timeout_sec, + "gpu_type": "mi300x", + } + ) + # The live state arrives on the context, the way the coordinator passes it. + ctx.extra["shared_state"] = _budgeted_state(remaining_sec=remaining_sec) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + return result, calls + + +class TestTheSessionBudgetReachesTheBaselineRound: + """The arm #1146 names as the largest hole, and the one that motivated it. + + A baseline is admitted on a catalogue cost of five minutes and given a + two-hour hang backstop (four, cold), so the round that runs first and + longest was the one round no wall-clock defence covered: no deadline + reached the reaper, and nothing clamped the cap to what was left. + """ + + def test_the_deadline_reaches_the_reaper(self, tmp_path): + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert calls and calls[0]["session_deadline_sec"] is not None + + def test_the_hang_backstop_is_clamped_to_what_is_left(self, tmp_path): + """A cap larger than the budget outlives the session it belongs to.""" + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=120.0, timeout_sec=7200) + + assert 1 <= calls[0]["timeout"] <= 120 + _SESSION_KILL_GRACE_SEC + + def test_an_unbounded_budget_leaves_the_cap_alone(self, tmp_path): + """No session context means no budget to respect, not a budget of zero.""" + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=None, timeout_sec=7200) + + assert calls[0]["timeout"] == 7200 + assert calls[0]["session_deadline_sec"] is None + + def test_a_budget_kill_is_not_recorded_as_a_broken_model(self, tmp_path): + """A reaped round leaves exactly what a broken server leaves behind. + + No workspace, no report, a non-zero returncode -- so without this branch + the run that ran out of time is filed as a fact about the model. + """ + result, _calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1.0, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + produces_workspace=False, + ) + + assert result["status"] == "failed" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + + def test_a_cancel_is_told_apart_from_a_spent_budget(self, tmp_path): + """A resume meets the spent budget again and does not meet the shutdown.""" + result, _calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + returncode=ORCHESTRATOR_CANCELLED_RETURNCODE, + produces_workspace=False, + ) + + assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + + def test_the_profile_arm_gets_all_of_it(self, tmp_path): + """Profile is the same executor with a four-hour default -- longer than + any session budget it could be given.""" + _result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=120.0, + timeout_sec=PROFILE_DEFAULT_TIMEOUT_SEC, + executor_cls=ProfileExecutor, + ) + + assert calls[0]["session_deadline_sec"] is not None + assert 1 <= calls[0]["timeout"] <= 120 + _SESSION_KILL_GRACE_SEC + + # _classify_subprocess_error unit tests from hyperloom.orchestrator.actions.executors.baseline import ( diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 92a38b17ab..47cd217bf6 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1256,39 +1256,98 @@ def _variant_progress_note( ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" -class _StoppedByTheRun(NamedTuple): - """How a returncode that stopped the round from outside is recorded. - - Neither cause is evidence about the variant, so both are recorded as - ``skipped`` rather than failed -- see the branch that consumes this. +class StoppedByTheRun(NamedTuple): + """How a sentinel returncode that stopped a round from outside is recorded. + + Neither cause is evidence about what was being measured, so a round that + ends this way is never graded as a failed measurement -- see the branches + that consume this, in the grid and in the baseline arm. + + Attributes: + error_class: The ledger class for the cause. + interrupted: What to report when the round was already running. + never_started: What to report when the round never began. + ends_the_batch: Whether the caller should stop launching further rounds + on its own, rather than leave that to a budget check it may not have. """ error_class: str interrupted: str never_started: str - ends_the_grid: bool + ends_the_batch: bool -# The two causes that stop a round without saying anything about the variant. -# The budget leaves the rest of the grid to the fit check at the top of the loop, -# which sees a deadline in the past and skips them all under the same label; a -# cancel has no such check, and nothing new should start under one anyway. -_STOPPED_BY_THE_RUN: dict[int, _StoppedByTheRun] = { - SESSION_TIME_EXHAUSTED_RETURNCODE: _StoppedByTheRun( +# The two causes that stop a round without saying anything about what it was +# measuring. The budget leaves the rest of a grid to the fit check at the top of +# its loop, which sees a deadline in the past and skips them all under the same +# label; a cancel has no such check, and nothing new should start under one. +_STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_RETURNCODE: StoppedByTheRun( error_class=SESSION_TIME_EXHAUSTED_CLASS, - interrupted="session wall-clock budget exhausted while this variant was running", - never_started="session wall-clock budget exhausted before this variant ran", - ends_the_grid=False, + interrupted="session wall-clock budget exhausted while this round was running", + never_started="session wall-clock budget exhausted before this round ran", + ends_the_batch=False, ), - ORCHESTRATOR_CANCELLED_RETURNCODE: _StoppedByTheRun( + ORCHESTRATOR_CANCELLED_RETURNCODE: StoppedByTheRun( error_class=ORCHESTRATOR_CANCELLED_CLASS, - interrupted="the orchestrator cancelled this action while this variant was running", - never_started="the orchestrator cancelled this action before this variant ran", - ends_the_grid=True, + interrupted="the orchestrator cancelled this action while this round was running", + never_started="the orchestrator cancelled this action before this round ran", + ends_the_batch=True, ), } +def stopped_by_the_run(returncode: int | None) -> StoppedByTheRun | None: + """Return how to record a round the run itself stopped, if it did. + + Args: + returncode: The round's returncode. + + Returns: + StoppedByTheRun | None: How to record it, or ``None`` when the + returncode says something about the round rather than about the run. + """ + if returncode is None: + return None + return _STOPPED_BY_THE_RUN.get(int(returncode)) + + +def session_clamped_timeout_sec( + cap: int, + session_deadline_sec: float | None, + *, + reserve_sec: float = 0.0, +) -> int: + """Reduce a hard timeout to what the session budget can still pay for. + + ``session_deadline_sec`` already excludes the close-window reserve, so no + further margin is taken for the session itself. Never returns less than 1 -- + a non-positive cap reads as "no timeout" to the subprocess layer, which is + the opposite of what an exhausted budget means. Whether the round should + start at all is the caller's decision, not this one's. + + A small grace is added past the session deadline so the in-process session + watchdog trips first: both fire at the same instant, and the hard cap raises + ``TimeoutExpired``, which the ledger records as a timeout of the thing being + measured. The watchdog's sentinel says the run ran out of time, which is what + actually happened. + + Args: + cap: The timeout the caller would grant with an unbounded budget. + session_deadline_sec: Monotonic-clock session deadline, or ``None`` when + the budget is unbounded, which leaves ``cap`` untouched. + reserve_sec: Seconds held back for rounds that must still follow this + one, so an early round cannot spend the budget a later one needs. + + Returns: + int: The hard timeout to grant this round, in seconds. + """ + if session_deadline_sec is None: + return int(cap) + usable = int(session_deadline_sec - time.monotonic() - max(0.0, reserve_sec)) + _SESSION_KILL_GRACE_SEC + return int(cap) if usable >= int(cap) else max(1, usable) + + def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call. @@ -1514,18 +1573,8 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl data point, so it is the round the budget is kept for; a discarded warmup that overruns is the cheaper thing to cut short. - ``session_deadline_sec`` already excludes the close-window reserve, so - no further margin is taken for the session itself. Never returns less - than 1 -- a non-positive cap would read as "no timeout" to the - subprocess layer, which is the opposite of what an exhausted budget - means. Deciding whether the round should start at all belongs to the - caller's fit check. - - A small grace is added past the session deadline so the in-process - session watchdog trips first: both fire at the same instant, and the hard - cap raises ``TimeoutExpired``, which the ledger records as a variant - timeout -- a claim about the variant. The watchdog's sentinel says the run - ran out of time, which is what actually happened. + The clamp itself is :func:`session_clamped_timeout_sec`; this adds the + grid's own log line. Args: idx (int): Zero-based variant index, for the log line. @@ -1538,12 +1587,9 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl int: The hard timeout to grant this round, in seconds. """ cap = int(variant_timeout_sec) - if session_deadline_sec is None: - return cap - usable = int(session_deadline_sec - time.monotonic() - max(0.0, reserve_sec)) + _SESSION_KILL_GRACE_SEC - if usable >= cap: + clamped = session_clamped_timeout_sec(cap, session_deadline_sec, reserve_sec=reserve_sec) + if clamped == cap: return cap - clamped = max(1, usable) log.info( "grid_runner: variant %d/%d name=%s %s cap clamped %ds -> %ds by the session budget", idx + 1, @@ -2297,7 +2343,7 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl ) ) await _pulse_after_variant(i) - if stopped.ends_the_grid: + if stopped.ends_the_batch: results.extend(_not_run_skip_result(rest, stopped) for rest in grid[i + 1 :]) break if not keep_going_on_failure: @@ -2523,7 +2569,7 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl return results -def _not_run_skip_result(variant: GridVariant, stopped: _StoppedByTheRun) -> VariantResult: +def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: """Build the ``skipped`` result for a variant the run never got to. Args: @@ -2679,6 +2725,7 @@ def _write_variant_abort_marker_impl( "MULTI_NODE_DEFAULT_KEEP_THRESHOLD_PCT", "ORCHESTRATOR_CANCELLED_CLASS", "SESSION_TIME_EXHAUSTED_CLASS", + "StoppedByTheRun", "SGLANG_WATCHDOG_TIMEOUT_ENV", "SINGLE_NODE_DEFAULT_KEEP_THRESHOLD_PCT", "VariantResult", @@ -2696,7 +2743,9 @@ def _write_variant_abort_marker_impl( "sanitize_result_dir", "sanitize_script_name", "server_args_env_name", + "session_clamped_timeout_sec", "session_grid_bounds", + "stopped_by_the_run", # Re-exported from the sibling modules to keep the namespace intact. "coerce_extra_envs", "compact_json_server_args", diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 9fb0b7958e..31b0ca8fba 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -50,12 +50,16 @@ _kill_stale_servers, sanitize_result_dir, sanitize_script_name, + session_clamped_timeout_sec, + session_grid_bounds, + stopped_by_the_run, ) from ._subprocess_kill import ( DETOKENIZER_STALL_RETURNCODE, SERVER_DEAD_RETURNCODE, run_with_session_kill, server_log_death_excerpt, + session_deadline_to_remaining_sec, ) from ._accuracy_gate import ( _RUN_EVAL_FALSE_VALUES, @@ -1625,6 +1629,10 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: ``INFERENCE_OPTIMIZER_COLD_START_TIMEOUT_SEC``) → warm default. Every path emits one log line for greppability. + All three are hang backstops sized in hours, with no reference to the + session budget; :meth:`_session_capped_timeout` reduces the result to + what is actually left, per round, at the point each round launches. + Args: params: Task params; an explicit ``timeout_sec`` overrides the probe-based selection. @@ -1701,6 +1709,39 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: ) return self.default_timeout_sec + @staticmethod + def _session_capped_timeout( + timeout_sec: int, + session_deadline_sec: float | None, + *, + output_dir: Path, + ) -> int: + """``timeout_sec`` reduced to what the session can still pay for. + + The baseline's own timeout is a catastrophic-hang backstop -- two hours + by default, four for a cold start -- chosen with no reference to how much + of the session is left. A round granted more than the budget has runs + past the end of the session and takes the closing phase with it. + + Args: + timeout_sec: The timeout this round would get on an unbounded budget. + session_deadline_sec: Monotonic-clock session deadline, or ``None`` + when there is no budget to respect. + output_dir: The round's workspace, for the log line. + + Returns: + int: The hard timeout to grant this round, in seconds. + """ + clamped = session_clamped_timeout_sec(timeout_sec, session_deadline_sec) + if clamped != timeout_sec: + log.info( + "baseline_executor: timeout clamped %ds -> %ds by the session budget (round=%s)", + timeout_sec, + clamped, + output_dir.name, + ) + return clamped + @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: """Resolve the InferenceX checkout the subprocess will ``cd`` into. @@ -3614,6 +3655,11 @@ async def _run_single_benchmark( ) ctx_extra = getattr(ctx, "extra", None) or {} + # The session's wall-clock budget, resolved per round rather than once + # per task: a baseline runs up to three of them, and a warmup that + # overran has already spent budget the ones after it were counting on. + session_deadline_sec, _ = session_grid_bounds(ctx_extra.get("shared_state") or self.shared_state) + timeout_sec = self._session_capped_timeout(timeout_sec, session_deadline_sec, output_dir=output_dir) if not ctx_extra.get("mn_round_restarted"): try: # Merge the reference base UNDER the per-task args (last-wins) so @@ -3699,6 +3745,7 @@ async def _run_single_benchmark( timeout=timeout_sec, server_log_path=_watchdog_server_log_path(_mn_warm_dir, framework), on_output=_mn_warm_activity.note, + session_deadline_sec=session_deadline_sec, ) log.info("baseline_executor: MN warmup pass done (discarded)") except Exception as exc: # noqa: BLE001 - warmup is best-effort @@ -3753,6 +3800,7 @@ async def _run_single_benchmark( cwd=str(output_dir), timeout=timeout_sec, server_log_path=watchdog_server_log, + session_remaining_sec=session_deadline_to_remaining_sec(session_deadline_sec), ) subprocess_runtime_sec = max(0.0, time.time() - subprocess_started_unix) else: @@ -3768,6 +3816,7 @@ async def _run_single_benchmark( timeout=timeout_sec, server_log_path=watchdog_server_log, on_output=activity.note, + session_deadline_sec=session_deadline_sec, ) subprocess_runtime_sec = max( 0.0, @@ -3792,6 +3841,32 @@ async def _run_single_benchmark( **capture_meta, } + # The run stopped this round rather than the round failing: the session + # budget elapsed mid-round, or the orchestrator cancelled the action. + # Classified apart from every measurement failure and checked before + # them, because the reap leaves the same evidence a broken server does -- + # no workspace, no report, a non-zero returncode -- and being graded as + # ``server_init_dead`` or ``subprocess_nonzero`` would put a verdict on + # the model that this round never reached. Nothing here arms a retry + # either: the cause is the run, and a resume meets it again. + stopped = stopped_by_the_run(proc_returncode) + if stopped is not None: + log.warning( + "baseline_executor: round reaped after %.1fs: %s; error_class=%s.", + subprocess_runtime_sec, + stopped.interrupted, + stopped.error_class, + ) + return { + "status": "failed", + "error_class": stopped.error_class, + "returncode": proc_returncode, + "error": stopped.interrupted, + "subprocess_runtime_sec": round(subprocess_runtime_sec, 2), + "output_dir": str(output_dir), + **capture_meta, + } + # Detokenizer-stall watchdog reap: the server came up healthy but went # silent for the stall grace window (hung engine / wedged detokenizer). # A stall reap leaves no benchmark_* workspace; a distinct error_class From cd2273f7c5d78425166afa25b9da8412fd545220 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 13:26:49 +0000 Subject: [PATCH 17/65] admission: price an action on this session's baseline, not the catalogue The first wall-clock layer asked the action catalogue what an action costs, and the catalogue's estimates are calibrated on small models: `baseline` 5 minutes, `roofline` 10. The two sessions that motivated #1146 measured 51 and 125 minutes of baseline and an 81-minute roofline, so the gate would have admitted every arm in both runs. PRELUDE's affordability gate already knew this -- its own docstring says so -- and anchors on the baseline round the session measured; layer 1 was the one still reading the catalogue. One measured round is now taken as a floor on any action that runs a benchmark round of its own, which is the same rule PRELUDE was applying, promoted into `expected_action_cost_minutes` so there is one definition and PRELUDE reads it rather than keeping a second. A floor rather than a replacement: an action that benches a grid costs more than one round and never less, and only the catalogue knows how many rounds an action runs. Which actions those are is read off `requires_lanes` rather than a list of names, so the rule follows what an action does: the benchmark and profile lanes are exactly the two that serialize GPU work. Writing a report costs what it costs no matter how large the model is. Also corrects `roofline.md`, which still documented the pre-rename `cost_minutes_p50=8` against a catalogue that carries `typical_runtime_min=10`, and now says which number the guards prefer. Co-authored-by: Cursor --- .../inference_optimizer/actions/roofline.md | 9 +- .../tests/test_session_time_budget.py | 65 +++++++++++++++ .../orchestrator/loop/coordinator_helpers.py | 83 +++++++++++++++++-- src/hyperloom/orchestrator/loop/dispatcher.py | 11 ++- src/hyperloom/orchestrator/phases/prelude.py | 32 ++++--- 5 files changed, 174 insertions(+), 26 deletions(-) diff --git a/src/hyperloom/inference_optimizer/actions/roofline.md b/src/hyperloom/inference_optimizer/actions/roofline.md index 9117f2215e..4ae92490fc 100644 --- a/src/hyperloom/inference_optimizer/actions/roofline.md +++ b/src/hyperloom/inference_optimizer/actions/roofline.md @@ -92,8 +92,11 @@ analysis task. ## Cost / runtime -* `cost_minutes_p50=8` / `cost_minutes_p75=15` — dominated by profile - (Magpie + torch profiler overhead) + trace_analyze (TraceLens - subprocess); `trace_split` inside TraceLens adds < 30s. +* `typical_runtime_min=10` — dominated by profile (Magpie + torch profiler + overhead) + trace_analyze (TraceLens subprocess); `trace_split` inside + TraceLens adds < 30s. The estimate is calibrated on small models: a field + session measured an 81-minute roofline, so the budget guards prefer this + session's own measured baseline round once one exists and fall back to this + number only before that. * `requires_lanes=[profile_lane]` — same lane as `profile` so we don't run two profile-class tasks concurrently against the server. diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 2c632a64fc..e46b579e4f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -28,6 +28,7 @@ import threading import time from collections.abc import Callable +from types import SimpleNamespace from typing import Any import pytest @@ -49,6 +50,7 @@ TIME_BUDGET_EXEMPT_ACTIONS, action_fits_time_budget, expected_action_cost_minutes, + measured_baseline_runtime_sec, ) from hyperloom.orchestrator.policy.gate import PolicyDenied from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan @@ -60,6 +62,10 @@ _EXPENSIVE_COST_MIN = 60.0 # Cheap enough to fit anything but a nearly-spent budget. _CHEAP_ACTION = "profile" +# An action the catalogue prices at five minutes, and what one of the two +# sessions that motivated the wall-clock work actually measured for it. +_BASELINE_ACTION = "baseline" +_MEASURED_BASELINE_SEC = 51 * 60.0 def _heartbeat() -> Intent: @@ -114,6 +120,52 @@ def test_no_catalogued_action_reads_as_free(self): assert free == [] +class TestTheCostIsAnchoredOnWhatThisSessionMeasured: + """The catalogue prices a baseline at five minutes; the field runs it in 51. + + Those estimates are calibrated on small models, so a gate anchored on them + admits arms a real model cannot pay for -- it would not have stopped either + of the two sessions that motivated the wall-clock work. PRELUDE's + affordability gate already anchors on the session's own baseline round; + admission now reads the same number through the same helper. + """ + + def test_a_measured_round_outprices_the_catalogue_for_an_action_that_benches(self): + cost = expected_action_cost_minutes( + ACTION_CATALOGUE["baseline"], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(51.0) + + def test_the_catalogue_wins_where_it_prices_more_than_one_round(self): + """The measurement is a floor, not a replacement: only the catalogue + knows an action benches a whole grid rather than a single variant.""" + cost = expected_action_cost_minutes( + ACTION_CATALOGUE[_EXPENSIVE_ACTION], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(_EXPENSIVE_COST_MIN) + + def test_an_action_that_never_benches_keeps_its_own_estimate(self): + """Writing the report costs what it costs; the model's size is not in it.""" + cost = expected_action_cost_minutes( + ACTION_CATALOGUE["report"], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(ACTION_CATALOGUE["report"].typical_runtime_min) + + def test_a_session_with_no_baseline_yet_falls_back_to_the_catalogue(self): + assert expected_action_cost_minutes(ACTION_CATALOGUE["baseline"]) == pytest.approx(5.0) + + def test_a_measurement_that_is_not_a_number_is_not_a_cost(self): + assert measured_baseline_runtime_sec(None) == 0.0 + assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec="not-a-number")) == 0.0 + assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec=-1.0)) == 0.0 + assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec=_MEASURED_BASELINE_SEC)) == ( + pytest.approx(_MEASURED_BASELINE_SEC) + ) + + class TestFitDecision: """The pure fit rule, independent of any Coordinator.""" @@ -251,6 +303,19 @@ def test_a_stopping_session_leaves_the_gate_to_the_stop_path(self, coord: Coordi coord.shared_state.stop_reason = "time_exhausted" assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None + def test_this_session_s_own_baseline_changes_the_answer(self, coord: Coordinator): + """Half an hour left admits a baseline the catalogue prices at five + minutes -- until this session has measured one and knows better.""" + _set_budget(coord, minutes=30) + assert coord._time_budget_denial_for_action(_BASELINE_ACTION) is None + + coord.shared_state.baseline_runtime_sec = _MEASURED_BASELINE_SEC + denied = coord._time_budget_denial_for_action(_BASELINE_ACTION) + + assert isinstance(denied, PolicyDenied) + assert denied.rule == "time_budget" + assert "51 min" in str(denied) + def test_the_budget_shrinks_the_gate_as_the_session_runs(self, coord: Coordinator): _set_budget(coord, minutes=120, elapsed_min=0.0) assert coord._time_budget_denial_for_action(_EXPENSIVE_ACTION) is None diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index e4bfb8209f..6061ab8e62 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -36,6 +36,8 @@ "_MIN_KERNEL_ENGAGED_GAIN_PCT", "action_fits_time_budget", "coerce_needs_gpu", + "expected_action_cost_minutes", + "measured_baseline_runtime_sec", ] @@ -229,27 +231,98 @@ def _positive_int(*keys: str) -> bool: } ) +# The lanes that serialize GPU work: an action requiring one of them spends its +# time running a benchmark round, so what this session measured says more about +# it than a catalogue estimate does. +_GPU_BENCH_LANES: frozenset[str] = frozenset( + { + "benchmark_lane", + "profile_lane", + } +) + + +def measured_baseline_runtime_sec(shared_state: Any | None) -> float: + """Read this session's own measured baseline round, in seconds. + + Args: + shared_state (Any | None): The session ``SharedState``, or ``None`` when + the caller has no session context. + + Returns: + float: The measured baseline runtime; ``0.0`` when the session has not + landed a baseline yet, which every caller reads as "no measurement". + """ + try: + return max(0.0, float(getattr(shared_state, "baseline_runtime_sec", 0.0) or 0.0)) + except (TypeError, ValueError): + return 0.0 + -def expected_action_cost_minutes(meta: Any | None) -> float: - """Read an action's expected cost out of its catalogue metadata. +def _action_benches_on_gpu(meta: Any | None) -> bool: + """Whether an action's cost is dominated by a benchmark round on the GPU. - Both budget guards go through here so the field is named once. Reading it + Read off the lanes the action must hold rather than off a list of names, so + an action added to the catalogue is classified by what it does. The + benchmark and profile lanes are exactly the two that serialize GPU work; an + action holding neither (``report``, ``target_analysis``, ``specialist``) + costs what its own bookkeeping costs and has nothing to do with model size. + + Args: + meta (Any | None): The action's catalogue metadata. + + Returns: + bool: ``True`` when the action runs at least one benchmark round. + """ + lanes = getattr(meta, "requires_lanes", ()) or () + try: + return any(str(lane) in _GPU_BENCH_LANES for lane in lanes) + except TypeError: + return False + + +def expected_action_cost_minutes( + meta: Any | None, + *, + measured_baseline_sec: float = 0.0, +) -> float: + """Read an action's expected cost, preferring what this session measured. + + Every budget guard goes through here so the field is named once. Reading it inline with a ``getattr`` default turned the catalogue's move off YAML — which renamed the field — into a gate that admitted everything without a word, because "no estimate on record" and "the field moved" look the same from a default. + The catalogue's estimates are calibrated on small models (``baseline`` 5 + min, ``roofline`` 10 min) while the two sessions that motivated the + wall-clock work measured 51 and 125 minutes of baseline and an 81-minute + roofline. A guard anchored on the catalogue alone therefore admits arms the + session cannot pay for — it would not have stopped either field run. So one + measured baseline round is taken as a *floor* on any action that runs a + benchmark round of its own: it is this model on this GPU under this + workload, which is what those actions spend their time doing. It is a floor + rather than a replacement because an action that benches several variants + costs more than one round, never less, and the catalogue is the only thing + that knows how many. + Args: meta (Any | None): The action's catalogue metadata, or ``None`` for an action the catalogue does not carry. + measured_baseline_sec (float): This session's measured baseline runtime + in seconds, from :func:`measured_baseline_runtime_sec`; ``0.0`` + before a baseline lands, which leaves the catalogue in charge. Returns: float: The expected cost in minutes; ``0.0`` when nothing is on record. """ try: - return float(getattr(meta, "typical_runtime_min", 0.0) or 0.0) + catalogue_min = float(getattr(meta, "typical_runtime_min", 0.0) or 0.0) except (TypeError, ValueError): - return 0.0 + catalogue_min = 0.0 + if measured_baseline_sec <= 0.0 or not _action_benches_on_gpu(meta): + return catalogue_min + return max(catalogue_min, measured_baseline_sec / 60.0) def action_fits_time_budget( diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index ebfe36f072..2847215983 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -39,6 +39,7 @@ action_fits_time_budget, coerce_needs_gpu, expected_action_cost_minutes, + measured_baseline_runtime_sec, ) from .coordinator import ( @@ -1365,6 +1366,11 @@ def _time_budget_denial_for_action( keeps the refusal out of the failure ledgers — no task row is created, so nothing teaches the KB that the action failed. + What the action is expected to cost is anchored on this session's own + baseline round once one exists, the way PRELUDE's affordability gate + already is; see :func:`expected_action_cost_minutes` for why a + catalogue-anchored gate admits arms a real model cannot pay for. + Args: action_name: The proposed/delegated/inline action name. @@ -1381,7 +1387,10 @@ def _time_budget_denial_for_action( meta = reg.get(action) if reg is not None else None if meta is None: return None - expected_min = expected_action_cost_minutes(meta) + expected_min = expected_action_cost_minutes( + meta, + measured_baseline_sec=measured_baseline_runtime_sec(self.shared_state), + ) usable_sec = self.shared_state.session_budget_usable_sec() if action_fits_time_budget( usable_sec=usable_sec, diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index 7cac792a77..a11d620f7e 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -24,7 +24,10 @@ from ..loop.coordinator import ( _DEFAULT_WARM_REPLAY_MIN_CONFIDENCE, ) -from ..loop.coordinator_helpers import expected_action_cost_minutes +from ..loop.coordinator_helpers import ( + expected_action_cost_minutes, + measured_baseline_runtime_sec, +) from .base import PhaseHandler log = _logging.getLogger(__name__) @@ -156,30 +159,25 @@ def _internal_analysis_kind(self) -> str: def _measured_analysis_cost_sec(self) -> float: """Expected cost of the initial roofline/profile arm, in seconds. - Anchored on this session's own baseline round rather than the action - catalog. The catalog's estimates (``baseline`` 5 min, ``roofline`` - 8 min) are calibrated on small models: the two sessions that motivated - this guard measured 51 and 125 minutes of baseline and an 81-minute - roofline, so a catalog-anchored guard admits an arm it cannot pay for. - The analysis arm boots its own server and runs the same benchmark under - a profiler, so one measured baseline round is a floor on its cost - rather than a guess at it. The catalog is the fallback for the first - analysis of a session that has no measurement yet. + a profiler, so one measured baseline round is a floor on its cost rather + than a guess at it; :func:`expected_action_cost_minutes` applies that + floor and falls back to the catalog for the first analysis of a session + that has no measurement yet. Returns: float: Expected cost in seconds; ``0.0`` when nothing is on record, which :func:`machine_state.prelude_can_afford` reads as free. """ - try: - measured = float(getattr(self.shared_state, "baseline_runtime_sec", 0.0) or 0.0) - except (TypeError, ValueError): - measured = 0.0 - if measured > 0.0: - return measured registry = getattr(self, "action_registry", None) meta = registry.get(self._internal_analysis_kind()) if registry is not None else None - return expected_action_cost_minutes(meta) * 60.0 + return ( + expected_action_cost_minutes( + meta, + measured_baseline_sec=measured_baseline_runtime_sec(self.shared_state), + ) + * 60.0 + ) def _record_prelude_arm_dropped(self, arm: str, evidence: dict[str, Any]) -> None: """Record a PRELUDE arm dropped for budget on the current phase record. From bd841b806dbe83409366bb55e0b7dc1180532b83 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 17:10:30 +0000 Subject: [PATCH 18/65] explore: a variant the run reaped is not a variant that failed run_grid already records a reaped round as skipped rather than failed, but explore did not consume the distinction: the result fell through the generic "not succeeded" branch and was written into the KB-facing tested ledger as FAILED with no error_class. That makes a resume skip a variant nothing ever measured, teaches the KB that these knobs are bad on the evidence of a clock, and leaves the round reporting a bare unattributed failure. Explore now recognises a stopped round in all three places it can appear -- warmup, decision, and the post-KEEP stack rebench -- and leaves the variant out of every ledger, exactly as it already did for one the admission gate refused. A reaped rebench also rolls back the in-batch stack fold and drops the decision entry, so a resume re-measures the variant and its confirmation together instead of the variant being evicted as KEEP_UNSTABLE by a round that measured nothing. The notion of "the run stopped this, the thing under test did not fail" moves out of _grid_runner into orchestrator/actions/stop_attribution, a leaf every ledger can import: the grid keys it by sentinel returncode, everything downstream carries the error class. The rebench result carries that class too, which also closes its gap on orchestrator_cancelled -- it recognised only the budget class before. Co-authored-by: Cursor --- .../tests/test_explore_executor.py | 220 ++++++++++++++++++ .../test_integrate_patch_coverage_unit.py | 11 +- .../actions/executors/_grid_runner.py | 61 +---- .../actions/executors/_stack_rebench.py | 31 +-- .../orchestrator/actions/executors/explore.py | 78 ++++++- .../orchestrator/actions/stop_attribution.py | 100 ++++++++ 6 files changed, 426 insertions(+), 75 deletions(-) create mode 100644 src/hyperloom/orchestrator/actions/stop_attribution.py diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index e5128eada3..8781228102 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -28,6 +28,14 @@ _SESSION_KILL_GRACE_SEC, apply_compatibility_filter, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, +) +from hyperloom.orchestrator.actions.stop_attribution import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, +) from hyperloom.orchestrator.actions.executors._accuracy_gate import ( _RUN_EVAL_FALSE_VALUES as _RUN_EVAL_FALSE, ) @@ -1741,6 +1749,218 @@ def _fake_run(cmd, *args, **kwargs): assert res.result["explore_search_update"]["tested"] == {} +@pytest.mark.asyncio +async def test_explore_leaves_a_variant_the_run_reaped_out_of_the_ledger( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """The common case: the budget expires while a variant is running, not before it. + + ``run_grid`` records such a variant as ``skipped`` because nothing was + measured. Explore has to consume that distinction: a variant written into + the KB-facing ``tested`` ledger as ``FAILED`` is one a resume will skip + forever, and one the KB learns is a bad idea, on the evidence of a clock. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 # admits both variants; the reap comes mid-round + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + ran: list[str] = [] + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + ran.append(slot.name) + if "v_reaped" not in str(slot): + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + # The session deadline elapsed while this round was running, so the + # reaper tore the tree down and named the cause. + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-reaped"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_measured", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + }, + { + "name": "v_reaped", + "extra_args": "--max-num-seqs 512", + "extra_envs": {}, + "provenance": "default_grid", + }, + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-reaped", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + tested = res.result["explore_search_update"]["tested"] + # A variant the run reaped was never measured; recording it keeps a resume + # from ever retrying it, and teaches the KB a clock's verdict. + assert [t["name"] for t in tested.values()] == ["v_measured"] + assert [lr["name"] for lr in res.result["losers"]] == [] + assert res.result["session_budget_untested"] == 1 + + +@pytest.mark.asyncio +async def test_explore_does_not_call_a_variant_unstable_when_the_run_reaped_its_rebench( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """A confirmation the run stopped is not a failed confirmation. + + The post-KEEP rebench is the second gate, so a reaped rebench used to evict + the variant as ``KEEP_UNSTABLE`` -- a stability verdict drawn from a round + that measured nothing. The decision round's own entry goes too, so a resume + re-measures the variant and its confirmation together. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + if "stack_rebench" in str(slot): + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + _fake_workspace(slot, tput=900.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-rebench-reaped"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_unconfirmed", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-rebench-reaped", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert res.result["explore_search_update"]["tested"] == {} + assert res.result["losers"] == [] + assert res.result["winners"] == [] + assert res.result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert res.result["session_budget_untested"] == 1 + + +@pytest.mark.asyncio +async def test_explore_attributes_a_round_the_run_reaped_before_anything_measured( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """With nothing measured the round is ``failed``, and must say who stopped it.""" + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + return subprocess.CompletedProcess( + args=cmd, + returncode=ORCHESTRATOR_CANCELLED_RETURNCODE, + stdout="", + stderr="cancelled", + ) + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-cancelled"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_cancelled", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-cancelled", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert res.result["status"] == "failed" + assert res.result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + assert res.result["explore_search_update"]["tested"] == {} + + @pytest.mark.asyncio async def test_explore_executor_empty_grid_returns_failed(sub_agent_runner, tmp_path): sub, tr, _ = sub_agent_runner diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py index 10d1851abb..19e02ce2a6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py @@ -491,7 +491,8 @@ async def _measure(**kwargs): @pytest.mark.asyncio -async def test_rebench_dropped_for_budget_is_not_reported_as_a_failed_measurement(tmp_path): +@pytest.mark.parametrize("error_class", ["session_time_exhausted", "orchestrator_cancelled"]) +async def test_rebench_the_run_stopped_is_not_reported_as_a_failed_measurement(tmp_path, error_class): """Not measuring a variant is not evidence that the variant is unstable.""" from unittest.mock import patch @@ -503,8 +504,8 @@ async def test_rebench_dropped_for_budget_is_not_reported_as_a_failed_measuremen extra_server_args="", extra_envs={}, status="skipped", - error="session wall-clock budget exhausted before this variant ran", - error_class="session_time_exhausted", + error="the run stopped this round before it measured anything", + error_class=error_class, ) async def _fake_run_grid(**_kwargs): @@ -521,8 +522,8 @@ async def _fake_run_grid(**_kwargs): variant_timeout_sec=600, ) - assert result.skipped_for_session_budget - assert result.warnings == ["stack_rebench_skipped:session_time_exhausted"] + assert result.error_class == error_class + assert result.warnings == [f"stack_rebench_skipped:{error_class}"] assert not any("stack_rebench_failed" in w for w in result.warnings) diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 47cd217bf6..7108361b45 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -34,6 +34,12 @@ from ...roles.robustness_pulse import pulse as _robustness_pulse from ...trace.task_progress import heartbeat_while_output_flows, report_progress +from ..stop_attribution import ( + ORCHESTRATOR_CANCELLED_CLASS, + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) from ._accuracy_gate import materialized_run_eval_disabled from ._subprocess_kill import ( AGENTX_PREFLIGHT_RETURNCODE, @@ -1244,56 +1250,13 @@ def _variant_progress_note( # that it cannot meaningfully eat the close-window reserve. _SESSION_KILL_GRACE_SEC: int = 15 -# Labels a round that never ran because the session wall-clock budget was spent. -# Distinct from any measurement failure: a round that was not run is not evidence -# about the variant, and the ledger must not read it as one. -SESSION_TIME_EXHAUSTED_CLASS = "session_time_exhausted" - -# Labels a round the orchestrator stopped from outside -- a shutdown, or a -# budget the dispatcher found spent. Kept apart from the class above for the -# same reason their returncodes are: a resume faces the spent budget again and -# does not face the shutdown. -ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" - - -class StoppedByTheRun(NamedTuple): - """How a sentinel returncode that stopped a round from outside is recorded. - - Neither cause is evidence about what was being measured, so a round that - ends this way is never graded as a failed measurement -- see the branches - that consume this, in the grid and in the baseline arm. - - Attributes: - error_class: The ledger class for the cause. - interrupted: What to report when the round was already running. - never_started: What to report when the round never began. - ends_the_batch: Whether the caller should stop launching further rounds - on its own, rather than leave that to a budget check it may not have. - """ - - error_class: str - interrupted: str - never_started: str - ends_the_batch: bool - - -# The two causes that stop a round without saying anything about what it was -# measuring. The budget leaves the rest of a grid to the fit check at the top of -# its loop, which sees a deadline in the past and skips them all under the same -# label; a cancel has no such check, and nothing new should start under one. +# The returncode side of :mod:`...stop_attribution`: the same two causes, keyed +# by the sentinel a subprocess comes back with. The classes themselves live in +# that leaf because the ledgers downstream of here carry the class, not the +# returncode. _STOPPED_BY_THE_RUN: dict[int, StoppedByTheRun] = { - SESSION_TIME_EXHAUSTED_RETURNCODE: StoppedByTheRun( - error_class=SESSION_TIME_EXHAUSTED_CLASS, - interrupted="session wall-clock budget exhausted while this round was running", - never_started="session wall-clock budget exhausted before this round ran", - ends_the_batch=False, - ), - ORCHESTRATOR_CANCELLED_RETURNCODE: StoppedByTheRun( - error_class=ORCHESTRATOR_CANCELLED_CLASS, - interrupted="the orchestrator cancelled this action while this round was running", - never_started="the orchestrator cancelled this action before this round ran", - ends_the_batch=True, - ), + SESSION_TIME_EXHAUSTED_RETURNCODE: STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + ORCHESTRATOR_CANCELLED_RETURNCODE: STOPPED_BY_THE_RUN[ORCHESTRATOR_CANCELLED_CLASS], } diff --git a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py index 7444321d9a..922203e82f 100644 --- a/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py +++ b/src/hyperloom/orchestrator/actions/executors/_stack_rebench.py @@ -14,7 +14,8 @@ from pathlib import Path from typing import Any -from ._grid_runner import SESSION_TIME_EXHAUSTED_CLASS, GridVariant, run_grid +from ..stop_attribution import stopped_by_the_run_class +from ._grid_runner import GridVariant, run_grid # Single source of truth for the post-KEEP confirmation floor shared by every @@ -37,21 +38,17 @@ class StackRebenchResult: workspace: str | None warnings: list[str] = field(default_factory=list) stable_floor: float = 0.0 + # Set to the ledger class from :mod:`..stop_attribution` when the run itself + # stopped the round, empty otherwise. Lets a caller tell "the confirmation + # did not happen" apart from "the confirmation failed": :attr:`stable` is + # ``False`` for both, and only one of them is evidence about the variant. + error_class: str = "" @property def stable(self) -> bool: """True when the measured throughput cleared the stability floor.""" return self.tput is not None and self.tput >= self.stable_floor - @property - def skipped_for_session_budget(self) -> bool: - """True when the round never ran because the session budget was spent. - - Lets a caller tell "the confirmation did not happen" apart from "the - confirmation failed", which are different facts about the variant. - """ - return any(w.startswith(f"stack_rebench_skipped:{SESSION_TIME_EXHAUSTED_CLASS}") for w in self.warnings) - async def measure_stack_rebench( *, @@ -119,18 +116,26 @@ async def measure_stack_rebench( tput: float | None = None workspace: str | None = None warnings: list[str] = [] + error_class = "" if rb is not None and rb.status == "succeeded": tput = rb.output_throughput workspace = rb.workspace warnings = list(rb.nonfatal_warnings) - elif rb is not None and getattr(rb, "error_class", "") == SESSION_TIME_EXHAUSTED_CLASS: - warnings.append(f"stack_rebench_skipped:{SESSION_TIME_EXHAUSTED_CLASS}") + elif rb is not None and stopped_by_the_run_class(getattr(rb, "error_class", "")) is not None: + error_class = rb.error_class + warnings.append(f"stack_rebench_skipped:{error_class}") elif rb is not None: warnings.append(f"stack_rebench_failed:{(rb.error or '')[-120:]}") else: warnings.append("stack_rebench_no_result") stable_floor = base_tput * (1.0 + stable_threshold_pct / 100.0) - return StackRebenchResult(tput=tput, workspace=workspace, warnings=warnings, stable_floor=stable_floor) + return StackRebenchResult( + tput=tput, + workspace=workspace, + warnings=warnings, + stable_floor=stable_floor, + error_class=error_class, + ) __all__ = ["DEFAULT_STACK_STABLE_PCT", "StackRebenchResult", "measure_stack_rebench"] diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 5a67692c69..490a979311 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -47,6 +47,12 @@ tail_excerpt, ) from ...state.shared_state import first_positive_tput, resolve_grading_anchor_tput, stack_base_params +from ..stop_attribution import ( + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, + stopped_by_the_run_class, +) from ._accuracy_gate import ( accuracy_passed, is_high_accuracy_risk, @@ -63,7 +69,6 @@ apply_aiter_moe_pin_filter, apply_multi_node_invalid_variants, reorder_grid_for_multi_node, - SESSION_TIME_EXHAUSTED_CLASS, run_grid, sanitize_result_dir, sanitize_script_name, @@ -1134,10 +1139,49 @@ async def __call__(self, ctx) -> dict[str, Any]: ) warmup_expected_sec = (baseline_runtime_sec if baseline_runtime_sec > 0 else None) or session_expected_sec decision_expected_sec = (decision_anchor_sec if decision_anchor_sec > 0 else None) or session_expected_sec - # Set when the loop stops early for lack of budget, so the round can say + # Set when the loop stops because the run stopped it -- the budget ran + # out, or the orchestrator cancelled the action -- so the round can say # so instead of reporting a bare, unattributed failure: a variant that - # never ran is not a variant that failed. + # never ran is not a variant that failed. ``run_stop_detail`` is the + # lead clause, which differs by whether a round was already under way. + run_stop: StoppedByTheRun | None = None + run_stop_detail = "" session_budget_untested = 0 + + def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_label: str) -> bool: + """Whether the run stopped this round, and record it if it did. + + A reaped round measured nothing, so the variant is left out of every + ledger exactly as an unadmitted one is: writing it as ``FAILED`` + would make a resume skip a variant nothing ever measured, and would + teach the KB that these knobs are bad because a clock ran out. + + Args: + result: The round's :class:`VariantResult`, or ``None``. + variant: The variant the round was measuring, for the log line. + idx: Its index in ``runnable``, for the untested count. + round_label: Which round was stopped, for the log line. + + Returns: + bool: ``True`` when the caller must stop testing variants. + """ + nonlocal run_stop, run_stop_detail, session_budget_untested + stopped = stopped_by_the_run_class(getattr(result, "error_class", "") if result is not None else "") + if stopped is None: + return False + run_stop = stopped + run_stop_detail = stopped.interrupted + session_budget_untested = len(runnable) - idx + log.warning( + "explore: the %s round of variant %s was stopped by the run (%s); it and the " + "%d variant(s) after it stay out of the ledger so a resume can retry them", + round_label, + variant.name, + stopped.error_class, + session_budget_untested - 1, + ) + return True + try: for idx, gv in enumerate(runnable): # A warm-decision variant pays for both rounds, so admitting it on @@ -1150,6 +1194,8 @@ async def __call__(self, ctx) -> dict[str, Any]: else: fit_required_sec = float(timeout_sec) if session_deadline_sec is not None and (session_deadline_sec - time.monotonic()) < fit_required_sec: + run_stop = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS] + run_stop_detail = run_stop.never_started session_budget_untested = len(runnable) - idx log.warning( "explore: session budget cannot fit another variant " @@ -1245,6 +1291,8 @@ async def __call__(self, ctx) -> dict[str, Any]: variant_expected_sec=warmup_expected_sec, ) w = warmup_results[0] if warmup_results else None + if _stopped_by_the_run(w, variant=gv, idx=idx, round_label="warmup"): + break if w is None or getattr(w, "status", "") != "succeeded": werr = (getattr(w, "error", "") or "")[-200:] if w is not None else "no_result" log.warning( @@ -1345,6 +1393,8 @@ async def __call__(self, ctx) -> dict[str, Any]: ) continue r = results[0] + if _stopped_by_the_run(r, variant=gv, idx=idx, round_label="decision"): + break # Overtime gate fired: record a ``KILLED_OVERTIME`` row (no # faked tput/gain), skip downstream gates, leave the stack @@ -1649,6 +1699,18 @@ async def __call__(self, ctx) -> dict[str, Any]: session_deadline_sec=session_deadline_sec, variant_expected_sec=decision_expected_sec, ) + # A confirmation the run stopped is not a failed + # confirmation: grading it would evict a variant as + # unstable on the strength of a round that never + # measured it. Undo the stack fold and drop the + # decision-round entry too, so a resume re-measures + # the variant and its confirmation together. + if _stopped_by_the_run(rebench, variant=gv, idx=idx, round_label="stack rebench"): + in_batch_keeps.pop() + tested_update.pop(fp, None) + if gv.name: + name_index.pop(gv.name, None) + break stack_rebench_tput = rebench.tput stack_rebench_workspace = rebench.workspace stack_rebench_warnings = rebench.warnings @@ -1956,16 +2018,16 @@ async def __call__(self, ctx) -> dict[str, Any]: if t.get("round_id") == round_id ) status = "succeeded" if produced_measurement or winners else "failed" - # A round that measured nothing because the budget ran out is not the same - # as one whose variants failed, and it used to be reported as a bare + # A round that measured nothing because the run stopped it is not the + # same as one whose variants failed, and it used to be reported as a bare # ``failed`` with no error_class at all -- nothing downstream could tell # the two apart, so the KB could learn that these variants are bad. budget_error: dict[str, Any] = {} - if status == "failed" and session_budget_untested > 0: + if status == "failed" and run_stop is not None: budget_error = { - "error_class": SESSION_TIME_EXHAUSTED_CLASS, + "error_class": run_stop.error_class, "error": ( - f"session wall-clock budget exhausted; {session_budget_untested} variant(s) never ran " + f"{run_stop_detail}; {session_budget_untested} variant(s) went unmeasured " "and stay out of the ledger so a resume can retry them" ), } diff --git a/src/hyperloom/orchestrator/actions/stop_attribution.py b/src/hyperloom/orchestrator/actions/stop_attribution.py new file mode 100644 index 0000000000..3bcd132fbd --- /dev/null +++ b/src/hyperloom/orchestrator/actions/stop_attribution.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""How work the run itself stopped is told apart from work that failed. + +Two causes stop a unit of work without saying anything about what it was +measuring: the session's wall-clock budget running out, and the orchestrator +cancelling the action (a shutdown, or a budget the dispatcher found spent). +Neither is evidence about the model under test, so nothing that ends this way +may be graded as a failed measurement. + +The distinction has to survive into every ledger, not just the one nearest the +subprocess. A grid records the variant, ``explore`` records the fingerprint the +KB reads, and the baseline arm records a failure streak that stops the session +-- three places that each decide what a round *meant*, and each of which would +otherwise file "the run ran out of time" as a verdict about the model. So the +notion lives here, in a leaf every one of them can import, rather than as three +local judgements that can drift apart. + +The returncode side of the same distinction lives in +:mod:`..executors._subprocess_kill`, which owns the sentinel space; this module +is the error-class side, which is what the ledgers carry. +""" + +from __future__ import annotations + +from typing import NamedTuple + +__all__ = [ + "ORCHESTRATOR_CANCELLED_CLASS", + "SESSION_TIME_EXHAUSTED_CLASS", + "STOPPED_BY_THE_RUN", + "StoppedByTheRun", + "stopped_by_the_run_class", +] + +# Labels work that never ran, or did not finish, because the session wall-clock +# budget was spent. Distinct from any measurement failure: work that was not run +# is not evidence about what it would have measured. +SESSION_TIME_EXHAUSTED_CLASS = "session_time_exhausted" + +# Labels work the orchestrator stopped from outside -- a shutdown, or a budget +# the dispatcher found spent. Kept apart from the class above for the same +# reason their returncodes are: a resume faces the spent budget again and does +# not face the shutdown. +ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" + + +class StoppedByTheRun(NamedTuple): + """How work that the run stopped from outside is recorded. + + Attributes: + error_class: The ledger class for the cause. + interrupted: What to report when the work was already running. + never_started: What to report when it never began. + ends_the_batch: Whether the caller should stop launching further work on + its own, rather than leave that to a budget check it may not have. + """ + + error_class: str + interrupted: str + never_started: str + ends_the_batch: bool + + +# The two causes, keyed by the class the ledgers carry. The budget leaves the +# rest of a batch to the fit check its caller runs, which sees a deadline in the +# past and skips them all under the same label; a cancel has no such check, and +# nothing new should start under one. +STOPPED_BY_THE_RUN: dict[str, StoppedByTheRun] = { + SESSION_TIME_EXHAUSTED_CLASS: StoppedByTheRun( + error_class=SESSION_TIME_EXHAUSTED_CLASS, + interrupted="session wall-clock budget exhausted while this round was running", + never_started="session wall-clock budget exhausted before this round ran", + ends_the_batch=False, + ), + ORCHESTRATOR_CANCELLED_CLASS: StoppedByTheRun( + error_class=ORCHESTRATOR_CANCELLED_CLASS, + interrupted="the orchestrator cancelled this action while this round was running", + never_started="the orchestrator cancelled this action before this round ran", + ends_the_batch=True, + ), +} + + +def stopped_by_the_run_class(error_class: str | None) -> StoppedByTheRun | None: + """Return how to record work the run itself stopped, if it did. + + Args: + error_class: The ``error_class`` a result carries; anything that is not + one of the two causes (including ``None`` and ``""``) reads as work + that has something to say about what it measured. + + Returns: + StoppedByTheRun | None: How to record it, or ``None`` when the class + names a failure of the thing under test rather than of the run. + """ + if not error_class: + return None + return STOPPED_BY_THE_RUN.get(str(error_class)) From d0cbf870ee7fb1753ffc20f5c9206512ab8f05c4 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 17:16:18 +0000 Subject: [PATCH 19/65] baseline: a round the run stopped is not a baseline that failed The unpromotable path charged every non-arg-error baseline failure to baseline_failure_streak, with no exemption for the two classes that mean the run stopped the round rather than the round finding something out. Three reaped or cancelled rounds therefore ended the session with stop_reason="baseline_failed" -- the run blaming the model for its own clock, the exact reading the grid executor already refuses to produce. Such a round now leaves both streaks and the combined backstop where they are, and says so in the log. It is still recorded in the action-failure log and the baseline_not_promoted event, carrying its class: not charging a round is not hiding it. A stop also does not forgive a real failure that came before it -- the streak is left untouched, not reset. Co-authored-by: Cursor --- .../tests/test_coordinator_runtime.py | 50 +++++++++++++++++++ src/hyperloom/orchestrator/loop/writeback.py | 21 +++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 3e21e6baad..cb5cd1fbc7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1179,6 +1179,56 @@ async def test_handle_unpromotable_baseline_third_failure_sets_stop_reason( await c.stop() +@pytest.mark.asyncio +@pytest.mark.parametrize("error_class", ["session_time_exhausted", "orchestrator_cancelled"]) +async def test_baseline_rounds_the_run_stopped_do_not_charge_the_failure_streak(session_dir, error_class): + """Three rounds the run stopped are not three baselines that failed. + + The executor refuses to grade a reaped round because it would put a verdict + on a model the round never reached; the streak has to agree, or the session + stops as ``baseline_failed`` on the evidence of its own clock. + """ + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + for i in range(3): + await c._handle_unpromotable_result( + _mk_task("baseline", f"t-stopped-{error_class}-{i}"), + { + "status": "failed", + "error_class": error_class, + "error": "the run stopped this round before it measured anything", + }, + ) + assert c.shared_state.baseline_failure_streak == 0 + assert c.shared_state.baseline_total_failures == 0 + assert c.shared_state.stop_reason in ("", None) + # The rounds are still recorded: not charging them is not hiding them. + assert len(c.shared_state.last_action_failures) == 3 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_a_stopped_baseline_round_does_not_clear_a_real_failure_streak(session_dir): + """A stop the run chose neither charges the streak nor forgives what preceded it.""" + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + await c._handle_unpromotable_result( + _mk_task("baseline", "t-real"), + {"status": "failed", "error_class": "no_report", "error": "missing"}, + ) + await c._handle_unpromotable_result( + _mk_task("baseline", "t-stopped"), + {"status": "failed", "error_class": "session_time_exhausted", "error": "reaped"}, + ) + assert c.shared_state.baseline_failure_streak == 1 + assert c.shared_state.baseline_total_failures == 1 + finally: + await c.stop() + + def _eval_failed_result() -> dict: return { "status": "failed", diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 924c45f41a..5bb4d6f448 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -25,6 +25,7 @@ summarize_change, ) from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON +from ..actions.stop_attribution import stopped_by_the_run_class from ..state.shared_state import SharedState, resolve_grading_anchor_tput from hyperloom.inference_optimizer.protocol.intent import Intent from ..bus.message_bus import Message @@ -891,6 +892,13 @@ async def _handle_unpromotable_result( # Only arm/streak while no baseline has succeeded yet (tput <= 0). if task.kind == "baseline" and self.shared_state.baseline_tput <= 0: err_class = result_payload.get("error_class", "") + # A round the run itself stopped -- the session budget reaped it, or + # the orchestrator cancelled the action -- measured nothing, so it + # says nothing about whether this baseline boots. Charging it to the + # streaks would let three stops the run chose end the session as + # ``baseline_failed``, blaming the model for the clock. The executor + # already refuses to grade such a round; the ledger has to agree. + stopped_by_the_run = stopped_by_the_run_class(err_class) is not None # While a serial enablement is actively engaged, baseline boots # re-fail on purpose (each round clears a deeper gap), so the # ``baseline_failed`` fast-fail must NOT fire here; the @@ -946,7 +954,15 @@ async def _handle_unpromotable_result( ) if eval_failed: self._persist_eval_failure(result_payload) - if err_class == "fast_exit_arg_error": + if stopped_by_the_run: + log.warning( + "baseline %s was stopped by the run (%s); the failure streak stays at %d " + "because nothing about the baseline was measured", + task.task_id, + err_class, + self.shared_state.baseline_failure_streak, + ) + elif err_class == "fast_exit_arg_error": self.shared_state.baseline_arg_error_streak += 1 if self.shared_state.baseline_arg_error_streak >= 2: self.shared_state.set_stop_reason("baseline_arg_error") @@ -957,7 +973,8 @@ async def _handle_unpromotable_result( self.shared_state.set_stop_reason("baseline_failed") # Combined backstop: count ALL baseline failures so mixed # error_classes that split the per-class streaks still fast-fail. - self.shared_state.baseline_total_failures += 1 + if not stopped_by_the_run: + self.shared_state.baseline_total_failures += 1 if ( self.shared_state.baseline_total_failures >= _BASELINE_MAX_TOTAL_FAILURES and not self.shared_state.stop_reason From 6cf096773e56dfa3ece2e07633eab3fa7c739aee Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 17:48:14 +0000 Subject: [PATCH 20/65] ray: the cancel reaches the round, not just the coroutine awaiting it Layer 4 had no reach on the path production actually takes. _run_magpie returns through ServingLease.run_session_kill, which blocked in ray.get and never entered run_with_session_kill, so no listener registered and the scope was never read. _should_use_ray_backend is off under PYTEST_CURRENT_TEST and on by default on a single node, so every test exercised the reachable branch while every real run took the other one. The budget trigger was partly covered by the remaining budget handed to the actor; shutdown_requested and coordinator_stop trip no deadline, so nothing in there stopped at all. The scope is a ContextVar and cannot cross a process boundary, so what crosses is the request. The lease listens on the scope while the round is in flight, polls with ray.wait, and forwards a cancel to the actor; the actor publishes a scope of its own around the round, which puts the round back under the same reaper as the local path and returns the same sentinel. The actor gets a second method slot, without which the cancel would queue behind the round it is meant to stop. An actor that does not return its round within the grace is killed, and the submitter attributes the stop on its behalf rather than leaving the ledger an unattributed failure. Releasing a lease now asks the actor to reap its server before ray.kill, which skips __ray_terminate__: the served process is deliberately in its own POSIX session, so nothing else reaches the tree once the actor's own reaper is gone. The tests drive a real ServingLease through cancel_inflight_actions over a Ray double that runs actor methods in real threads, sized from the max_concurrency the production code passes -- dropping that option fails them. Co-authored-by: Cursor --- .../inference_optimizer/tests/conftest.py | 99 +++++++ .../tests/test_ray_backend_unit.py | 21 ++ .../tests/test_session_time_budget.py | 95 +++++++ .../orchestrator/actions/cancel_channel.py | 10 +- .../actions/executors/_ray_serving.py | 263 ++++++++++++++++-- 5 files changed, 458 insertions(+), 30 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/conftest.py b/src/hyperloom/inference_optimizer/tests/conftest.py index 3b0b9d4fcd..b801dc0e7a 100644 --- a/src/hyperloom/inference_optimizer/tests/conftest.py +++ b/src/hyperloom/inference_optimizer/tests/conftest.py @@ -333,3 +333,102 @@ async def _no_restart(*_args, **_kwargs) -> None: monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) monkeypatch.setattr(mnl, "restart_server_for_round", _no_restart) + + +class _RayDoubleActorClass: + """The ``@ray.remote`` class: ``.options(...)`` then ``.remote()`` for a handle.""" + + def __init__(self, cls: type) -> None: + self._cls = cls + self._options: dict = {} + + def options(self, **opts): + self._options = dict(opts) + return self + + def remote(self, *args, **kwargs) -> "_RayDoubleActorHandle": + return _RayDoubleActorHandle(self._cls(*args, **kwargs), self._options) + + +class _RayDoubleActorHandle: + """An actor handle whose methods run in a pool sized like the real actor's. + + ``max_concurrency`` is read from the options the production code passed, not + assumed: an actor left at Ray's single method slot gets a single-worker pool + here too, so a method that has to reach a call already running blocks behind + it exactly as it would on a real cluster. + """ + + def __init__(self, obj, options: dict) -> None: + from concurrent.futures import ThreadPoolExecutor + + self._obj = obj + self._pool = ThreadPoolExecutor(max_workers=int(options.get("max_concurrency", 1) or 1)) + self.killed = False + + def __getattr__(self, name: str): + from types import SimpleNamespace + + method = getattr(self._obj, name) + return SimpleNamespace(remote=lambda *a, **kw: self._pool.submit(method, *a, **kw)) + + +class RayDouble: + """A ``ray`` module stand-in that runs actor methods in real threads. + + Ray is not a test dependency, and what these tests are about is what a lease + and its actor do to each other while work is in flight. So the transport is + the only thing faked: the actor is the real class, running real subprocesses, + and an ``ObjectRef`` is a :class:`~concurrent.futures.Future`. + """ + + class exceptions: # noqa: N801 — mirrors the ray.exceptions namespace + class RayActorError(Exception): + pass + + class RayTaskError(Exception): + pass + + class GetTimeoutError(Exception): + pass + + def __init__(self) -> None: + self.killed: list = [] + + def remote(self, cls: type) -> _RayDoubleActorClass: + return _RayDoubleActorClass(cls) + + def cluster_resources(self) -> dict: + return {"CPU": 8.0, "GPU": 8.0, "serving_slot": 1.0} + + def get(self, ref, timeout: float | None = None): + return ref.result(timeout) + + def wait(self, refs: list, *, num_returns: int = 1, timeout: float | None = None): + from concurrent.futures import FIRST_COMPLETED + from concurrent.futures import wait as futures_wait + + done, not_done = futures_wait(refs, timeout=timeout, return_when=FIRST_COMPLETED) + return list(done)[:num_returns], list(not_done) + + def kill(self, actor) -> None: + actor.killed = True + self.killed.append(actor) + + +@pytest.fixture +def serving_lease_on_a_ray_double(monkeypatch): + """A real :class:`ServingLease` over :class:`RayDouble`, closed on teardown.""" + import sys + from types import SimpleNamespace + + from hyperloom.orchestrator.actions.executors import _ray_backend as rb + from hyperloom.orchestrator.actions.executors import _ray_serving as rs + + monkeypatch.setitem(sys.modules, "ray", RayDouble()) + monkeypatch.setattr(rb, "get_ray_backend", lambda: SimpleNamespace(ensure=lambda **_kw: None)) + lease = rs.ServingLease(num_gpus=1) + try: + yield lease + finally: + lease.close() diff --git a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py index 76b592a995..75260e735d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py @@ -1647,6 +1647,27 @@ def test_serving_lease_close_swallows_kill_error(monkeypatch: pytest.MonkeyPatch assert lease._actor is None +def test_releasing_a_lease_reaps_the_served_process(serving_lease_on_a_ray_double): + """``ray.kill`` skips ``__ray_terminate__``, so the actor must be asked first. + + The served process is deliberately started in its own POSIX session, which + is exactly what a process-group teardown does not reach, so a lease released + without asking can leave a GPU held by a process nothing owns any more. + """ + lease = serving_lease_on_a_ray_double + lease.ensure() + pid = lease._actor.start.remote(["sleep", "60"]).result(timeout=10) + assert _pid_alive(pid) + + lease.close() + + deadline = time.time() + 10.0 + while time.time() < deadline and _pid_alive(pid): + time.sleep(0.05) + assert not _pid_alive(pid), "the served process outlived the lease that owned it" + assert lease._actor is None + + def test_managed_process_start_with_log_path(tmp_path: Path): """start(log_path=...) opens the log file (covers the log_path branch).""" log = tmp_path / "nested" / "server.log" diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index e46b579e4f..6365babedf 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -25,6 +25,7 @@ from __future__ import annotations import asyncio +import sys import threading import time from collections.abc import Callable @@ -753,6 +754,100 @@ async def test_a_thread_with_nothing_listening_is_still_not_waited_for(self, coo assert atask.cancelled() +def _runs_a_round_in_a_lease(started: asyncio.Event, *, outcome: dict[str, Any], lease: Any): + """An executor shaped like the production default: a round inside a Ray lease. + + ``_should_use_ray_backend`` is off under pytest and on by default on a single + node, so this is the branch every real run takes and no test did. + """ + + async def _run(_ctx) -> dict: + started.set() + rc, _out, _err = await asyncio.to_thread( + lease.run_session_kill, + [sys.executable, "-c", f"import time; time.sleep({_BLOCKING_SEC})"], + timeout=_BLOCKING_SEC * 4, + ) + outcome["returncode"] = rc + return {"returncode": rc} + + return _run + + +class TestCancellingARoundInsideARayLease: + """The production default routes rounds through a Ray actor, not a local child. + + The scope is a ContextVar, so it does not exist in the actor's process: the + lease has to notice the cancel on this side and forward it, or the four-layer + defence has no reach at all on the path every real single-node run takes. + """ + + @pytest.fixture + def lease(self, serving_lease_on_a_ray_double: Any) -> Any: + return serving_lease_on_a_ray_double + + @pytest.mark.asyncio + async def test_the_round_in_the_actor_stops_before_the_cancel_returns( + self, + coord: Coordinator, + lease: Any, + ): + outcome: dict[str, Any] = {} + task, atask = await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-ray", + make_executor=lambda started: _runs_a_round_in_a_lease(started, outcome=outcome, lease=lease), + ) + began = time.monotonic() + + assert await coord.dispatcher.cancel_inflight_actions(reason="test") == [task.task_id] + + assert outcome, "the cancel returned while the round was still running in the actor" + assert time.monotonic() - began < _BLOCKING_SEC + await _settle(atask) + + @pytest.mark.asyncio + async def test_the_stop_is_attributed_to_the_orchestrator(self, coord: Coordinator, lease: Any): + """The actor reaps its own tree, so the sentinel is the same one the local path returns.""" + outcome: dict[str, Any] = {} + await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-ray-rc", + make_executor=lambda started: _runs_a_round_in_a_lease(started, outcome=outcome, lease=lease), + ) + + await coord.dispatcher.cancel_inflight_actions(reason="test_reason") + + assert outcome["returncode"] == ORCHESTRATOR_CANCELLED_RETURNCODE + + @pytest.mark.asyncio + async def test_an_actor_that_will_not_answer_is_killed_and_the_stop_still_named( + self, + coord: Coordinator, + lease: Any, + monkeypatch: pytest.MonkeyPatch, + ): + """A wedged actor must not hold the lease open, and must not go unattributed.""" + from hyperloom.orchestrator.actions.executors import _ray_serving as rs + + monkeypatch.setattr(rs, "_CANCEL_ROUND_GRACE_SEC", 0.5) + monkeypatch.setattr(rs.ServingLease, "_ask_actor_to_cancel", lambda _self, _reason: False) + outcome: dict[str, Any] = {} + await _start_action( + coord, + kind=_CHEAP_ACTION, + key="c-ray-wedged", + make_executor=lambda started: _runs_a_round_in_a_lease(started, outcome=outcome, lease=lease), + ) + + await coord.dispatcher.cancel_inflight_actions(reason="test_reason") + + assert outcome["returncode"] == ORCHESTRATOR_CANCELLED_RETURNCODE + assert lease._actor is None, "the lease must be released when its actor is killed" + + class TestTheRunnerRecordsACancellation: """``CancelledError`` is not an ``Exception``, so the runner must name it.""" diff --git a/src/hyperloom/orchestrator/actions/cancel_channel.py b/src/hyperloom/orchestrator/actions/cancel_channel.py index 4a243dbcac..cb6beb9d56 100644 --- a/src/hyperloom/orchestrator/actions/cancel_channel.py +++ b/src/hyperloom/orchestrator/actions/cancel_channel.py @@ -20,9 +20,13 @@ stopped through it, which is why a scope also counts its listeners -- the canceller waits only for work that can hear it. -The channel is in-process. A round running inside a Ray actor is in another -process where this ContextVar is unset, and stops on the session deadline it -was handed instead. +The channel is in-process: this ContextVar is unset inside a Ray actor, so a +round running there cannot read the scope. What crosses is the request, not the +channel -- :class:`..executors._ray_serving.ServingLease` watches the scope on +the submitter's side and forwards a cancel to the actor, which publishes a scope +of its own around the round so the same reaper stops it. Work put on Ray by +anything that does not do that forwarding is out of reach, and stops only on the +session deadline it was handed. """ from __future__ import annotations diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py index 0cf5387f3c..8f262f1582 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py @@ -27,6 +27,31 @@ # Timeout for ray.get probes on specialist actor methods (is_alive/exit_code/stop). _LEASE_PROBE_TIMEOUT_SEC: float = 30.0 +# How often the submitter of a round looks up from ``ray.wait`` to see whether +# the action it belongs to has been cancelled. Short enough that the hop through +# Ray costs the cancel almost nothing on top of what stopping the round costs +# anyway, long enough not to spin. +_CANCEL_POLL_SEC: float = 0.25 + +# How long the submitter waits for a cancelled round to come back on its own +# before killing the actor out from under it. The round stops itself the same +# way the local path does -- notice the scope at its poll, SIGTERM the tree, +# wait out the grace, drain the pipes -- and that is what this is sized on. It +# stays under the dispatcher's cooperative window so the honest stop is the one +# that usually happens, and the kill is what a wedged actor gets. +_CANCEL_ROUND_GRACE_SEC: float = 8.0 + +# How long releasing a lease waits for the actor to reap its served process +# before killing the actor anyway. Sized on what that reap costs -- SIGTERM, the +# grace, SIGKILL -- and deliberately short: teardown often runs inside the +# closing window, which is reserved for the report, not for waiting on a server. +_CLOSE_STOP_TIMEOUT_SEC: float = 10.0 + +# Method slots the serving actor runs at once: the round, plus room for the +# cancel that has to reach it. A single-slot actor would queue the cancel behind +# the very round it is meant to stop. +_SERVING_ACTOR_CONCURRENCY: int = 2 + _VISIBLE_DEVICE_ENV_KEYS: tuple[str, ...] = ( "ROCR_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", @@ -221,6 +246,11 @@ class ServingActor: def __init__(self) -> None: self._mgr = ManagedServerProcess() + # The cancel scope of the round currently in flight, if any. The + # dispatcher's scope is a ContextVar in the submitter's process and + # cannot cross into this one, so the actor keeps its own and + # :meth:`cancel_round` is the wire between them. + self._round_scope: Any = None def start( self, @@ -284,24 +314,57 @@ def run_blocking( ``session_remaining_sec`` is a duration, not the submitter's absolute session deadline: this actor is a separate process with its own ``time.monotonic()`` origin, so only a duration survives the trip. + + The round runs under a cancel scope published in this process, which + is what gives :meth:`cancel_round` something to raise: the reaper + inside ``run_with_session_kill`` then stops the tree and names the + stop exactly as it does on the local path. """ import subprocess as _sp # noqa: PLC0415 + from ..cancel_channel import CancelScope, use_cancel_scope # noqa: PLC0415 from ._ray_backend import _run_subprocess_worker # noqa: PLC0415 + scope = CancelScope() + self._round_scope = scope try: - return _run_subprocess_worker( - cmd=cmd, - env=env, - cwd=cwd, - timeout_s=timeout, - soft_deadline_sec=soft_deadline_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - session_remaining_sec=session_remaining_sec, - ) + with use_cancel_scope(scope): + return _run_subprocess_worker( + cmd=cmd, + env=env, + cwd=cwd, + timeout_s=timeout, + soft_deadline_sec=soft_deadline_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + session_remaining_sec=session_remaining_sec, + ) except _sp.TimeoutExpired as exc: return _ACTOR_TIMEOUT_RC, "", f"TimeoutExpired: {exc}" + finally: + self._round_scope = None + + def cancel_round(self, reason: str) -> bool: + """Ask the round in flight to stop itself; return whether there was one. + + Runs in a second method slot (see ``_SERVING_ACTOR_CONCURRENCY``) so + it is not queued behind the round it is cancelling. Returns as soon + as the flag is raised: the round is what decides how to stop, and the + submitter waits for it to come back. + + Args: + reason: Short cause from the canceller, carried into the stopped + round's message. + + Returns: + ``True`` when a round was asked to stop, ``False`` when the actor + was idle -- which the submitter reads as "nothing to wait for". + """ + scope = self._round_scope + if scope is None: + return False + scope.cancel(reason=reason) + return True def is_alive(self) -> bool: """Return whether the serving process is still up. @@ -342,10 +405,19 @@ def __ray_terminate__(self) -> None: # pragma: no cover - Ray teardown hook def make_serving_actor(num_gpus: float, *, serving_slot: bool = True): - """Create a ServingActor handle holding ``num_gpus`` (+ optional ``serving_slot``).""" + """Create a ServingActor handle holding ``num_gpus`` (+ optional ``serving_slot``). + + Given more than one method slot so ``cancel_round`` can reach a round that is + already running; with the default single slot it would wait for the round to + finish, which is the one thing a cancel cannot do. + """ actor_cls: Any = _serving_actor_body() resources = {"serving_slot": 1} if serving_slot else None - return actor_cls.options(num_gpus=num_gpus, resources=resources).remote() + return actor_cls.options( + num_gpus=num_gpus, + resources=resources, + max_concurrency=_SERVING_ACTOR_CONCURRENCY, + ).remote() def make_gpu_specialist_actor(num_gpus: float, *, serving_slot: bool = False): @@ -407,6 +479,11 @@ def run_session_kill( on hard timeout. Cluster-ensure failures and Ray worker errors degrade to a benchmark failure (rc=1) rather than crashing the session. + The cancel scope published by the dispatcher is watched for as long as + the round is in flight, the same as on the local path -- the difference + is that the round is in another process, so the scope cannot be read + there and the cancel is forwarded to the actor instead. + Args: cmd: The benchmark command to run inside the actor. env: Caller env for the subprocess. @@ -424,25 +501,56 @@ def run_session_kill( Returns: ``(returncode, stdout, stderr)`` from the round. """ - import subprocess as _sp # noqa: PLC0415 - - import ray # noqa: PLC0415 + from ..cancel_channel import cancel_scope_listener # noqa: PLC0415 try: self.ensure() except (RayInfeasibleError, RuntimeError) as exc: log.warning("ServingLease.run_session_kill: cluster ensure failed: %r", exc) return 1, "", f"ray_ensure_error: {exc}"[:2000] - ref = self._actor.run_blocking.remote( - cmd, - env=env, - cwd=cwd, - timeout=timeout, - soft_deadline_sec=soft_deadline_sec, - server_log_path=server_log_path, - server_already_ready=server_already_ready, - session_remaining_sec=session_remaining_sec, - ) + # Registered before the round is submitted, so a cancel that arrives + # while Ray is still scheduling it is one this call is counted as able + # to hear -- the same window the local path opens around its spawn. + with cancel_scope_listener() as cancel_scope: + ref = self._actor.run_blocking.remote( + cmd, + env=env, + cwd=cwd, + timeout=timeout, + soft_deadline_sec=soft_deadline_sec, + server_log_path=server_log_path, + server_already_ready=server_already_ready, + session_remaining_sec=session_remaining_sec, + ) + return self._collect_round(ref, cmd=cmd, timeout=timeout, cancel_scope=cancel_scope) + + def _collect_round( + self, + ref: Any, + *, + cmd: list[str], + timeout: int | float | None, + cancel_scope: Any, + ) -> tuple[int, str, str]: + """Wait for a submitted round, forwarding a cancel to the actor if one comes. + + Args: + ref: The ``ObjectRef`` for the round in flight. + cmd: The round's command, for the ``TimeoutExpired`` it may raise. + timeout: The round's hard timeout, for the same reason. + cancel_scope: The scope to watch, or ``None`` when the caller is not + running under one, in which case this is a plain blocking wait. + + Returns: + ``(returncode, stdout, stderr)`` from the round. + + Raises: + subprocess.TimeoutExpired: When the actor reports a hard timeout. + """ + import subprocess as _sp # noqa: PLC0415 + + import ray # noqa: PLC0415 + # Resolve Ray's exception classes defensively. Real ray always exposes # both, but this is a failure hot-path: a partial test double or a future # ray rename must never turn a benchmark failure into an AttributeError @@ -452,7 +560,10 @@ def run_session_kill( _actor_err: Any = getattr(_ray_exc, "RayActorError", ()) if _ray_exc else () _task_err: Any = getattr(_ray_exc, "RayTaskError", ()) if _ray_exc else () try: - rc, out, err = ray.get(ref) + if cancel_scope is None: + rc, out, err = ray.get(ref) + else: + rc, out, err = self._await_or_cancel(ref, cancel_scope=cancel_scope) except _actor_err as exc: # type: ignore[misc] # The actor (worker) itself died — e.g. its server OOM-killed the # worker, or raylet reaped it. Drop the dead handle so the NEXT round @@ -479,8 +590,106 @@ def run_session_kill( raise _sp.TimeoutExpired(cmd, timeout or 0, output=out or None, stderr=err or None) return rc, out, err + def _await_or_cancel(self, ref: Any, *, cancel_scope: Any) -> tuple[int, str, str]: + """Block on a round, asking the actor to stop it if the scope is cancelled. + + The round attributes its own stop, exactly as the local path does, so + the sentinel this returns is the actor's whenever the actor answers. + Only a wedged actor -- one that has not come back within + ``_CANCEL_ROUND_GRACE_SEC`` of being asked -- is killed, and only then + does the submitter attribute the stop on its behalf, because otherwise + an unattributed failure is what the ledger would read. + + Args: + ref: The ``ObjectRef`` for the round in flight. + cancel_scope: The scope this call is listening on. + + Returns: + ``(returncode, stdout, stderr)`` from the round, or an + ``ORCHESTRATOR_CANCELLED_RETURNCODE`` triple when the actor had to + be killed. + """ + import ray # noqa: PLC0415 + + from ._subprocess_kill import ORCHESTRATOR_CANCELLED_RETURNCODE # noqa: PLC0415 + + asked_at: float | None = None + while True: + ready, _ = ray.wait([ref], num_returns=1, timeout=_CANCEL_POLL_SEC) + if ready: + return ray.get(ref) + if asked_at is None: + if not cancel_scope.cancelled: + continue + reason = cancel_scope.reason or "orchestrator_cancelled" + asked_at = time.monotonic() + log.warning( + "ServingLease: asking the actor to stop the round in flight (%s)", + reason, + ) + if not self._ask_actor_to_cancel(reason): + # The actor never took the round, or cannot be reached to be + # told about it. Either way nothing in there will stop on its + # own, so go straight to the kill. + asked_at -= _CANCEL_ROUND_GRACE_SEC + elif time.monotonic() - asked_at >= _CANCEL_ROUND_GRACE_SEC: + log.warning( + "ServingLease: the actor did not return its cancelled round within %.0fs; " + "killing it to release the lease", + _CANCEL_ROUND_GRACE_SEC, + ) + # Straight to the kill: an actor that has not answered is not + # going to answer a graceful stop either, and waiting for one + # would spend the rest of the window the caller is owed. + self._kill_actor() + return ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + "", + "the orchestrator cancelled this action; its Ray actor was killed after " + f"{_CANCEL_ROUND_GRACE_SEC:.0f}s without returning the round", + ) + + def _ask_actor_to_cancel(self, reason: str) -> bool: + """Tell the actor to stop the round it is running. Never raises. + + Args: + reason: Short cause, carried into the stopped round's message. + + Returns: + ``True`` when the actor confirmed it had a round to stop. + """ + import ray # noqa: PLC0415 + + actor = self._actor + if actor is None: + return False + try: + return bool(ray.get(actor.cancel_round.remote(reason), timeout=_LEASE_PROBE_TIMEOUT_SEC)) + except Exception as exc: # noqa: BLE001 — an unreachable actor gets killed instead + log.warning("ServingLease: could not reach the actor to cancel its round: %r", exc) + return False + def close(self) -> None: - """Kill the actor, releasing the GPU lease. Idempotent, never raises.""" + """Release the GPU lease: stop the server, then kill the actor. Idempotent. + + The stop comes first because ``ray.kill`` skips ``__ray_terminate__``, + so the actor's own reaper never runs on that path; the served process is + deliberately in its own POSIX session, which is exactly what a + process-group teardown does not reach. Never raises, and the kill still + happens when the stop does not. + """ + if self._actor is None: + return + try: + import ray # noqa: PLC0415 + + ray.get(self._actor.stop.remote(), timeout=_CLOSE_STOP_TIMEOUT_SEC) + except Exception as exc: # noqa: BLE001 — the kill below is the backstop + log.warning("ServingLease.close: the actor did not stop its server: %r", exc) + self._kill_actor() + + def _kill_actor(self) -> None: + """Kill the actor handle without waiting for it. Idempotent, never raises.""" if self._actor is None: return try: From a46eeb4750cb7d46b0fe8016c45b11eef20baa21 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 18:05:59 +0000 Subject: [PATCH 21/65] docs(executors): say what crossing the process boundary actually costs session_remaining_to_deadline_sec's docstring claimed the transit is charged to the receiver; the code forgives it and hands over a fresh full window. The behaviour is the safe one -- the alternative is guessing at the trip and charging a round for time it never had -- so the doc moves to the code. Also joins the two f-string concatenations _subprocess_kill.py added, which were the file's only ruff format --check failures. The other files this branch touches were already unformatted at the merge base and stay that way. Co-authored-by: Cursor --- .../actions/executors/_subprocess_kill.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 6cf10e7b6a..52c672843e 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -719,9 +719,12 @@ def session_deadline_to_remaining_sec(session_deadline_sec: float | None) -> flo def session_remaining_to_deadline_sec(session_remaining_sec: float | None) -> float | None: """Re-anchor a remaining session budget onto this process's monotonic clock. - The inverse of :func:`session_deadline_to_remaining_sec`. Whatever the - budget spent in transit is charged to the receiver, since no clock is shared - across the boundary to measure it with. + The inverse of :func:`session_deadline_to_remaining_sec`. Whatever the trip + itself cost is forgiven: no clock is shared across the boundary to measure it + with, so the receiver starts a fresh window of the full duration. The + receiver can therefore run marginally past the sender's deadline, which is + the safe direction -- the alternative is guessing at the transit and charging + a round for time it never had. Args: session_remaining_sec: Seconds left on the session budget as measured by @@ -904,8 +907,7 @@ def __init__(self, *, overrun_sec: float, elapsed_sec: float) -> None: elapsed_sec (float): Wall-clock elapsed for this round at trip time. """ super().__init__( - f"the session wall-clock budget was exhausted {overrun_sec:.1f}s ago " - f"(round elapsed={elapsed_sec:.1f}s)" + f"the session wall-clock budget was exhausted {overrun_sec:.1f}s ago (round elapsed={elapsed_sec:.1f}s)" ) self.overrun_sec = float(overrun_sec) self.elapsed_sec = float(elapsed_sec) @@ -929,8 +931,7 @@ def __init__(self, *, reason: str, elapsed_sec: float) -> None: elapsed_sec (float): Wall-clock elapsed for this round at trip time. """ super().__init__( - f"the orchestrator cancelled this action ({reason or 'no reason given'}; " - f"round elapsed={elapsed_sec:.1f}s)" + f"the orchestrator cancelled this action ({reason or 'no reason given'}; round elapsed={elapsed_sec:.1f}s)" ) self.reason = str(reason) self.elapsed_sec = float(elapsed_sec) From 0d9a8d719a88e92cf061cbe989e7ade9e2c4d6aa Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 18:06:07 +0000 Subject: [PATCH 22/65] fix(close): bound the wait on a running step by the work, not the budget The closing reserve answers "how much of the session do we hold back for CLOSE"; it scales with the session and is twelve seconds for a ten-minute one. Using it as the wait for a step the deadline path already dispatched answered a different question with that number, and no report is written in twelve seconds -- so the step was declared failed while it was still running, in exactly the session whose report is worth the most. The wait now comes from the step's own expected runtime, with a floor for the steps the catalogue prices at almost nothing and a ceiling so a wedged task cannot hold the process open. Co-authored-by: Cursor --- .../tests/test_close_phase_sequencer.py | 107 +++++++++++++++--- src/hyperloom/orchestrator/phases/close.py | 49 +++++++- 2 files changed, 137 insertions(+), 19 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py index 4ef6b60368..a53a2688fb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py +++ b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py @@ -5,6 +5,7 @@ from __future__ import annotations +import time from dataclasses import dataclass, field from pathlib import Path from types import SimpleNamespace @@ -12,6 +13,7 @@ import pytest +from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE from hyperloom.orchestrator.knowledge.config import KnowledgeConfig, KnowledgeStoreMode from hyperloom.orchestrator.roles.agent_role import default_role_registry from hyperloom.orchestrator.roles.mock_backend import ( @@ -20,6 +22,10 @@ ScriptedPlan, ) from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.phases.close import ( + _CLOSE_STEP_WAIT_CEILING_SEC, + _CLOSE_STEP_WAIT_FLOOR_SEC, +) from hyperloom.orchestrator.policy.gate import CORE_STATE_FIELDS from hyperloom.orchestrator.state.shared_state import effective_closing_grace_sec @@ -338,26 +344,27 @@ async def test_a_terminal_task_is_reported_not_run(coord): class _FinishesWhileWaiting(_StubTaskRegistry): - """Registry whose running row lands terminal once the sequencer looks again.""" + """Registry whose running row lands terminal after ``lands_on`` lookups.""" - def __init__(self, terminal_state: str): + def __init__(self, terminal_state: str, *, lands_on: int = 2): super().__init__() self._terminal_state = terminal_state + self._lands_on = lands_on self.gets = 0 async def get(self, task_id): row = await super().get(task_id) self.gets += 1 - if self.gets >= 2: + if self.gets >= self._lands_on: row.state = self._terminal_state return row -def _running_report_row(coord) -> _StubTaskRow: - """Register a report the wall-clock deadline path already enqueued AND dispatched.""" +def _running_report_row(coord, *, kind: str = "report") -> _StubTaskRow: + """Register a close-step task the wall-clock deadline path already enqueued AND dispatched.""" row = _StubTaskRow( task_id="wallclock-report", - kind="report", + kind=kind, state="running", params={}, idempotency_key="closing-report-1234", @@ -366,6 +373,29 @@ def _running_report_row(coord) -> _StubTaskRow: return row +def _clock_advancing_by(monkeypatch: pytest.MonkeyPatch, step_sec: float) -> None: + """Give the CLOSE module a monotonic clock that jumps ``step_sec`` per read. + + The wait under test is measured in minutes, so a test that spent it would + be a test nobody runs. Only the CLOSE module's view of the clock is + replaced, which leaves the event loop's own timekeeping alone. + """ + from hyperloom.orchestrator.phases import close as close_mod + + now = 0.0 + + def _monotonic() -> float: + nonlocal now + now += step_sec + return now + + monkeypatch.setattr( + close_mod, + "time", + SimpleNamespace(monotonic=_monotonic, time=time.time), + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("terminal_state", ["succeeded", "failed"]) async def test_a_running_task_is_waited_for_not_re_run(coord, terminal_state: str): @@ -386,11 +416,14 @@ async def test_a_running_task_is_waited_for_not_re_run(coord, terminal_state: st @pytest.mark.asyncio -async def test_a_running_task_that_never_lands_is_reported_not_waited_on_forever(coord): - """The closing grace window is the budget for this work, so it is also the bound.""" - coord._dispatcher_poll_sec = 0.01 - # Resolves to a 1.2s grace window, the same arithmetic a real session uses. - coord.shared_state.max_minutes = 1 +async def test_a_running_task_that_never_lands_is_reported_not_waited_on_forever( + coord, + monkeypatch: pytest.MonkeyPatch, +): + """The wait is patient, not unbounded: a step that never lands is recorded, not awaited forever.""" + coord._dispatcher_poll_sec = 0.0 + coord.shared_state.max_minutes = 60 + _clock_advancing_by(monkeypatch, step_sec=30.0) state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") @@ -399,16 +432,60 @@ async def test_a_running_task_that_never_lands_is_reported_not_waited_on_forever @pytest.mark.asyncio -async def test_a_zero_grace_window_buys_no_wait_at_all(coord): - """The row is still looked at once: no window is not the same as no answer.""" - coord.shared_state.max_minutes = 0 # no budget -> no grace window +async def test_the_wait_for_a_running_report_outlives_a_short_session_reserve( + coord, + monkeypatch: pytest.MonkeyPatch, +): + """A ten-minute session reserves twelve seconds for CLOSE; no report is written in twelve seconds. + + Bounding the wait by the reserve made it a wait on paper only — the step + the deadline path dispatched was declared failed while it was still + running, and the session that ran out of time is the one whose report is + worth the most. The bound belongs to the work, so it is the report's own + expected runtime. + """ + coord.tasks = _FinishesWhileWaiting("succeeded", lands_on=5) + coord._dispatcher_poll_sec = 0.0 + coord.shared_state.max_minutes = 10 + assert coord.shared_state.closing_reserve_sec() == pytest.approx(12.0) + # Five looks at five simulated seconds apiece: past the reserve, inside the + # two minutes the catalogue prices a report at. + _clock_advancing_by(monkeypatch, step_sec=5.0) state = await coord._run_close_task(_running_report_row(coord), step="1 (report)") - assert state == "running" + assert state == "succeeded" assert coord.sub.run_calls == [] +def test_the_wait_is_the_step_s_own_expected_runtime(coord): + bound = coord.phase_close._close_step_wait_sec(_running_report_row(coord)) + + assert bound == pytest.approx(ACTION_CATALOGUE["report"].typical_runtime_min * 60.0) + + +def test_a_step_the_catalogue_prices_at_almost_nothing_still_gets_the_floor(coord): + """``session_breakdown`` is priced at 12s; giving up on it after 12s is giving up on it.""" + row = _running_report_row(coord, kind="session_breakdown") + + assert coord.phase_close._close_step_wait_sec(row) == pytest.approx(_CLOSE_STEP_WAIT_FLOOR_SEC) + + +def test_an_uncatalogued_step_gets_the_floor_too(coord): + row = _running_report_row(coord, kind="not_an_action") + + assert coord.phase_close._close_step_wait_sec(row) == pytest.approx(_CLOSE_STEP_WAIT_FLOOR_SEC) + + +def test_an_extravagantly_priced_step_is_capped(coord): + """A wedged step must not hold the process open for as long as its action might legitimately run.""" + coord.action_registry = {"report": SimpleNamespace(typical_runtime_min=1000.0)} + + bound = coord.phase_close._close_step_wait_sec(_running_report_row(coord)) + + assert bound == pytest.approx(_CLOSE_STEP_WAIT_CEILING_SEC) + + @pytest.mark.asyncio async def test_the_sequencer_records_the_state_a_running_report_ended_in(coord): """End to end: the waited-for report is reported like any other outcome.""" diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index f2e8c77dab..9133861d22 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -36,6 +36,17 @@ # Fallback registry poll interval, for a caller with no dispatcher poll set. _DEFAULT_TASK_POLL_SEC: float = 10.0 +# Floor on how long CLOSE waits for a step it found already running, for a step +# the catalogue prices at almost nothing or does not carry at all. Long enough +# that a step which is merely slow to be written is not abandoned one poll in. +_CLOSE_STEP_WAIT_FLOOR_SEC: float = 60.0 + +# Ceiling on the same wait. The step is the last thing standing between the run +# and having nothing to show for itself, so the wait is generous -- but a task +# wedged forever must not hold the process open, and a report that has taken +# five times its typical runtime is not about to land. +_CLOSE_STEP_WAIT_CEILING_SEC: float = 600.0 + def _task_is_dead(task: Task | None) -> bool: """True when ``task`` reached a terminal state without producing its artifact. @@ -512,6 +523,32 @@ async def _enqueue_internal_session_breakdown_task( idempotency_key=f"internal-session_breakdown-{reason}", ) + def _close_step_wait_sec(self, task: Task) -> float: + """How long to wait for a close-step task that is already running. + + The bound is the step's own expected runtime, clamped into + ``[_CLOSE_STEP_WAIT_FLOOR_SEC, _CLOSE_STEP_WAIT_CEILING_SEC]``. This is + deliberately not the closing reserve: the reserve answers "how much of + the session do we hold back for CLOSE", which scales with the session + and is a handful of seconds for a short one, while this answers "how + long is it reasonable to wait for work that is already under way", + which scales with the work. Bounding a two-minute report by a + twelve-second reserve is a wait only on paper. + + Args: + task: The close-step task found in ``running``. + + Returns: + The bound in seconds. + """ + from ..loop.coordinator_helpers import expected_action_cost_minutes + + registry = getattr(self, "action_registry", None) + kind = str(getattr(task, "kind", "") or "") + meta = registry.get(kind) if registry is not None else None + typical_sec = expected_action_cost_minutes(meta) * 60.0 + return min(_CLOSE_STEP_WAIT_CEILING_SEC, max(_CLOSE_STEP_WAIT_FLOOR_SEC, typical_sec)) + async def _await_running_close_task(self, task: Task, *, step: str) -> str: """Wait for an already-dispatched close-step task to reach a terminal state. @@ -522,9 +559,13 @@ async def _await_running_close_task(self, task: Task, *, step: str) -> str: with it — and the session that ran out of time is the session whose report is worth the most. - The wait is bounded by the closing grace window, which is the budget - reserved for exactly this work, so a task that never lands costs CLOSE - that window and no more. + How long to wait is a question about the work, not about the budget: + the closing reserve says how much of the session to hold back for + CLOSE, which for a short session is a few seconds — less than any + report takes to write, so bounding the wait by it is the same as not + waiting. :func:`_close_step_wait_sec` bounds it by what the step's own + action typically takes instead, so a task that never lands costs CLOSE + that bound and no more. Args: task: The close-step task found in ``running``. @@ -534,7 +575,7 @@ async def _await_running_close_task(self, task: Task, *, step: str) -> str: The state the task ended in, or ``running`` when the bound elapsed first — which the caller records the same way it records a failure. """ - bound_sec = self.shared_state.closing_reserve_sec() + bound_sec = self._close_step_wait_sec(task) poll_sec = float(getattr(self, "_dispatcher_poll_sec", _DEFAULT_TASK_POLL_SEC)) deadline = time.monotonic() + bound_sec log.info( From 2b84b1cd996f744e1d91a4b6040f71ebe7453124 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 18:09:03 +0000 Subject: [PATCH 23/65] test(budget): record why a warm replay is priced like a baseline round replay_warm_recipe looks like it should be cheap -- the recipe is already known -- so the measured-baseline floor reads as excess caution that can refuse a warm replay near the tail. It is not: the action is dispatched to BaselineExecutor with the recipe's server args and patches, so it boots its own server and runs the same benchmark. The recipe changes what is measured, not how long measuring takes. Pinning it here keeps the reasoning next to the price, and fails if warm replay ever learns to re-attach to a server that is already hot. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 6365babedf..2fa2a110a0 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -158,6 +158,24 @@ def test_an_action_that_never_benches_keeps_its_own_estimate(self): def test_a_session_with_no_baseline_yet_falls_back_to_the_catalogue(self): assert expected_action_cost_minutes(ACTION_CATALOGUE["baseline"]) == pytest.approx(5.0) + def test_a_warm_replay_is_priced_as_the_baseline_round_it_is(self): + """Warm replay is not a cheap re-attach to a server that is already hot. + + ``replay_warm_recipe`` is dispatched to ``BaselineExecutor`` with the + recipe's ``extra_server_args``/``extra_envs``/``patches``, so it boots + its own server and runs the same benchmark the baseline ran; the recipe + changes what is measured, not how long measuring takes. Being refused + near the tail on a 51-minute price is therefore the gate working, not + the gate being timid — and if warm replay ever does learn to re-attach, + this is where the price stops being right. + """ + cost = expected_action_cost_minutes( + ACTION_CATALOGUE["replay_warm_recipe"], + measured_baseline_sec=_MEASURED_BASELINE_SEC, + ) + assert cost == pytest.approx(_MEASURED_BASELINE_SEC / 60.0) + assert ACTION_CATALOGUE["replay_warm_recipe"].requires_lanes == ACTION_CATALOGUE["baseline"].requires_lanes + def test_a_measurement_that_is_not_a_number_is_not_a_cost(self): assert measured_baseline_runtime_sec(None) == 0.0 assert measured_baseline_runtime_sec(SimpleNamespace(baseline_runtime_sec="not-a-number")) == 0.0 From 97943dff2496859ac43063190581fa3616d2b9f2 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 19:26:40 +0000 Subject: [PATCH 24/65] fix(policy): the model cannot rewrite the budget its closing reserve leaves closing_grace_sec sizes the reserve session_budget_usable_sec subtracts, so it decides how much of max_minutes a run may still spend. max_minutes is locked against update_state and this field was not, which left the same forgery one field over: a large value reports the budget as spent and the admission gate then cancels every in-flight and queued action, a zero one erases the window the CLOSE report is written in. Co-authored-by: Cursor --- .../agents/robustness/role/envelope.py | 1 + .../tests/test_agent_roles_and_policy.py | 26 +++++++++++++++++++ src/hyperloom/orchestrator/policy/gate.py | 5 ++++ 3 files changed, 32 insertions(+) diff --git a/src/hyperloom/agents/robustness/role/envelope.py b/src/hyperloom/agents/robustness/role/envelope.py index a9a8512163..4931f5b01d 100644 --- a/src/hyperloom/agents/robustness/role/envelope.py +++ b/src/hyperloom/agents/robustness/role/envelope.py @@ -114,6 +114,7 @@ class IntentType(str, Enum): "start_ts", "resumed_ts", "max_minutes", + "closing_grace_sec", "optimization_stack", "gain_per_stack_entry", "schema_version", diff --git a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py index b6474f8f5c..cd64866643 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py @@ -5,8 +5,11 @@ from __future__ import annotations +from datetime import datetime, timedelta, timezone + import pytest +from hyperloom.common.timeutil import iso_z from hyperloom.orchestrator.roles.agent_role import ( BackendType, DEFAULT_CLAUDE_MODEL, @@ -711,6 +714,16 @@ def test_gate_update_state_cannot_move_the_resume_boundary(gate): assert exc.value.rule == "state_field" +def test_the_model_cannot_rewrite_the_budget_the_closing_reserve_leaves_it(gate): + """The reserve decides how much of ``max_minutes`` is spendable, so it is budget too.""" + with pytest.raises(PolicyDenied) as exc: + gate.validate_intent( + "orchestration", + Intent(type=IntentType.UPDATE_STATE, payload={"changes": {"closing_grace_sec": 0.0}}), + ) + assert exc.value.rule == "state_field" + + def test_gate_update_state_cannot_move_a_session_end_time(gate): # stop_ts is the timestamp half of stop_reason, written by the same setter: # locking only the reason lets a model post-date the session's end. @@ -726,6 +739,19 @@ def test_gate_update_state_cannot_move_a_session_end_time(gate): assert exc.value.rule == "state_field" +def test_a_forged_closing_reserve_would_have_spent_the_session_outright(): + """Names what the lock prevents: one field, and the run has no usable time left.""" + state = SharedState(session_id="s", max_minutes=100) + state.start_ts = iso_z(datetime.now(timezone.utc) - timedelta(minutes=90)) + honest = state.session_budget_usable_sec() + + applied = state.apply_changes({"closing_grace_sec": 1e9}, allow_core=False) + + assert applied == {} + assert honest > 0.0 + assert state.session_budget_usable_sec() == honest + + def test_core_state_fields_synced_with_robustness_envelope(): # gate.CORE_STATE_FIELDS and the robustness # envelope copy must stay byte-identical. This direct assertion never skips diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index b751ee3567..de11f5cdb7 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -556,6 +556,11 @@ def _source_file_candidates(value: str) -> tuple[str, ...]: # leg's CLOSE transition back the right to speak for this one. "resumed_ts", "max_minutes", + # Sizes the closing reserve, so it decides how much of ``max_minutes`` + # is still usable: locking the budget without locking this one leaves + # the same forgery one field over -- a large value spends the session + # outright, a zero one erases the window the CLOSE report needs. + "closing_grace_sec", # fact-layer KEEP ledger; Coordinator is the sole writer. "optimization_stack", "gain_per_stack_entry", From 9bf3afc13654dc385023ac0aa1e219fa0587a87b Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 19:45:10 +0000 Subject: [PATCH 25/65] integrate: a cancelled gate takes its patch back out of the tree Nothing cancelled a dispatched action's task before this branch, so the integrate gate's REVERT paths only ever had to handle a bench that failed. Now that ``cancel_inflight_actions`` reaches running work on shutdown and on a spent wall-clock budget, they have to handle a bench that was stopped -- and ``CancelledError`` is not an ``Exception``, so every one of those handlers unwinds straight through. The patch stays in the framework tree, the operator's auto-stash stays on the stack, and the budget case does not even end the process: CLOSE goes on to report against a tree carrying a patch that nothing ever graded. The undo goes at the ``__call__`` seam rather than into the six handlers because that is the scope the mutation actually spans -- ``_stage_apply`` puts the candidate in the tree and the gate is what takes it back out -- so one guard covers the bench, both confirmation legs and every verdict branch, and it holds for an ordinary exception escaping the gate too. The stash restore reuses the wrapper's own logging, extracted for the purpose. The cancel is re-raised, not turned into a ``reverted`` result. Swallowing it would hand SubAgentRunner a normal return, which records the row ``succeeded`` with a REVERT verdict -- the run's own stop filed as evidence that the patch failed a bench it never finished. Re-raising keeps the dispatcher's cancellation contract and lands the row on ``cancelled``, which is what stop_attribution asks of every ledger. Co-authored-by: Cursor --- .../test_integrate_patch_coverage_unit.py | 52 +++++++++ .../actions/executors/integrate_patch.py | 110 ++++++++++++++---- 2 files changed, 142 insertions(+), 20 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py index 19e02ce2a6..20a0313af9 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio import json import os import subprocess @@ -1209,3 +1210,54 @@ def _fake_apply(self, specs, *, backup_root): # in the stash. This is the regression the fix guards against. assert scratch.exists(), "user auto-stash was not restored after artifact_install_failed" assert scratch.read_text(encoding="utf-8") == "user work in progress\n" + + +@pytest.mark.asyncio +async def test_cancelled_gate_reverts_the_patch_and_re_raises(tmp_path, monkeypatch): + """A cancel unwinds the gate, so the tree it mutated must not outlive it. + + The dispatcher cancels in-flight actions when the run is shutting down or + the session wall-clock budget is spent. ``CancelledError`` is not an + ``Exception``, so the gate's own revert handlers never see it: the patch + would stay in the framework tree, the operator's auto-stash would stay + unpopped, and the session would run its CLOSE phase against a tree carrying + an ungraded patch. + + The cancel is re-raised rather than graded as a REVERT: SubAgentRunner + records a cancelled executor as ``cancelled``, and work the run stopped is + not work that failed. + """ + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + async def _cancel(self, **kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(IntegratePatchExecutor, "_bench_patch", _cancel) + ex = IntegratePatchExecutor(session_dir=session) + with pytest.raises(asyncio.CancelledError): + await ex( + _make_ctx( + "t", + {"specialist_task_id": "spec", "framework_source_root": str(repo)}, + ) + ) + + assert (repo / "src.py").read_text(encoding="utf-8").endswith("return 1\n"), ( + "the cancelled candidate was left applied in the framework tree" + ) + assert scratch.exists(), "user auto-stash was not restored after the cancel" + assert scratch.read_text(encoding="utf-8") == "user work in progress\n" + stash_list = subprocess.run( + ["git", "-C", str(repo), "stash", "list"], + check=True, + capture_output=True, + text=True, + ) + assert stash_list.stdout.strip() == "", "the auto-stash was left on the stack" diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 4c280db011..df4e59c330 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -980,6 +980,27 @@ def _git_restore_stash_if_needed( return f"git stash pop {ref} rc={cp.returncode}: {detail}; user changes remain in git stash" +def _restore_stash_logged( + framework_root: Path, + stash_state: str, + stash_ref: str, +) -> str: + """Restore a pre-candidate stash, reporting a failure to do so. + + Args: + framework_root (Path): The tree the stash was taken from. + stash_state (str): The state :func:`_git_stash_if_dirty` returned. + stash_ref (str): The stash ref to pop. + + Returns: + str: The failure note, or ``""`` when nothing was left in the stash. + """ + note = _git_restore_stash_if_needed(framework_root, stash_state, stash_ref) + if note: + log.warning("integrate_patch: user-change stash restore failed: %s", note) + return note + + def _with_stash_restore( framework_root: Path, stash_state: str, @@ -987,10 +1008,9 @@ def _with_stash_restore( result: dict[str, Any], ) -> dict[str, Any]: """Restore a pre-candidate stash before returning an executor result.""" - note = _git_restore_stash_if_needed(framework_root, stash_state, stash_ref) + note = _restore_stash_logged(framework_root, stash_state, stash_ref) if not note: return result - log.warning("integrate_patch: user-change stash restore failed: %s", note) out = dict(result) out["stash_restore_error"] = note return out @@ -1668,24 +1688,37 @@ async def __call__(self, ctx) -> dict[str, Any]: extra_envs_applied: dict[str, str] = ctx._ip_extra_envs_applied # type: ignore[attr-defined] setup_result: dict[str, Any] = ctx._ip_setup_result # type: ignore[attr-defined] - return await self._stage_gate( - ctx, - params, - extra, - specialist_task_id=specialist_task_id, - shared_state=shared_state, - done_payload=done_payload, - output_root=output_root, - framework_root=framework_root, - stash_state=stash_state, - stash_note=stash_note, - applied=applied, - applied_artifacts=applied_artifacts, - config_changes_applied=config_changes_applied, - extra_server_args_applied=extra_server_args_applied, - extra_envs_applied=extra_envs_applied, - setup_result=setup_result, - ) + try: + return await self._stage_gate( + ctx, + params, + extra, + specialist_task_id=specialist_task_id, + shared_state=shared_state, + done_payload=done_payload, + output_root=output_root, + framework_root=framework_root, + stash_state=stash_state, + stash_note=stash_note, + applied=applied, + applied_artifacts=applied_artifacts, + config_changes_applied=config_changes_applied, + extra_server_args_applied=extra_server_args_applied, + extra_envs_applied=extra_envs_applied, + setup_result=setup_result, + ) + except BaseException: + # The gate either returns a verdict or leaves the tree as it found + # it; there is no third outcome where the candidate stays applied + # with nothing having graded it. + self._undo_ungraded_candidate( + framework_root=framework_root, + stash_state=stash_state, + stash_note=stash_note, + applied=applied, + applied_artifacts=applied_artifacts, + ) + raise # --------------------------------------------------------------------------- # Stage helpers (called sequentially by __call__) @@ -3757,6 +3790,43 @@ async def _maybe_write_framework_kb_record( exc, ) + def _undo_ungraded_candidate( + self, + *, + framework_root: Path | None, + stash_state: str, + stash_note: str, + applied: list[Path], + applied_artifacts: list[dict[str, Any]], + ) -> None: + """Take the candidate back out when the gate unwound instead of returning. + + Every REVERT the gate itself decides hangs off an ``except Exception``, + and the stop that matters most here is not one of those: the dispatcher + cancels in-flight actions on shutdown and on a spent wall-clock budget, + and ``CancelledError`` derives from ``BaseException``. Unhandled, it + leaves the patch in the framework tree and the operator's auto-stash on + the stack — and the budget case does not end the process, so CLOSE would + report against a tree carrying a patch nothing ever graded. + + The cancel itself is re-raised by the caller rather than turned into a + REVERT verdict, so the run records it the way + :mod:`..stop_attribution` requires: work the run stopped, not work that + failed. Every step here is synchronous, so no second cancel can be + delivered part-way through the undo. + + Args: + framework_root: The source root the candidate was applied to. + stash_state: The state :func:`_git_stash_if_dirty` returned. + stash_note: The auto-stash ref to restore. + applied: The patches applied to the tree. + applied_artifacts: The artifact records to undo. + """ + self._revert_artifacts(applied_artifacts) + self._revert_patches(framework_root, applied) + if framework_root is not None: + _restore_stash_logged(framework_root, stash_state, stash_note) + def _revert_patches( self, framework_root: Path | None, From 57488682821081a0b4a53c8a11714edc9168a631 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 19:45:19 +0000 Subject: [PATCH 26/65] framework: the candidate agent has the same blind spot on a cancel It is the other executor that applies patches to the live framework tree behind an auto-stash, and its REVERT hangs off the same ``except Exception`` beside the same kind of multi-minute bench. This branch made it cancellable along with the rest, so a cancel there strands the candidate exactly as it did in the integrate gate. Fixed at the bench site rather than at the executor's entry: ``__call__`` here is one long function with the apply, the gate and the verdicts inline, so a guard spanning the whole thing would mean re-indenting it for a fix that is about one await. The KB writebacks after the verdict are still a window in which a cancel leaves the stash unpopped -- the patches by then are already reverted or already committed -- and closing it is a restructuring of this executor, not part of this fix. Co-authored-by: Cursor --- .../tests/test_framework_agent_executor.py | 46 +++++++++++++++++++ .../actions/executors/framework_agent.py | 10 ++++ 2 files changed, 56 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py index fb96cd66a6..f790a207d5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio import json import os import subprocess @@ -718,6 +719,51 @@ async def boom(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 assert (repo / "src.py").read_text().endswith("return 1\n") +@pytest.mark.asyncio +async def test_executor_cancelled_bench_reverts_and_re_raises(tmp_path: Path): + """A cancelled bench must not leave the candidate in the framework tree. + + The dispatcher cancels in-flight actions on shutdown and on a spent + wall-clock budget. ``CancelledError`` is not an ``Exception``, so the REVERT + handler beside it never sees the stop, and the candidate would stay applied + with the operator's auto-stash still on the stack. The cancel is re-raised + rather than graded as a REVERT: work the run stopped is not work that + failed, and SubAgentRunner records it as ``cancelled``. + """ + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + + async def cancelled(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 + raise asyncio.CancelledError + + ctx = _make_ctx( + "t-fp-cancel", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + }, + ) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=cancelled): + with pytest.raises(asyncio.CancelledError): + await executor(ctx) + + assert (repo / "src.py").read_text().endswith("return 1\n"), ( + "the cancelled candidate was left applied in the framework tree" + ) + assert scratch.exists(), "user auto-stash was not restored after the cancel" + assert scratch.read_text(encoding="utf-8") == "user work in progress\n" + + _PATCH_B_ADDS_FILE = """\ diff --git a/new.py b/new.py new file mode 100644 diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 29babb7d54..178788bfd3 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -41,6 +41,7 @@ _accuracy_delta_pct, _git_apply_collect_feedback, _git_stash_if_dirty, + _restore_stash_logged, _with_stash_restore, _resolve_framework_root, ) @@ -857,6 +858,15 @@ async def __call__(self, ctx) -> dict[str, Any]: "workspace": str(output_root), }, ) + except BaseException: + # A stop, not a verdict: the dispatcher cancels in-flight actions on + # shutdown and on a spent wall-clock budget, and ``CancelledError`` + # is not an ``Exception``, so the REVERT above never sees it. Undo + # the candidate here and let the stop through -- graded as a REVERT + # it would read as the patch having failed a bench that never ran. + self._revert_patches(framework_root, applied, pre_apply_sha=pre_apply_sha) + _restore_stash_logged(framework_root, stash_state, stash_note) + raise # KEEP / REVERT decision. base_tput = float(params.get("base_tput") or 0.0) From 933659d12ea1a4e6538dcb39636eb1694658b611 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 20:13:42 +0000 Subject: [PATCH 27/65] grid/baseline: every round carries the stop that ended it, not just the measured one A variant's warmup pass is a full benchmark round, so the session deadline and an orchestrator cancel reach it exactly as they reach the measured round. Only the measured round consulted the sentinels: a reaped warmup was graded warmup_round_failed, which files a verdict about a variant the run never measured, and because the cancel's ends_the_batch was never read the grid went on booting a server per remaining variant after the action was asked to stop. One helper now decides what a stopped round means, called after each launch and before any grading, so the grid's three rounds cannot drift apart. The multi-node warmup stops discarding its returncode along with its report, and the baseline's discarded multi-node pass no longer runs the measured round after a cancel. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 52 +++++ .../tests/test_explore_executor.py | 84 ++++++++ .../tests/test_grid_runner.py | 154 ++++++++++++++- .../actions/executors/_grid_runner.py | 179 +++++++++++++----- .../actions/executors/baseline.py | 103 +++++++--- 5 files changed, 501 insertions(+), 71 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 757bb8627e..fe0995a972 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -1946,6 +1946,58 @@ def test_a_cancel_is_told_apart_from_a_spent_budget(self, tmp_path): assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + def test_a_cancelled_multi_node_warmup_does_not_go_on_to_the_measured_round( + self, + tmp_path, + monkeypatch, + ): + """The discarded warmup is a full pass, so a cancel there ends the round. + + Running the measured round anyway spends a second pass of GPU time the + run has already been told to stop spending -- and grades the baseline on + a round started after the stop. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl + + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", "2") + + async def fake_restart_server_for_round(*_args, **_kwargs): + return None + + monkeypatch.setattr(mnl, "restart_server_for_round", fake_restart_server_for_round) + launched: list[str] = [] + + def fake_run(cmd, *args, **kwargs): + slot = Path(cmd[cmd.index("--output-dir") + 1]) + launched.append(slot.name) + if slot.name == "mn_warmup": + return subprocess.CompletedProcess(cmd, ORCHESTRATOR_CANCELLED_RETURNCODE, "", "") + _fake_workspace(slot, tput=_HOT_TPUT) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + ) + ctx = _make_ctx( + { + "output_dir": str(tmp_path / "ws"), + "timeout_sec": 600, + "gpu_type": "mi300x", + } + ) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert launched == ["mn_warmup"], f"the measured round ran after the cancel: {launched}" + assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + def test_the_profile_arm_gets_all_of_it(self, tmp_path): """Profile is the same executor with a four-hour default -- longer than any session budget it could be given.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 8781228102..ce1a7591b7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -1831,6 +1831,90 @@ def _fake_run(cmd, *args, **kwargs): assert res.result["session_budget_untested"] == 1 +@pytest.mark.asyncio +async def test_explore_leaves_a_variant_out_when_the_run_reaped_its_grid_warmup( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """The stop has to survive ``run_grid``'s own discarded warmup round. + + With server-lifecycle reuse ineligible, explore's decision round is a plain + ``run_grid`` call, and ``run_grid`` runs its own warmup pass before it (on by + default outside pytest). Explore reads the stop off the result's + ``error_class``, so a warmup reap graded as ``warmup_round_failed`` reaches + this ledger as a measured verdict about the variant. + """ + _force_cold_decision(monkeypatch) + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "1") + monkeypatch.setattr( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + lambda _config_path: { + "eligible": True, + "framework": "sglang", + "port": 30000, + "reason": "", + }, + ) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + ran: list[str] = [] + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + ran.append(slot.name) + if slot.name == "warmup_round": + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + _fake_workspace(slot, tput=840.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-warmup-reaped"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_warmup_reaped", + "extra_args": "--max-num-seqs 256", + "extra_envs": {}, + "provenance": "default_grid", + } + ], + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + }, + idempotency_key="ex-budget-warmup-reaped", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + assert ran == ["warmup_round"], f"the decision round ran after the warmup was reaped: {ran}" + assert res.result["explore_search_update"]["tested"] == {} + assert res.result["losers"] == [] + assert res.result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert res.result["session_budget_untested"] == 1 + + @pytest.mark.asyncio async def test_explore_does_not_call_a_variant_unstable_when_the_run_reaped_its_rebench( sub_agent_runner, diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index e80cfdad13..07a5a79d05 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -19,6 +19,10 @@ from hyperloom.orchestrator.actions.executors import _grid_runner from hyperloom.orchestrator.actions.executors import _grid_runner as gr +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, + SESSION_TIME_EXHAUSTED_RETURNCODE, +) from hyperloom.orchestrator.actions.executors._grid_runner import ( _MN_BACKENDS_PRIORITY, _MN_PARAMS_PRIORITY, @@ -1617,10 +1621,6 @@ class TestSessionKillAttribution: @pytest.mark.asyncio async def test_mid_round_budget_kill_is_recorded_as_skipped_not_failed(self, tmp_path): - from hyperloom.orchestrator.actions.executors._subprocess_kill import ( - SESSION_TIME_EXHAUSTED_RETURNCODE, - ) - base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) @@ -1711,6 +1711,152 @@ def fake_run(cmd, *args, **kwargs): assert seen == [deadline] +def _reaping_round(returncode: int, *, slot_name: str): + """A ``run_with_session_kill`` double that reaps one named round of a variant. + + Args: + returncode: The sentinel the reaped round comes back with. + slot_name: Output-slot directory name identifying the round to reap; + every other round succeeds with a valid report. + + Returns: + tuple: The ``side_effect`` callable, and the list of slot names it + appends to as rounds are launched. + """ + launched: list[str] = [] + + def fake_run(cmd, *args, **kwargs): + # The module-memoized interpreter probe is not a benchmark round. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + launched.append(slot.name) + if slot.name == slot_name: + return subprocess.CompletedProcess(cmd, returncode, "", "") + _fake_workspace(slot) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + + return fake_run, launched + + +class TestEveryRoundCarriesTheStopThatEndedIt: + """A round the run stopped is a stop whichever round it was. + + The measured round is not the only full benchmark pass a variant costs: the + discarded warmup runs the same workload, and so does the multi-node client + warmup. A reap in either has exactly as much to say about the variant as one + in the measured round -- nothing -- so grading it as ``warmup_round_failed`` + files a verdict the run never reached, and ignoring the returncode entirely + keeps launching benchmark rounds after the orchestrator asked the action to + stop. + """ + + @pytest.mark.asyncio + async def test_a_warmup_reaped_by_the_budget_is_skipped_not_a_failed_variant(self, tmp_path): + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + fake_run, launched = _reaping_round(SESSION_TIME_EXHAUSTED_RETURNCODE, slot_name="warmup_round") + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("cand0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 600.0, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + + assert launched == ["warmup_round"], "the measured round must not run after the warmup was reaped" + assert [r.status for r in results] == ["skipped"] + assert results[0].error_class == "session_time_exhausted" + assert _read_marker(tmp_path / "out" / "variant_00_cand0")["error_class"] == "session_time_exhausted" + + @pytest.mark.asyncio + async def test_a_cancelled_warmup_ends_the_grid_instead_of_booting_the_next_variant(self, tmp_path): + """Every remaining variant would boot its own server on the Ray path. + + ``run_session_kill`` re-``ensure()``s a lease whose actor the cancel just + killed, so a grid that keeps going after a cancel starts a fresh actor + and a fresh GPU server per remaining variant. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + fake_run, launched = _reaping_round(ORCHESTRATOR_CANCELLED_RETURNCODE, slot_name="warmup_round") + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("c0"), GridVariant("c1"), GridVariant("c2")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + keep_going_on_failure=True, + session_deadline_sec=time.monotonic() + 600.0, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + + assert launched == ["warmup_round"], f"a cancelled action kept launching rounds: {launched}" + assert [r.status for r in results] == ["skipped"] * 3 + assert [r.error_class for r in results] == ["orchestrator_cancelled"] * 3 + + @pytest.mark.asyncio + async def test_a_cancelled_multi_node_warmup_ends_the_grid(self, tmp_path, monkeypatch): + """The multi-node warmup discarded its returncode along with its report.""" + from hyperloom.orchestrator.actions.executors import _multi_node_env as mne + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnsl + + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + monkeypatch.setattr(mne, "is_multi_node", lambda: True) + monkeypatch.setattr(mne, "mn_bench_warmup_enabled", lambda: True) + + async def fake_restart_server_for_round(**_kwargs): + return None + + monkeypatch.setattr(mnsl, "restart_server_for_round", fake_restart_server_for_round) + fake_run, launched = _reaping_round(ORCHESTRATOR_CANCELLED_RETURNCODE, slot_name="mn_warmup") + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("c0"), GridVariant("c1")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + keep_going_on_failure=True, + session_deadline_sec=time.monotonic() + 600.0, + variant_expected_sec=30.0, + ) + + assert launched == ["mn_warmup"], f"a cancelled action kept launching rounds: {launched}" + assert [r.error_class for r in results] == ["orchestrator_cancelled"] * 2 + + class TestSessionBudgetWarmupRounds: """A warmup round costs a full pass, and the measured round is paid first.""" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 7108361b45..bdb2301fc6 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1564,6 +1564,83 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl ) return clamped + def _record_round_stop( + stopped: StoppedByTheRun, + *, + idx: int, + variant: GridVariant, + slot: Path, + round_label: str, + returncode: int | None, + started_unix: float, + server_log: Path, + ) -> bool: + """Record a round the run stopped and say whether the grid is over. + + Every round a variant runs is a full benchmark pass -- the discarded + warmup and the multi-node client warmup as much as the measured one -- so + any of them can be reaped by the session deadline or by an orchestrator + cancel, and none of them says anything about the variant when it is. + Hence one place decides what such a round means, called after each launch + and before any grading: the row is ``skipped``, exactly like a variant the + budget refused to start, because nothing was measured and there is no + verdict to record. Grading it as a failure -- or worse as + ``killed_overtime``, which asserts the variant is abnormally slow -- puts + a conclusion the run never reached into the ledger and the KB. + + A cause that ``ends_the_batch`` also fills in every variant after this + one: nothing new may start under it, and their rows have to say why they + were never tested rather than be absent. + + Args: + stopped (StoppedByTheRun): How to record the cause. + idx (int): Zero-based index of the variant whose round was stopped. + variant (GridVariant): That variant. + slot (Path): Its slot directory, where the abort marker is written. + round_label (str): Which round was stopped, for the log line. + returncode (int | None): The round's returncode, kept on the row. + started_unix (float): ``time.time()`` when the round was launched. + server_log (Path): The stopped round's ``server.log``, if it wrote one. + + Returns: + bool: ``True`` when the caller must stop testing variants. + """ + runtime_sec = round(max(0.0, time.time() - started_unix), 2) + log.warning( + "grid_runner: variant %d/%d name=%s %s round reaped after %.1fs: %s; recorded as skipped, not failed", + idx + 1, + len(grid), + variant.name, + round_label, + runtime_sec, + stopped.interrupted, + ) + _write_variant_abort_marker( + slot, + variant_name=variant.name, + error_class=stopped.error_class, + error_summary=f"{stopped.interrupted}; tree reaped", + extra_args=variant.extra_server_args, + ) + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="skipped", + returncode=returncode, + runtime_sec=runtime_sec, + error=stopped.interrupted, + error_class=stopped.error_class, + server_log_path=_existing_log_path(server_log), + note=variant.note, + ) + ) + if stopped.ends_the_batch: + results.extend(_not_run_skip_result(rest, stopped) for rest in grid[idx + 1 :]) + return True + return not keep_going_on_failure + # How many full benchmark passes one variant costs. Both warmups run the same # workload as the measured pass -- neither is a reduced one -- so a variant # that warms up costs twice what its measured round does. Admitting on a @@ -1851,6 +1928,30 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl break continue + warmup_stopped = stopped_by_the_run(warmup_rc) + if warmup_stopped is not None: + from ._server_lifecycle import teardown_lifecycle_server + + teardown_lifecycle_server( + pid_dir=slot, + framework=str(lifecycle.get("framework") or ""), + port=int(lifecycle.get("port") or 0), + ) + grid_is_over = _record_round_stop( + warmup_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="warmup", + returncode=warmup_rc, + started_unix=warmup_started_unix, + server_log=warmup_server_log, + ) + await _pulse_after_variant(i) + if grid_is_over: + break + continue + warmup_run_ws = select_run_workspace(warmup_slot, known_before=warmup_workspaces_before) warmup_workspace = warmup_run_ws if warmup_run_ws is not None else warmup_slot warmup_harvested = harvest_leaked_artifacts( @@ -2016,8 +2117,13 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl if _mn_imn() and _mn_warm(): _mn_warm_slot = slot / "mn_warmup" + _mn_warm_started_unix = time.time() + # The measurement is discarded, but the returncode is not: a warmup + # the run stopped is the same stop as one in the measured round, and + # discarding it launches the measured round after the cancel. + _mn_warm_rc: int | None = None try: - await _reported_magpie( + _mn_warm_rc, _, _ = await _reported_magpie( i, "mn_warmup", magpie_python=magpie_python, @@ -2037,10 +2143,11 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl session_deadline_sec=session_deadline_sec, ) log.info( - "grid_runner: MN warmup pass done (discarded) %d/%d name=%s", + "grid_runner: MN warmup pass done (discarded) %d/%d name=%s rc=%s", i + 1, len(grid), variant.name, + _mn_warm_rc, ) except Exception as exc: # noqa: BLE001 - warmup is best-effort log.warning( @@ -2048,6 +2155,22 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl variant.name, exc, ) + _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) + if _mn_warm_stopped is not None: + grid_is_over = _record_round_stop( + _mn_warm_stopped, + idx=i, + variant=variant, + slot=slot, + round_label="mn_warmup", + returncode=_mn_warm_rc, + started_unix=_mn_warm_started_unix, + server_log=_mn_warm_slot / "server.log", + ) + await _pulse_after_variant(i) + if grid_is_over: + break + continue # Snapshot wall-clock before launch so the salvage path can mtime-gate # leak destinations per-variant. @@ -2266,50 +2389,20 @@ def _round_timeout_sec(idx: int, name: str, *, round_label: str, reserve_sec: fl break continue - # The round was stopped by the run rather than by anything about the - # variant, and the tree was reaped. Recorded as ``skipped``, exactly like - # a variant the budget refused to start: in both cases nothing was - # measured, so there is no verdict to record about the variant. Grading it - # as a failure -- or worse as ``killed_overtime``, which asserts the - # variant is abnormally slow -- would put a conclusion the run never - # reached into the ledger and the KB. - stopped = _STOPPED_BY_THE_RUN.get(rc) + stopped = stopped_by_the_run(rc) if stopped is not None: - variant_runtime_sec = round(max(0.0, time.time() - variant_started_unix), 2) - log.warning( - "grid_runner: variant %d/%d name=%s reaped after %.1fs: %s; recorded as skipped, not failed", - i + 1, - len(grid), - variant.name, - variant_runtime_sec, - stopped.interrupted, - ) - _write_variant_abort_marker( - slot, - variant_name=variant.name, - error_class=stopped.error_class, - error_summary=f"{stopped.interrupted}; tree reaped", - extra_args=variant.extra_server_args, - ) - results.append( - VariantResult( - name=variant.name, - extra_server_args=variant.extra_server_args, - extra_envs=dict(variant.extra_envs), - status="skipped", - returncode=rc, - runtime_sec=variant_runtime_sec, - error=stopped.interrupted, - error_class=stopped.error_class, - server_log_path=_existing_log_path(server_log), - note=variant.note, - ) + grid_is_over = _record_round_stop( + stopped, + idx=i, + variant=variant, + slot=slot, + round_label="measured", + returncode=rc, + started_unix=variant_started_unix, + server_log=server_log, ) await _pulse_after_variant(i) - if stopped.ends_the_batch: - results.extend(_not_run_skip_result(rest, stopped) for rest in grid[i + 1 :]) - break - if not keep_going_on_failure: + if grid_is_over: break continue diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 31b0ca8fba..248b35cd03 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -37,6 +37,7 @@ from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import heartbeat_while_output_flows, report_progress from ...phases import machine_state as _phase_state +from ..stop_attribution import StoppedByTheRun from . import _server_lifecycle as _lifecycle from ._file_lock import best_effort_file_lock from ._aiter_jit import ( @@ -269,6 +270,58 @@ def _watchdog_server_log_path(output_dir: Path, framework: str) -> str | None: return str(output_dir / "server.log") +def _stopped_round_result( + stopped: StoppedByTheRun, + *, + round_label: str, + returncode: int | None, + runtime_sec: float, + output_dir: Path, + capture_meta: dict[str, Any], +) -> dict[str, Any]: + """Build the result for a round the run itself stopped. + + The session budget elapsed mid-round, or the orchestrator cancelled the + action. Classified apart from every measurement failure -- and checked before + them, because the reap leaves exactly the evidence a broken server does (no + workspace, no report, a non-zero returncode) and being graded as + ``server_init_dead`` or ``subprocess_nonzero`` would put a verdict on the + model that this round never reached. Every round the baseline runs goes + through here, the discarded multi-node warmup pass included: a stop in the + round that warms the server means the round is over, and going on to the + measured pass would spend GPU time the run has been told to stop spending. + Nothing here arms a retry either: the cause is the run, and a resume meets it + again. + + Args: + stopped: How to record the cause. + round_label: Which round was stopped, for the log line. + returncode: The stopped round's returncode. + runtime_sec: Wall-clock seconds the round had run for. + output_dir: The task workspace, echoed onto the result. + capture_meta: Config/eval-contract facts every failure result carries. + + Returns: + dict[str, Any]: The failed result carrying the stop's own error class. + """ + log.warning( + "baseline_executor: %s reaped after %.1fs: %s; error_class=%s.", + round_label, + runtime_sec, + stopped.interrupted, + stopped.error_class, + ) + return { + "status": "failed", + "error_class": stopped.error_class, + "returncode": returncode, + "error": stopped.interrupted, + "subprocess_runtime_sec": round(runtime_sec, 2), + "output_dir": str(output_dir), + **capture_meta, + } + + def _disable_cuda_graph_flag(framework: str) -> str: """Return the framework-correct flag that disables cuda-graph capture. @@ -3725,6 +3778,12 @@ async def _run_single_benchmark( if _mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted"): _mn_warm_dir = output_dir / "mn_warmup" + _mn_warm_started_unix = time.time() + # The measurement is discarded, but the returncode is not: this pass + # is a full benchmark round, so a stop here ends the baseline round. + # Going on to the measured pass would spend a second round of GPU + # time the run has already been told to stop spending. + _mn_warm_rc: int | None = None try: _mn_warm_dir.mkdir(parents=True, exist_ok=True) _mn_warm_cmd = [str(_mn_warm_dir) if c == str(output_dir) else c for c in cmd] @@ -3737,7 +3796,7 @@ async def _run_single_benchmark( unit="baseline_round", label="mn_warmup", ) as _mn_warm_activity: - await asyncio.to_thread( + _mn_warm_proc = await asyncio.to_thread( run_with_session_kill, _mn_warm_cmd, env=_mn_warm_env, @@ -3747,9 +3806,20 @@ async def _run_single_benchmark( on_output=_mn_warm_activity.note, session_deadline_sec=session_deadline_sec, ) - log.info("baseline_executor: MN warmup pass done (discarded)") + _mn_warm_rc = _mn_warm_proc.returncode + log.info("baseline_executor: MN warmup pass done (discarded) rc=%s", _mn_warm_rc) except Exception as exc: # noqa: BLE001 - warmup is best-effort log.warning("baseline_executor: MN warmup pass failed (ignored): %r", exc) + _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) + if _mn_warm_stopped is not None: + return _stopped_round_result( + _mn_warm_stopped, + round_label="multi-node warmup pass", + returncode=_mn_warm_rc, + runtime_sec=max(0.0, time.time() - _mn_warm_started_unix), + output_dir=output_dir, + capture_meta=capture_meta, + ) workspaces_before = snapshot_workspaces(output_dir) subprocess_started_unix = time.time() @@ -3841,31 +3911,16 @@ async def _run_single_benchmark( **capture_meta, } - # The run stopped this round rather than the round failing: the session - # budget elapsed mid-round, or the orchestrator cancelled the action. - # Classified apart from every measurement failure and checked before - # them, because the reap leaves the same evidence a broken server does -- - # no workspace, no report, a non-zero returncode -- and being graded as - # ``server_init_dead`` or ``subprocess_nonzero`` would put a verdict on - # the model that this round never reached. Nothing here arms a retry - # either: the cause is the run, and a resume meets it again. stopped = stopped_by_the_run(proc_returncode) if stopped is not None: - log.warning( - "baseline_executor: round reaped after %.1fs: %s; error_class=%s.", - subprocess_runtime_sec, - stopped.interrupted, - stopped.error_class, + return _stopped_round_result( + stopped, + round_label="measured round", + returncode=proc_returncode, + runtime_sec=subprocess_runtime_sec, + output_dir=output_dir, + capture_meta=capture_meta, ) - return { - "status": "failed", - "error_class": stopped.error_class, - "returncode": proc_returncode, - "error": stopped.interrupted, - "subprocess_runtime_sec": round(subprocess_runtime_sec, 2), - "output_dir": str(output_dir), - **capture_meta, - } # Detokenizer-stall watchdog reap: the server came up healthy but went # silent for the stall grace window (hung engine / wedged detokenizer). From 168eefc0a20ef460438fd6429c66a59e91d3274f Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 20:18:17 +0000 Subject: [PATCH 28/65] writeback: a revalidation the run reaped is not an enablement that stalled The baseline failure streak already exempts a round the run stopped, because nothing about the baseline was measured. The enablement revalidation branch charged the same round to its own stall streak and ended the session as enablement_stalled -- one round exempt from one ledger and charged to the other, with the session's terminal reason and stop_ts drawn from a clock. The reaped round now leaves the revalidation window open instead of closing it: validation_pending is set only by an eval-origin KEEP, so clearing it strands a patch nothing ever revalidated. The generation advances so the next enqueue's idempotency key does not resolve to the row the run just stopped. Co-authored-by: Cursor --- .../tests/test_coordinator_runtime.py | 64 ++++++++++++++ src/hyperloom/orchestrator/loop/writeback.py | 88 ++++++++++++++----- 2 files changed, 129 insertions(+), 23 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index cb5cd1fbc7..93e7caa123 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1510,6 +1510,70 @@ async def test_revalidation_boot_failure_clears_pending_and_rearmes(session_dir, await c.stop() +@pytest.mark.asyncio +@pytest.mark.parametrize("error_class", ["session_time_exhausted", "orchestrator_cancelled"]) +async def test_a_revalidation_the_run_stopped_does_not_burn_the_stall_streak( + session_dir, + monkeypatch, + error_class, +): + """The same round cannot be exempt from one ledger and charged to the other. + + A reaped revalidation baseline is exempted from the baseline failure streak + because nothing about the baseline was measured. Charging it to the + enablement stall streak reaches the cap on the evidence of a clock, and the + session's terminal reason becomes ``enablement_stalled`` for rounds nobody + ever ran. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = "t-reval-stopped" + st.enablement.stall_streak = 4 + await c._handle_unpromotable_result( + _mk_task("baseline", "t-reval-stopped"), + {"status": "failed", "error_class": error_class, "error": "reaped"}, + ) + assert st.enablement.stall_streak == 4 + assert st.stop_reason in ("", None) + assert st.baseline_failure_streak == 0 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_a_reaped_revalidation_leaves_the_window_open_for_a_resume(session_dir, monkeypatch): + """Nothing else reopens the window, so the stop must not close it. + + ``validation_pending`` is set only by an eval-origin KEEP, and the + revalidation enqueue is gated on it, so clearing it strands a KEEP'd patch + that was never revalidated. The generation is bumped for the same reason + opening the window bumps it: the idempotency key must not resolve to the row + the run just stopped. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = "t-reval-open" + st.enablement.revalidation_generation = 2 + await c._handle_unpromotable_result( + _mk_task("baseline", "t-reval-open"), + {"status": "failed", "error_class": "session_time_exhausted", "error": "reaped"}, + ) + assert st.enablement.validation_pending is True + assert st.enablement.revalidation_task_id == "" + assert st.enablement.revalidation_generation == 3 + assert st.enablement.inflight_task_id == "" + finally: + await c.stop() + + @pytest.mark.asyncio async def test_handle_unpromotable_records_for_non_baseline_kinds(session_dir): c = Coordinator(session_dir, backends=_silent_backends()) diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 5bb4d6f448..95e5b23cc6 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -762,6 +762,66 @@ def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: state.enablement.baseline_eval_evidence = evidence[:4000] state.enablement.launch_log = evidence + def _record_revalidation_not_promoted( + self, + *, + task: Task, + result_payload: dict[str, Any], + err_class: str, + stopped_by_the_run: bool, + ) -> None: + """Close out an enablement revalidation baseline that did not promote. + + A genuine failure -- boot, OOM, timeout, eval -- is a no-progress round: + it closes the revalidation window, reopens the authoring loop, and counts + toward the ``enablement_stalled`` cap so repeated KEEP-then-fail cycles + terminate. + + A round the run stopped is none of those things. It measured nothing, so + it is no evidence that the KEEP'd patch fails to revalidate, and charging + it to the stall streak reaches the cap on the evidence of a clock -- the + same round the baseline failure streak deliberately exempts. The window + therefore stays open, because only an eval-origin KEEP ever opens one and + closing it here would strand a patch nothing revalidated. The generation + advances for the same reason opening a window does: the next enqueue's + idempotency key must not resolve to the row the run just stopped. + + Args: + task: The revalidation baseline task that came back unpromotable. + result_payload: Its result, read for the launch/traceback text. + err_class: Its ``error_class``, for the log line. + stopped_by_the_run: Whether the run stopped the round rather than the + round saying anything about the baseline. + """ + state = self.shared_state + state.enablement.revalidation_task_id = "" + if stopped_by_the_run: + state.enablement.revalidation_generation = int(state.enablement.revalidation_generation or 0) + 1 + state.enablement.inflight_task_id = "" + else: + state.enablement.validation_pending = False + state.enablement.stall_streak = int(state.enablement.stall_streak or 0) + 1 + try: + from ..phases.framework import _ENABLEMENT_MAX_STALL as _max_stall + except ImportError: + _max_stall = 5 + if state.enablement.stall_streak >= _max_stall and not state.stop_reason: + state.set_stop_reason("enablement_stalled") + else: + state.enablement.inflight_task_id = "" + launch_log = _extract_enablement_launch_log(result_payload) + if launch_log: + state.enablement.launch_log = launch_log + log.warning( + "enablement revalidation task %s %s (error_class=%s); stall_streak=%d pending=%s rearm=%s", + task.task_id, + "was stopped by the run" if stopped_by_the_run else "failed", + err_class, + int(state.enablement.stall_streak or 0), + bool(state.enablement.validation_pending), + not bool(state.stop_reason), + ) + async def _handle_unpromotable_result( self, task: Task, @@ -916,29 +976,11 @@ async def _handle_unpromotable_result( or (reval_tid and reval_tid == str(task.task_id or "")) ) if is_revalidation and bool(getattr(self.shared_state.enablement, "validation_pending", False)): - self.shared_state.enablement.validation_pending = False - self.shared_state.enablement.revalidation_task_id = "" - self.shared_state.enablement.stall_streak = ( - int(getattr(self.shared_state.enablement, "stall_streak", 0) or 0) + 1 - ) - try: - from ..phases.framework import _ENABLEMENT_MAX_STALL as _max_stall - except ImportError: - _max_stall = 5 - if self.shared_state.enablement.stall_streak >= _max_stall and not self.shared_state.stop_reason: - self.shared_state.set_stop_reason("enablement_stalled") - else: - self.shared_state.enablement.inflight_task_id = "" - launch_log = _extract_enablement_launch_log(result_payload) - if launch_log: - self.shared_state.enablement.launch_log = launch_log - log.warning( - "enablement revalidation task %s failed (error_class=%s); " - "stall_streak=%d rearm=%s", - task.task_id, - err_class, - self.shared_state.enablement.stall_streak, - not bool(self.shared_state.stop_reason), + self._record_revalidation_not_promoted( + task=task, + result_payload=result_payload, + err_class=err_class, + stopped_by_the_run=stopped_by_the_run, ) any_changed = True from ..actions.executors._accuracy_gate import eval_enablement_allowed # noqa: PLC0415 From 5f9c69da05c7c1a6995958486a890e078de4e5c8 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 20:24:08 +0000 Subject: [PATCH 29/65] explore: rolling back a reaped rebench must not delete a prior round's row The reaped-rebench rollback popped the fingerprint out of a dict initialised from the persisted ledger, and a fingerprint may be re-run across rounds -- so undoing this round's write deleted an earlier round's measured row along with it, and the name index entry pointing at it. The model was then free to re-propose a variant already measured and failed, at the cost of a full benchmark round. This round's writes now live in their own dict, merged over the inherited ledger after the loop. The rollback structurally cannot reach a row it did not write. Co-authored-by: Cursor --- .../tests/test_explore_executor.py | 102 ++++++++++++++++++ .../orchestrator/actions/executors/explore.py | 49 +++++---- 2 files changed, 132 insertions(+), 19 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index ce1a7591b7..c4a5901ab7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -1986,6 +1986,108 @@ def _fake_run(cmd, *args, **kwargs): assert res.result["session_budget_untested"] == 1 +@pytest.mark.asyncio +async def test_the_reaped_rebench_rollback_spares_a_prior_rounds_ledger_row( + sub_agent_runner, + tmp_path, + monkeypatch, +): + """Rolling back this round's write must not delete an earlier round's. + + A fingerprint may be re-run across rounds, so the row a rerun overwrites is a + real measurement from a previous round. Undoing the rerun by deleting the key + takes that measurement out of the negative ledger with it, and the model is + free to re-propose a variant already measured and failed -- at the cost of a + full benchmark round. + """ + _force_cold_decision(monkeypatch) + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.max_minutes = 600.0 + state.baseline_runtime_sec = 20.0 + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + fp_rerun = canonical_fingerprint("--rerun-flag", {}) + + def _fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + # Match on path segments: ``tmp_path`` is named after the test, so a + # substring check would fire on every round. + parts = set(slot.parts) + rerun = "v01_v_ok" in parts + if rerun and "stack_rebench" in parts: + return subprocess.CompletedProcess( + args=cmd, + returncode=SESSION_TIME_EXHAUSTED_RETURNCODE, + stdout="", + stderr="reaped", + ) + _fake_workspace(slot, tput=1000.0 if rerun else 900.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-rollback"), + "base_tput": 800.0, + "grid": [ + { + "name": "v_measured", + "extra_args": "--other-flag", + "extra_envs": {}, + "provenance": "default_grid", + }, + { + "name": "v_ok", + "extra_args": "--rerun-flag", + "extra_envs": {}, + "provenance": "default_grid", + }, + ], + "explore_search": { + "tested": { + fp_rerun: { + "fingerprint": fp_rerun, + "name": "v_ok", + "extra_server_args": "--rerun-flag", + "extra_envs": {}, + "outcome": "FAILED", + "round_id": "explore-001", + } + }, + "rejected": [], + "name_index": {"v_ok": fp_rerun}, + }, + "variant_timeout_sec": 3600, + "baseline_runtime_sec": 20.0, + "enable_stack_rebench": True, + }, + idempotency_key="ex-rollback-prior-row", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + update = res.result["explore_search_update"] + prior = update["tested"].get(fp_rerun) + assert prior is not None, "prior round's ledger entry was deleted by the rollback" + assert prior["round_id"] == "explore-001" + assert prior["outcome"] == "FAILED" + assert update["name_index"]["v_ok"] == fp_rerun + # The round did measure something, so the update replaces the persisted + # ledger wholesale -- which is what makes a deletion here durable. + assert {t["name"] for t in update["tested"].values()} == {"v_measured", "v_ok"} + + @pytest.mark.asyncio async def test_explore_attributes_a_round_the_run_reaped_before_anything_measured( sub_agent_runner, diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 490a979311..1f2b08ff7c 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -985,7 +985,7 @@ async def __call__(self, ctx) -> dict[str, Any]: search.setdefault(key, default) tested_dict = search.get("tested") or {} - name_index = dict(search.get("name_index") or {}) + inherited_name_index: dict[str, Any] = dict(search.get("name_index") or {}) # Attach the per-variant fingerprint as an attribute so the result # loop needn't recompute. @@ -1064,7 +1064,13 @@ async def __call__(self, ctx) -> dict[str, Any]: winners: list[dict[str, Any]] = [] losers: list[dict[str, Any]] = [] keep_unstable: list[dict[str, Any]] = [] - tested_update: dict[str, dict[str, Any]] = dict(tested_dict) + # This round's own ledger writes, kept apart from the ledger it inherited + # and merged over it once the loop is done. A variant whose confirmation + # round the run reaps has to be rolled back, and a fingerprint may be + # re-run across rounds -- so rolling back against a merged dict deletes + # the earlier round's measured row along with this round's write. + round_tested: dict[str, dict[str, Any]] = {} + round_name_index: dict[str, Any] = {} rejected_update: list[dict[str, Any]] = list(search.get("rejected") or []) winners_history_update: list[dict[str, Any]] = list(search.get("winners_history") or []) @@ -1300,7 +1306,7 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la gv.name, werr, ) - tested_update[fp] = { + round_tested[fp] = { "fingerprint": fp, "name": gv.name, "extra_server_args": gv.extra_server_args, @@ -1326,7 +1332,7 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la "raw_result_path": w.raw_result_path if w is not None else None, } if gv.name: - name_index[gv.name] = fp + round_name_index[gv.name] = fp rejected_update.append( { "fingerprint": fp, @@ -1408,7 +1414,7 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la # Informational only: ``tput`` stays None so this never # enters winner selection or gain math. est_tput = getattr(r, "estimated_output_throughput", None) - tested_update[fp] = { + round_tested[fp] = { "fingerprint": fp, "name": gv.name, "extra_server_args": gv.extra_server_args, @@ -1444,7 +1450,7 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la "error_class": "killed_overtime", } if gv.name: - name_index[gv.name] = fp + round_name_index[gv.name] = fp rejected_update.append( { "fingerprint": fp, @@ -1556,7 +1562,7 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la outcome = "KEEP" decision_tput = r.output_throughput - tested_update[fp] = { + round_tested[fp] = { "fingerprint": fp, "name": gv.name, "extra_server_args": gv.extra_server_args, @@ -1581,7 +1587,7 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la "stage": FAILURE_STAGE_DECISION, } if gv.name: - name_index[gv.name] = fp + round_name_index[gv.name] = fp # ---- KEEP path (with warm round-2 rebench) ---- if outcome == "KEEP": @@ -1707,9 +1713,9 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la # the variant and its confirmation together. if _stopped_by_the_run(rebench, variant=gv, idx=idx, round_label="stack rebench"): in_batch_keeps.pop() - tested_update.pop(fp, None) + round_tested.pop(fp, None) if gv.name: - name_index.pop(gv.name, None) + round_name_index.pop(gv.name, None) break stack_rebench_tput = rebench.tput stack_rebench_workspace = rebench.workspace @@ -1727,10 +1733,10 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la running_base_tput, stack_stable_threshold_pct, ) - tested_update[fp]["outcome"] = "KEEP_UNSTABLE" - tested_update[fp]["stack_rebench_tput"] = stack_rebench_tput - tested_update[fp]["stack_rebench_workspace"] = stack_rebench_workspace - tested_update[fp]["stack_rebench_warnings"] = stack_rebench_warnings + round_tested[fp]["outcome"] = "KEEP_UNSTABLE" + round_tested[fp]["stack_rebench_tput"] = stack_rebench_tput + round_tested[fp]["stack_rebench_workspace"] = stack_rebench_workspace + round_tested[fp]["stack_rebench_warnings"] = stack_rebench_warnings keep_unstable.append( { **keep_entry, @@ -1775,11 +1781,11 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la keep_entry["stack_rebench_tput"] = stack_rebench_tput keep_entry["stack_rebench_workspace"] = stack_rebench_workspace keep_entry["stack_rebench_warnings"] = stack_rebench_warnings - tested_update[fp]["tput"] = stack_rebench_tput - tested_update[fp]["gain_pct"] = gain - tested_update[fp]["stack_rebench_tput"] = stack_rebench_tput - tested_update[fp]["stack_rebench_workspace"] = stack_rebench_workspace - tested_update[fp]["stack_rebench_warnings"] = stack_rebench_warnings + round_tested[fp]["tput"] = stack_rebench_tput + round_tested[fp]["gain_pct"] = gain + round_tested[fp]["stack_rebench_tput"] = stack_rebench_tput + round_tested[fp]["stack_rebench_workspace"] = stack_rebench_workspace + round_tested[fp]["stack_rebench_warnings"] = stack_rebench_warnings else: # Round 2 disabled — KEEP on the cold round-1 # measurement, advance the running baseline naively. @@ -1861,6 +1867,11 @@ def _stopped_by_the_run(result: Any, *, variant: GridVariant, idx: int, round_la round_serving_lease.close() # ----- Ledger compaction (per-fingerprint last-wins) ---------------- + # This round's writes over the ledger it inherited: a re-run fingerprint + # replaces its earlier row, which is what a fresh measurement means, and a + # variant this round rolled back leaves the earlier row standing. + tested_update: dict[str, dict[str, Any]] = {**tested_dict, **round_tested} + name_index: dict[str, Any] = {**inherited_name_index, **round_name_index} rejected_dedup: dict[str, dict[str, Any]] = {} for entry in rejected_update: fp = str(entry.get("fingerprint") or "") From a2326c44d80b70523352985f4afa87f4d685c9c6 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 20:41:14 +0000 Subject: [PATCH 30/65] cancel: derive the three cooperative-stop windows from what stopping costs The dispatcher's cooperative window, the Ray submitter's cancel grace and the SIGTERM grace were each sized to look reasonable beside the others -- ten, eight and five seconds -- and composed they said the dispatcher gives up a good five seconds before the work it is waiting for can finish. A window a hair short of what stopping costs does not expire occasionally: it expires every time, and what it discards is the attributed sentinel the round was about to return, replaced by a hard CancelledError. That is precisely what the cooperative channel exists to avoid. The reap sequence's components are now named and summed into one budget, and both graces derive from it. A round in a Ray actor is allowed the reap plus the poll its answer is seen at; the dispatcher is allowed the slower of the two paths plus the release of whatever the round held. The window is only ever spent when work is still unwinding -- the wait ends the moment the last victim is done. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 44 ++++++++++++++++- .../actions/executors/_ray_serving.py | 29 +++++++----- .../actions/executors/_server_lifecycle.py | 4 +- .../actions/executors/_subprocess_kill.py | 47 +++++++++++++++---- src/hyperloom/orchestrator/loop/dispatcher.py | 44 +++++++++++++---- 5 files changed, 134 insertions(+), 34 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 2fa2a110a0..346070d1cb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -41,12 +41,18 @@ current_cancel_scope, use_cancel_scope, ) +from hyperloom.orchestrator.actions.executors._ray_serving import CANCEL_ROUND_GRACE_SEC from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + COOPERATIVE_REAP_BUDGET_SEC, ORCHESTRATOR_CANCELLED_RETURNCODE, + STOP_GATE_POLL_SECONDS, run_with_session_kill, ) from hyperloom.orchestrator.loop.coordinator import Coordinator -from hyperloom.orchestrator.loop.dispatcher import _COOPERATIVE_CANCEL_GRACE_SEC +from hyperloom.orchestrator.loop.dispatcher import ( + _CANCEL_NOTICE_SEC, + _COOPERATIVE_CANCEL_GRACE_SEC, +) from hyperloom.orchestrator.loop.coordinator_helpers import ( TIME_BUDGET_EXEMPT_ACTIONS, action_fits_time_budget, @@ -678,6 +684,40 @@ async def _run(_ctx) -> dict: return _run +class TestTheCooperativeStopWindowsCompose: + """Three waits on the same stop, which only mean anything together. + + Each was picked to look reasonable beside the others -- ten seconds at the + dispatcher, eight for a round in a Ray actor, five for the SIGTERM grace -- + and composed they said the dispatcher gives up before the work it is waiting + for can finish. A window a hair short of what stopping costs does not expire + occasionally: it expires every time, and what it discards is the attributed + sentinel the round was about to return. + + The components are spelled out here rather than re-derived from the constants + under test, so a change to one of them has to be argued for. + """ + + def test_the_reap_budget_is_what_stopping_a_round_costs(self): + # Notice at the 0.5s poll, SIGTERM and wait out the 5s grace, collect the + # SIGKILL'd child for 1s, drain its pipes for 2s. + assert COOPERATIVE_REAP_BUDGET_SEC == 0.5 + 5.0 + 1.0 + 2.0 + + def test_the_ray_grace_outlasts_a_round_stopping_itself(self): + """A round in an actor stops the same way; the grace has to cover it.""" + assert CANCEL_ROUND_GRACE_SEC >= COOPERATIVE_REAP_BUDGET_SEC + + def test_the_dispatcher_outlasts_the_slowest_honest_stop(self): + # The Ray path is the long one: 8.5s for the round to stop itself, 0.25s + # for the answer to be seen, then up to 10s to release the lease it held. + assert _COOPERATIVE_CANCEL_GRACE_SEC >= 8.5 + 0.25 + 10.0 + + def test_the_notice_window_is_the_poll_the_scope_is_checked_at(self): + """Nothing is listening yet is a claim about the poll, not about the work.""" + assert _CANCEL_NOTICE_SEC >= STOP_GATE_POLL_SECONDS + assert _CANCEL_NOTICE_SEC < COOPERATIVE_REAP_BUDGET_SEC + + class TestTheCancelChannel: """The channel itself: what it carries, and how far it reaches.""" @@ -850,7 +890,7 @@ async def test_an_actor_that_will_not_answer_is_killed_and_the_stop_still_named( """A wedged actor must not hold the lease open, and must not go unattributed.""" from hyperloom.orchestrator.actions.executors import _ray_serving as rs - monkeypatch.setattr(rs, "_CANCEL_ROUND_GRACE_SEC", 0.5) + monkeypatch.setattr(rs, "CANCEL_ROUND_GRACE_SEC", 0.5) monkeypatch.setattr(rs.ServingLease, "_ask_actor_to_cancel", lambda _self, _reason: False) outcome: dict[str, Any] = {} await _start_action( diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py index 8f262f1582..635d9558eb 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py @@ -14,6 +14,8 @@ from hyperloom.common.env_safety import scrub_benchmark_process_env +from ._subprocess_kill import COOPERATIVE_REAP_BUDGET_SEC + log = logging.getLogger(__name__) # Ray-side sentinel returncodes, allocated out of the same space as @@ -34,18 +36,19 @@ _CANCEL_POLL_SEC: float = 0.25 # How long the submitter waits for a cancelled round to come back on its own -# before killing the actor out from under it. The round stops itself the same -# way the local path does -- notice the scope at its poll, SIGTERM the tree, -# wait out the grace, drain the pipes -- and that is what this is sized on. It -# stays under the dispatcher's cooperative window so the honest stop is the one -# that usually happens, and the kill is what a wedged actor gets. -_CANCEL_ROUND_GRACE_SEC: float = 8.0 +# before killing the actor out from under it. The round in the actor stops itself +# exactly the way a local child does, so this is that cost plus the poll the +# answer is seen at -- derived, not picked, because a grace even slightly short of +# the reap expires every single time and throws away the sentinel the round was +# about to hand back. The kill is what a wedged actor gets, not what a working one +# gets for being ordinary. +CANCEL_ROUND_GRACE_SEC: float = COOPERATIVE_REAP_BUDGET_SEC + _CANCEL_POLL_SEC # How long releasing a lease waits for the actor to reap its served process # before killing the actor anyway. Sized on what that reap costs -- SIGTERM, the # grace, SIGKILL -- and deliberately short: teardown often runs inside the # closing window, which is reserved for the report, not for waiting on a server. -_CLOSE_STOP_TIMEOUT_SEC: float = 10.0 +CLOSE_STOP_TIMEOUT_SEC: float = 10.0 # Method slots the serving actor runs at once: the round, plus room for the # cancel that has to reach it. A single-slot actor would queue the cancel behind @@ -596,7 +599,7 @@ def _await_or_cancel(self, ref: Any, *, cancel_scope: Any) -> tuple[int, str, st The round attributes its own stop, exactly as the local path does, so the sentinel this returns is the actor's whenever the actor answers. Only a wedged actor -- one that has not come back within - ``_CANCEL_ROUND_GRACE_SEC`` of being asked -- is killed, and only then + ``CANCEL_ROUND_GRACE_SEC`` of being asked -- is killed, and only then does the submitter attribute the stop on its behalf, because otherwise an unattributed failure is what the ledger would read. @@ -631,12 +634,12 @@ def _await_or_cancel(self, ref: Any, *, cancel_scope: Any) -> tuple[int, str, st # The actor never took the round, or cannot be reached to be # told about it. Either way nothing in there will stop on its # own, so go straight to the kill. - asked_at -= _CANCEL_ROUND_GRACE_SEC - elif time.monotonic() - asked_at >= _CANCEL_ROUND_GRACE_SEC: + asked_at -= CANCEL_ROUND_GRACE_SEC + elif time.monotonic() - asked_at >= CANCEL_ROUND_GRACE_SEC: log.warning( "ServingLease: the actor did not return its cancelled round within %.0fs; " "killing it to release the lease", - _CANCEL_ROUND_GRACE_SEC, + CANCEL_ROUND_GRACE_SEC, ) # Straight to the kill: an actor that has not answered is not # going to answer a graceful stop either, and waiting for one @@ -646,7 +649,7 @@ def _await_or_cancel(self, ref: Any, *, cancel_scope: Any) -> tuple[int, str, st ORCHESTRATOR_CANCELLED_RETURNCODE, "", "the orchestrator cancelled this action; its Ray actor was killed after " - f"{_CANCEL_ROUND_GRACE_SEC:.0f}s without returning the round", + f"{CANCEL_ROUND_GRACE_SEC:.0f}s without returning the round", ) def _ask_actor_to_cancel(self, reason: str) -> bool: @@ -683,7 +686,7 @@ def close(self) -> None: try: import ray # noqa: PLC0415 - ray.get(self._actor.stop.remote(), timeout=_CLOSE_STOP_TIMEOUT_SEC) + ray.get(self._actor.stop.remote(), timeout=CLOSE_STOP_TIMEOUT_SEC) except Exception as exc: # noqa: BLE001 — the kill below is the backstop log.warning("ServingLease.close: the actor did not stop its server: %r", exc) self._kill_actor() diff --git a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py index 32ce3342ad..6e2a38e1a1 100644 --- a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py +++ b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py @@ -24,7 +24,7 @@ import yaml -from ._subprocess_kill import _process_group_alive, _signal_group +from ._subprocess_kill import TERM_GRACE_SECONDS, _process_group_alive, _signal_group log = logging.getLogger(__name__) @@ -278,7 +278,7 @@ def teardown_lifecycle_server( # Server is setsid'd, so pgid == pid unless the pid file gave one. pgid = server_pgid if server_pgid is not None else server_pid _signal_group(pgid, signal.SIGTERM) - deadline = time.monotonic() + 5.0 + deadline = time.monotonic() + TERM_GRACE_SECONDS while time.monotonic() < deadline: if not _process_group_alive(pgid): break diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 52c672843e..32a4b6338e 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -33,8 +33,35 @@ log = logging.getLogger(__name__) -# Grace window between SIGTERM and SIGKILL. -_TERM_GRACE_SECONDS = 5.0 +# Grace window between SIGTERM and SIGKILL, here and for a driver-side teardown +# of a server a round left behind: the same signal, the same thing being waited +# for. +TERM_GRACE_SECONDS: float = 5.0 + +# How long the reaper waits to collect the SIGKILL'd child before giving up on it. +_REAP_COLLECT_SECONDS: float = 1.0 + +# How long draining a reaped child's capture threads is given. +_CAPTURE_DRAIN_SECONDS: float = 2.0 + +# How often the blocking side looks up from the child to check its stop gates -- +# the session deadline and the cancel scope among them. Short enough that the +# check costs a stop almost nothing on top of the reap, long enough not to spin. +STOP_GATE_POLL_SECONDS: float = 0.5 + +# What stopping a running round costs, end to end, from the moment something asks +# it to: noticing at the poll, SIGTERM'ing the tree, waiting out the grace before +# SIGKILL, collecting the child, and draining its pipes. +# +# Every window that waits for a round to stop itself is derived from this rather +# than picked next to it -- the Ray submitter's grace and the dispatcher's +# cooperative window both are. A window shorter than what stopping costs looks +# generous in isolation and still expires every time, and what it discards is the +# honest sentinel the round was about to return, replacing it with a hard kill +# and an unattributed failure. +COOPERATIVE_REAP_BUDGET_SEC: float = ( + STOP_GATE_POLL_SECONDS + TERM_GRACE_SECONDS + _REAP_COLLECT_SECONDS + _CAPTURE_DRAIN_SECONDS +) def new_session_kwargs() -> dict: @@ -99,7 +126,7 @@ def _signal_group(pgid: int, sig: int) -> None: def kill_my_spawned_server( proc: subprocess.Popen | None, *, - grace_seconds: float = _TERM_GRACE_SECONDS, + grace_seconds: float = TERM_GRACE_SECONDS, ) -> None: """Tear down the entire process tree rooted at ``proc``. @@ -171,12 +198,13 @@ def kill_my_spawned_server( _signal_group(pgid, signal.SIGKILL) try: - proc.wait(timeout=1.0) + proc.wait(timeout=_REAP_COLLECT_SECONDS) except subprocess.TimeoutExpired: log.warning( - "_subprocess_kill: proc.wait() did not return within 1s " + "_subprocess_kill: proc.wait() did not return within %.0fs " "after SIGKILL'ing pgid=%d (pid=%d). The reaper may be " "wedged; leaving the zombie for init to collect.", + _REAP_COLLECT_SECONDS, pgid, proc.pid, ) @@ -830,7 +858,7 @@ def run_with_session_kill( except subprocess.TimeoutExpired: kill_my_spawned_server(proc) if capture is not None: - capture.finish(timeout=2.0) + capture.finish(timeout=_CAPTURE_DRAIN_SECONDS) raise except _ReapedByWatchdog as exc: kill_my_spawned_server(proc) @@ -873,7 +901,7 @@ def _finish_capture(capture: _StreamCapture | None, *, text: bool) -> tuple[str empty: str | bytes = "" if text else b"" if capture is None: return empty, empty - stdout, stderr = capture.finish(timeout=2.0) + stdout, stderr = capture.finish(timeout=_CAPTURE_DRAIN_SECONDS) return ( stdout if stdout is not None else empty, stderr if stderr is not None else empty, @@ -1068,7 +1096,7 @@ def _communicate_with_soft_deadline( # any soft deadline with a log present scans. soft_watches_log = soft_active and bool(server_log_path) scan_active = stall_active or soft_watches_log - poll_interval = 0.5 + poll_interval = STOP_GATE_POLL_SECONDS start = time.monotonic() dead_marker_since: float | None = None # Detokenizer-stall watchdog state: per-log byte offsets consumed so far, @@ -1201,12 +1229,15 @@ def _communicate_with_soft_deadline( __all__ = [ "AGENTX_PREFLIGHT_RETURNCODE", + "COOPERATIVE_REAP_BUDGET_SEC", "DETOKENIZER_STALL_RETURNCODE", "EVAL_PROBE_UNPATCHABLE_RETURNCODE", "ORCHESTRATOR_CANCELLED_RETURNCODE", "OVERTIME_KILL_RETURNCODE", "SERVER_DEAD_RETURNCODE", "SESSION_TIME_EXHAUSTED_RETURNCODE", + "STOP_GATE_POLL_SECONDS", + "TERM_GRACE_SECONDS", "kill_my_spawned_server", "new_session_kwargs", "run_with_session_kill", diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 2847215983..aca6dc6279 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -17,6 +17,15 @@ KERNEL_AGENT_OWNED_ACTIONS, ) from ..actions.cancel_channel import CancelScope, use_cancel_scope +from ..actions.executors._ray_serving import ( + CANCEL_ROUND_GRACE_SEC, + CLOSE_STOP_TIMEOUT_SEC, +) +from ..actions.executors._subprocess_kill import ( + COOPERATIVE_REAP_BUDGET_SEC, + STOP_GATE_POLL_SECONDS, + TERM_GRACE_SECONDS, +) from ..phases import machine_state as _phase_state from ..bus.message_bus import Message from ..kernel.request_handlers import get_handler @@ -50,19 +59,36 @@ log = _logging.getLogger(__name__) # How long a cancel waits for work that is listening on its cancel scope to stop -# itself. Sized on what stopping actually costs: the poll the blocking side -# checks the scope at, plus the SIGTERM grace before the tree is SIGKILLed, plus -# the drain of the child's pipes. Past that the coroutine is cancelled anyway, -# which is where this started -- the wait buys the guarantee when the work can -# give it, and never turns a shutdown into a hang when it cannot. -_COOPERATIVE_CANCEL_GRACE_SEC: float = 10.0 +# itself, composed from the two things an action still has to do after it is +# asked: +# +# * stop the round in flight -- locally that is +# :data:`COOPERATIVE_REAP_BUDGET_SEC`, through a Ray actor it is +# :data:`CANCEL_ROUND_GRACE_SEC`, and whichever path this action took, only the +# longer of the two bounds it; +# * release what the round held -- the driver-side teardown of a server it left +# behind (:data:`TERM_GRACE_SECONDS`) or the release of the Ray lease it ran in +# (:data:`CLOSE_STOP_TIMEOUT_SEC`). Alternatives, not a sequence: a round's +# server is reaped by whichever of the two owned it. +# +# Derived rather than picked, because these three windows only mean anything +# together. Each was plausible on its own at ten, eight and five seconds, and +# composed they said the dispatcher gives up a good five seconds before the work +# it is waiting for can finish -- so the honest sentinel the round was about to +# return was discarded for a hard ``CancelledError`` every time, which is exactly +# what the cooperative channel exists to avoid. Past this window the coroutine is +# cancelled anyway, and the window is only ever spent when work is still +# unwinding: the wait ends the moment the last victim is done. +_COOPERATIVE_CANCEL_GRACE_SEC: float = max(COOPERATIVE_REAP_BUDGET_SEC, CANCEL_ROUND_GRACE_SEC) + max( + TERM_GRACE_SECONDS, CLOSE_STOP_TIMEOUT_SEC +) # How long a cancel waits for anything to start listening before deciding # nothing will. Work that is already blocking is listening before the cancel is # raised; the window is for the gap between an action starting and reaching the -# call that watches the scope, so it is sized on the same poll interval rather -# than on how long the work takes. -_CANCEL_NOTICE_SEC: float = 0.5 +# call that watches the scope, so it is the poll that call checks the scope at +# rather than anything about how long the work takes. +_CANCEL_NOTICE_SEC: float = STOP_GATE_POLL_SECONDS class _InflightAction(NamedTuple): From 24083834f5e18cc1a9e8c6659ea8a46c7ef23111 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 20:50:52 +0000 Subject: [PATCH 31/65] dispatcher/integrate_patch: split the three functions this branch pushed over 100 lines The wall-clock work added a self-heal preamble, an inline cancel handle and a budget-bounded confirmation round, and three functions crossed the limit as a result. Each split at a seam that already existed rather than at a line count: reclaiming state a previous tick left stuck, deciding whether an inline action is admitted, deriving a confirmation's stability floor, and grading a confirmation that finished. Co-authored-by: Cursor --- .../actions/executors/integrate_patch.py | 60 +++++++--- src/hyperloom/orchestrator/loop/dispatcher.py | 112 ++++++++++++------ 2 files changed, 122 insertions(+), 50 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index df4e59c330..b5616e415c 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -59,7 +59,11 @@ ) from . import _framework_switch_manifest as _switch_manifest from ._grid_server_args import compose_server_args -from ._stack_rebench import DEFAULT_STACK_STABLE_PCT, measure_stack_rebench +from ._stack_rebench import ( + DEFAULT_STACK_STABLE_PCT, + StackRebenchResult, + measure_stack_rebench, +) from ._workload_envs import ( FrameworkScriptMismatchError, default_baseline_config, @@ -4294,24 +4298,12 @@ async def _confirm_stack_rebench( _rt_rb = params.get("runtime_override") if isinstance(_rt_rb, dict) and _rt_rb: variant.runtime_override = dict(_rt_rb) - # A confirmation rebench must remain a stability check, not become a - # stricter second discovery gate as the per-cycle KEEP threshold decays. - # An explicit lower per-task floor remains valid, but it cannot exceed - # half of the threshold that admitted this patch. - keep_threshold_pct = float(params.get("keep_threshold_pct", self.keep_threshold_pct)) - requested_stable_threshold_pct = float( - params.get("rebench_stable_threshold_pct", DEFAULT_STACK_STABLE_PCT) - ) - stable_threshold_pct = min( - requested_stable_threshold_pct, - max(0.0, keep_threshold_pct / 2.0), - ) rebench = await measure_stack_rebench( config_path=config_path, base_extra_args=base_extra_args, variant=variant, base_tput=base_tput, - stable_threshold_pct=stable_threshold_pct, + stable_threshold_pct=self._rebench_stable_threshold_pct(params), output_slot=output_root / "stack_rebench", variant_timeout_sec=int(params.get("variant_timeout_sec", self.variant_timeout_sec)), model_path=resolved_model or None, @@ -4323,6 +4315,46 @@ async def _confirm_stack_rebench( session_deadline_sec=session_deadline_sec, variant_expected_sec=variant_expected_sec, ) + return self._graded_rebench(rebench, params=params, override_result_dir=override_result_dir) + + def _rebench_stable_threshold_pct(self, params: dict[str, Any]) -> float: + """The floor a confirmation rebench's throughput is graded against. + + A confirmation must remain a stability check rather than become a + stricter second discovery gate as the per-cycle KEEP threshold decays. An + explicit lower per-task floor stays valid, but it cannot exceed half of + the threshold that admitted this patch in the first place. + + Args: + params: The task parameters, read for the per-task overrides. + + Returns: + float: The stability floor as a percentage over the base throughput. + """ + keep_threshold_pct = float(params.get("keep_threshold_pct", self.keep_threshold_pct)) + requested_stable_threshold_pct = float(params.get("rebench_stable_threshold_pct", DEFAULT_STACK_STABLE_PCT)) + return min(requested_stable_threshold_pct, max(0.0, keep_threshold_pct / 2.0)) + + def _graded_rebench( + self, + rebench: StackRebenchResult, + *, + params: dict[str, Any], + override_result_dir: str | None, + ) -> dict[str, Any]: + """Grade a finished confirmation round and shape its verdict for the caller. + + Args: + rebench: The confirmation round's measurement. + params: The task parameters, read for the accuracy baseline and + framework. + override_result_dir: An explicit ``$RESULT_DIR``, which wins over the + round's own workspace when grading accuracy. + + Returns: + dict[str, Any]: ``stable`` / ``tput`` / ``workspace`` / ``warnings`` / + ``stable_floor`` / ``accuracy_pass``. + """ # See ``_bench_patch``: lm-eval writes to the grid slot (the parent of # ``rebench.workspace``), so grade from there, honoring ``result_dir``. rebench_eval_root = override_result_dir or (str(Path(rebench.workspace).parent) if rebench.workspace else "") diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index aca6dc6279..f7c9b04503 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -305,24 +305,24 @@ async def _cancel_inflight_that_outlived_the_session(self) -> bool: ) return False - async def _pump_dispatcher_once(self) -> None: - """Dispatch queued tasks respecting per-lane capacity, re-scanning for - newly-fittable tasks while in-flight tasks run. - - Re-scans the queue whenever an in-flight task completes - (FIRST_COMPLETED) or a short poll elapses, so a queued GPU task starts - the moment its lane frees. The pump still fully drains all currently - dispatchable work before returning. Each GPU lease is bound to its - task_id and released by the runner. - - Budget guard: once the phase's cyclic budget is spent - (:meth:`_dispatch_paused_for_phase_budget`), stop spawning NEW - phase-scoped variants — drain in-flight, then return so the tick can - advance the phase. + async def _reclaim_stale_dispatch_state(self) -> None: + """Free rows and leases a previous tick left stuck, before scanning the queue. + + Runs every tick and is idempotent. Four independent claims a crashed or + vanished worker can leave behind, each reclaimed on its own so one + failure does not hide the next -- and every one of them best-effort, + because a self-heal that raises would stop the pump it exists to keep + running: + + * a running row whose holder PID is dead, so its lanes free and the task + fails while it is still retry-eligible, this same tick; + * the failure that reclaim implies, charged once; + * the lane leases that holder still held; + * a running row past its TTL, covering a recycled holder PID or a + missing holder record, which the dead-PID check cannot see; + * an ``integrate_patch`` row cancelled at dispatch whose critic verdict + was restored afterwards by a resume. """ - # Dead-holder self-heal (runs every tick, before scanning the queue): - # detect a crashed worker's dead PID so its leased lanes free and the - # stuck task fails (retry-eligible) this same tick. dead_tasks: list[str] = [] try: dead_tasks = await self.tasks.reclaim_dead_running(reason="dead_holder_pump") @@ -343,8 +343,6 @@ async def _pump_dispatcher_once(self) -> None: await self.locks.reap_dead_holders() except Exception: # noqa: BLE001 log.exception("dispatcher: dead-holder lease reap failed") - # TTL-expiry self-heal (runs every tick): covers tasks whose holder PID - # was recycled or whose holder record is missing. Idempotent. try: expired_tasks = await self.tasks.reclaim_expired_running(reason="pump_watchdog") if expired_tasks: @@ -359,6 +357,23 @@ async def _pump_dispatcher_once(self) -> None: await self._reconcile_cancelled_policy_denied_integrate_tasks() except Exception: # noqa: BLE001 — reconcile must not abort the pump log.exception("dispatcher: cancelled policy-denied integrate_patch reconcile failed") + + async def _pump_dispatcher_once(self) -> None: + """Dispatch queued tasks respecting per-lane capacity, re-scanning for + newly-fittable tasks while in-flight tasks run. + + Re-scans the queue whenever an in-flight task completes + (FIRST_COMPLETED) or a short poll elapses, so a queued GPU task starts + the moment its lane frees. The pump still fully drains all currently + dispatchable work before returning. Each GPU lease is bound to its + task_id and released by the runner. + + Budget guard: once the phase's cyclic budget is spent + (:meth:`_dispatch_paused_for_phase_budget`), stop spawning NEW + phase-scoped variants — drain in-flight, then return so the tick can + advance the phase. + """ + await self._reclaim_stale_dispatch_state() inflight: list[tuple[Task, asyncio.Task[SubAgentResult], Any]] = [] # Cumulative across the whole pump, not just the live in-flight set, so a # fast task reaped before its queued->running transition is visible is @@ -1716,23 +1731,28 @@ def _run_action_now_sync( log.exception("run_action_now: inline run of %r failed", name) return f"(run_action_now: {name!r} errored: {exc!r})" - async def _run_action_now( + async def _inline_action_denial( self, action_name: str, params: dict[str, Any], - ) -> str: - """Coordinator-loop coroutine that runs a whitelisted action inline through PolicyGate + SubAgentRunner, publishing a delegated_result for audit/inbox parity. + ) -> str | None: + """Run the admission gates an inline action shares with a dispatched one. + + The synthetic delegate goes through PolicyGate so the phase, role and + path gates reach an inline call exactly as they reach one that went + through the queue; the sequence gate is asked separately because it is + about what the run has already done rather than about the intent. Either + denial is recorded before it is reported, so an inline call that never + ran is still auditable. Args: - action_name: Name of the action to execute. - params: Parameter mapping forwarded to the task/executor. + action_name: Name of the action about to run. + params: Parameters it would run with. Returns: - A status string: a policy/sequence denial message, an - already-in-flight notice, or the rendered delegated_result line. + str | None: The message for the caller when the action is denied, or + ``None`` when it may run. """ - - # PolicyGate parity: validate the synthetic delegate so phase/role/path gates apply. intent = Intent( type=IntentType.DELEGATE, payload={"action_name": action_name, "params": dict(params or {})}, @@ -1747,14 +1767,34 @@ async def _run_action_now( f"{str(getattr(denied, 'hint', denied))[:200]})" ) seq_denied = self._admission_denial_for_action(action_name) - if seq_denied is not None: - await self._record_policy_denied( - "orchestration", - intent, - seq_denied, - action_name=action_name, - ) - return f"(run_action_now: {action_name!r} denied: {str(getattr(seq_denied, 'hint', seq_denied))[:200]})" + if seq_denied is None: + return None + await self._record_policy_denied( + "orchestration", + intent, + seq_denied, + action_name=action_name, + ) + return f"(run_action_now: {action_name!r} denied: {str(getattr(seq_denied, 'hint', seq_denied))[:200]})" + + async def _run_action_now( + self, + action_name: str, + params: dict[str, Any], + ) -> str: + """Coordinator-loop coroutine that runs a whitelisted action inline through PolicyGate + SubAgentRunner, publishing a delegated_result for audit/inbox parity. + + Args: + action_name: Name of the action to execute. + params: Parameter mapping forwarded to the task/executor. + + Returns: + A status string: a policy/sequence denial message, an + already-in-flight notice, or the rendered delegated_result line. + """ + denial = await self._inline_action_denial(action_name, params) + if denial is not None: + return denial lanes, ttl = self._registry_lanes_ttl(action_name) content_fp = hashlib.sha1( json.dumps(params or {}, sort_keys=True, default=str).encode(), From c744315b85c6254f1a8be37401db17ca7586d4f9 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:03:44 +0000 Subject: [PATCH 32/65] cancel: reaping a round's server and dropping its lease are a sequence The cooperative-stop window took the longer of the two release waits, on the reasoning that a round's server is reaped by whichever of the server teardown and the Ray lease owned it. They are not alternatives. On the Ray path both run on the same unwind and in that order: the explore executor's per-variant `finally` calls `teardown_lifecycle_server` and the enclosing one then closes the round's lease, and the baseline executor makes the same two calls in one `finally` with a comment stating the order as a requirement -- the server is reaped before the lease is dropped so that no GPU process outlives it. Maxing the two therefore reproduced, on the path that pays the most, the exact shortfall this derivation exists to remove: 18.75s of window against 23.75s of unwind. Sum them. The wait is a cap it leaves the moment the last victim is done, so a round that stops promptly pays nothing for the extra five seconds. What the budget case pays is five more seconds of overshoot before the run crosses its deadline, since this wait runs inside the reserve the admission gate holds back; the closing phase is not charged for it, because its grace window is measured from when it starts. An attributed sentinel is worth five seconds. No live consequence today -- both teardowns are synchronous, so the dispatcher cannot observe its own deadline mid-unwind -- but wrapping either in `to_thread`, the natural fix for blocking the loop, would let the hard cancel land on the ledger compaction that writes the sentinel. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 11 +++++++ src/hyperloom/orchestrator/loop/dispatcher.py | 32 +++++++++++++------ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 346070d1cb..73b412424d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -712,6 +712,17 @@ def test_the_dispatcher_outlasts_the_slowest_honest_stop(self): # for the answer to be seen, then up to 10s to release the lease it held. assert _COOPERATIVE_CANCEL_GRACE_SEC >= 8.5 + 0.25 + 10.0 + def test_reaping_a_server_and_dropping_its_lease_are_both_paid(self): + """The two release waits are a sequence, so the window has to cover both. + + A Ray round's unwind reaps the server it left behind and only then closes + the lease it ran in -- that order is a requirement, not an accident, so no + GPU process outlives the lease. Taking the longer of the two leaves the + window five seconds short of what that unwind costs, which is the same + shortfall these windows were derived to remove. + """ + assert _COOPERATIVE_CANCEL_GRACE_SEC >= 8.5 + 0.25 + 5.0 + 10.0 + def test_the_notice_window_is_the_poll_the_scope_is_checked_at(self): """Nothing is listening yet is a claim about the poll, not about the work.""" assert _CANCEL_NOTICE_SEC >= STOP_GATE_POLL_SECONDS diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index f7c9b04503..3ff8e8b751 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -66,21 +66,35 @@ # :data:`COOPERATIVE_REAP_BUDGET_SEC`, through a Ray actor it is # :data:`CANCEL_ROUND_GRACE_SEC`, and whichever path this action took, only the # longer of the two bounds it; -# * release what the round held -- the driver-side teardown of a server it left -# behind (:data:`TERM_GRACE_SECONDS`) or the release of the Ray lease it ran in -# (:data:`CLOSE_STOP_TIMEOUT_SEC`). Alternatives, not a sequence: a round's -# server is reaped by whichever of the two owned it. +# * release what the round held -- the driver-side teardown of the server it left +# behind (:data:`TERM_GRACE_SECONDS`) and then the release of the Ray lease it +# ran in (:data:`CLOSE_STOP_TIMEOUT_SEC`). A sequence, not alternatives: the +# server is reaped BEFORE the lease is dropped so that no GPU process outlives +# it (§4.2), and on the Ray path a single unwind pays both -- in the explore +# executor the per-variant ``finally`` tears the server down and the enclosing +# one then closes the round's lease; the baseline executor does the same two +# calls in one ``finally``. # # Derived rather than picked, because these three windows only mean anything # together. Each was plausible on its own at ten, eight and five seconds, and # composed they said the dispatcher gives up a good five seconds before the work # it is waiting for can finish -- so the honest sentinel the round was about to # return was discarded for a hard ``CancelledError`` every time, which is exactly -# what the cooperative channel exists to avoid. Past this window the coroutine is -# cancelled anyway, and the window is only ever spent when work is still -# unwinding: the wait ends the moment the last victim is done. -_COOPERATIVE_CANCEL_GRACE_SEC: float = max(COOPERATIVE_REAP_BUDGET_SEC, CANCEL_ROUND_GRACE_SEC) + max( - TERM_GRACE_SECONDS, CLOSE_STOP_TIMEOUT_SEC +# what the cooperative channel exists to avoid. Taking the longer of the two +# release terms instead of both reproduced that shortfall exactly, on the path +# that pays the most: the teardown a Ray round owes is not an alternative to +# closing its lease, it is what it does first. +# +# Past this window the coroutine is cancelled anyway, and the window is only ever +# spent when work is still unwinding: the wait ends the moment the last victim is +# done, so covering the teardown term costs a round that stops promptly nothing. +# What the budget case pays for it is five more seconds before the run crosses its +# deadline, because this wait runs inside the reserve the admission gate holds +# back. It does not come out of the closing phase, whose grace window is measured +# from the moment it starts, and five seconds of overshoot is cheaper than the +# attributed sentinel a hard cancel destroys. +_COOPERATIVE_CANCEL_GRACE_SEC: float = ( + max(COOPERATIVE_REAP_BUDGET_SEC, CANCEL_ROUND_GRACE_SEC) + TERM_GRACE_SECONDS + CLOSE_STOP_TIMEOUT_SEC ) # How long a cancel waits for anything to start listening before deciding From 8ab077fd36cef787e6ab68dcd14f0a5a89a81855 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:03:45 +0000 Subject: [PATCH 33/65] executors: a cancel at the post-verdict KB write owes the stash back Both patch executors decide a verdict, write a KB record, and only then pop the auto-stash they took from the operator. The write crosses the event loop, and a cancel arrives at whatever await the action happens to be at -- budget exhaustion is precisely what makes it arrive at an arbitrary one rather than never. The bench guard added earlier does not span these awaits, so the candidate's stash stayed on the stack for the rest of the session, and nothing in the logs said so. framework_agent: the undo a stop owes is now one closure the bench handler and the writeback share, and both verdicts record through one guarded recorder rather than repeating the six-argument call. Reverting past an already-committed KEEP is deliberate: the result carrying it never reaches the Coordinator, so the tree must not claim a win the session has no record of. integrate_patch: the gate's awaits were already covered, the apply stage's were not -- it stashes, applies, and then writes a KB record on each of its failure verdicts outside the guard. Rather than guard those two awaits, the existing guard now spans both stages and `_undo_ungraded_candidate` reads what the tree was given from `ctx`, which the apply stage publishes as it mutates. That covers every await in the stage, including ones added later. Every stash restore is still the expression of a return statement, so no await follows one and a stop cannot reach a handler that would pop twice. Co-authored-by: Cursor --- .../tests/test_framework_agent_executor.py | 74 +++++++++++ .../test_integrate_patch_coverage_unit.py | 63 +++++++++ .../actions/executors/framework_agent.py | 78 ++++++++---- .../actions/executors/integrate_patch.py | 120 ++++++++++-------- 4 files changed, 256 insertions(+), 79 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py index f790a207d5..023e575fd5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py @@ -764,6 +764,80 @@ async def cancelled(self, *, params, output_root, slug, **_kwargs): # noqa: ARG assert scratch.read_text(encoding="utf-8") == "user work in progress\n" +def _stash_list(repo: Path) -> str: + """What ``git stash list`` reports for ``repo``.""" + return subprocess.run( + ["git", "-C", str(repo), "stash", "list"], + capture_output=True, + text=True, + check=False, + ).stdout + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bench_tput,verdict", [(900.0, "revert"), (1100.0, "keep")]) +async def test_a_cancel_at_the_kb_writeback_still_hands_the_stash_back( + tmp_path: Path, + bench_tput: float, + verdict: str, +): + """The last await a candidate crosses is the KB writeback, not the bench. + + Both verdicts record their outcome after the verdict is decided and before + the stash restore that returns it. A cancel arrives at whatever await the + action happens to be at, and a spent wall-clock budget is exactly what makes + it arrive at an arbitrary one -- so if that window is unguarded the + operator's uncommitted work stays in ``git stash`` for the rest of the + session with nothing saying so. + """ + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + executor = FrameworkAgentExecutor(session_dir=session_dir) + + async def fake_bench(self, *, params, output_root, slug, **_kwargs): # noqa: ARG001 + return ( + {"status": "succeeded", "output_throughput": bench_tput}, + {"accuracy_pass": None}, + ) + + async def cancelled_writeback(self, **_kwargs): # noqa: ARG001 + raise asyncio.CancelledError + + ctx = _make_ctx( + f"t-fp-kb-cancel-{verdict}", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + ) + with ( + patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench), + patch.object(FrameworkAgentExecutor, "_write_kb_record", new=cancelled_writeback), + ): + with pytest.raises(asyncio.CancelledError): + await executor(ctx) + + assert scratch.read_text(encoding="utf-8") == "user work in progress\n", ( + "the user's uncommitted work was left in the stash" + ) + assert _stash_list(repo) == "", "the auto-stash was never popped" + # A KEEP that is cancelled before its result reaches the Coordinator is a + # KEEP the session does not record, so the commit must not survive either. + assert (repo / "src.py").read_text().endswith("return 1\n"), ( + "the ungraded candidate was left in the framework tree" + ) + + _PATCH_B_ADDS_FILE = """\ diff --git a/new.py b/new.py new file mode 100644 diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py index 20a0313af9..c83070f63d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_coverage_unit.py @@ -1261,3 +1261,66 @@ async def _cancel(self, **kwargs): text=True, ) assert stash_list.stdout.strip() == "", "the auto-stash was left on the stack" + + +@pytest.mark.asyncio +async def test_a_cancel_in_the_apply_stage_still_hands_the_stash_back(tmp_path, monkeypatch): + """The apply stage stashes and mutates the tree, then awaits, same as the gate. + + Each of its failure verdicts writes a KB record before the stash restore that + returns it, and a cancel arrives at whatever await the action happens to be + at -- a spent wall-clock budget is what makes it arrive at an arbitrary one. + Only the gate was guarded, so this window left the operator's uncommitted + work in ``git stash`` for the rest of the session. + """ + session = tmp_path / "s" + session.mkdir() + repo = tmp_path / "fw" + _init_git_repo(repo) + _write_workspace(session, "spec") + + scratch = repo / "user_scratch.txt" + scratch.write_text("user work in progress\n", encoding="utf-8") + + def _fake_resolve(*args, **kwargs): + spec = ip._ArtifactSpec( + source=tmp_path / "tuned.json", + target=repo / "tuned.json", + rel_target="tuned.json", + kind="config_json", + ) + return [spec], [] + + async def _cancel(self, **kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(ip, "_resolve_artifact_specs", _fake_resolve) + monkeypatch.setattr( + IntegratePatchExecutor, + "_apply_artifacts", + lambda self, specs, *, backup_root: ([], [{"artifact": "tuned.json", "error": "disk full"}]), + ) + monkeypatch.setattr(IntegratePatchExecutor, "_maybe_write_framework_kb_record", _cancel) + + ex = IntegratePatchExecutor(session_dir=session) + with pytest.raises(asyncio.CancelledError): + await ex( + _make_ctx( + "t", + {"specialist_task_id": "spec", "framework_source_root": str(repo)}, + ) + ) + + assert scratch.read_text(encoding="utf-8") == "user work in progress\n", ( + "the user's uncommitted work was left in the stash" + ) + stash_list = subprocess.run( + ["git", "-C", str(repo), "stash", "list"], + check=True, + capture_output=True, + text=True, + ) + assert stash_list.stdout.strip() == "", "the auto-stash was left on the stack" + assert (repo / "src.py").read_text(encoding="utf-8").endswith("return 1\n"), ( + "the ungraded candidate was left applied in the framework tree" + ) diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 178788bfd3..44d2c556cb 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -712,6 +712,27 @@ async def __call__(self, ctx) -> dict[str, Any]: applied: list[Path] = [] apply_errors: list[dict[str, str]] = [] apply_feedbacks: list[ApplyFeedback] = [] + + def _undo_candidate() -> None: + """Take the candidate back out of the tree and hand the stash back. + + What a stop owes, as opposed to a verdict. The dispatcher cancels + in-flight actions on shutdown and on a spent wall-clock budget, and + ``CancelledError`` is not an ``Exception``, so none of the REVERT + handlers below see one. Unhandled it leaves the candidate applied and + the operator's uncommitted work in ``git stash`` indefinitely -- the + budget case does not end the process, so CLOSE would go on to report + against a tree carrying a patch nothing ever graded. + + Reverting past a KEEP that was already committed is deliberate: the + result carrying that KEEP never reaches the Coordinator, so leaving + the commit would leave the tree claiming a win the session does not + record. Every step is synchronous, so no second cancel can be + delivered part-way through the undo. + """ + self._revert_patches(framework_root, applied, pre_apply_sha=pre_apply_sha) + _restore_stash_logged(framework_root, stash_state, stash_note) + # Structural safety gate on the (remote / untrusted) diff before it is # applied to the live framework tree: reject non-diff blobs and any # header path that escapes the tree (absolute / ``..``). Stale / @@ -859,13 +880,10 @@ async def __call__(self, ctx) -> dict[str, Any]: }, ) except BaseException: - # A stop, not a verdict: the dispatcher cancels in-flight actions on - # shutdown and on a spent wall-clock budget, and ``CancelledError`` - # is not an ``Exception``, so the REVERT above never sees it. Undo - # the candidate here and let the stop through -- graded as a REVERT - # it would read as the patch having failed a bench that never ran. - self._revert_patches(framework_root, applied, pre_apply_sha=pre_apply_sha) - _restore_stash_logged(framework_root, stash_state, stash_note) + # A stop, not a verdict: let it through rather than grading it, since + # as a REVERT it would read as the patch having failed a bench that + # never ran. See :func:`_undo_candidate` for what the stop owes. + _undo_candidate() raise # KEEP / REVERT decision. @@ -901,6 +919,34 @@ async def __call__(self, ctx) -> dict[str, Any]: gate_pass = tput_ok and not acc_block acc_delta_pct = _accuracy_delta_pct(gate_evidence.get("accuracy"), params.get("accuracy_baseline")) + async def _record_outcome(outcome: str) -> None: + """Write this candidate's KB record, undoing it if the write is stopped. + + Both verdicts record the same measurements and differ only in the + outcome label, and both record them after the verdict is decided and + before the ``_with_stash_restore`` that returns it. That await is the + last one the candidate crosses while the auto-stash is still on the + stack, and no handler stands between it and the caller: a cancel + delivered here -- which is what a spent budget delivers, at whatever + await the action happens to be at -- would strand the operator's work + in the stash with nothing in the session saying so. + + Args: + outcome: The KB outcome label for the verdict just decided. + """ + try: + await self._write_kb_record( + candidate=candidate, + outcome=outcome, + tps_delta_pct=float(delta_pct or 0.0), + patch_path=str(applied[0]) if applied else "", + extra=extra, + accuracy_delta_pct=acc_delta_pct, + ) + except BaseException: + _undo_candidate() + raise + if not gate_pass: reverted = self._revert_patches( framework_root, @@ -919,14 +965,7 @@ async def __call__(self, ctx) -> dict[str, Any]: revert_status = ( "accuracy_unavailable_reject" if (acc_block and accuracy_pass is None and tput_ok) else "reverted" ) - await self._write_kb_record( - candidate=candidate, - outcome=OUTCOME_REVERTED_SMOKE_FAIL, - tps_delta_pct=float(delta_pct or 0.0), - patch_path=str(applied[0]) if applied else "", - extra=extra, - accuracy_delta_pct=acc_delta_pct, - ) + await _record_outcome(OUTCOME_REVERTED_SMOKE_FAIL) return _with_stash_restore( framework_root, stash_state, @@ -986,14 +1025,7 @@ async def __call__(self, ctx) -> dict[str, Any]: }, ) - await self._write_kb_record( - candidate=candidate, - outcome=OUTCOME_INTEGRATED, - tps_delta_pct=float(delta_pct or 0.0), - patch_path=str(applied[0]) if applied else "", - extra=extra, - accuracy_delta_pct=acc_delta_pct, - ) + await _record_outcome(OUTCOME_INTEGRATED) return _with_stash_restore( framework_root, stash_state, diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index b5616e415c..dbdb8925f5 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -1676,23 +1676,33 @@ async def __call__(self, ctx) -> dict[str, Any]: if localize_early is not None: return localize_early - apply_result = await self._stage_apply(ctx, params, extra, specialist_task_id, shared_state, done_payload) - if apply_result is not None: - return apply_result - - # _stage_apply populates these. - output_root: Path = ctx._ip_output_root # type: ignore[attr-defined] - framework_root: Path | None = ctx._ip_framework_root # type: ignore[attr-defined] - stash_state: str = ctx._ip_stash_state # type: ignore[attr-defined] - stash_note: str = ctx._ip_stash_note # type: ignore[attr-defined] - applied: list[Path] = ctx._ip_applied # type: ignore[attr-defined] - applied_artifacts: list[dict[str, Any]] = ctx._ip_applied_artifacts # type: ignore[attr-defined] - config_changes_applied: dict[str, str] = ctx._ip_config_changes_applied # type: ignore[attr-defined] - extra_server_args_applied: str = ctx._ip_extra_server_args_applied # type: ignore[attr-defined] - extra_envs_applied: dict[str, str] = ctx._ip_extra_envs_applied # type: ignore[attr-defined] - setup_result: dict[str, Any] = ctx._ip_setup_result # type: ignore[attr-defined] - + # The apply and the gate both mutate the framework tree behind the + # operator's auto-stash, and both cross awaits while it is on the stack -- + # the apply stage writes a KB record on each of its failure verdicts. So + # the guard spans both: whichever stage was running, the candidate is + # taken back out and the stash handed back, and the stop is re-raised + # rather than graded. Each stage publishes its tree-mutation bookkeeping + # to ``ctx`` as it becomes real, because that is all the handler can see + # when the stop arrives mid-stage. try: + apply_result = await self._stage_apply( + ctx, params, extra, specialist_task_id, shared_state, done_payload + ) + if apply_result is not None: + return apply_result + + # _stage_apply populates these. + output_root: Path = ctx._ip_output_root # type: ignore[attr-defined] + framework_root: Path | None = ctx._ip_framework_root # type: ignore[attr-defined] + stash_state: str = ctx._ip_stash_state # type: ignore[attr-defined] + stash_note: str = ctx._ip_stash_note # type: ignore[attr-defined] + applied: list[Path] = ctx._ip_applied # type: ignore[attr-defined] + applied_artifacts: list[dict[str, Any]] = ctx._ip_applied_artifacts # type: ignore[attr-defined] + config_changes_applied: dict[str, str] = ctx._ip_config_changes_applied # type: ignore[attr-defined] + extra_server_args_applied: str = ctx._ip_extra_server_args_applied # type: ignore[attr-defined] + extra_envs_applied: dict[str, str] = ctx._ip_extra_envs_applied # type: ignore[attr-defined] + setup_result: dict[str, Any] = ctx._ip_setup_result # type: ignore[attr-defined] + return await self._stage_gate( ctx, params, @@ -1712,16 +1722,7 @@ async def __call__(self, ctx) -> dict[str, Any]: setup_result=setup_result, ) except BaseException: - # The gate either returns a verdict or leaves the tree as it found - # it; there is no third outcome where the candidate stays applied - # with nothing having graded it. - self._undo_ungraded_candidate( - framework_root=framework_root, - stash_state=stash_state, - stash_note=stash_note, - applied=applied, - applied_artifacts=applied_artifacts, - ) + self._undo_ungraded_candidate(ctx) raise # --------------------------------------------------------------------------- @@ -2395,6 +2396,14 @@ async def _stage_apply( applied_artifacts: list[dict[str, Any]] = [] apply_errors: list[dict[str, str]] = [] apply_feedbacks: list[ApplyFeedback] = [] + # From here on the tree is mutable and the stash is on the stack, so + # ``__call__``'s undo has to be able to see both before this stage + # returns: ``applied`` is published by identity and appended to in place. + ctx._ip_framework_root = framework_root # type: ignore[attr-defined] + ctx._ip_stash_state = stash_state # type: ignore[attr-defined] + ctx._ip_stash_note = stash_note # type: ignore[attr-defined] + ctx._ip_applied = applied # type: ignore[attr-defined] + ctx._ip_applied_artifacts = applied_artifacts # type: ignore[attr-defined] for patch in patch_paths: if git_tree: ok, err, fb = _git_apply_collect_feedback(framework_root, patch, three_way=False) @@ -2450,6 +2459,7 @@ async def _stage_apply( artifact_specs, backup_root=output_root / "artifact_backups", ) + ctx._ip_applied_artifacts = applied_artifacts # type: ignore[attr-defined] if artifact_apply_errors: self._revert_artifacts(applied_artifacts) reverted = self._revert_patches(framework_root, applied) @@ -2499,12 +2509,9 @@ async def _stage_apply( }, ) + # The tree-mutation values are already published above, as the tree took + # them; what is left is what only the gate reads. ctx._ip_output_root = output_root # type: ignore[attr-defined] - ctx._ip_framework_root = framework_root # type: ignore[attr-defined] - ctx._ip_stash_state = stash_state # type: ignore[attr-defined] - ctx._ip_stash_note = stash_note # type: ignore[attr-defined] - ctx._ip_applied = applied # type: ignore[attr-defined] - ctx._ip_applied_artifacts = applied_artifacts # type: ignore[attr-defined] ctx._ip_config_changes_applied = config_changes_applied # type: ignore[attr-defined] ctx._ip_extra_server_args_applied = extra_server_args_applied # type: ignore[attr-defined] ctx._ip_extra_envs_applied = extra_envs_applied # type: ignore[attr-defined] @@ -3794,24 +3801,17 @@ async def _maybe_write_framework_kb_record( exc, ) - def _undo_ungraded_candidate( - self, - *, - framework_root: Path | None, - stash_state: str, - stash_note: str, - applied: list[Path], - applied_artifacts: list[dict[str, Any]], - ) -> None: - """Take the candidate back out when the gate unwound instead of returning. + def _undo_ungraded_candidate(self, ctx: Any) -> None: + """Take the candidate back out when a stage unwound instead of returning. - Every REVERT the gate itself decides hangs off an ``except Exception``, - and the stop that matters most here is not one of those: the dispatcher - cancels in-flight actions on shutdown and on a spent wall-clock budget, - and ``CancelledError`` derives from ``BaseException``. Unhandled, it - leaves the patch in the framework tree and the operator's auto-stash on - the stack — and the budget case does not end the process, so CLOSE would - report against a tree carrying a patch nothing ever graded. + Every REVERT the stages themselves decide hangs off an ``except + Exception``, and the stop that matters most here is not one of those: the + dispatcher cancels in-flight actions on shutdown and on a spent + wall-clock budget, and ``CancelledError`` derives from ``BaseException``. + Unhandled, it leaves the patch in the framework tree and the operator's + auto-stash on the stack — and the budget case does not end the process, + so CLOSE would report against a tree carrying a patch nothing ever + graded. The cancel itself is re-raised by the caller rather than turned into a REVERT verdict, so the run records it the way @@ -3819,17 +3819,25 @@ def _undo_ungraded_candidate( failed. Every step here is synchronous, so no second cancel can be delivered part-way through the undo. + Read from ``ctx`` rather than from arguments because a stop can arrive + mid-stage, before the stage has returned anything to the caller: what the + undo owes is exactly what the tree has already been given, and each stage + publishes that as it happens. A stage that has not stashed yet leaves + ``clean`` behind, which makes the whole undo a no-op. + Args: - framework_root: The source root the candidate was applied to. - stash_state: The state :func:`_git_stash_if_dirty` returned. - stash_note: The auto-stash ref to restore. - applied: The patches applied to the tree. - applied_artifacts: The artifact records to undo. + ctx: The runner context the stages publish their ``_ip_*`` + tree-mutation bookkeeping onto. """ - self._revert_artifacts(applied_artifacts) - self._revert_patches(framework_root, applied) + framework_root: Path | None = getattr(ctx, "_ip_framework_root", None) + self._revert_artifacts(list(getattr(ctx, "_ip_applied_artifacts", None) or [])) + self._revert_patches(framework_root, list(getattr(ctx, "_ip_applied", None) or [])) if framework_root is not None: - _restore_stash_logged(framework_root, stash_state, stash_note) + _restore_stash_logged( + framework_root, + str(getattr(ctx, "_ip_stash_state", "") or "clean"), + str(getattr(ctx, "_ip_stash_note", "") or ""), + ) def _revert_patches( self, From ab0f7831da05c17123c105de8b8371220e65b037 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:03:45 +0000 Subject: [PATCH 34/65] enablement: a revalidation window the run stopped stays usable, and free The exemption the reap grants was charged back one resume later, and in between the window was stuck. Four steps composed into it: the reap leaves the window open on a fresh generation; the pump then enqueues that generation into a session whose budget is spent; the queue scan cancels the row at dispatch; and a row cancelled at dispatch never produces a result to route, so nothing advances the generation past it. `create_or_return_existing` resolved the window to that terminal row for the rest of the session, and the resume-time recovery -- which keys off a tracked id and could not tell a cancel from a failure -- closed the window with exactly the stall-streak increment the reap went out of its way not to charge. Closed at both ends, and at the root rather than after the fact: * the pump does not open a row the dispatcher would cancel on sight. It asks the same budget gate the queue scan asks, and holds the window shut for this tick when a baseline no longer fits. Nothing is enqueued, so no key is spent, and the resume that has budget again enqueues it. * a key that resolves to a terminal row is recognised as a spent generation and the next one is opened, so a row cancelled before this change -- or by any other dispatch-time refusal -- cannot wedge the window either. * the resume recovery now asks how the row ended. A cancelled row gets the same verdict the writeback gives the same round, through the same method both now call: window open, generation advanced, stall streak untouched. Anything else had its chance and is still charged. Not generalised from `_reconcile_cancelled_policy_denied_integrate_tasks`, which re-queues a row whose *denial* has since been lifted by a restored critic verdict. A budget denial has not been lifted -- the budget is still spent -- so re-queueing here would recreate the row the scan just cancelled. The shapes rhyme; the remedies are opposites. Co-authored-by: Cursor --- .../tests/test_coordinator_runtime.py | 133 ++++++++++++++++++ ...test_enablement_coordinator_wiring_unit.py | 4 + .../orchestrator/loop/coordinator.py | 1 + src/hyperloom/orchestrator/loop/writeback.py | 88 ++++++++---- .../orchestrator/phases/framework.py | 63 +++++++-- 5 files changed, 257 insertions(+), 32 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 93e7caa123..4f88a60951 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1574,6 +1574,139 @@ async def test_a_reaped_revalidation_leaves_the_window_open_for_a_resume(session await c.stop() +async def _cancelled_revalidation_row(c: Coordinator, *, gen: int) -> Task: + """A revalidation row for ``gen`` that the queue scan cancelled before dispatch.""" + task, _existing = await c.tasks.create_or_return_existing( + kind="baseline", + params={"reason": "enablement_eval_revalidation"}, + idempotency_key=f"enablement_revalidation:gen{gen}", + ) + await c.tasks.transition(task.task_id, "cancelled", evidence={"reason": "time_budget"}) + return task + + +@pytest.mark.asyncio +async def test_a_revalidation_the_budget_cannot_fit_is_not_enqueued(session_dir, monkeypatch): + """Opening a row the dispatcher would cancel on sight is what wedges the window. + + A revalidation is a full baseline, and the queue scan drops a queued one the + wall-clock budget can no longer fit. That leaves a cancelled row owning this + window's idempotency key -- and a row cancelled at dispatch never produces a + result to route, so nothing advances the generation past it and every later + tick resolves the window to a row that measured nothing. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_generation = 3 + st.max_minutes = 60 + st.elapsed_minutes = lambda **_kw: 60.0 # type: ignore[method-assign] + + assert await c._maybe_enqueue_enablement_baseline_revalidation() == "" + + # The window survives the stop: same generation, still pending, and no + # row for the key a resume with budget left will need. + assert st.enablement.validation_pending is True + assert st.enablement.revalidation_generation == 3 + assert st.enablement.revalidation_task_id == "" + assert await c.tasks.by_state("cancelled") == [] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_a_revalidation_key_spent_on_a_cancelled_row_opens_the_next_one(session_dir, monkeypatch): + """A terminal row is a spent generation, not an enqueue. + + ``create_or_return_existing`` hands back the cancelled row for as long as the + key names it, so without recognising that the window stays open resolving to + it for the rest of the session. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + st.enablement.validation_pending = True + st.enablement.revalidation_generation = 3 + spent = await _cancelled_revalidation_row(c, gen=3) + + tid = await c._maybe_enqueue_enablement_baseline_revalidation() + + assert tid and tid != spent.task_id, "the window resolved to the cancelled row" + assert st.enablement.revalidation_generation == 4 + assert (await c.tasks.get(tid)).state == "queued" + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_resume_does_not_charge_a_revalidation_the_run_cancelled( + session_dir, + monkeypatch, +): + """The exemption the reap grants must not be charged back by the resume. + + The reap path leaves the window open without charging the stall streak, + because a round the run stopped measured nothing. The resume-time recovery saw + only "tracked row is terminal" and closed the window with the increment the + reap went out of its way to avoid -- reaching the ``enablement_stalled`` cap on + the evidence of a clock, one resume later. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + cancelled = await _cancelled_revalidation_row(c, gen=3) + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = cancelled.task_id + st.enablement.revalidation_generation = 3 + st.enablement.stall_streak = 4 + + report: dict[str, Any] = {"fixes": []} + await c.writeback._resume_recover_pending_revalidation(report) + + assert st.enablement.stall_streak == 4 + assert st.stop_reason in ("", None) + # And the window is left usable rather than merely uncharged. + assert st.enablement.validation_pending is True + assert st.enablement.revalidation_task_id == "" + assert st.enablement.revalidation_generation == 4 + assert [f["kind"] for f in report["fixes"]] == ["reopened_revalidation_the_run_cancelled"] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_resume_still_closes_a_revalidation_window_that_had_its_chance(session_dir, monkeypatch): + """A row that is terminal for any other reason is evidence, and still charged.""" + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + try: + st = c.shared_state + task, _existing = await c.tasks.create_or_return_existing( + kind="baseline", + params={"reason": "enablement_eval_revalidation"}, + idempotency_key="enablement_revalidation:gen0", + ) + await c.tasks.transition(task.task_id, "running") + await c.tasks.transition(task.task_id, "succeeded") + st.enablement.validation_pending = True + st.enablement.revalidation_task_id = task.task_id + st.enablement.stall_streak = 1 + + report: dict[str, Any] = {"fixes": []} + await c.writeback._resume_recover_pending_revalidation(report) + + assert st.enablement.validation_pending is False + assert st.enablement.revalidation_task_id == "" + assert st.enablement.stall_streak == 2 + assert [f["kind"] for f in report["fixes"]] == ["cleared_orphaned_revalidation_pending"] + finally: + await c.stop() + + @pytest.mark.asyncio async def test_handle_unpromotable_records_for_non_baseline_kinds(session_dir): c = Coordinator(session_dir, backends=_silent_backends()) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index a95404bc24..99977afca8 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -296,6 +296,10 @@ async def _record_obs(_source, _topic, payload): fake._maybe_enqueue_enablement_baseline_revalidation = types.MethodType( Coordinator._maybe_enqueue_enablement_baseline_revalidation, fake ) + fake._open_revalidation_row = types.MethodType(Coordinator._open_revalidation_row, fake) + # Admission on the session wall-clock is exercised in test_coordinator_runtime + # against a real coordinator; here nothing is ever denied for want of budget. + fake._time_budget_denial_for_action = lambda _action: None from hyperloom.orchestrator.phases.framework import FrameworkPhase fake._enablement_in_flight = types.MethodType(FrameworkPhase._enablement_in_flight, fake) diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index dfb5bd8cc4..d008831686 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1086,6 +1086,7 @@ def router(self) -> IntentRouter: "_pump_framework_agent_phase_safely": "phase_framework", "_pump_enablement_safely": "phase_framework", "_maybe_enqueue_enablement_baseline_revalidation": "phase_framework", + "_open_revalidation_row": "phase_framework", "_record_framework_agent_authored_outcome": "phase_framework", "_recover_framework_agent_authoring_outcome": "phase_framework", "_record_framework_agent_authoring_empty_outcome": "phase_framework", diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 95e5b23cc6..3e30d239da 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -762,6 +762,31 @@ def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: state.enablement.baseline_eval_evidence = evidence[:4000] state.enablement.launch_log = evidence + def _reopen_revalidation_window(self) -> None: + """Leave an enablement revalidation window open for a round the run stopped. + + A round the run stopped measured nothing, so it says nothing about whether + the KEEP'd patch still revalidates. The window therefore stays open -- + only an eval-origin KEEP ever opens one, and closing it here would strand + a patch nothing revalidated -- and the stall streak is not charged, + because reaching the ``enablement_stalled`` cap on the evidence of a clock + is exactly what the baseline failure streak already exempts this round + from. + + The generation advances for the same reason opening a window does: the + next enqueue's idempotency key must not resolve to the row the run + stopped. The tracked id goes with it, since the row it names is spent and + the next enqueue records its own. + + Two callers reach the same round by different routes: the writeback, when + the reaped row's result is routed, and the resume recovery, when the row + was cancelled at dispatch and so produced no result to route at all. + """ + state = self.shared_state + state.enablement.revalidation_generation = int(state.enablement.revalidation_generation or 0) + 1 + state.enablement.revalidation_task_id = "" + state.enablement.inflight_task_id = "" + def _record_revalidation_not_promoted( self, *, @@ -777,14 +802,8 @@ def _record_revalidation_not_promoted( toward the ``enablement_stalled`` cap so repeated KEEP-then-fail cycles terminate. - A round the run stopped is none of those things. It measured nothing, so - it is no evidence that the KEEP'd patch fails to revalidate, and charging - it to the stall streak reaches the cap on the evidence of a clock -- the - same round the baseline failure streak deliberately exempts. The window - therefore stays open, because only an eval-origin KEEP ever opens one and - closing it here would strand a patch nothing revalidated. The generation - advances for the same reason opening a window does: the next enqueue's - idempotency key must not resolve to the row the run just stopped. + A round the run stopped is none of those things; what it gets instead, and + why, is :meth:`_reopen_revalidation_window`. Args: task: The revalidation baseline task that came back unpromotable. @@ -794,11 +813,10 @@ def _record_revalidation_not_promoted( round saying anything about the baseline. """ state = self.shared_state - state.enablement.revalidation_task_id = "" if stopped_by_the_run: - state.enablement.revalidation_generation = int(state.enablement.revalidation_generation or 0) + 1 - state.enablement.inflight_task_id = "" + self._reopen_revalidation_window() else: + state.enablement.revalidation_task_id = "" state.enablement.validation_pending = False state.enablement.stall_streak = int(state.enablement.stall_streak or 0) + 1 try: @@ -4421,8 +4439,9 @@ async def _resume_consistency_pass(self) -> dict[str, Any]: # restart, so restore both Recipe and Kernel trees before continuing. await self._resume_recover_pending_warm_replay(report) # (1d) Orphaned revalidation tasks: if enablement_validation_pending is set - # but the tracked revalidation task is already terminal, clear the pending - # flag and rearm the stall counter so a fresh revalidation can be enqueued. + # but the tracked revalidation task is already terminal, unstick the window + # so a fresh revalidation can be enqueued -- closed and charged to the + # stall counter, or reopened uncharged when the run cancelled the row. await self._resume_recover_pending_revalidation(report) # (2) Orphaned KEEPs: replay integrate_patch KEEPs # that crashed before the append landed; surface ambiguous ones loudly. @@ -4873,12 +4892,20 @@ async def _resume_recover_pending_targeted_build(self, report: dict[str, Any]) - report["fixes"].append(summary) async def _resume_recover_pending_revalidation(self, report: dict[str, Any]) -> None: - """Clear stale enablement_validation_pending when the tracked revalidation task is terminal. + """Unstick enablement_validation_pending when the tracked revalidation task is terminal. If the coordinator died while a revalidation baseline was running, the task row may already be in a terminal state on resume. Without this recovery the pending flag stays set indefinitely and the next revalidation cannot be enqueued (tracked_tid is still the old row). + + Which recovery depends on how the row ended, not on this being the resume + path. A row the run cancelled -- which is what the queue scan does to a + revalidation the wall-clock budget can no longer fit -- measured nothing + and produced no result to route, so it is no evidence about the baseline + and gets :meth:`_reopen_revalidation_window`, the same verdict the + writeback reaches for the same round. Anything else ended having had its + chance: the window closes and the round is charged to the stall streak. """ state = self.shared_state if not bool(state.enablement.validation_pending): @@ -4889,25 +4916,38 @@ async def _resume_recover_pending_revalidation(self, report: dict[str, Any]) -> try: from ..state.task_registry import TERMINAL_STATES, TaskNotFound + row_state = "" try: row = await self.tasks.get(tracked_tid) - is_terminal = row.state in TERMINAL_STATES + row_state = str(getattr(row, "state", "") or "") + is_terminal = row_state in TERMINAL_STATES except TaskNotFound: is_terminal = True - if is_terminal: - state.enablement.validation_pending = False - state.enablement.revalidation_task_id = "" - state.enablement.stall_streak = ( - int(state.enablement.stall_streak or 0) + 1 - ) - state.enablement.inflight_task_id = "" + if not is_terminal: + return + if row_state == "cancelled": + self._reopen_revalidation_window() report["fixes"].append( - {"kind": "cleared_orphaned_revalidation_pending", "task_id": tracked_tid} + {"kind": "reopened_revalidation_the_run_cancelled", "task_id": tracked_tid} ) log.info( - "resume: cleared stale enablement_validation_pending for terminal revalidation task %s", + "resume: revalidation task %s was cancelled by the run; window left open " + "at generation %d without charging the stall streak", tracked_tid, + int(state.enablement.revalidation_generation or 0), ) + return + state.enablement.validation_pending = False + state.enablement.revalidation_task_id = "" + state.enablement.stall_streak = int(state.enablement.stall_streak or 0) + 1 + state.enablement.inflight_task_id = "" + report["fixes"].append( + {"kind": "cleared_orphaned_revalidation_pending", "task_id": tracked_tid} + ) + log.info( + "resume: cleared stale enablement_validation_pending for terminal revalidation task %s", + tracked_tid, + ) except Exception: # noqa: BLE001 — best-effort log.debug("resume: revalidation pending recovery check failed", exc_info=True) diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 5ea0a2b61f..06a6cd2277 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -4549,6 +4549,17 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: return tracked_tid except Exception: # noqa: BLE001 — defensive pass + # Do not open a row the dispatcher would cancel on sight. A revalidation + # is a full baseline, and the queue scan drops a queued one the session + # budget can no longer fit -- which leaves a cancelled row owning this + # window's idempotency key, and a row cancelled at dispatch never + # produces a result to route, so nothing would advance the generation + # past it. Holding the window shut for now costs nothing: it stays open, + # and the resume that has budget again enqueues it. + denied = self._time_budget_denial_for_action("baseline") + if denied is not None: + log.info("ENABLEMENT revalidation: window held open, not enqueued -- %s", denied) + return "" params: dict[str, Any] = { "source": "coordinator_internal", "reason": ENABLEMENT_REVALIDATION_REASON, @@ -4574,14 +4585,9 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: rt_override = rt_obj.to_runtime_override() if rt_override: params["runtime_override"] = rt_override - # Use generation in the idempotency key so each revalidation window gets - # a fresh row even when a prior window's row is in a terminal state. - gen = int(state.enablement.revalidation_generation or 0) - task, _existing = await self.tasks.create_or_return_existing( - kind="baseline", - params=params, - idempotency_key=f"enablement_revalidation:gen{gen}", - ) + task = await self._open_revalidation_row(params) + if task is None: + return "" task_id = str(getattr(task, "task_id", "") or "") # Persist the task_id so _promote_baseline can verify identity. if task_id and task_id != str(state.enablement.revalidation_task_id or ""): @@ -4592,6 +4598,47 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: log.debug("enablement revalidation: save of task_id failed", exc_info=True) return task_id + async def _open_revalidation_row(self, params: dict[str, Any]) -> "Task | None": + """Resolve this revalidation window's task row, on a generation it can use. + + The generation is in the idempotency key so each window gets a fresh row + even when a prior window's row is in a terminal state. A key that resolves + to a terminal row is a spent generation rather than an enqueue, and it has + to be recognised as one: a row cancelled at dispatch never produces a + result to route, so nothing downstream advances the generation past it, + and every later tick would resolve this window to a row that measured + nothing. + + Args: + params: The baseline params for the revalidation row. + + Returns: + The row to track, or ``None`` when this tick found only spent + generations -- the window stays open and the next tick tries again. + """ + from ..state.task_registry import TERMINAL_STATES + + state = self.shared_state + for _attempt in range(2): + gen = int(state.enablement.revalidation_generation or 0) + task, _existing = await self.tasks.create_or_return_existing( + kind="baseline", + params=params, + idempotency_key=f"enablement_revalidation:gen{gen}", + ) + if str(getattr(task, "state", "") or "") not in TERMINAL_STATES: + return task + log.warning( + "ENABLEMENT revalidation: gen%d resolves to terminal task %s (%s); " + "opening generation %d so the window is not stuck on it", + gen, + getattr(task, "task_id", ""), + getattr(task, "state", ""), + gen + 1, + ) + state.enablement.revalidation_generation = gen + 1 + return None + async def _pump_enablement_safely(self, *, caller: str) -> None: """Phase-independent enablement pump — runs every tick. From de7e3218551380729d9493651e41fec15ee3274d Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:36:45 +0000 Subject: [PATCH 35/65] enablement: a build whose launch probe the run stopped is still unprobed The launch probe is what declares KEEP for a targeted build -- an artifact that compiles is not a runtime that boots -- and this branch's queue scan cancels any queued row the wall-clock budget can no longer fit. A probe dropped that way is worse than one diagnostic lost: the routing pass marks the build accounted for the moment it opens a probe, and that mark is persisted, so the verified build is never read again, in this session or any resume of it. The build's whole cost is discarded on the evidence of a clock. Closed the way the revalidation window was, at the root and then behind it: * the probe is not opened into a session that cannot run it. It asks the same gate the scan asks, and a denial enqueues nothing and records nothing, so the build stays owed a probe and the tick or resume that can afford one opens it. * a probe that was cancelled before it ran is recognised as no evidence about the build, so the build is routed again -- and on a fresh generation, since the cancelled row owns the key it was opened on. That covers the cancel a phase boundary hands out too, which predates this branch. * a probe that ran and failed is evidence, and is still final. The spent-generation recognition is the one the revalidation window already uses, lifted into a method both call: the caller passes a key builder and the generation it has, and gets back the row and the generation to store, because only the caller knows where its generation lives. No flag distinguishes them, so nothing is duplicated and nothing is bent to fit. Splitting the succeeded-build branch out of the routing pass is what makes room for this; it also takes that function back under 100 lines and orphaned an unused TERMINAL_STATES import, now deleted. Co-authored-by: Cursor --- .../tests/test_enablement_build_routing.py | 152 +++++++-- ...test_enablement_coordinator_wiring_unit.py | 3 + .../orchestrator/loop/coordinator.py | 5 + .../orchestrator/phases/framework.py | 294 ++++++++++++++---- 4 files changed, 359 insertions(+), 95 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py b/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py index 7259d62cd4..00f300ffa5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_build_routing.py @@ -14,6 +14,7 @@ import pytest +from hyperloom.inference_optimizer.protocol.action_surfaces import ACTION_CATALOGUE from hyperloom.orchestrator.framework.build_actions import TargetedBuildAction, BuildResult, FrameworkRuntime from hyperloom.orchestrator.loop.build_lifecycle import BuildLifecycleCollaborator from hyperloom.orchestrator.loop.coordinator import Coordinator @@ -34,9 +35,19 @@ def coord(build_coord): phase delegates to (launch-probe enqueue, rearm capture, build lifecycle). """ build_coord._rearm_calls = [] - build_coord._enqueue_build_launch_probe = _types.MethodType( - Coordinator._enqueue_build_launch_probe, build_coord - ) + for name in ( + "_enqueue_build_launch_probe", + "_route_succeeded_build", + "_build_routing_record", + "_note_build_routed", + "_build_probe_was_cancelled", + "_open_row_past_spent_generations", + "_time_budget_denial_for_action", + ): + setattr(build_coord, name, _types.MethodType(getattr(Coordinator, name), build_coord)) + # The real wall-clock gate, on the real catalogue: with no budget set it + # admits everything, so a test that wants a denial sets one. + build_coord.action_registry = ACTION_CATALOGUE def _maybe_rearm_enablement(res): build_coord._rearm_calls.append(dict(res) if isinstance(res, dict) else {}) @@ -362,26 +373,51 @@ async def _enqueue_and_transition(coord, action, state): return task_id -@pytest.mark.asyncio -async def test_route_succeeded_row_enqueues_launch_probe(coord, tmp_path): - """A succeeded build must enqueue an integrate_patch launch probe, not call rearm directly.""" - root = tmp_path / "attempt_s" +async def _verified_build(coord, root, *, gap_id, framework="vllm", ref="v1", runtime_env=None): + """A succeeded ``targeted_build`` row whose result.json carries a usable runtime.""" root.mkdir(parents=True, exist_ok=True) - - rt = FrameworkRuntime(pythonpath_prefixes=(str(root),), runtime_env={"X": "1"}) + rt = FrameworkRuntime(pythonpath_prefixes=(str(root),), runtime_env=runtime_env or {}) br = BuildResult(ok=True, attempt_root=str(root), runtime=rt) (root / "result.json").write_text(json.dumps(br.to_state()), encoding="utf-8") + action = TargetedBuildAction( + gap_id=gap_id, + framework=framework, + component="aiter", + capability="fp4_moe", + ref=ref, + attempt_root=str(root), + ) + return await _enqueue_and_transition(coord, action, "succeeded") - action = TargetedBuildAction(gap_id="g2", framework="vllm", component="aiter", - capability="fp4_moe", ref="v1", attempt_root=str(root)) - await _enqueue_and_transition(coord, action, "succeeded") + +async def _queued_probes(coord): + """The launch-probe rows currently waiting to be dispatched.""" + return [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] + + +def _spend_the_budget(coord, *, minutes=60): + """Leave the session with no wall-clock budget for a probe to run in.""" + coord.shared_state.max_minutes = minutes + coord.shared_state.elapsed_minutes = lambda **_kw: float(minutes) + + +def _restore_the_budget(coord): + """Give the session a budget a probe fits in again, as a resume would.""" + coord.shared_state.max_minutes = 600 + coord.shared_state.elapsed_minutes = lambda **_kw: 0.0 + + +@pytest.mark.asyncio +async def test_route_succeeded_row_enqueues_launch_probe(coord, tmp_path): + """A succeeded build must enqueue an integrate_patch launch probe, not call rearm directly.""" + await _verified_build(coord, tmp_path / "attempt_s", gap_id="g2", runtime_env={"X": "1"}) await Coordinator._maybe_route_build_outcomes(coord) # Must NOT directly rearm with "kept" — KEEP comes from the probe. assert not any(r.get("status") == "kept" for r in coord._rearm_calls) # Must have queued a launch-probe task. - probes = [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] + probes = await _queued_probes(coord) assert len(probes) == 1 probe_params = probes[0].params assert probe_params.get("enablement_launch_only") is True @@ -393,21 +429,18 @@ async def test_route_succeeded_row_enqueues_launch_probe(coord, tmp_path): @pytest.mark.asyncio async def test_route_succeeded_row_probe_carries_config_path(coord, tmp_path): """Launch probe inherits baseline_config_path from shared state.""" - root = tmp_path / "attempt_cfg" - root.mkdir(parents=True, exist_ok=True) - - rt = FrameworkRuntime(pythonpath_prefixes=(str(root),)) - br = BuildResult(ok=True, attempt_root=str(root), runtime=rt) - (root / "result.json").write_text(json.dumps(br.to_state()), encoding="utf-8") - coord.shared_state.baseline_config_path = "/cfg/bench.yaml" - action = TargetedBuildAction(gap_id="g3", framework="sglang", component="aiter", - capability="fp4_moe", ref="v2", attempt_root=str(root)) - await _enqueue_and_transition(coord, action, "succeeded") + await _verified_build( + coord, + tmp_path / "attempt_cfg", + gap_id="g3", + framework="sglang", + ref="v2", + ) await Coordinator._maybe_route_build_outcomes(coord) - probes = [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] + probes = await _queued_probes(coord) assert len(probes) == 1 assert probes[0].params.get("config_path") == "/cfg/bench.yaml" @@ -456,22 +489,73 @@ async def test_route_succeeded_empty_runtime_override_calls_reverted(coord, tmp_ @pytest.mark.asyncio async def test_route_succeeded_probe_idempotent(coord, tmp_path): """Calling _maybe_route_build_outcomes twice for the same row only enqueues one probe.""" - root = tmp_path / "attempt_idem" - root.mkdir(parents=True, exist_ok=True) + await _verified_build(coord, tmp_path / "attempt_idem", gap_id="g6") - rt = FrameworkRuntime(pythonpath_prefixes=(str(root),)) - br = BuildResult(ok=True, attempt_root=str(root), runtime=rt) - (root / "result.json").write_text(json.dumps(br.to_state()), encoding="utf-8") + await Coordinator._maybe_route_build_outcomes(coord) + await Coordinator._maybe_route_build_outcomes(coord) - action = TargetedBuildAction(gap_id="g6", framework="vllm", component="aiter", - capability="fp4_moe", ref="v1", attempt_root=str(root)) - await _enqueue_and_transition(coord, action, "succeeded") + probes = await _queued_probes(coord) + assert len(probes) == 1 # idempotent + + +@pytest.mark.asyncio +async def test_a_launch_probe_the_budget_cannot_fit_is_not_enqueued(coord, tmp_path): + """A probe opened into a spent budget is cancelled at dispatch and lost. + + The probe is what declares KEEP for a build, and the queue scan drops a + queued row the wall-clock budget can no longer fit. Opening one anyway spends + the build's one routing pass on a row that will never run, so the build stays + verified and unlaunched with nothing left to notice it. + """ + build_tid = await _verified_build(coord, tmp_path / "attempt_broke", gap_id="g_budget") + _spend_the_budget(coord) + + await Coordinator._maybe_route_build_outcomes(coord) + assert await _queued_probes(coord) == [] + # Nothing was routed, so the build is still owed a probe. + assert Coordinator._build_routing_record(coord, build_tid) is None + assert coord._rearm_calls == [] + + +@pytest.mark.asyncio +async def test_a_build_whose_probe_the_run_cancelled_is_still_unprobed(coord, tmp_path): + """A cancelled probe is no evidence about the build, so the build gets another. + + The gate above narrows the window but cannot close it: a probe that fits when + it is opened can still be dropped before it is dispatched, and a probe row + cancelled that way owns this build's idempotency key for the rest of the + session. Both halves have to hold -- the build is routed again, and the key it + is routed on is a fresh generation rather than the cancelled row. + """ + build_tid = await _verified_build(coord, tmp_path / "attempt_again", gap_id="g_again") await Coordinator._maybe_route_build_outcomes(coord) + first = (await _queued_probes(coord))[0] + await coord.tasks.transition(first.task_id, "cancelled", evidence={"reason": "time_budget"}) + _restore_the_budget(coord) + await Coordinator._maybe_route_build_outcomes(coord) - probes = [t for t in await coord.tasks.queued() if t.kind == "integrate_patch"] - assert len(probes) == 1 # idempotent + probes = await _queued_probes(coord) + assert [p.task_id for p in probes] != [], "the build was left accounted for by a probe that never ran" + assert first.task_id not in {p.task_id for p in probes}, "the window resolved to the cancelled row" + record = Coordinator._build_routing_record(coord, build_tid) or {} + assert record.get("probe_task_id") == probes[0].task_id + assert int(record.get("probe_generation") or 0) == 1 + + +@pytest.mark.asyncio +async def test_a_build_whose_probe_ran_and_failed_is_not_probed_again(coord, tmp_path): + """A probe that ran said something about the build; only a cancel says nothing.""" + await _verified_build(coord, tmp_path / "attempt_failed", gap_id="g_failed") + await Coordinator._maybe_route_build_outcomes(coord) + probe = (await _queued_probes(coord))[0] + await coord.tasks.transition(probe.task_id, "running") + await coord.tasks.transition(probe.task_id, "failed") + + await Coordinator._maybe_route_build_outcomes(coord) + + assert await _queued_probes(coord) == [] @pytest.mark.asyncio diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index 99977afca8..ca17f52297 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -297,6 +297,9 @@ async def _record_obs(_source, _topic, payload): Coordinator._maybe_enqueue_enablement_baseline_revalidation, fake ) fake._open_revalidation_row = types.MethodType(Coordinator._open_revalidation_row, fake) + fake._open_row_past_spent_generations = types.MethodType( + Coordinator._open_row_past_spent_generations, fake + ) # Admission on the session wall-clock is exercised in test_coordinator_runtime # against a real coordinator; here nothing is ever denied for want of budget. fake._time_budget_denial_for_action = lambda _action: None diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index d008831686..89844563e3 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1054,6 +1054,10 @@ def router(self) -> IntentRouter: "_maybe_escalate_to_targeted_build": "phase_framework", "_maybe_enqueue_specialist_requested_build": "phase_framework", "_maybe_route_build_outcomes": "phase_framework", + "_route_succeeded_build": "phase_framework", + "_build_routing_record": "phase_framework", + "_note_build_routed": "phase_framework", + "_build_probe_was_cancelled": "phase_framework", "_enqueue_build_launch_probe": "phase_framework", "_maybe_rearm_authored_lane": "phase_framework", "_enqueue_author_specialist": "phase_framework", @@ -1087,6 +1091,7 @@ def router(self) -> IntentRouter: "_pump_enablement_safely": "phase_framework", "_maybe_enqueue_enablement_baseline_revalidation": "phase_framework", "_open_revalidation_row": "phase_framework", + "_open_row_past_spent_generations": "phase_framework", "_record_framework_agent_authored_outcome": "phase_framework", "_recover_framework_agent_authoring_outcome": "phase_framework", "_record_framework_agent_authoring_empty_outcome": "phase_framework", diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 06a6cd2277..3c055e2989 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -12,6 +12,7 @@ import subprocess import time import tempfile +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any @@ -4369,7 +4370,9 @@ async def _maybe_route_build_outcomes(self) -> None: A succeeded build no longer synthesises status='kept' directly. Instead it enqueues an integrate_patch launch probe so the runtime must actually - boot the model before KEEP is declared. + boot the model before KEEP is declared. Each row is read once, except that + a build whose probe was cancelled before it ran is read again: nothing + launched the runtime, so nothing has decided anything about that build. """ try: all_tasks = [] @@ -4382,45 +4385,23 @@ async def _maybe_route_build_outcomes(self) -> None: # Pick the most recent terminal row (by updated_at). task = sorted(all_tasks, key=lambda t: str(getattr(t, "updated_at", "") or ""))[-1] task_id = str(getattr(task, "task_id", "") or "") - # Skip rows already accounted for (tracked by enablement_build_manifest). + # Skip rows already accounted for (tracked by enablement_build_manifest), + # unless what they were routed to was cancelled before it ran, which + # leaves the build no more launched than an unrouted one. state = self.shared_state - manifest = list(state.enablement.build_manifest or []) - seen_ids = {str(m.get("task_id") or "") for m in manifest if isinstance(m, dict)} - if task_id in seen_ids: + routed = self._build_routing_record(task_id) + if routed is not None and not await self._build_probe_was_cancelled(routed): return - # Mark as seen immediately so we don't process the same row twice. - manifest.append({"task_id": task_id, "routed": True}) - state.enablement.build_manifest = manifest - fc = "" if task.state == "succeeded": - # Load the BuildResult from result.json for the rich runtime. - attempt_root = str((getattr(task, "params", {}) or {}).get("attempt_root") or "") - # The build's attempt_root is resolved at pump time and is NOT - # written back into the task params (they keep the enqueue-time - # default ""). Fall back to the deterministic build path keyed by - # task_id so a *successful* build is not wrongly rejected as - # ``artifact_unreadable`` (mirrors BuildLifecycle._attempt_root). - if not attempt_root and task_id: - attempt_root = str(self.session_dir / "enablement" / "builds" / task_id) - br = None - if attempt_root: - from ..framework.targeted_build import _load_result_json - - br = _load_result_json(attempt_root) - - # If the runtime can't be read, it can't be launched → reverted. - if br is None or not br.ok or not br.runtime.to_runtime_override(): - res: dict = {"enablement": True, "status": "reverted", "reason": "artifact_unreadable"} - log.info("ENABLEMENT: targeted_build artifact-unreadable task=%s", task_id) - self._maybe_rearm_enablement(res) - return - - # Enqueue a launch probe; KEEP is declared by the probe result. - log.info("ENABLEMENT: targeted_build artifact-verified → enqueue launch probe task=%s", task_id) - await self._enqueue_build_launch_probe(task_id, br) + await self._route_succeeded_build(task, routed) return + # A failed build is accounted for the moment it is read: the rearm + # below is the whole outcome, so re-reading the row would charge the + # stall streak twice for one build. + self._note_build_routed(task_id) + fc = "" # failed row — read failure_class from history or last_build_failure history = getattr(task, "history", None) or [] if isinstance(history, (list, tuple)) and history: @@ -4476,7 +4457,128 @@ async def _maybe_route_build_outcomes(self) -> None: except Exception: # noqa: BLE001 — never wedge the tick log.debug("enablement: route_build_outcomes failed", exc_info=True) - async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None: + async def _route_succeeded_build(self, task: "Task", routed: dict[str, Any] | None) -> None: + """Turn a succeeded targeted build into a launch probe, or a no-progress round. + + KEEP is declared by the probe, never here: an artifact that builds is not + a runtime that boots. So this either opens a probe -- and remembers which + one, because a probe cancelled before it ran leaves the build unlaunched + and worth another -- or, when the built runtime cannot even be read, ends + the round as a revert. + + The build is recorded as accounted for only once there is something to + account for. A probe the budget refused leaves it unrouted on purpose: it + is still a verified build nothing has launched, and the manifest saying + otherwise is what would strand it for the rest of the session. + + Args: + task: The succeeded ``targeted_build`` row. + routed: What this build was routed to before, when it is being routed + again after its probe was cancelled; ``None`` on the first pass. + """ + task_id = str(getattr(task, "task_id", "") or "") + attempt_root = str((getattr(task, "params", {}) or {}).get("attempt_root") or "") + # The build's attempt_root is resolved at pump time and is NOT written + # back into the task params (they keep the enqueue-time default ""). Fall + # back to the deterministic build path keyed by task_id so a *successful* + # build is not wrongly rejected as ``artifact_unreadable`` (mirrors + # BuildLifecycle._attempt_root). + if not attempt_root and task_id: + attempt_root = str(self.session_dir / "enablement" / "builds" / task_id) + br = None + if attempt_root: + from ..framework.targeted_build import _load_result_json + + br = _load_result_json(attempt_root) + + # If the runtime can't be read, it can't be launched → reverted. + if br is None or not br.ok or not br.runtime.to_runtime_override(): + self._note_build_routed(task_id) + log.info("ENABLEMENT: targeted_build artifact-unreadable task=%s", task_id) + self._maybe_rearm_enablement( + {"enablement": True, "status": "reverted", "reason": "artifact_unreadable"} + ) + return + + log.info("ENABLEMENT: targeted_build artifact-verified → enqueue launch probe task=%s", task_id) + probe_tid, generation = await self._enqueue_build_launch_probe( + task_id, + br, + generation=int((routed or {}).get("probe_generation") or 0), + ) + if not probe_tid: + return + self._note_build_routed(task_id, probe_task_id=probe_tid, probe_generation=generation) + + def _build_routing_record(self, build_task_id: str) -> dict[str, Any] | None: + """The record of what a build's outcome was already routed to, if any. + + Only the routing sentinels carry a ``task_id``; the build attempts the + lifecycle appends to the same manifest carry ``ok`` instead. + + Args: + build_task_id: The ``targeted_build`` row to look for. + + Returns: + The sentinel dict, or ``None`` when this build has not been routed. + """ + for entry in reversed(list(self.shared_state.enablement.build_manifest or [])): + if isinstance(entry, dict) and str(entry.get("task_id") or "") == build_task_id: + return entry + return None + + def _note_build_routed(self, build_task_id: str, **fields: Any) -> None: + """Record that a build's outcome has been routed, and to what. + + Args: + build_task_id: The ``targeted_build`` row that was routed. + fields: What it was routed to, for a build whose outcome is a probe. + """ + manifest = list(self.shared_state.enablement.build_manifest or []) + for idx, entry in enumerate(manifest): + if isinstance(entry, dict) and str(entry.get("task_id") or "") == build_task_id: + manifest[idx] = {**entry, **fields} + break + else: + manifest.append({"task_id": build_task_id, "routed": True, **fields}) + self.shared_state.enablement.build_manifest = manifest + + async def _build_probe_was_cancelled(self, routed: dict[str, Any]) -> bool: + """Whether the probe a build was routed to was stopped before it ran. + + A cancelled probe is no evidence about the build: the queue scan drops a + queued row the wall-clock budget can no longer fit, and a phase boundary + drops one the new phase does not allow. Either way the built runtime was + never launched, so the build is still owed a probe -- and without noticing + that, the manifest entry written when the first one was opened keeps this + build accounted for permanently, across resumes included. + + Args: + routed: The build's routing record. + + Returns: + ``True`` when the recorded probe exists and was cancelled. + """ + from ..state.task_registry import TaskNotFound + + probe_tid = str(routed.get("probe_task_id") or "").strip() + if not probe_tid: + return False + try: + probe = await self.tasks.get(probe_tid) + except TaskNotFound: + # Pruned rather than cancelled; re-probing on a row that is gone + # would re-probe on every later tick too. + return False + return str(getattr(probe, "state", "") or "") == "cancelled" + + async def _enqueue_build_launch_probe( + self, + build_task_id: str, + br: Any, + *, + generation: int = 0, + ) -> tuple[str, int]: """Enqueue an integrate_patch launch probe for a verified build. Runs the built runtime through the enablement runnable gate without @@ -4485,9 +4587,34 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None _maybe_rearm_authored_lane → _maybe_rearm_enablement, producing a genuine KEEP/advanced/reverted outcome. The whole-machine GPU pool is acquired via _framework_gpu_params. + + The probe is what declares KEEP for a build, so it must not be opened + into a session that cannot run it: the queue scan drops a queued row the + wall-clock budget can no longer fit, and a probe cancelled that way + leaves the build verified but never launched. So the same gate the scan + asks is asked here first, and a denial enqueues nothing -- the build + stays unrouted, and the tick or resume that can afford a probe opens one. + + Args: + build_task_id: The verified build this probe launches. + br: Its ``BuildResult``, read for the runtime override. + generation: The probe generation to try first, from what this build + was routed to before. + + Returns: + The probe ``task_id`` and the generation it sits on; the id is empty + when nothing was enqueued. """ from hyperloom.agents.framework.enablement import classify_failure + denied = self._time_budget_denial_for_action("integrate_patch") + if denied is not None: + log.info( + "ENABLEMENT: build launch probe held for build=%s, not enqueued -- %s", + build_task_id, + denied, + ) + return "", generation state = self.shared_state runtime_override = br.runtime.to_runtime_override() launch_log = str(state.enablement.launch_log or "") @@ -4509,22 +4636,27 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None ) if cfg: params["config_path"] = cfg - idem = f"build_launch_probe:{build_task_id}" lanes, ttl = self._framework_authoring_lanes_ttl(params, base_ttl_sec=3600) - probe_task, existing = await self.tasks.create_or_return_existing( + probe_task, generation = await self._open_row_past_spent_generations( kind="integrate_patch", params=params, - idempotency_key=idem, + key_for=lambda gen: f"build_launch_probe:{build_task_id}:gen{gen}", + generation=generation, + label="build launch probe", requires_lanes=lanes, lease_ttl_sec=ttl, ) + if probe_task is None: + return "", generation probe_tid = str(getattr(probe_task, "task_id", "") or "") log.info( - "ENABLEMENT: build launch probe %s task=%s (existing=%s)", - "re-used" if existing else "enqueued", + "ENABLEMENT: build launch probe task=%s gen%d state=%s (build=%s)", probe_tid, - existing, + generation, + getattr(probe_task, "state", ""), + build_task_id, ) + return probe_tid, generation async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: """Enqueue one genuine baseline to revalidate a KEEP'd eval-origin patch. @@ -4601,14 +4733,6 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: async def _open_revalidation_row(self, params: dict[str, Any]) -> "Task | None": """Resolve this revalidation window's task row, on a generation it can use. - The generation is in the idempotency key so each window gets a fresh row - even when a prior window's row is in a terminal state. A key that resolves - to a terminal row is a spent generation rather than an enqueue, and it has - to be recognised as one: a row cancelled at dispatch never produces a - result to route, so nothing downstream advances the generation past it, - and every later tick would resolve this window to a row that measured - nothing. - Args: params: The baseline params for the revalidation row. @@ -4616,28 +4740,76 @@ async def _open_revalidation_row(self, params: dict[str, Any]) -> "Task | None": The row to track, or ``None`` when this tick found only spent generations -- the window stays open and the next tick tries again. """ + state = self.shared_state + task, generation = await self._open_row_past_spent_generations( + kind="baseline", + params=params, + key_for=lambda gen: f"enablement_revalidation:gen{gen}", + generation=int(state.enablement.revalidation_generation or 0), + label="revalidation", + ) + state.enablement.revalidation_generation = generation + return task + + async def _open_row_past_spent_generations( + self, + *, + kind: str, + params: dict[str, Any], + key_for: Callable[[int], str], + generation: int, + label: str, + attempts: int = 2, + **create_kwargs: Any, + ) -> tuple["Task | None", int]: + """Create or re-use a task row, skipping generations already spent. + + A generation in the idempotency key is what lets one piece of work get a + fresh row after an earlier attempt at it went terminal. That only holds if + a key resolving to a terminal row is recognised as a spent generation + rather than an enqueue: a row cancelled at dispatch -- which is what the + queue scan does to work the wall-clock budget can no longer fit -- never + produces a result to route, so nothing downstream advances the generation + past it, and every later attempt resolves to a row that measured nothing. + + Args: + kind: The task kind to create. + params: The task params. + key_for: Builds the idempotency key for a generation number. + generation: The generation to try first. + label: How this work is named in the log when a generation is spent. + attempts: How many generations to try before giving up this pass. + **create_kwargs: Passed through to + :meth:`TaskRegistry.create_or_return_existing` (lanes, TTL). + + Returns: + The row to use and the generation it sits on, or ``None`` with the + generation to try next when every attempt this pass found a spent + one. The caller persists the generation, since only the caller knows + where it lives. + """ from ..state.task_registry import TERMINAL_STATES - state = self.shared_state - for _attempt in range(2): - gen = int(state.enablement.revalidation_generation or 0) + for _attempt in range(max(1, attempts)): task, _existing = await self.tasks.create_or_return_existing( - kind="baseline", + kind=kind, params=params, - idempotency_key=f"enablement_revalidation:gen{gen}", + idempotency_key=key_for(generation), + **create_kwargs, ) if str(getattr(task, "state", "") or "") not in TERMINAL_STATES: - return task + return task, generation log.warning( - "ENABLEMENT revalidation: gen%d resolves to terminal task %s (%s); " - "opening generation %d so the window is not stuck on it", - gen, + "ENABLEMENT %s: gen%d resolves to terminal task %s (%s); opening " + "generation %d so the work is not stuck on it", + label, + generation, getattr(task, "task_id", ""), getattr(task, "state", ""), - gen + 1, + generation + 1, ) - state.enablement.revalidation_generation = gen + 1 - return None + generation += 1 + return None, generation async def _pump_enablement_safely(self, *, caller: str) -> None: """Phase-independent enablement pump — runs every tick. From 4a9b65a4bd876b67567ce654bca2aa7b87f5e4de Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 23:50:30 +0000 Subject: [PATCH 36/65] tests: every pass of a round is asserted to carry the session deadline The deadline is what lets the reaper attribute a budget kill to the run instead of to the variant, and each launch site hands it over on its own. Only the measured round was asserted on, so deleting the kwarg from either multi-node-warmup launch left both suites green -- and those two lines are also where this branch conflicts with the progress-heartbeat branch, so a merge resolution that dropped this side would have shipped green too. Parameterize the two deadline tests over the round slot, so a variant's discarded warmup, its client warmup and its measured round are each covered by construction rather than one at a time, and add the catch-all a pass added later trips without anyone remembering to assert on it. The grid's launch-recording double now records the whole call keyed by the slot it ran in, which the ad-hoc double in the deadline test was already doing by hand. Co-authored-by: Cursor --- .../inference_optimizer/tests/conftest.py | 17 ++ .../tests/test_baseline_warmup_double_run.py | 72 +++++-- .../tests/test_grid_runner.py | 182 ++++++++++++------ 3 files changed, 200 insertions(+), 71 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/conftest.py b/src/hyperloom/inference_optimizer/tests/conftest.py index b801dc0e7a..e694072a46 100644 --- a/src/hyperloom/inference_optimizer/tests/conftest.py +++ b/src/hyperloom/inference_optimizer/tests/conftest.py @@ -46,6 +46,23 @@ def _bootstrap_kernel_agent_env() -> None: _bootstrap_kernel_agent_env() +def launches_by_round_slot(recorded: list[dict]) -> dict[str, dict]: + """Index recorded benchmark launches by the output slot each round ran in. + + A round is identified by the slot it writes into rather than by its position + in the launch order, so a test can assert on one pass of a round without + encoding how many passes precede it. + + Args: + recorded: Launch records, each carrying the ``round_slot`` name the + subprocess doubles stamp on every round they see. + + Returns: + dict[str, dict]: The last launch recorded per slot name. + """ + return {launch["round_slot"]: launch for launch in recorded} + + def seed_target_analysis_marker(session_dir: Path) -> Path: """Write a ``no_target_gpu_configured`` marker JSON at the session dir.""" from hyperloom.inference_optimizer.session.session_paths import target_baseline_json diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index fe0995a972..031133acb7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -48,6 +48,8 @@ from .conftest import chatty_child, enable_multi_node, suppression_window_s +from .conftest import launches_by_round_slot + @pytest.fixture(autouse=True) def _isolate_leak_root(tmp_path_factory, monkeypatch): @@ -1834,6 +1836,15 @@ def test_teardown_lifecycle_server_removes_state_files(tmp_path): assert not (pid_dir / "vllm_8888.json").exists() +# Every output slot a multi-node baseline round launches a benchmark process +# into, in launch order. The discarded client warmup is a full pass and costs the +# same wall-clock as the measured round it precedes, so both need the deadline. +# ``mn_warmup`` is production's name for the warmup slot; the measured round runs +# in the task's own output dir, which these tests name. +_MEASURED_ROUND_SLOT = "measured_round" +_BASELINE_ROUND_SLOTS = ("mn_warmup", _MEASURED_ROUND_SLOT) + + def _budgeted_state(*, remaining_sec: float | None) -> SimpleNamespace: """A session state whose only content is how much wall-clock is left.""" return SimpleNamespace( @@ -1844,19 +1855,40 @@ def _budgeted_state(*, remaining_sec: float | None) -> SimpleNamespace: def _capturing_fake_run(returncode: int = 0, *, produces_workspace: bool = True): - """A ``run_with_session_kill`` stand-in that records how it was called.""" + """A ``run_with_session_kill`` stand-in that records how each round was launched. + + Every record carries the ``round_slot`` the round wrote into, so a launch can + be looked up by which pass it was rather than by the order it happened in. + """ calls: list[dict] = [] def fake_run(cmd, *args, **kwargs): - calls.append(dict(kwargs)) + slot = Path(cmd[cmd.index("--output-dir") + 1]) + calls.append({"round_slot": slot.name, **kwargs}) if produces_workspace: - out_idx = cmd.index("--output-dir") - _fake_workspace(Path(cmd[out_idx + 1]), tput=_HOT_TPUT) + _fake_workspace(slot, tput=_HOT_TPUT) return subprocess.CompletedProcess(cmd, returncode, "ok", "") return fake_run, calls +def _enable_multi_node(monkeypatch) -> None: + """Put the executor on the multi-node path with the per-round restart stubbed. + + Multi-node is what adds the discarded client-warmup pass in front of the + measured round, so it is the only mode in which a baseline round launches + more than one benchmark process. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl + + monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", "2") + + async def fake_restart_server_for_round(*_args, **_kwargs): + return None + + monkeypatch.setattr(mnl, "restart_server_for_round", fake_restart_server_for_round) + + def _run_baseline_under_budget( tmp_path, *, @@ -1877,7 +1909,7 @@ def _run_baseline_under_budget( ) ctx = _make_ctx( { - "output_dir": str(tmp_path / "ws"), + "output_dir": str(tmp_path / _MEASURED_ROUND_SLOT), "timeout_sec": timeout_sec, "gpu_type": "mi300x", } @@ -1901,10 +1933,27 @@ class TestTheSessionBudgetReachesTheBaselineRound: reached the reaper, and nothing clamped the cap to what was left. """ - def test_the_deadline_reaches_the_reaper(self, tmp_path): + @pytest.mark.parametrize("round_slot", _BASELINE_ROUND_SLOTS) + def test_the_deadline_reaches_the_reaper(self, tmp_path, monkeypatch, round_slot): + """Parameterized over the passes a round launches, not just the measured one. + + The reaper is the only thing that attributes a budget kill correctly, and + it only knows about the deadline it was handed. A pass launched without + one runs until its own hard cap and comes back looking like a variant + that timed out. + """ + _enable_multi_node(monkeypatch) + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert launches_by_round_slot(calls)[round_slot]["session_deadline_sec"] is not None + + def test_no_pass_of_a_round_is_launched_without_the_deadline(self, tmp_path, monkeypatch): + """The net for a pass added later, which no per-slot test would know about.""" + _enable_multi_node(monkeypatch) _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) - assert calls and calls[0]["session_deadline_sec"] is not None + assert set(launches_by_round_slot(calls)) >= set(_BASELINE_ROUND_SLOTS) + assert [c["round_slot"] for c in calls if c.get("session_deadline_sec") is None] == [] def test_the_hang_backstop_is_clamped_to_what_is_left(self, tmp_path): """A cap larger than the budget outlives the session it belongs to.""" @@ -1957,16 +2006,9 @@ def test_a_cancelled_multi_node_warmup_does_not_go_on_to_the_measured_round( run has already been told to stop spending -- and grades the baseline on a round started after the stop. """ - from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl - base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") - monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", "2") - - async def fake_restart_server_for_round(*_args, **_kwargs): - return None - - monkeypatch.setattr(mnl, "restart_server_for_round", fake_restart_server_for_round) + _enable_multi_node(monkeypatch) launched: list[str] = [] def fake_run(cmd, *args, **kwargs): diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index 07a5a79d05..7a165f53a6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -40,6 +40,8 @@ run_grid, ) +from .conftest import launches_by_round_slot + # Section 1: _write_variant_abort_marker @@ -1397,15 +1399,16 @@ def fake_run(cmd, *args, **kwargs): assert [r.status for r in results] == ["succeeded", "succeeded"] -def _capture_timeouts(recorded: list[tuple[str, int]]): - """A ``run_with_session_kill`` double that records each round's granted timeout. +def _capture_launches(recorded: list[dict]): + """A ``run_with_session_kill`` double that records how each round was launched. Only benchmark rounds are recorded: the interpreter probe carries no ``--output-dir`` and is module-memoized, so counting it would make these assertions depend on which test ran first. Args: - recorded: Appended to as ``(slot_name, timeout)`` per launched round. + recorded: Appended to per launched round, each record carrying the + ``round_slot`` the round wrote into alongside the launch kwargs. Returns: A callable usable as ``side_effect``. @@ -1415,13 +1418,81 @@ def fake_run(cmd, *args, **kwargs): if "--output-dir" not in cmd: return subprocess.CompletedProcess(cmd, 0, "ok", "") slot = Path(cmd[cmd.index("--output-dir") + 1]) - recorded.append((slot.name, int(kwargs["timeout"]))) + recorded.append({"round_slot": slot.name, **kwargs}) _fake_workspace(slot) return subprocess.CompletedProcess(cmd, 0, "ok", "") return fake_run +def _granted_timeouts(recorded: list[dict]) -> list[int]: + """The hard timeout each recorded round was granted, in launch order.""" + return [int(launch["timeout"]) for launch in recorded] + + +# Every output slot one variant launches a benchmark process into, in launch +# order: the discarded warmup, the multi-node client warmup, and the measured +# round. All three are full benchmark passes on the GPU. +_GRID_ROUND_SLOTS = ("warmup_round", "mn_warmup", "variant_00_v0") + + +async def _launch_every_pass_of_one_variant( + tmp_path, + monkeypatch, + *, + session_deadline_sec: float | None, +) -> list[dict]: + """Run one variant with every optional pass enabled, recording each launch. + + The passes are independently gated -- the discarded warmup on lifecycle + eligibility, the client warmup on multi-node -- so a test that wants to reach + every launch site the grid has must turn all of them on at once. + + Args: + tmp_path: Test-scoped directory for the config and the output root. + monkeypatch: Used to put the grid on the multi-node path. + session_deadline_sec: The session deadline handed to ``run_grid``. + + Returns: + list[dict]: One record per launched round, as ``_capture_launches`` makes + them. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_env as mne + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnsl + + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + monkeypatch.setattr(mne, "is_multi_node", lambda: True) + monkeypatch.setattr(mne, "mn_bench_warmup_enabled", lambda: True) + + async def fake_restart_server_for_round(**_kwargs): + return None + + monkeypatch.setattr(mnsl, "restart_server_for_round", fake_restart_server_for_round) + recorded: list[dict] = [] + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + ): + await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=session_deadline_sec, + variant_expected_sec=30.0, + warmup_before_measure=True, + ) + return recorded + + class TestSessionGridBounds: """One definition of the two numbers every benching arm needs. @@ -1468,11 +1539,11 @@ class TestSessionBudgetAdmission: async def test_variant_runs_when_budget_fits_expected_but_not_the_backstop(self, tmp_path): base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): results = await run_grid( base_yaml_path=base, @@ -1491,11 +1562,11 @@ async def test_variant_runs_when_budget_fits_expected_but_not_the_backstop(self, async def test_variant_skipped_when_budget_cannot_fit_the_expected_runtime(self, tmp_path): base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): results = await run_grid( base_yaml_path=base, @@ -1515,11 +1586,11 @@ async def test_without_an_estimate_the_stricter_backstop_check_is_kept(self, tmp """Callers that cannot estimate keep the pre-existing, stricter gate.""" base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): results = await run_grid( base_yaml_path=base, @@ -1547,11 +1618,11 @@ class TestSessionBudgetTimeoutClamp: async def test_granted_cap_is_clamped_to_the_remaining_budget(self, tmp_path): base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): await run_grid( base_yaml_path=base, @@ -1564,7 +1635,7 @@ async def test_granted_cap_is_clamped_to_the_remaining_budget(self, tmp_path): ) assert len(recorded) == 1 - granted = recorded[0][1] + granted = _granted_timeouts(recorded)[0] # The cap is allowed a small grace past the deadline so the in-process # session watchdog trips first and attributes the kill correctly. assert 60 <= granted <= 120 + _SESSION_KILL_GRACE_SEC, ( @@ -1575,11 +1646,11 @@ async def test_granted_cap_is_clamped_to_the_remaining_budget(self, tmp_path): async def test_declared_cap_is_kept_when_the_budget_is_larger(self, tmp_path): base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): await run_grid( base_yaml_path=base, @@ -1591,17 +1662,17 @@ async def test_declared_cap_is_kept_when_the_budget_is_larger(self, tmp_path): variant_expected_sec=30.0, ) - assert [t for _, t in recorded] == [600] + assert _granted_timeouts(recorded) == [600] @pytest.mark.asyncio async def test_no_deadline_leaves_the_declared_cap_untouched(self, tmp_path): base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): await run_grid( base_yaml_path=base, @@ -1613,7 +1684,7 @@ async def test_no_deadline_leaves_the_declared_cap_untouched(self, tmp_path): variant_expected_sec=30.0, ) - assert [t for _, t in recorded] == [600] + assert _granted_timeouts(recorded) == [600] class TestSessionKillAttribution: @@ -1656,11 +1727,11 @@ async def test_the_hard_cap_leaves_room_for_the_session_watchdog_to_win(self, tm """ base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ): await run_grid( base_yaml_path=base, @@ -1673,42 +1744,41 @@ async def test_the_hard_cap_leaves_room_for_the_session_watchdog_to_win(self, tm ) assert len(recorded) == 1 - granted = recorded[0][1] + granted = _granted_timeouts(recorded)[0] assert granted > 60, f"hard cap {granted}s must sit past the ~60s deadline, not on it" assert granted <= 90, f"the grace must stay small, got {granted}s" + @pytest.mark.parametrize("round_slot", _GRID_ROUND_SLOTS) @pytest.mark.asyncio - async def test_the_session_deadline_reaches_the_subprocess_layer(self, tmp_path): - """Regression: the clamped cap alone bounds the round but mislabels the kill.""" - base = tmp_path / "base.yaml" - _write_baseline_yaml_overrides(base) + async def test_the_session_deadline_reaches_the_subprocess_layer(self, tmp_path, monkeypatch, round_slot): + """Regression: the clamped cap alone bounds the round but mislabels the kill. + + Parameterized over every pass a variant costs, because each launch site + hands the deadline over on its own. A site that leaves it out is still + bounded by its clamped cap, so the round still ends -- as a + ``TimeoutExpired`` the ledger reads as a variant too slow to measure. + """ deadline = time.monotonic() + 120.0 - seen: list[float | None] = [] + recorded = await _launch_every_pass_of_one_variant( + tmp_path, + monkeypatch, + session_deadline_sec=deadline, + ) - def fake_run(cmd, *args, **kwargs): - # The module-memoized interpreter probe is not a benchmark round and - # correctly carries no session deadline; only rounds are of interest. - if "--output-dir" not in cmd: - return subprocess.CompletedProcess(cmd, 0, "ok", "") - seen.append(kwargs.get("session_deadline_sec")) - _fake_workspace(Path(cmd[cmd.index("--output-dir") + 1])) - return subprocess.CompletedProcess(cmd, 0, "ok", "") + assert launches_by_round_slot(recorded)[round_slot]["session_deadline_sec"] == deadline - with patch( - "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=fake_run, - ): - await run_grid( - base_yaml_path=base, - base_extra_args="", - grid=[GridVariant("v0")], - output_root=tmp_path / "out", - variant_timeout_sec=600, - session_deadline_sec=deadline, - variant_expected_sec=30.0, - ) + @pytest.mark.asyncio + async def test_no_pass_of_a_variant_is_launched_without_the_deadline(self, tmp_path, monkeypatch): + """The net for a pass added later, which no per-slot test would know about.""" + deadline = time.monotonic() + 120.0 + recorded = await _launch_every_pass_of_one_variant( + tmp_path, + monkeypatch, + session_deadline_sec=deadline, + ) - assert seen == [deadline] + assert set(launches_by_round_slot(recorded)) >= set(_GRID_ROUND_SLOTS) + assert [c["round_slot"] for c in recorded if c.get("session_deadline_sec") != deadline] == [] def _reaping_round(returncode: int, *, slot_name: str): @@ -1870,12 +1940,12 @@ async def test_admission_accounts_for_the_warmup_pass(self, tmp_path): """ base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with ( patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ), patch( "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", @@ -1900,12 +1970,12 @@ async def test_admission_accounts_for_the_warmup_pass(self, tmp_path): async def test_warmup_cap_reserves_budget_for_the_measured_round(self, tmp_path): base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - recorded: list[tuple[str, int]] = [] + recorded: list[dict] = [] with ( patch( "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=_capture_timeouts(recorded), + side_effect=_capture_launches(recorded), ), patch( "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", @@ -1923,9 +1993,9 @@ async def test_warmup_cap_reserves_budget_for_the_measured_round(self, tmp_path) warmup_before_measure=True, ) - by_round = dict(recorded) - warmup = next(t for slot, t in recorded if "warmup" in slot) - measure = next(t for slot, t in recorded if "warmup" not in slot) + by_round = launches_by_round_slot(recorded) + warmup = next(int(c["timeout"]) for c in recorded if "warmup" in c["round_slot"]) + measure = next(int(c["timeout"]) for c in recorded if "warmup" not in c["round_slot"]) assert warmup <= 240 + _SESSION_KILL_GRACE_SEC, ( f"warmup cap must hold back the measured round's 60s, got {warmup}" ) From 524179446127a44806596258db27373afee1c0ea Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Thu, 13 Aug 2026 23:55:09 +0000 Subject: [PATCH 37/65] baseline: the multi-node warmup holds the measured round's budget back The clamp was computed once and handed to both passes, so each could be granted the whole remaining budget. The absolute deadline still stops the session, so nothing overruns it -- but a slow warmup spends what the measured round needed, that round is admitted with nothing left, and the pass that gets reaped is the only one that would have produced a data point. The baseline is then reported as session_time_exhausted although its warmup succeeded, after a round of GPU time that could never have finished. Reserve the measured round's cap when clamping the warmup, which is how the grid already prices the passes that follow its own warmups, so the two arms spell the same idea the same way. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 25 +++++++++++++++++++ .../actions/executors/baseline.py | 25 +++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 031133acb7..83b190d3a4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -2040,6 +2040,31 @@ def fake_run(cmd, *args, **kwargs): assert launched == ["mn_warmup"], f"the measured round ran after the cancel: {launched}" assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + def test_the_multi_node_warmup_cap_reserves_budget_for_the_measured_round( + self, + tmp_path, + monkeypatch, + ): + """One clamp handed to both passes grants each of them the whole budget. + + The absolute deadline still stops the session, so nothing overruns it -- + but a warmup granted the measured round's cap can spend the budget that + round needed, which is then admitted with nothing left and reaped. The + result is a baseline reported as ``session_time_exhausted`` whose warmup + succeeded, after a round of GPU time that could never have finished. + """ + _enable_multi_node(monkeypatch) + _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=1000.0, timeout_sec=600) + launches = launches_by_round_slot(calls) + + warmup = launches["mn_warmup"]["timeout"] + measured = launches[_MEASURED_ROUND_SLOT]["timeout"] + assert measured == 600, f"the measured round keeps its declared cap, got {measured}" + assert warmup <= 1000 - 600 + _SESSION_KILL_GRACE_SEC, ( + f"the warmup cap must hold back the measured round's {measured}s, got {warmup}" + ) + assert warmup < measured, "the warmup must be granted less than the round it reserves budget for" + def test_the_profile_arm_gets_all_of_it(self, tmp_path): """Profile is the same executor with a four-hour default -- longer than any session budget it could be given.""" diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 248b35cd03..68577878be 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1768,6 +1768,7 @@ def _session_capped_timeout( session_deadline_sec: float | None, *, output_dir: Path, + reserve_sec: float = 0.0, ) -> int: """``timeout_sec`` reduced to what the session can still pay for. @@ -1776,16 +1777,25 @@ def _session_capped_timeout( of the session is left. A round granted more than the budget has runs past the end of the session and takes the closing phase with it. + ``reserve_sec`` holds back what the passes still to come need, so an + earlier pass cannot spend the budget a later one was counting on. This is + the baseline's spelling of :func:`~._grid_runner._round_timeout_sec`'s + reserve, and it is kept for the same pass: the measured round is the only + one that yields a data point, so a discarded warmup that overruns is the + cheaper thing to cut short. + Args: timeout_sec: The timeout this round would get on an unbounded budget. session_deadline_sec: Monotonic-clock session deadline, or ``None`` when there is no budget to respect. output_dir: The round's workspace, for the log line. + reserve_sec: Seconds held back for the passes that must still follow + this one. Zero for the last pass of a round. Returns: int: The hard timeout to grant this round, in seconds. """ - clamped = session_clamped_timeout_sec(timeout_sec, session_deadline_sec) + clamped = session_clamped_timeout_sec(timeout_sec, session_deadline_sec, reserve_sec=reserve_sec) if clamped != timeout_sec: log.info( "baseline_executor: timeout clamped %ds -> %ds by the session budget (round=%s)", @@ -3778,6 +3788,17 @@ async def _run_single_benchmark( if _mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted"): _mn_warm_dir = output_dir / "mn_warmup" + # The measured round's whole cap is held back from the warmup. Handed + # the same cap, a slow warmup can spend the budget the measured round + # is then admitted without -- so the round that could never finish is + # the one launched and reaped, and the baseline is reported as + # ``session_time_exhausted`` after a warmup that actually succeeded. + _mn_warm_timeout_sec = self._session_capped_timeout( + timeout_sec, + session_deadline_sec, + output_dir=_mn_warm_dir, + reserve_sec=timeout_sec, + ) _mn_warm_started_unix = time.time() # The measurement is discarded, but the returncode is not: this pass # is a full benchmark round, so a stop here ends the baseline round. @@ -3801,7 +3822,7 @@ async def _run_single_benchmark( _mn_warm_cmd, env=_mn_warm_env, cwd=str(_mn_warm_dir), - timeout=timeout_sec, + timeout=_mn_warm_timeout_sec, server_log_path=_watchdog_server_log_path(_mn_warm_dir, framework), on_output=_mn_warm_activity.note, session_deadline_sec=session_deadline_sec, From 4177de8d806f5a60828a0b3e330919b1bd4556df Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:01:01 +0000 Subject: [PATCH 38/65] stop_attribution: say where the returncode side of the distinction lives The module docstring pointed a reader at the sentinel space for "the returncode side", where the two codes are but the function that decodes them is not, and the baseline's import of six names from the private grid module said nothing about why. Record both: the codes stay with the allocation map so a collision stays visible in one file, the decoder stays with the session-budget helpers both arms share, and neither can join its class-side sibling in the leaf because the executors package imports that leaf. Co-authored-by: Cursor --- .../orchestrator/actions/executors/baseline.py | 6 ++++++ .../orchestrator/actions/stop_attribution.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 68577878be..96def09c78 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -47,6 +47,12 @@ probe_aiter_jit_cache as _probe_aiter_jit_cache, sweep_stale_aiter_locks_if_dead, ) +# The grid module is the namespace the helpers both benching arms share ended up +# in: how a sentinel returncode reads back, how a round's cap is clamped to the +# budget, how the two session bounds are resolved, and the hygiene every launch +# needs. Imported rather than restated here so a baseline round and a grid round +# are priced the same way. The returncode decoder's class-side sibling is in +# ``..stop_attribution``, which says why the two sides sit where they do. from ._grid_runner import ( _kill_stale_servers, sanitize_result_dir, diff --git a/src/hyperloom/orchestrator/actions/stop_attribution.py b/src/hyperloom/orchestrator/actions/stop_attribution.py index 3bcd132fbd..c207f63b33 100644 --- a/src/hyperloom/orchestrator/actions/stop_attribution.py +++ b/src/hyperloom/orchestrator/actions/stop_attribution.py @@ -17,9 +17,16 @@ notion lives here, in a leaf every one of them can import, rather than as three local judgements that can drift apart. -The returncode side of the same distinction lives in -:mod:`..executors._subprocess_kill`, which owns the sentinel space; this module -is the error-class side, which is what the ledgers carry. +This module is the error-class side, which is what the ledgers carry. The +returncode side of the same distinction is deliberately not here, and is itself +in two pieces: the two sentinel codes are allocated in +:mod:`..executors._subprocess_kill`, beside every other code that space hands +out so that a collision is visible in one file, and ``stopped_by_the_run``, the +function that reads a code back, sits with the session-budget helpers in +``..executors._grid_runner`` that both benching arms already share. Neither +piece can move here: this leaf is imported by the executors package, so naming +a returncode would close an import cycle. A reader following a sentinel +returncode therefore starts there and arrives at the classes below. """ from __future__ import annotations From 8111ba6b45fee62067f4cfcf6c42eca9d92dca39 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:07:31 +0000 Subject: [PATCH 39/65] tests: one multi-node setup for both benching arms' suites Both suites now need a round that launches more than one benchmark pass, and each had its own way of getting there -- the baseline's set the node count, the grid's patched the two multi-node predicates. Neither is about what those tests assert, so keep one helper next to the other shared subprocess doubles and let both arms reach the same mode the same way. Co-authored-by: Cursor --- .../inference_optimizer/tests/conftest.py | 20 ++++++++++++++ .../tests/test_baseline_warmup_double_run.py | 27 ++++--------------- .../tests/test_grid_runner.py | 13 ++------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/conftest.py b/src/hyperloom/inference_optimizer/tests/conftest.py index e694072a46..a894bf54c9 100644 --- a/src/hyperloom/inference_optimizer/tests/conftest.py +++ b/src/hyperloom/inference_optimizer/tests/conftest.py @@ -46,6 +46,26 @@ def _bootstrap_kernel_agent_env() -> None: _bootstrap_kernel_agent_env() +def enable_multi_node(monkeypatch, nodes: int = 2) -> None: + """Put the executors in multi-node mode with a no-op per-round server restart. + + Multi-node is what puts a discarded client-warmup pass in front of a measured + round, so it is the mode in which one round launches more than one benchmark + process -- and the restart between them is the part that needs a cluster. + + Args: + monkeypatch: The requesting test's monkeypatch fixture. + nodes: How many nodes to claim, which is what the executors read. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl + + async def _no_restart(*_args, **_kwargs) -> None: + return None + + monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) + monkeypatch.setattr(mnl, "restart_server_for_round", _no_restart) + + def launches_by_round_slot(recorded: list[dict]) -> dict[str, dict]: """Index recorded benchmark launches by the output slot each round ran in. diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 83b190d3a4..e5c1c0312a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -48,7 +48,7 @@ from .conftest import chatty_child, enable_multi_node, suppression_window_s -from .conftest import launches_by_round_slot +from .conftest import enable_multi_node, launches_by_round_slot @pytest.fixture(autouse=True) @@ -1872,23 +1872,6 @@ def fake_run(cmd, *args, **kwargs): return fake_run, calls -def _enable_multi_node(monkeypatch) -> None: - """Put the executor on the multi-node path with the per-round restart stubbed. - - Multi-node is what adds the discarded client-warmup pass in front of the - measured round, so it is the only mode in which a baseline round launches - more than one benchmark process. - """ - from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl - - monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", "2") - - async def fake_restart_server_for_round(*_args, **_kwargs): - return None - - monkeypatch.setattr(mnl, "restart_server_for_round", fake_restart_server_for_round) - - def _run_baseline_under_budget( tmp_path, *, @@ -1942,14 +1925,14 @@ def test_the_deadline_reaches_the_reaper(self, tmp_path, monkeypatch, round_slot one runs until its own hard cap and comes back looking like a variant that timed out. """ - _enable_multi_node(monkeypatch) + enable_multi_node(monkeypatch) _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) assert launches_by_round_slot(calls)[round_slot]["session_deadline_sec"] is not None def test_no_pass_of_a_round_is_launched_without_the_deadline(self, tmp_path, monkeypatch): """The net for a pass added later, which no per-slot test would know about.""" - _enable_multi_node(monkeypatch) + enable_multi_node(monkeypatch) _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) assert set(launches_by_round_slot(calls)) >= set(_BASELINE_ROUND_SLOTS) @@ -2008,7 +1991,7 @@ def test_a_cancelled_multi_node_warmup_does_not_go_on_to_the_measured_round( """ base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") - _enable_multi_node(monkeypatch) + enable_multi_node(monkeypatch) launched: list[str] = [] def fake_run(cmd, *args, **kwargs): @@ -2053,7 +2036,7 @@ def test_the_multi_node_warmup_cap_reserves_budget_for_the_measured_round( result is a baseline reported as ``session_time_exhausted`` whose warmup succeeded, after a round of GPU time that could never have finished. """ - _enable_multi_node(monkeypatch) + enable_multi_node(monkeypatch) _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=1000.0, timeout_sec=600) launches = launches_by_round_slot(calls) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index 7a165f53a6..14b92d0287 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -40,7 +40,7 @@ run_grid, ) -from .conftest import launches_by_round_slot +from .conftest import enable_multi_node, launches_by_round_slot # Section 1: _write_variant_abort_marker @@ -1457,18 +1457,9 @@ async def _launch_every_pass_of_one_variant( list[dict]: One record per launched round, as ``_capture_launches`` makes them. """ - from hyperloom.orchestrator.actions.executors import _multi_node_env as mne - from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnsl - base = tmp_path / "base.yaml" _write_baseline_yaml_overrides(base) - monkeypatch.setattr(mne, "is_multi_node", lambda: True) - monkeypatch.setattr(mne, "mn_bench_warmup_enabled", lambda: True) - - async def fake_restart_server_for_round(**_kwargs): - return None - - monkeypatch.setattr(mnsl, "restart_server_for_round", fake_restart_server_for_round) + enable_multi_node(monkeypatch) recorded: list[dict] = [] with ( patch( From 073e0086e8741c02e6f928063a204b9ba1a81f0d Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 00:21:29 +0000 Subject: [PATCH 40/65] tests: the Ray lease is a launch site of its own A single-node round goes through a serving lease by default, and that door is different: the lease is handed what is left of the budget as a duration, because an absolute monotonic instant means nothing in the actor's process. Deleting that argument left every suite green, so the round that ran out of time would have come back from the actor labelled a variant timeout. The site takes a different call and a different keyword from the local one, so the round-slot parameterization cannot reach it; assert on it directly, reusing the same recording double behind the lease's own signature. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index e5c1c0312a..d81036dce9 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -1872,6 +1872,28 @@ def fake_run(cmd, *args, **kwargs): return fake_run, calls +class _CapturingLease: + """A serving-lease stand-in that records how a round reached the Ray actor. + + The Ray path is the same round through a different door, and the door matters + here: the lease is handed what is left of the budget as a duration, because + the absolute deadline is a ``time.monotonic()`` instant that means nothing in + the actor's process. So it is a launch site of its own, with its own way of + losing the reaper. + """ + + def __init__(self) -> None: + self._run, self.calls = _capturing_fake_run() + + def run_session_kill(self, cmd, **kwargs) -> tuple[int | None, str, str]: + """Record the launch and answer as the actor does, with a bare triple.""" + proc = self._run(cmd, **kwargs) + return proc.returncode, proc.stdout, proc.stderr + + def close(self) -> None: + """The executor closes the lease it was given; nothing is held here.""" + + def _run_baseline_under_budget( tmp_path, *, @@ -1938,6 +1960,25 @@ def test_no_pass_of_a_round_is_launched_without_the_deadline(self, tmp_path, mon assert set(launches_by_round_slot(calls)) >= set(_BASELINE_ROUND_SLOTS) assert [c["round_slot"] for c in calls if c.get("session_deadline_sec") is None] == [] + def test_the_ray_path_is_handed_the_budget_as_a_duration(self, tmp_path, monkeypatch): + """The fourth launch site, and the one a parameterization cannot reach. + + Production runs a single-node round through a Ray lease, which is handed a + remaining duration rather than the deadline, by a different call. The + reaper in the actor's process has nothing else to go on. + """ + from hyperloom.orchestrator.actions.executors import _ray_serving + + lease = _CapturingLease() + monkeypatch.setattr(_ray_serving, "maybe_serving_lease", lambda **_kwargs: lease) + result, _calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert result["status"] == "succeeded" + launch = launches_by_round_slot(lease.calls)[_MEASURED_ROUND_SLOT] + remaining = launch.get("session_remaining_sec") + assert remaining is not None, f"the budget did not cross the process boundary: {sorted(launch)}" + assert 0 < remaining <= 3600.0 + def test_the_hang_backstop_is_clamped_to_what_is_left(self, tmp_path): """A cap larger than the budget outlives the session it belongs to.""" _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=120.0, timeout_sec=7200) From 9c03b67b56d688c519824f795263bd42a6dc1457 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 04:19:34 +0000 Subject: [PATCH 41/65] baseline: the warmup holds back the measured round's expected runtime, not its backstop Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 207 ++++++++++++++++-- .../tests/test_grid_runner.py | 115 +++++++++- .../actions/executors/_grid_runner.py | 78 ++++++- .../actions/executors/baseline.py | 122 ++++++++--- 4 files changed, 469 insertions(+), 53 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index d81036dce9..1d8dd6f396 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -1845,26 +1845,47 @@ def test_teardown_lifecycle_server_removes_state_files(tmp_path): _BASELINE_ROUND_SLOTS = ("mn_warmup", _MEASURED_ROUND_SLOT) -def _budgeted_state(*, remaining_sec: float | None) -> SimpleNamespace: - """A session state whose only content is how much wall-clock is left.""" +def _budgeted_state(*, remaining_sec: float | None, measured_expected_sec: float = 0.0) -> SimpleNamespace: + """A session state carrying how much wall-clock is left, and what a round costs. + + ``measured_expected_sec`` is this session's measured baseline round, which is + what a normally-behaving round of the same workload is expected to need; zero + is "no round measured yet", the only state a first baseline can be in. + """ return SimpleNamespace( baseline_double_run=False, grid_session_deadline_sec=lambda: (None if remaining_sec is None else time.monotonic() + remaining_sec), - baseline_runtime_sec=0.0, + baseline_runtime_sec=measured_expected_sec, ) -def _capturing_fake_run(returncode: int = 0, *, produces_workspace: bool = True): +def _capturing_fake_run( + returncode: int = 0, + *, + produces_workspace: bool = True, + pass_duration_sec: float = 0.0, +): """A ``run_with_session_kill`` stand-in that records how each round was launched. Every record carries the ``round_slot`` the round wrote into, so a launch can be looked up by which pass it was rather than by the order it happened in. + + ``pass_duration_sec`` makes the double honour the cap it was handed the way a + real benchmark pass does: a round granted less than the workload takes raises + ``TimeoutExpired`` instead of returning a report, which is the only way to see + what a cap too small to survive costs the round after it. """ calls: list[dict] = [] def fake_run(cmd, *args, **kwargs): + # The memoized interpreter probe is not a benchmark round. + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") slot = Path(cmd[cmd.index("--output-dir") + 1]) calls.append({"round_slot": slot.name, **kwargs}) + granted = float(kwargs.get("timeout") or 0.0) + if pass_duration_sec and granted < pass_duration_sec: + raise subprocess.TimeoutExpired(cmd, granted) if produces_workspace: _fake_workspace(slot, tput=_HOT_TPUT) return subprocess.CompletedProcess(cmd, returncode, "ok", "") @@ -1901,12 +1922,18 @@ def _run_baseline_under_budget( timeout_sec: int = 7200, returncode: int = 0, produces_workspace: bool = True, + measured_expected_sec: float = 0.0, + pass_duration_sec: float = 0.0, executor_cls=BaselineExecutor, ) -> tuple[dict, list[dict]]: """Run one baseline round against a session with ``remaining_sec`` left.""" base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") - fake_run, calls = _capturing_fake_run(returncode, produces_workspace=produces_workspace) + fake_run, calls = _capturing_fake_run( + returncode, + produces_workspace=produces_workspace, + pass_duration_sec=pass_duration_sec, + ) executor = executor_cls( magpie_python=sys.executable, default_config_path=base, @@ -1920,7 +1947,10 @@ def _run_baseline_under_budget( } ) # The live state arrives on the context, the way the coordinator passes it. - ctx.extra["shared_state"] = _budgeted_state(remaining_sec=remaining_sec) + ctx.extra["shared_state"] = _budgeted_state( + remaining_sec=remaining_sec, + measured_expected_sec=measured_expected_sec, + ) with patch( "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", side_effect=fake_run, @@ -1929,6 +1959,52 @@ def _run_baseline_under_budget( return result, calls +def _mn_warmup_cap_sec(calls: list[dict]) -> int | None: + """The cap the multi-node warmup pass was granted, or ``None`` when it never ran. + + Both are outcomes a round is allowed to have -- a skipped warmup is what a + single-node baseline does -- so they are told apart here rather than by a + ``KeyError`` at the assertion. + """ + launch = launches_by_round_slot(calls).get("mn_warmup") + return None if launch is None else int(launch["timeout"]) + + +def _launch_one_grid_variant_under_budget( + tmp_path, + *, + remaining_sec: float, + variant_timeout_sec: int, + variant_expected_sec: float, +) -> list[dict]: + """Run one grid variant against the same budget a baseline round would get. + + The other arm that benches on the GPU, driven through its own entry point so + the two can be compared on what they grant the passes they both run. + """ + tmp_path.mkdir(parents=True, exist_ok=True) + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + fake_run, calls = _capturing_fake_run() + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ): + _run( + run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant(name="candidate")], + output_root=tmp_path / "out", + magpie_python=sys.executable, + variant_timeout_sec=variant_timeout_sec, + session_deadline_sec=time.monotonic() + remaining_sec, + variant_expected_sec=variant_expected_sec, + ) + ) + return calls + + class TestTheSessionBudgetReachesTheBaselineRound: """The arm #1146 names as the largest hole, and the one that motivated it. @@ -2064,30 +2140,123 @@ def fake_run(cmd, *args, **kwargs): assert launched == ["mn_warmup"], f"the measured round ran after the cancel: {launched}" assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS + @pytest.mark.parametrize( + ("remaining_sec", "declared_cap_sec"), + [ + pytest.param(1000.0, 600, id="the-budget-outlasts-the-declared-cap"), + pytest.param(3600.0, 7200, id="less-budget-left-than-the-declared-cap"), + ], + ) def test_the_multi_node_warmup_cap_reserves_budget_for_the_measured_round( self, tmp_path, monkeypatch, + remaining_sec, + declared_cap_sec, ): - """One clamp handed to both passes grants each of them the whole budget. + """The warmup holds back one measured pass, and is granted one of its own. + + Both directions are the point. Reserve too little and a warmup granted the + measured round's cap can spend the budget that round needed, which is then + admitted with nothing left and reaped. Reserve too much -- the measured + round's *backstop* rather than its expected runtime -- and nothing is left + behind it: the warmup is launched on a cap it cannot survive, killed, + swallowed as best-effort, and the measured round runs cold against a server + nothing ever drove. + + Parameterized over both sides of ``remaining < declared cap``, because + that is the boundary the two spellings of the reserve disagree on: they + agree while the whole cap fits and diverge the moment it does not. + """ + enable_multi_node(monkeypatch) + expected_sec = 300.0 + _result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=remaining_sec, + timeout_sec=declared_cap_sec, + measured_expected_sec=expected_sec, + ) + warmup = _mn_warmup_cap_sec(calls) + measured = launches_by_round_slot(calls)[_MEASURED_ROUND_SLOT]["timeout"] + assert warmup is not None, "this budget fits both passes, so the warmup must have been launched" + assert measured == pytest.approx( + min(declared_cap_sec, remaining_sec + _SESSION_KILL_GRACE_SEC), + abs=2, + ), f"the measured round keeps what the budget can pay for, got {measured}" + assert warmup >= expected_sec, ( + f"a warmup capped under the {expected_sec}s a pass takes is launched to be killed, got {warmup}" + ) + assert warmup == pytest.approx( + min(declared_cap_sec, remaining_sec - expected_sec + _SESSION_KILL_GRACE_SEC), + abs=2, + ), f"the warmup must hold back exactly one measured pass of {expected_sec}s, got {warmup}" - The absolute deadline still stops the session, so nothing overruns it -- - but a warmup granted the measured round's cap can spend the budget that - round needed, which is then admitted with nothing left and reaped. The - result is a baseline reported as ``session_time_exhausted`` whose warmup - succeeded, after a round of GPU time that could never have finished. + def test_a_warmup_that_does_not_fit_is_skipped_rather_than_launched_to_be_killed( + self, + tmp_path, + monkeypatch, + ): + """Below two passes of budget the warmup is dropped, not starved. + + A skipped warmup is a shape the round already supports -- it is what a + single-node baseline does, and what a profile round that claimed the + restart does. A warmup launched on a cap smaller than the pass it runs is + not: it raises ``TimeoutExpired`` into the best-effort handler, which + swallows it, so the measured round proceeds cold and its throughput + becomes the session's baseline anchor. Explore variants do get their + warmups, so every later gain is measured against a depressed anchor. """ enable_multi_node(monkeypatch) - _result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=1000.0, timeout_sec=600) + pass_sec = 600.0 + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=800.0, + timeout_sec=7200, + measured_expected_sec=pass_sec, + pass_duration_sec=pass_sec, + ) launches = launches_by_round_slot(calls) - warmup = launches["mn_warmup"]["timeout"] - measured = launches[_MEASURED_ROUND_SLOT]["timeout"] - assert measured == 600, f"the measured round keeps its declared cap, got {measured}" - assert warmup <= 1000 - 600 + _SESSION_KILL_GRACE_SEC, ( - f"the warmup cap must hold back the measured round's {measured}s, got {warmup}" + starved = [(slot, launch["timeout"]) for slot, launch in launches.items() if launch["timeout"] < pass_sec] + assert starved == [], f"a round was launched on a cap it cannot survive: {starved}" + assert _mn_warmup_cap_sec(calls) is None, "the warmup was launched and killed instead of being skipped" + assert _MEASURED_ROUND_SLOT in launches, "dropping the warmup must not drop the measured round" + assert result["status"] == "succeeded" + + def test_both_benching_arms_reserve_the_same_seconds_for_the_measured_round( + self, + tmp_path, + monkeypatch, + ): + """The grid and the baseline run the same warmup, so it must cost the same. + + ``session_grid_bounds`` exists so a deadline cannot be derived one way in + one executor and another way in the next. The hold-back is the other half + of that contract: an arm reserving the measured round's hang backstop + where the other reserves its expected runtime abandons a different amount + of the tail budget -- and, past ``remaining < backstop``, all of it. + """ + enable_multi_node(monkeypatch) + remaining_sec, declared_cap_sec, expected_sec = 3600.0, 7200, 600.0 + _result, baseline_calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=remaining_sec, + timeout_sec=declared_cap_sec, + measured_expected_sec=expected_sec, + ) + grid_calls = _launch_one_grid_variant_under_budget( + tmp_path / "grid_arm", + remaining_sec=remaining_sec, + variant_timeout_sec=declared_cap_sec, + variant_expected_sec=expected_sec, + ) + + baseline_warmup = _mn_warmup_cap_sec(baseline_calls) + grid_warmup = _mn_warmup_cap_sec(grid_calls) + assert grid_warmup is not None, "the grid arm ran no warmup, so there is nothing to compare" + assert baseline_warmup == pytest.approx(grid_warmup, abs=2), ( + f"the baseline arm reserved a different measured round: {baseline_warmup}s vs the grid's {grid_warmup}s" ) - assert warmup < measured, "the warmup must be granted less than the round it reserves budget for" def test_the_profile_arm_gets_all_of_it(self, tmp_path): """Profile is the same executor with a four-hour default -- longer than diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index 14b92d0287..cf79d9cf66 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -1518,6 +1518,69 @@ def test_state_without_the_deadline_accessor_is_tolerated(self): assert _grid_runner.session_grid_bounds(state) == (None, 600.0) +class TestTheWarmupHoldBack: + """The other half of ``session_grid_bounds``' contract: what a pass holds back. + + Both benching arms discard a warmup pass in front of a measured round, so both + have to hold back the same seconds for it. And because a reserve is the only + thing that can pull a cap before the session deadline, the same number decides + whether the warmup is worth launching at all. + """ + + def test_the_measured_rounds_expected_runtime_is_held_back(self): + cap = _grid_runner.session_warmup_cap_sec( + 7800, + time.monotonic() + 3600.0, + measured_expected_sec=600.0, + ) + assert cap == pytest.approx(3600 - 600 + _SESSION_KILL_GRACE_SEC, abs=2) + + def test_a_budget_that_fits_the_declared_cap_leaves_it_alone(self): + assert ( + _grid_runner.session_warmup_cap_sec( + 600, + time.monotonic() + 36000.0, + measured_expected_sec=600.0, + ) + == 600 + ) + + def test_a_warmup_that_cannot_fit_is_refused_rather_than_starved(self): + """Two passes do not fit in 800s, and the 1s floor is not a warmup.""" + assert ( + _grid_runner.session_warmup_cap_sec( + 7800, + time.monotonic() + 800.0, + measured_expected_sec=600.0, + ) + is None + ) + + def test_an_unmeasured_session_holds_nothing_back(self): + """Zero is "no round measured yet", not "the next round needs no time". + + With nothing known to reserve the cap stays past the deadline, which leaves + the session watchdog -- the only thing that attributes a budget kill + correctly -- as what stops the pass. + """ + cap = _grid_runner.session_warmup_cap_sec( + 7800, + time.monotonic() + 800.0, + measured_expected_sec=0.0, + ) + assert cap == pytest.approx(800 + _SESSION_KILL_GRACE_SEC, abs=2) + + def test_an_unbounded_budget_leaves_the_cap_alone(self): + """Not even a cap smaller than the pass is this function's business here. + + Nothing is being held back from anyone, so there is nothing for the warmup + to be short of; a declared cap under the measured runtime is a fact about + the cap, and refusing the pass over it would be this helper deciding + something the budget never asked it to. + """ + assert _grid_runner.session_warmup_cap_sec(600, None, measured_expected_sec=900.0) == 600 + + class TestSessionBudgetAdmission: """A variant is admitted on what it is expected to need, not on its backstop. @@ -1987,12 +2050,62 @@ async def test_warmup_cap_reserves_budget_for_the_measured_round(self, tmp_path) by_round = launches_by_round_slot(recorded) warmup = next(int(c["timeout"]) for c in recorded if "warmup" in c["round_slot"]) measure = next(int(c["timeout"]) for c in recorded if "warmup" not in c["round_slot"]) + assert warmup >= 60, f"a warmup capped under the 60s a pass takes is launched to be killed, got {warmup}" assert warmup <= 240 + _SESSION_KILL_GRACE_SEC, ( f"warmup cap must hold back the measured round's 60s, got {warmup}" ) - assert warmup < measure, "the warmup must be granted less than the round it reserves budget for" + assert warmup <= measure, "the warmup is never granted more than the round it holds budget back for" assert len(by_round) == 2, f"expected a warmup and a measured round, got {list(by_round)}" + @pytest.mark.asyncio + async def test_a_warmup_killed_at_a_clamped_cap_is_logged_with_the_cap_it_got(self, tmp_path, caplog): + """The abort line is the only record of how long the round was allowed. + + The declared cap is a hang backstop; what the warmup was granted is that + cap minus the reserve, and a round killed after four minutes logged as a + two-hour timeout reads as a variant that hangs rather than a budget that + ran out. + """ + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + recorded: list[dict] = [] + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + recorded.append({"round_slot": slot.name, **kwargs}) + raise subprocess.TimeoutExpired(cmd, float(kwargs.get("timeout") or 0.0)) + + with ( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=fake_run, + ), + patch( + "hyperloom.orchestrator.actions.executors._server_lifecycle.resolve_lifecycle_params", + return_value={"eligible": True, "framework": "sglang", "port": 30000, "reason": ""}, + ), + caplog.at_level("WARNING"), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=7800, + session_deadline_sec=time.monotonic() + 300.0, + variant_expected_sec=60.0, + warmup_before_measure=True, + ) + + granted = int(recorded[0]["timeout"]) + assert granted < 7800, f"this test needs a clamped cap to be about, got {granted}" + aborts = [r.message for r in caplog.records if "warmup timeout" in r.message] + assert aborts, f"the warmup abort was not logged: {[r.message for r in caplog.records]}" + assert f"timeout_sec={granted}" in aborts[0], f"the abort line reports a cap the round never had: {aborts[0]}" + assert results[0].error_class == "warmup_magpie_timeout" + class TestCompactJsonServerArgs: """JSON-valued flags must be space-free to survive Magpie's unquoted diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index bdb2301fc6..67122a2c16 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1295,6 +1295,16 @@ def session_clamped_timeout_sec( measured. The watchdog's sentinel says the run ran out of time, which is what actually happened. + A non-zero ``reserve_sec`` is the only thing that can move the returned cap + earlier than the session deadline: with no reserve the grace puts the cap at + least ``_SESSION_KILL_GRACE_SEC`` past it, so the watchdog always gets there + first. Every caller that reserves therefore re-opens the spurious-timeout + window the grace exists to close, and owes a reason its round cannot be reaped + by its own cap -- either an admission gate built from the same number + (:func:`run_grid` refuses a variant whose rounds do not fit before it reserves + for them) or the decision to skip the round instead of starting one that will + be killed (:func:`session_warmup_cap_sec`). + Args: cap: The timeout the caller would grant with an unbounded budget. session_deadline_sec: Monotonic-clock session deadline, or ``None`` when @@ -1311,6 +1321,52 @@ def session_clamped_timeout_sec( return int(cap) if usable >= int(cap) else max(1, usable) +def session_warmup_cap_sec( + cap: int, + session_deadline_sec: float | None, + *, + measured_expected_sec: float, +) -> int | None: + """The hard cap for a discarded warmup pass, or ``None`` when it must be skipped. + + A warmup is the pass whose measurement is thrown away, so it is the pass whose + budget is given up first: it holds back what the measured round after it is + expected to need. That reserve is also the only thing that can pull a cap + before the session deadline (see :func:`session_clamped_timeout_sec`), so the + same number decides whether to launch at all. A warmup granted less than one + full pass warms nothing: it is killed at its cap, both arms swallow that as + best-effort, and the measured round then runs against a server nothing ever + drove -- and its throughput is what the session anchors every later gain on. + A skipped warmup is a shape both arms already support; a warmup killed at its + cap is not. + + ``measured_expected_sec`` is the measured baseline runtime from + :func:`session_grid_bounds` -- what a normally-behaving pass of the same + workload needs, and what the admission gates upstream judge on. Zero means the + session has not measured a round yet, which leaves the warmup with no + hold-back: there is no number to reserve, and without a reserve the cap cannot + be pulled before the deadline, so the watchdog stays the thing that stops the + pass and attributes it to the budget. An unbounded budget is the same case for + the same reason -- nothing is being held back from anyone, so there is nothing + for the warmup to be short of, whatever its declared cap. + + Args: + cap: The timeout the caller would grant with an unbounded budget. + session_deadline_sec: Monotonic-clock session deadline, or ``None`` when + the budget is unbounded, which leaves ``cap`` untouched. + measured_expected_sec: Seconds one measured pass of this workload is + expected to take; ``0`` when the session has not measured one. + + Returns: + int | None: The hard timeout for the warmup pass, or ``None`` when what is + left cannot fit both it and the measured round it reserves for. + """ + if session_deadline_sec is None or measured_expected_sec <= 0: + return session_clamped_timeout_sec(cap, session_deadline_sec) + clamped = session_clamped_timeout_sec(cap, session_deadline_sec, reserve_sec=measured_expected_sec) + return clamped if clamped >= measured_expected_sec else None + + def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call. @@ -1866,6 +1922,19 @@ def _record_round_stop( warmup_workspaces_before = snapshot_workspaces(warmup_slot) warmup_started_unix = time.time() + # Held in a local because the abort line below has to name the cap the + # round was actually granted: the declared one is a hang backstop, and + # a round killed at the reserved cap logged as a two-hour timeout reads + # as a variant that hangs rather than a budget that ran out. The + # admission gate above is what keeps this reserve from starving the + # round -- it refuses a variant whose passes do not fit before any of + # them is reserved for. + warmup_cap_sec = _round_timeout_sec( + i, + variant.name, + round_label="warmup", + reserve_sec=float(variant_expected_sec or 0.0) * (1 + _mn_warmup_rounds), + ) try: warmup_rc, warmup_stdout, warmup_stderr = await _reported_magpie( i, @@ -1873,12 +1942,7 @@ def _record_round_stop( magpie_python=magpie_python, config_path=warmup_cfg_path, output_dir=warmup_slot, - timeout_sec=_round_timeout_sec( - i, - variant.name, - round_label="warmup", - reserve_sec=float(variant_expected_sec or 0.0) * (1 + _mn_warmup_rounds), - ), + timeout_sec=warmup_cap_sec, cwd=cwd, result_dir=result_dir, soft_deadline_sec=None, @@ -1899,7 +1963,7 @@ def _record_round_stop( i + 1, len(grid), variant.name, - variant_timeout_sec, + warmup_cap_sec, exc, ) _write_variant_abort_marker( diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 96def09c78..52bc9b81c2 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -59,6 +59,7 @@ sanitize_script_name, session_clamped_timeout_sec, session_grid_bounds, + session_warmup_cap_sec, stopped_by_the_run, ) from ._subprocess_kill import ( @@ -276,6 +277,31 @@ def _watchdog_server_log_path(output_dir: Path, framework: str) -> str | None: return str(output_dir / "server.log") +def _logged_session_clamp(timeout_sec: int, clamped: int, *, output_dir: Path) -> int: + """Announce a round's cap being cut by the session budget, and return the cut. + + Every pass of a baseline round derives its cap from the same budget but on its + own terms, so the line names the round it belongs to. + + Args: + timeout_sec: The cap the round would have had on an unbounded budget. + clamped: The cap the budget leaves it; equal to ``timeout_sec`` when the + budget was never the binding constraint, which logs nothing. + output_dir: The round's workspace, whose name identifies the pass. + + Returns: + int: ``clamped``, unchanged. + """ + if clamped != timeout_sec: + log.info( + "baseline_executor: timeout clamped %ds -> %ds by the session budget (round=%s)", + timeout_sec, + clamped, + output_dir.name, + ) + return clamped + + def _stopped_round_result( stopped: StoppedByTheRun, *, @@ -1774,7 +1800,6 @@ def _session_capped_timeout( session_deadline_sec: float | None, *, output_dir: Path, - reserve_sec: float = 0.0, ) -> int: """``timeout_sec`` reduced to what the session can still pay for. @@ -1783,33 +1808,69 @@ def _session_capped_timeout( of the session is left. A round granted more than the budget has runs past the end of the session and takes the closing phase with it. - ``reserve_sec`` holds back what the passes still to come need, so an - earlier pass cannot spend the budget a later one was counting on. This is - the baseline's spelling of :func:`~._grid_runner._round_timeout_sec`'s - reserve, and it is kept for the same pass: the measured round is the only - one that yields a data point, so a discarded warmup that overruns is the - cheaper thing to cut short. + Nothing is held back here, which is what keeps the cap sitting past the + session deadline: the measured round is the last pass of the round, and a + cap the session watchdog reaches first is a cap whose kill is attributed + to the budget rather than to the model. The pass that does hold budget + back is the discarded warmup, in :meth:`_mn_warmup_capped_timeout`. Args: timeout_sec: The timeout this round would get on an unbounded budget. session_deadline_sec: Monotonic-clock session deadline, or ``None`` when there is no budget to respect. output_dir: The round's workspace, for the log line. - reserve_sec: Seconds held back for the passes that must still follow - this one. Zero for the last pass of a round. Returns: int: The hard timeout to grant this round, in seconds. """ - clamped = session_clamped_timeout_sec(timeout_sec, session_deadline_sec, reserve_sec=reserve_sec) - if clamped != timeout_sec: - log.info( - "baseline_executor: timeout clamped %ds -> %ds by the session budget (round=%s)", - timeout_sec, - clamped, + return _logged_session_clamp( + timeout_sec, + session_clamped_timeout_sec(timeout_sec, session_deadline_sec), + output_dir=output_dir, + ) + + @staticmethod + def _mn_warmup_capped_timeout( + timeout_sec: int, + session_deadline_sec: float | None, + *, + output_dir: Path, + measured_expected_sec: float | None, + ) -> int | None: + """The multi-node warmup pass's cap, or ``None`` when it should be skipped. + + The decision itself is :func:`~._grid_runner.session_warmup_cap_sec`, so + both benching arms hold back the same seconds for the measured round they + discard a pass in front of; this adds the baseline's log lines. + + Args: + timeout_sec: The timeout this pass would get on an unbounded budget. + session_deadline_sec: Monotonic-clock session deadline, or ``None`` + when there is no budget to respect. + output_dir: The warmup pass's workspace, for the log line. + measured_expected_sec: Seconds the measured round is expected to take, + or ``None`` when this session has not measured one yet. + + Returns: + int | None: The hard timeout for the warmup pass, or ``None`` when the + remaining budget cannot fit it and the measured round after it. + """ + expected_sec = float(measured_expected_sec or 0.0) + clamped = session_warmup_cap_sec( + timeout_sec, + session_deadline_sec, + measured_expected_sec=expected_sec, + ) + if clamped is None: + log.warning( + "baseline_executor: skipping the MN warmup pass — what is left of the " + "session budget cannot fit both it and the measured round's expected " + "%.0fs, and a warmup capped under one pass warms nothing (round=%s)", + expected_sec, output_dir.name, ) - return clamped + return None + return _logged_session_clamp(timeout_sec, clamped, output_dir=output_dir) @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: @@ -3727,7 +3788,9 @@ async def _run_single_benchmark( # The session's wall-clock budget, resolved per round rather than once # per task: a baseline runs up to three of them, and a warmup that # overran has already spent budget the ones after it were counting on. - session_deadline_sec, _ = session_grid_bounds(ctx_extra.get("shared_state") or self.shared_state) + session_deadline_sec, measured_expected_sec = session_grid_bounds( + ctx_extra.get("shared_state") or self.shared_state + ) timeout_sec = self._session_capped_timeout(timeout_sec, session_deadline_sec, output_dir=output_dir) if not ctx_extra.get("mn_round_restarted"): try: @@ -3792,19 +3855,26 @@ async def _run_single_benchmark( mn_bench_warmup_enabled as _mn_warm, ) - if _mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted"): - _mn_warm_dir = output_dir / "mn_warmup" - # The measured round's whole cap is held back from the warmup. Handed - # the same cap, a slow warmup can spend the budget the measured round - # is then admitted without -- so the round that could never finish is - # the one launched and reaped, and the baseline is reported as - # ``session_time_exhausted`` after a warmup that actually succeeded. - _mn_warm_timeout_sec = self._session_capped_timeout( + _mn_warm_dir = output_dir / "mn_warmup" + # What the measured round is expected to need is held back from the + # warmup, the same reserve the grid builds from the same number. Handed + # the whole cap, a slow warmup can spend the budget the measured round is + # then admitted without -- so the round that could never finish is the one + # launched and reaped, and the baseline is reported as + # ``session_time_exhausted`` after a warmup that actually succeeded. + # ``None`` means what is left cannot fit both passes, and the warmup is + # the one to drop. + _mn_warm_timeout_sec = ( + self._mn_warmup_capped_timeout( timeout_sec, session_deadline_sec, output_dir=_mn_warm_dir, - reserve_sec=timeout_sec, + measured_expected_sec=measured_expected_sec, ) + if (_mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted")) + else None + ) + if _mn_warm_timeout_sec is not None: _mn_warm_started_unix = time.time() # The measurement is discarded, but the returncode is not: this pass # is a full benchmark round, so a stop here ends the baseline round. From a81d52b748f018a25aaf2a9578feba99b4d9b25e Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 06:15:29 +0000 Subject: [PATCH 42/65] cancel: a cancelled action's unwind does not stop to observe itself Co-authored-by: Cursor --- .../tests/test_grid_runner_behavior_lock.py | 186 +++++++++++++----- .../tests/test_session_time_budget.py | 39 +++- .../orchestrator/actions/cancel_channel.py | 15 ++ .../actions/executors/_grid_runner.py | 26 ++- src/hyperloom/orchestrator/loop/dispatcher.py | 10 + 5 files changed, 221 insertions(+), 55 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py index 6549e7314e..a895860cf6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner_behavior_lock.py @@ -17,6 +17,7 @@ import pytest import yaml +from hyperloom.orchestrator.actions.cancel_channel import CancelScope, use_cancel_scope from hyperloom.orchestrator.actions.executors import _grid_runner as gr from hyperloom.orchestrator.actions.executors import _multi_node_env as mne from hyperloom.orchestrator.actions.executors import ( @@ -27,6 +28,9 @@ GridVariant, run_grid, ) +from hyperloom.orchestrator.actions.executors._subprocess_kill import ( + ORCHESTRATOR_CANCELLED_RETURNCODE, +) from hyperloom.orchestrator.trace.task_progress import progress_scope from .conftest import chatty_child, suppression_window_s @@ -107,59 +111,65 @@ def _invalid_rc0_workspace(slot: Path) -> Path: # --------------------------------------------------------------------------- +def _run_with_pulse_capture( + *, + multi_node, + run_side_effect, + base, + out, + restart=None, + keep_going=True, + grid_n=1, + scope=None, + notes=None, +): + pulse_calls: list = [] + + async def fake_pulse(**kwargs): + pulse_calls.append(kwargs) + + async def collect(**note): + notes.append(note) + + with ExitStack() as st: + st.enter_context(patch.object(mne, "is_multi_node", lambda: multi_node)) + st.enter_context(patch.object(gr, "_robustness_pulse", side_effect=fake_pulse)) + st.enter_context( + patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=run_side_effect, + ) + ) + if restart is not None: + st.enter_context(patch.object(mnsl, "restart_server_for_round", restart)) + # Entered outside ``asyncio.run`` on purpose: a task copies the context + # at creation, which is how the dispatcher's scope reaches the action it + # publishes it for. + if scope is not None: + st.enter_context(use_cancel_scope(scope)) + if notes is not None: + st.enter_context(progress_scope(collect)) + grid = [GridVariant(name=f"c{i}") for i in range(grid_n)] + results = asyncio.run( + run_grid( + base_yaml_path=base, + base_extra_args="", + grid=grid, + output_root=out, + magpie_python=sys.executable, + variant_timeout_sec=10, + gpu_type="mi300x", + keep_going_on_failure=keep_going, + ) + ) + return results, pulse_calls + + class TestPulseMatrix: """``_pulse_after_variant`` (progress note plus ``_robustness_pulse``) fires on every variant outcome, including the multi-node ``mn_server_restart_failed`` path that used to leave before reaching it.""" - def _run_with_pulse_capture( - self, - *, - multi_node, - run_side_effect, - base, - out, - restart=None, - keep_going=True, - grid_n=1, - notes=None, - ): - pulse_calls: list = [] - - async def fake_pulse(**kwargs): - pulse_calls.append(kwargs) - - async def collect(**note): - notes.append(note) - - with ExitStack() as st: - st.enter_context(patch.object(mne, "is_multi_node", lambda: multi_node)) - st.enter_context(patch.object(gr, "_robustness_pulse", side_effect=fake_pulse)) - st.enter_context( - patch( - "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", - side_effect=run_side_effect, - ) - ) - if notes is not None: - st.enter_context(progress_scope(collect)) - if restart is not None: - st.enter_context(patch.object(mnsl, "restart_server_for_round", restart)) - grid = [GridVariant(name=f"c{i}") for i in range(grid_n)] - results = asyncio.run( - run_grid( - base_yaml_path=base, - base_extra_args="", - grid=grid, - output_root=out, - magpie_python=sys.executable, - variant_timeout_sec=10, - gpu_type="mi300x", - keep_going_on_failure=keep_going, - ) - ) - return results, pulse_calls - def test_mn_server_restart_failed_reaches_the_variant_boundary(self, tmp_path, monkeypatch): """A variant whose remote server never came back still ends its own row. @@ -177,7 +187,7 @@ def test_mn_server_restart_failed_reaches_the_variant_boundary(self, tmp_path, m async def _restart_fail(**_kwargs): raise mnsl.ServerRestartFailed("server /health did not return 200") - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=True, run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 0, "ok", ""), base=base, @@ -200,7 +210,7 @@ def test_mn_server_restart_failed_reports_each_variant_it_ends(self, tmp_path, m async def _restart_fail(**_kwargs): raise mnsl.ServerRestartFailed("health probe timed out") - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=True, run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 0, "ok", ""), base=base, @@ -223,7 +233,7 @@ def test_no_benchmark_workspace_failure_pulses(self, tmp_path, monkeypatch): base = tmp_path / "base.yaml" _write_base_yaml(base) - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=False, run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 1, "stdout", "boom"), base=base, @@ -298,7 +308,7 @@ def _ok(cmd, *a, **k): _valid_workspace(Path(cmd[out_idx + 1])) return subprocess.CompletedProcess(cmd, 0, "ok", "") - results, pulse_calls = self._run_with_pulse_capture( + results, pulse_calls = _run_with_pulse_capture( multi_node=False, run_side_effect=_ok, base=base, @@ -308,6 +318,78 @@ def _ok(cmd, *a, **k): assert len(pulse_calls) == 1 +class TestThePulseStaysOutOfACancelledActionsUnwind: + """A cancel is answered by unwinding, not by observing what was cancelled. + + A cooperative stop *returns* its sentinel, so ``run_grid`` walks its ordinary + stop path and would spend the pulse's whole budget between recording the row + and releasing the lease -- serially, inside the window the dispatcher gives + the action to finish. That window is derived from the terms of the unwind and + this is not one of them, so the pulse is skipped rather than budgeted for: + eight seconds of observing a tree the orchestrator just reaped is what the + rows already built are traded away for when the window expires. + + The gate is the cancel scope and not the sentinel returncode, because a + variant can be failing for its own reasons when the cancel lands -- its row + is a genuine failure and its pulse would run in the same window. + """ + + def _cancelled_scope(self) -> CancelScope: + scope = CancelScope() + scope.cancel(reason="session_time_exhausted") + return scope + + def test_the_round_the_run_stopped_is_not_pulsed(self, tmp_path, monkeypatch): + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "0") + base = tmp_path / "base.yaml" + _write_base_yaml(base) + + results, pulse_calls = _run_with_pulse_capture( + multi_node=False, + run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess( + cmd, ORCHESTRATOR_CANCELLED_RETURNCODE, "", "" + ), + base=base, + out=tmp_path / "out", + scope=self._cancelled_scope(), + ) + assert results[0].status == "skipped" + assert results[0].error_class == "orchestrator_cancelled" + assert pulse_calls == [], "the pulse must not run inside the cancel window" + + def test_a_variant_failing_on_its_own_when_the_cancel_lands_is_not_pulsed(self, tmp_path, monkeypatch): + """The row is a real failure; the eight seconds are still not affordable.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "0") + base = tmp_path / "base.yaml" + _write_base_yaml(base) + + results, pulse_calls = _run_with_pulse_capture( + multi_node=False, + run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 1, "stdout", "boom"), + base=base, + out=tmp_path / "out", + scope=self._cancelled_scope(), + ) + assert results[0].error_class == "no_benchmark_workspace" + assert pulse_calls == [] + + def test_a_variant_that_failed_under_a_live_scope_is_still_pulsed(self, tmp_path, monkeypatch): + """Only a cancel silences the tick, so #1177's terminal-row tick survives.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_RUN_GRID_WARMUP", "0") + base = tmp_path / "base.yaml" + _write_base_yaml(base) + + results, pulse_calls = _run_with_pulse_capture( + multi_node=False, + run_side_effect=lambda cmd, *a, **k: subprocess.CompletedProcess(cmd, 1, "stdout", "boom"), + base=base, + out=tmp_path / "out", + scope=CancelScope(), + ) + assert results[0].error_class == "no_benchmark_workspace" + assert len(pulse_calls) == 1 + + # --------------------------------------------------------------------------- # keep_going_on_failure asymmetry # --------------------------------------------------------------------------- diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 73b412424d..7919d8ad77 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -39,6 +39,7 @@ from hyperloom.orchestrator.actions.cancel_channel import ( CancelScope, current_cancel_scope, + stop_was_asked_for, use_cancel_scope, ) from hyperloom.orchestrator.actions.executors._ray_serving import CANCEL_ROUND_GRACE_SEC @@ -61,6 +62,7 @@ ) from hyperloom.orchestrator.policy.gate import PolicyDenied from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan +from hyperloom.orchestrator.roles.robustness_pulse import _PULSE_TIMEOUT_SEC from hyperloom.orchestrator.state.shared_state import SharedState, effective_closing_grace_sec from hyperloom.orchestrator.state.task_registry import Task @@ -695,7 +697,9 @@ class TestTheCooperativeStopWindowsCompose: sentinel the round was about to return. The components are spelled out here rather than re-derived from the constants - under test, so a change to one of them has to be argued for. + under test, so a change to one of them has to be argued for -- and the sum is + spelled out too, so a serial step the unwind takes and no term covers has to + be argued for as well, rather than quietly making the window short again. """ def test_the_reap_budget_is_what_stopping_a_round_costs(self): @@ -723,6 +727,39 @@ def test_reaping_a_server_and_dropping_its_lease_are_both_paid(self): """ assert _COOPERATIVE_CANCEL_GRACE_SEC >= 8.5 + 0.25 + 5.0 + 10.0 + def test_the_window_is_exactly_the_terms_it_names(self): + """An upper bound, so a term the unwind pays and the sum omits is a bug. + + Spelled as a total and not only as a floor: a fifth serial step was found + in the unwind that no term covered, and a floor would have gone on passing + while the sum stayed short of what stopping costs. + """ + assert _COOPERATIVE_CANCEL_GRACE_SEC == 8.5 + 0.25 + 5.0 + 10.0 + + def test_the_variant_boundary_tick_is_skipped_rather_than_budgeted_for(self): + """The one step in the unwind this window deliberately does not cover. + + A cooperative stop returns its sentinel, so ``run_grid`` reaches its + variant boundary the ordinary way and would spend the robustness tick's + whole budget there -- between recording the stopped round's row and + releasing what the round held, which are the terms above. Counting it + would take the window past what an operator's own ``SIGTERM`` grace + allows, so the tick gives way instead: the scope it reads is cancelled for + the rest of the action's life, so once the cancel is out no boundary + spends it. + """ + assert _PULSE_TIMEOUT_SEC == 8.0 + assert 8.5 + 0.25 + 5.0 + 10.0 + _PULSE_TIMEOUT_SEC > _COOPERATIVE_CANCEL_GRACE_SEC + scope = CancelScope() + with use_cancel_scope(scope): + assert not stop_was_asked_for() + scope.cancel(reason="session_time_exhausted") + assert stop_was_asked_for() + + def test_work_outside_an_action_is_never_told_to_skip(self): + """No scope means no cancel, so a bare call keeps every step it had.""" + assert not stop_was_asked_for() + def test_the_notice_window_is_the_poll_the_scope_is_checked_at(self): """Nothing is listening yet is a claim about the poll, not about the work.""" assert _CANCEL_NOTICE_SEC >= STOP_GATE_POLL_SECONDS diff --git a/src/hyperloom/orchestrator/actions/cancel_channel.py b/src/hyperloom/orchestrator/actions/cancel_channel.py index cb6beb9d56..1c132b9317 100644 --- a/src/hyperloom/orchestrator/actions/cancel_channel.py +++ b/src/hyperloom/orchestrator/actions/cancel_channel.py @@ -40,6 +40,7 @@ "CancelScope", "cancel_scope_listener", "current_cancel_scope", + "stop_was_asked_for", "use_cancel_scope", ] @@ -125,6 +126,20 @@ def current_cancel_scope() -> CancelScope | None: return _CURRENT_SCOPE.get() +def stop_was_asked_for() -> bool: + """Whether the action running in this context has already been asked to stop. + + For code deciding whether a step is still worth taking, rather than for code + stopping something in flight: no scope, or an uncancelled one, both read as + "carry on", so a caller outside an action behaves exactly as it always did. + + Returns: + bool: ``True`` when a scope is published and cancelled. + """ + scope = _CURRENT_SCOPE.get() + return scope is not None and scope.cancelled + + @contextmanager def use_cancel_scope(scope: CancelScope | None) -> Iterator[CancelScope | None]: """Publish ``scope`` for the duration of the block. diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 67122a2c16..9dee2458b2 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -34,6 +34,7 @@ from ...roles.robustness_pulse import pulse as _robustness_pulse from ...trace.task_progress import heartbeat_while_output_flows, report_progress +from ..cancel_channel import stop_was_asked_for from ..stop_attribution import ( ORCHESTRATOR_CANCELLED_CLASS, SESSION_TIME_EXHAUSTED_CLASS, @@ -1567,14 +1568,35 @@ async def _pulse_after_variant(idx: int) -> None: """Report the finished variant and run a best-effort robustness pulse. Called once the variant's result has been appended, so the progress - note carries what actually landed. Exceptions from the pulse are - swallowed (logged at debug) so a pulse failure never aborts the grid. + note carries what actually landed. The robustness tick is skipped once + the action has been asked to stop: a cooperative stop returns its + sentinel rather than raising, so this is reached on the ordinary path + with the cancel already outstanding, and the tick is a subprocess + spawned and waited out for its whole timeout -- serially, between + recording the row and releasing what the round held, inside the one + window the canceller allows the entire unwind. What it would observe + is the reap the orchestrator just ordered, and what waiting for it costs + is every row this grid has already built. + + The gate is the cancel scope rather than the round's returncode because a + variant can be failing for its own reasons when the cancel lands: that + row is a genuine failure and its tick is just as unaffordable. + + Exceptions from the pulse are swallowed (logged at debug) so a pulse + failure never aborts the grid. Args: idx (int): Zero-based index of the just-finished variant, passed through as the pulse ``tick_index``. """ await report_progress(**_variant_progress_note(grid, results, idx)) + if stop_was_asked_for(): + log.info( + "grid_runner: variant %d/%d robustness pulse skipped; the orchestrator cancelled this action", + idx + 1, + len(grid), + ) + return try: await _robustness_pulse(tick_index=idx) except Exception as exc: # noqa: BLE001 diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 3ff8e8b751..a6b99bb493 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -85,6 +85,16 @@ # that pays the most: the teardown a Ray round owes is not an alternative to # closing its lease, it is what it does first. # +# The enumeration above has to stay exhaustive, so a serial step the unwind takes +# and this sum does not name has to be removed from the unwind instead. One is: +# ``run_grid`` fires a robustness tick at each variant boundary, and a cooperative +# stop reaches that boundary the ordinary way, so the tick would sit between +# recording the round's row and releasing what the round held -- eight more +# seconds, spent observing the reap this cancel just ordered. It is skipped +# whenever the scope is already cancelled rather than budgeted for, because the +# rows the grid has already built are what a window short by those eight seconds +# throws away, and they are worth more than one tick. +# # Past this window the coroutine is cancelled anyway, and the window is only ever # spent when work is still unwinding: the wait ends the moment the last victim is # done, so covering the teardown term costs a round that stops promptly nothing. From 4c495f892f3679e008be004c7ab3e7738f6253f7 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 08:43:24 +0000 Subject: [PATCH 43/65] baseline: a warmup may claim half of what the round has, and a budget under one whole round produces no anchor at all Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 401 ++++++++++---- .../tests/test_coordinator_runtime.py | 44 ++ .../actions/executors/baseline.py | 491 +++++++++++++----- .../orchestrator/actions/executors/report.py | 6 +- .../orchestrator/actions/stop_attribution.py | 19 + src/hyperloom/orchestrator/loop/writeback.py | 24 +- .../orchestrator/phases/machine_state.py | 66 ++- 7 files changed, 807 insertions(+), 244 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 1d8dd6f396..50af5c8dc3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -26,7 +26,13 @@ import yaml from hyperloom.orchestrator.actions.executors.baseline import ( + BASELINE_COLD_START_TIMEOUT_SEC, + BASELINE_DEFAULT_TIMEOUT_SEC, BaselineExecutor, + warmup_pass_cap_sec, +) +from hyperloom.orchestrator.actions.stop_attribution import ( + SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, ) from hyperloom.orchestrator.actions.executors.profile import ( PROFILE_DEFAULT_TIMEOUT_SEC, @@ -377,23 +383,63 @@ def _run_double_run_baseline(tmp_path, shared_state) -> dict: return result -def test_measured_round_is_dropped_when_preparation_has_spent_the_budget(tmp_path): - """The MiniMax-M2 shape: a 66-minute warmup, then a second round the session cannot use. +def test_a_budget_below_one_round_produces_no_anchor_at_all(tmp_path): + """The MiniMax-M2 shape: preparation has spent the share the round needs. - The warmup already carries accuracy and a throughput figure, so dropping - the measured round leaves the single-round baseline the codebase supports - rather than a half-measured state. + The warmup carries a throughput figure and it is tempting to keep it -- it + is the number a single-round baseline would have produced. But the round runs + twice precisely because that number is cold-contaminated, and every later + comparison the session makes is computed against whatever is anchored here. + Promoting the discarded pass would depress the anchor and inflate every gain + reported against it, for the whole run, to save one round. So the round + produces nothing, and says why. """ result = _run_double_run_baseline( tmp_path, _prelude_shared_state(spent_sec=10_000.0, usable_sec=500.0), ) - assert result["status"] == "succeeded" - assert result["_rounds_run"] == 1 - assert result["output_throughput"] == pytest.approx(_COLD_TPUT) - assert "baseline_measure_round_dropped_low_budget" in result["nonfatal_warnings"] - assert result["measure_round_dropped"]["bound"] == "prelude_ceiling" + assert result["status"] == "failed" + assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS + assert result["_rounds_run"] == 0, "GPU time was spent on a round known not to fit" + assert result.get("output_throughput") is None, "the cold warmup was promoted as the anchor" + assert result["budget_shortfall"]["bound"] == "prelude_ceiling" + + +def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_path): + """The cap bounds the warmup pass; the gate after it bounds the whole round. + + A cap of half the phase's headroom is exactly the pre-image of that gate -- + both bounds behind the headroom fall by the wall-clock the warmup burns, so a + pass that survives its cap is one the gate passes. What the cap does not + bound is the rest of the round: the server boot in front of the pass and the + teardown behind it are wall-clock too. Here the pass keeps well inside its + cap and the round still spends the share, which the gate sees because it + prices on what was actually spent rather than on what was allowed. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + state = _BudgetedState(remaining_sec=7200.0, double_run=True) + fake_run, calls = _capturing_fake_run(state=state, charge_sec=3000.0) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=state, + ) + ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert [c["round_slot"] for c in calls] == ["warmup_round"], ( + f"the measured round ran on a budget the round had already spent: {[c['round_slot'] for c in calls]}" + ) + assert result["status"] == "failed" + assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS + assert result.get("output_throughput") is None def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): @@ -405,7 +451,7 @@ def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): assert result["_rounds_run"] == 2 assert result["output_throughput"] == pytest.approx(_HOT_TPUT) - assert "measure_round_dropped" not in result + assert "budget_shortfall" not in result def test_deferred_accuracy_skips_eval_when_hot_throughput_regresses( @@ -1845,18 +1891,70 @@ def test_teardown_lifecycle_server_removes_state_files(tmp_path): _BASELINE_ROUND_SLOTS = ("mn_warmup", _MEASURED_ROUND_SLOT) -def _budgeted_state(*, remaining_sec: float | None, measured_expected_sec: float = 0.0) -> SimpleNamespace: - """A session state carrying how much wall-clock is left, and what a round costs. +# ``--max-hours`` defaults to 2.0 (``cli/parser.py``), so this is the session +# shape almost every run has. It matters here because PRELUDE's share of it is +# 48 minutes while a baseline round's declared cap is 130 minutes warm and 150 +# cold: any rule that prices the pair at the declared cap refuses every default +# session, and any rule that prices it at nothing lets the warmup eat the round. +_DEFAULT_SESSION_MINUTES = 120.0 + - ``measured_expected_sec`` is this session's measured baseline round, which is - what a normally-behaving round of the same workload is expected to need; zero - is "no round measured yet", the only state a first baseline can be in. +class _BudgetedState: + """A session state whose budget accounting moves as the passes spend it. + + Production reads one session through two accessors -- the deadline the round + reaper is handed and the usable-seconds figure the phase policy reads -- and + a double that lets them drift cannot see the regime where they disagree. + Both are derived from one deadline here, and PRELUDE's spend is whatever the + session has already used, which is what a run that has not left PRELUDE has + spent it on. + + A pass calls :meth:`charge` for the wall-clock it burned, which is the only + way a test can reach the case this whole mechanism exists for: a round whose + first pass leaves the second one nothing. + + ``max_minutes`` defaults to a session that has just started with + ``remaining_sec`` on the clock, so a test that only cares how much is left + says only that. Give it explicitly to place the round part-way through a + session, which is what decides how much of PRELUDE's own share is gone. """ - return SimpleNamespace( - baseline_double_run=False, - grid_session_deadline_sec=lambda: (None if remaining_sec is None else time.monotonic() + remaining_sec), - baseline_runtime_sec=measured_expected_sec, - ) + + def __init__( + self, + *, + remaining_sec: float | None, + measured_expected_sec: float = 0.0, + max_minutes: float | None = None, + phase: str = "PRELUDE", + double_run: bool = False, + ) -> None: + self.baseline_double_run = double_run + self.baseline_runtime_sec = measured_expected_sec + self.phase = phase + if remaining_sec is None: + self.max_minutes = 0.0 + else: + self.max_minutes = remaining_sec / 60.0 if max_minutes is None else max_minutes + self.phase_started_unix = 0.0 + self._deadline = None if remaining_sec is None else time.monotonic() + remaining_sec + + def charge(self, seconds: float) -> None: + """Spend ``seconds`` of the session, as a pass that ran that long does.""" + if self._deadline is not None: + self._deadline -= seconds + + def grid_session_deadline_sec(self) -> float | None: + return self._deadline + + def session_budget_usable_sec(self) -> float | None: + return None if self._deadline is None else self._deadline - time.monotonic() + + @property + def phase_elapsed_totals(self) -> dict[str, float]: + usable = self.session_budget_usable_sec() + if usable is None: + return {} + return {"PRELUDE": max(0.0, self.max_minutes * 60.0 - usable)} def _capturing_fake_run( @@ -1864,6 +1962,8 @@ def _capturing_fake_run( *, produces_workspace: bool = True, pass_duration_sec: float = 0.0, + state: _BudgetedState | None = None, + charge_sec: float | None = None, ): """A ``run_with_session_kill`` stand-in that records how each round was launched. @@ -1871,9 +1971,14 @@ def _capturing_fake_run( be looked up by which pass it was rather than by the order it happened in. ``pass_duration_sec`` makes the double honour the cap it was handed the way a - real benchmark pass does: a round granted less than the workload takes raises - ``TimeoutExpired`` instead of returning a report, which is the only way to see - what a cap too small to survive costs the round after it. + real benchmark pass does, and it charges the budget for what it ran: a pass + granted less than the workload takes is killed rather than reporting, and it + is killed by whichever of the two limits it meets first -- the session + watchdog, which comes back with the sentinel returncode that says the run ran + out of time, or its own hard cap, which raises ``TimeoutExpired``. + + ``charge_sec`` charges the session more than the pass itself ran, which is + what a round with a server restart and a teardown around the pass costs. """ calls: list[dict] = [] @@ -1884,7 +1989,18 @@ def fake_run(cmd, *args, **kwargs): slot = Path(cmd[cmd.index("--output-dir") + 1]) calls.append({"round_slot": slot.name, **kwargs}) granted = float(kwargs.get("timeout") or 0.0) - if pass_duration_sec and granted < pass_duration_sec: + deadline = kwargs.get("session_deadline_sec") + ran_sec = min(granted, pass_duration_sec) if pass_duration_sec else 0.0 + reaped_by_the_session = False + if deadline is not None and pass_duration_sec: + until_deadline = max(0.0, deadline - time.monotonic()) + reaped_by_the_session = until_deadline < ran_sec + ran_sec = min(ran_sec, until_deadline) + if state is not None: + state.charge(charge_sec if charge_sec is not None else ran_sec) + if pass_duration_sec and ran_sec < pass_duration_sec: + if reaped_by_the_session: + return subprocess.CompletedProcess(cmd, SESSION_TIME_EXHAUSTED_RETURNCODE, "", "") raise subprocess.TimeoutExpired(cmd, granted) if produces_workspace: _fake_workspace(slot, tput=_HOT_TPUT) @@ -1924,15 +2040,24 @@ def _run_baseline_under_budget( produces_workspace: bool = True, measured_expected_sec: float = 0.0, pass_duration_sec: float = 0.0, + max_minutes: float | None = None, + phase: str = "PRELUDE", executor_cls=BaselineExecutor, ) -> tuple[dict, list[dict]]: """Run one baseline round against a session with ``remaining_sec`` left.""" base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") + state = _BudgetedState( + remaining_sec=remaining_sec, + measured_expected_sec=measured_expected_sec, + max_minutes=max_minutes, + phase=phase, + ) fake_run, calls = _capturing_fake_run( returncode, produces_workspace=produces_workspace, pass_duration_sec=pass_duration_sec, + state=state, ) executor = executor_cls( magpie_python=sys.executable, @@ -1947,10 +2072,7 @@ def _run_baseline_under_budget( } ) # The live state arrives on the context, the way the coordinator passes it. - ctx.extra["shared_state"] = _budgeted_state( - remaining_sec=remaining_sec, - measured_expected_sec=measured_expected_sec, - ) + ctx.extra["shared_state"] = state with patch( "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", side_effect=fake_run, @@ -2005,6 +2127,38 @@ def _launch_one_grid_variant_under_budget( return calls +class TestHowMuchAWarmupPassMayClaim: + """The arithmetic behind the round-level behaviour, at its boundaries.""" + + def test_the_warmup_may_claim_half_of_what_the_round_has(self): + """Two passes of comparable cost, so the discarded one gets half.""" + assert warmup_pass_cap_sec(7800, headroom_sec=2880.0) == 1440 + + def test_a_declared_cap_under_the_share_is_left_alone(self): + """The share is a ceiling on the pass, not a grant to spend up to.""" + assert warmup_pass_cap_sec(600, headroom_sec=2880.0) == 600 + + def test_an_overspent_share_starts_nothing(self): + assert warmup_pass_cap_sec(7800, headroom_sec=-720.0) is None + + def test_a_known_pass_that_does_not_fit_the_share_is_refused_before_it_runs(self): + """History turns the cap into a prediction, which is cheaper than finding out.""" + assert warmup_pass_cap_sec(7800, headroom_sec=800.0, measured_expected_sec=600.0) is None + + def test_a_known_pass_that_fits_is_not_refused(self): + assert warmup_pass_cap_sec(7800, headroom_sec=2880.0, measured_expected_sec=600.0) == 1440 + + def test_a_declared_cap_under_a_known_pass_is_not_the_budgets_business(self): + """The first baseline runs under the 9000s cold cap and promotes its runtime; + every later one is given the 7800s warm cap, so an anchor in between sits + above the cap forever. Refusing the warmup over that would disable it for + the rest of the run and name the session budget as the reason.""" + assert warmup_pass_cap_sec(7800, headroom_sec=1_000_000.0, measured_expected_sec=8000.0) == 7800 + + def test_no_budget_at_all_leaves_the_cap_alone(self): + assert warmup_pass_cap_sec(600, headroom_sec=None, measured_expected_sec=900.0) == 600 + + class TestTheSessionBudgetReachesTheBaselineRound: """The arm #1146 names as the largest hole, and the one that motivated it. @@ -2140,104 +2294,157 @@ def fake_run(cmd, *args, **kwargs): assert launched == ["mn_warmup"], f"the measured round ran after the cancel: {launched}" assert result["error_class"] == ORCHESTRATOR_CANCELLED_CLASS - @pytest.mark.parametrize( - ("remaining_sec", "declared_cap_sec"), - [ - pytest.param(1000.0, 600, id="the-budget-outlasts-the-declared-cap"), - pytest.param(3600.0, 7200, id="less-budget-left-than-the-declared-cap"), - ], - ) - def test_the_multi_node_warmup_cap_reserves_budget_for_the_measured_round( + def test_a_first_baseline_on_a_default_session_runs_both_of_its_passes( self, tmp_path, monkeypatch, - remaining_sec, - declared_cap_sec, ): - """The warmup holds back one measured pass, and is granted one of its own. - - Both directions are the point. Reserve too little and a warmup granted the - measured round's cap can spend the budget that round needed, which is then - admitted with nothing left and reaped. Reserve too much -- the measured - round's *backstop* rather than its expected runtime -- and nothing is left - behind it: the warmup is launched on a cap it cannot survive, killed, - swallowed as best-effort, and the measured round runs cold against a server - nothing ever drove. - - Parameterized over both sides of ``remaining < declared cap``, because - that is the boundary the two spellings of the reserve disagree on: they - agree while the whole cap fits and diverge the moment it does not. + """The regime three rounds of this mechanism have failed in, pinned directly. + + A session's first baseline has measured nothing, so there is no history to + predict a pass from -- and the only other number available, the round's + declared cap, is a hang backstop of 7800s warm and 9000s cold, each longer + than the whole two-hour default session PRELUDE gets 40% of. Pricing the + pair at that cap refuses a baseline in essentially every run, which turns + "produce nothing rather than a cold anchor" into "produce nothing". + + So the warmup is sized at half of what the phase can still spend, which + needs no history, and this ten-minute workload is nowhere near it: both + passes run and the round yields the warm anchor it exists to produce. """ enable_multi_node(monkeypatch) - expected_sec = 300.0 - _result, calls = _run_baseline_under_budget( + pass_sec = 600.0 + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=_DEFAULT_SESSION_MINUTES * 60.0, + timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, + measured_expected_sec=0.0, + pass_duration_sec=pass_sec, + ) + launches = launches_by_round_slot(calls) + + assert result["status"] == "succeeded", f"a first baseline was refused a default session: {result}" + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + assert set(launches) >= set(_BASELINE_ROUND_SLOTS), f"the round did not run both passes: {list(launches)}" + warmup = _mn_warmup_cap_sec(calls) + assert warmup is not None and warmup >= pass_sec, ( + f"the warmup was capped under the {pass_sec}s this pass takes, so it could only be killed: {warmup}" + ) + + def test_a_first_baseline_too_big_for_its_budget_costs_one_share_to_find_out( + self, + tmp_path, + monkeypatch, + ): + """With nothing measured, the shortfall is discovered by the cap, not predicted. + + Nothing this session measured says what a pass of this workload costs, and + the two numbers that could stand in for it both lie: the declared cap is a + hang backstop longer than the session, and the static per-action estimates + are calibrated on small models. So the warmup is launched with its share + and the cap is what finds out -- which bounds the cost of finding out to + that share. + + The alternative is what a warmup with no hold-back does: it is granted + more than the whole remaining budget, runs to completion, and leaves the + measured round to be admitted with nothing and reaped, so the session pays + for both passes and keeps neither. + """ + enable_multi_node(monkeypatch) + remaining_sec = 3600.0 + result, calls = _run_baseline_under_budget( tmp_path, remaining_sec=remaining_sec, - timeout_sec=declared_cap_sec, - measured_expected_sec=expected_sec, + timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, + measured_expected_sec=0.0, + pass_duration_sec=2000.0, ) + launches = launches_by_round_slot(calls) + + assert _MEASURED_ROUND_SLOT not in launches, "a measured round was launched into a budget that cannot pay" warmup = _mn_warmup_cap_sec(calls) - measured = launches_by_round_slot(calls)[_MEASURED_ROUND_SLOT]["timeout"] - assert warmup is not None, "this budget fits both passes, so the warmup must have been launched" - assert measured == pytest.approx( - min(declared_cap_sec, remaining_sec + _SESSION_KILL_GRACE_SEC), - abs=2, - ), f"the measured round keeps what the budget can pay for, got {measured}" - assert warmup >= expected_sec, ( - f"a warmup capped under the {expected_sec}s a pass takes is launched to be killed, got {warmup}" + assert warmup is not None and warmup <= remaining_sec / 2.0, ( + f"the warmup was allowed more than its share of what is left: {warmup}s of {remaining_sec}s" + ) + assert result["status"] == "failed" + assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, ( + f"the shortfall was reported as something else: {result.get('error_class')}" ) - assert warmup == pytest.approx( - min(declared_cap_sec, remaining_sec - expected_sec + _SESSION_KILL_GRACE_SEC), - abs=2, - ), f"the warmup must hold back exactly one measured pass of {expected_sec}s, got {warmup}" + assert result["budget_shortfall"]["bound"] == "prelude_ceiling" - def test_a_warmup_that_does_not_fit_is_skipped_rather_than_launched_to_be_killed( + def test_a_warmup_that_does_not_fit_leaves_no_anchor_rather_than_a_cold_one( self, tmp_path, monkeypatch, ): - """Below two passes of budget the warmup is dropped, not starved. - - A skipped warmup is a shape the round already supports -- it is what a - single-node baseline does, and what a profile round that claimed the - restart does. A warmup launched on a cap smaller than the pass it runs is - not: it raises ``TimeoutExpired`` into the best-effort handler, which - swallows it, so the measured round proceeds cold and its throughput - becomes the session's baseline anchor. Explore variants do get their - warmups, so every later gain is measured against a depressed anchor. + """Dropping the warmup does not drop the harm the warmup exists to prevent. + + The per-round server restart runs under the same condition as the warmup, + so whenever the warmup would have run, the server was just restarted and + the warmup is the only thing that drives it. Running the measured pass + without it makes that pass the first traffic against a cold server, and + its throughput becomes the anchor every later gain is reported against -- + recorded as a plain success, with nothing saying so. """ enable_multi_node(monkeypatch) pass_sec = 600.0 result, calls = _run_baseline_under_budget( tmp_path, remaining_sec=800.0, - timeout_sec=7200, + timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, measured_expected_sec=pass_sec, pass_duration_sec=pass_sec, ) - launches = launches_by_round_slot(calls) - starved = [(slot, launch["timeout"]) for slot, launch in launches.items() if launch["timeout"] < pass_sec] - assert starved == [], f"a round was launched on a cap it cannot survive: {starved}" - assert _mn_warmup_cap_sec(calls) is None, "the warmup was launched and killed instead of being skipped" - assert _MEASURED_ROUND_SLOT in launches, "dropping the warmup must not drop the measured round" - assert result["status"] == "succeeded" + assert calls == [], ( + f"the round it already knows does not fit was launched anyway: {[c['round_slot'] for c in calls]}" + ) + assert result["status"] == "failed" + assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS + assert result.get("output_throughput") is None, "a cold anchor was persisted anyway" - def test_both_benching_arms_reserve_the_same_seconds_for_the_measured_round( + def test_a_declared_cap_below_one_pass_is_not_blamed_on_the_budget( self, tmp_path, monkeypatch, ): - """The grid and the baseline run the same warmup, so it must cost the same. + """A cap smaller than the workload is a fact about the cap, not the clock. + + The first baseline of a session runs under the 9000s cold-start cap and + promotes its runtime; every later one runs under the 7800s warm cap. An + anchor runtime in between therefore sits above the cap the next round is + given, permanently -- and a rule that refuses the warmup whenever its cap + is under one measured pass disables it for the rest of the run while + naming the session budget as the reason, with a week of clock left. + """ + enable_multi_node(monkeypatch) + _result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=7.0 * 24 * 3600, + timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, + measured_expected_sec=BASELINE_COLD_START_TIMEOUT_SEC - 1000.0, + ) + + assert _mn_warmup_cap_sec(calls) == BASELINE_DEFAULT_TIMEOUT_SEC, ( + "the warmup was refused over its own declared cap while the session had a week left" + ) - ``session_grid_bounds`` exists so a deadline cannot be derived one way in - one executor and another way in the next. The hold-back is the other half - of that contract: an arm reserving the measured round's hang backstop - where the other reserves its expected runtime abandons a different amount - of the tail budget -- and, past ``remaining < backstop``, all of it. + def test_neither_benching_arm_launches_a_pass_its_measured_round_cannot_follow( + self, + tmp_path, + monkeypatch, + ): + """The contract the two arms share, which is about outcomes, not seconds. + + They answer from different information -- the grid always knows what a + pass of this workload costs, a session's first baseline never does -- so + pinning them to the same number would pin one of them to a rule it has no + basis for. What must hold either way is that a discarded pass is never + started when the round it warms for cannot follow it: that is the only + shape in which the GPU time buys nothing at all. """ enable_multi_node(monkeypatch) - remaining_sec, declared_cap_sec, expected_sec = 3600.0, 7200, 600.0 + remaining_sec, declared_cap_sec, expected_sec = 800.0, 7200, 600.0 _result, baseline_calls = _run_baseline_under_budget( tmp_path, remaining_sec=remaining_sec, @@ -2251,12 +2458,8 @@ def test_both_benching_arms_reserve_the_same_seconds_for_the_measured_round( variant_expected_sec=expected_sec, ) - baseline_warmup = _mn_warmup_cap_sec(baseline_calls) - grid_warmup = _mn_warmup_cap_sec(grid_calls) - assert grid_warmup is not None, "the grid arm ran no warmup, so there is nothing to compare" - assert baseline_warmup == pytest.approx(grid_warmup, abs=2), ( - f"the baseline arm reserved a different measured round: {baseline_warmup}s vs the grid's {grid_warmup}s" - ) + assert _mn_warmup_cap_sec(baseline_calls) is None, "the baseline arm launched a doomed warmup" + assert _mn_warmup_cap_sec(grid_calls) is None, "the grid arm launched a doomed warmup" def test_the_profile_arm_gets_all_of_it(self, tmp_path): """Profile is the same executor with a four-hour default -- longer than diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 4f88a60951..56f480154a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1209,6 +1209,50 @@ async def test_baseline_rounds_the_run_stopped_do_not_charge_the_failure_streak( await c.stop() +@pytest.mark.asyncio +async def test_a_baseline_the_budget_cannot_fit_closes_the_run_naming_the_budget(session_dir): + """One shortfall is terminal, and it is not the model's fault. + + A baseline round is two passes and what is left cannot buy both. That is a + fact about the clock, not about this attempt, and the clock only shrinks -- + so a retry re-derives the same answer having spent more on the asking. The + two shapes already here both get it wrong. Left as a stop the run chose, the + streak stays at zero and the reactor keeps proposing baselines until the + session clock genuinely dies, which it need not do for hours after PRELUDE's + own share is gone. Counted as a failure, three of them close the run as + ``baseline_failed``, which reads as a model that cannot boot. + + So it closes on the first one, with the word the vocabulary already has for + preparation running out of clock -- and ``abort_prelude`` already routes that + word to CLOSE, so nothing new had to be taught how to end a run. + """ + from hyperloom.orchestrator.phases.machine_state import PHASE_CLOSE, compute_next_phase + + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + await c._handle_unpromotable_result( + _mk_task("baseline", "t-below-one-round"), + { + "status": "failed", + "error_class": "session_budget_below_one_round", + "error": "the remaining budget cannot fit both passes of this round", + }, + ) + assert c.shared_state.stop_reason == "time_exhausted_during_prelude" + assert c.shared_state.baseline_failure_streak == 0, "the clock was charged to the model" + assert c.shared_state.baseline_total_failures == 0 + assert len(c.shared_state.last_action_failures) == 1, "the round was not recorded" + + c.shared_state.phase = "PRELUDE" + transition = compute_next_phase(c.shared_state) + assert transition is not None, "the run had no way to reach a terminal state" + assert transition[0] == PHASE_CLOSE + assert transition[1] == "time_exhausted_during_prelude" + finally: + await c.stop() + + @pytest.mark.asyncio async def test_a_stopped_baseline_round_does_not_clear_a_real_failure_streak(session_dir): """A stop the run chose neither charges the streak nor forgives what preceded it.""" diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 52bc9b81c2..3bb692901d 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -37,7 +37,11 @@ from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import heartbeat_while_output_flows, report_progress from ...phases import machine_state as _phase_state -from ..stop_attribution import StoppedByTheRun +from ..stop_attribution import ( + SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) from . import _server_lifecycle as _lifecycle from ._file_lock import best_effort_file_lock from ._aiter_jit import ( @@ -59,7 +63,6 @@ sanitize_script_name, session_clamped_timeout_sec, session_grid_bounds, - session_warmup_cap_sec, stopped_by_the_run, ) from ._subprocess_kill import ( @@ -109,6 +112,10 @@ ) # Bounded per-file read so log scanning never slurps a multi-GB server.log. _LOG_SCAN_MAX_BYTES = 262_144 +# The measured pass ran as the first traffic against a freshly restarted server, +# because the warmup that exists to drive it did not. Its throughput is a cold +# number, and the session anchors every later gain on it. +_MN_WARMUP_DID_NOT_WARM_WARNING = "baseline_mn_warmup_did_not_run" # The cold-start guard's two round directories. The warmup round is the only one # that measures accuracy (``RUN_EVAL=true``); the measured round is hot @@ -310,6 +317,8 @@ def _stopped_round_result( runtime_sec: float, output_dir: Path, capture_meta: dict[str, Any], + never_started: bool = False, + evidence: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build the result for a round the run itself stopped. @@ -332,26 +341,118 @@ def _stopped_round_result( runtime_sec: Wall-clock seconds the round had run for. output_dir: The task workspace, echoed onto the result. capture_meta: Config/eval-contract facts every failure result carries. + never_started: Whether the round was refused before it launched, which + is the other half of every cause here and reads differently in a + ledger: no GPU time was spent, so there is not even a partial round + to look for. + evidence: The numbers behind the stop, when the cause is one the run + computed rather than observed. Returns: dict[str, Any]: The failed result carrying the stop's own error class. """ + detail = stopped.never_started if never_started else stopped.interrupted log.warning( - "baseline_executor: %s reaped after %.1fs: %s; error_class=%s.", + "baseline_executor: %s %s after %.1fs: %s; error_class=%s.", round_label, + "refused" if never_started else "reaped", runtime_sec, - stopped.interrupted, + detail, stopped.error_class, ) - return { + result = { "status": "failed", "error_class": stopped.error_class, "returncode": returncode, - "error": stopped.interrupted, + "error": detail, "subprocess_runtime_sec": round(runtime_sec, 2), "output_dir": str(output_dir), **capture_meta, } + if evidence is not None: + result["budget_shortfall"] = evidence + return result + + +def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple[float | None, dict[str, Any]]: + """Seconds this round's budget may still spend, and the numbers behind it. + + In PRELUDE that is the phase's own share + (:func:`~...phases.machine_state.prelude_affordable_seconds`), which is the + same figure the post-warmup gate judges the measured round against. Outside + it a re-baseline answers to what is left before the session deadline, the + only bound the round is under there. + + Args: + state: The session ``SharedState``, or ``None`` when the caller has no + session context (direct executor invocation, tests). + session_deadline_sec: Monotonic-clock session deadline, or ``None``. + + Returns: + tuple[float | None, dict[str, Any]]: The headroom, or ``None`` when the + round is under no budget at all, plus the evidence behind it. + """ + if state is None: + outside: dict[str, Any] = {"reason": "no_session_state"} + else: + phase = str(getattr(state, "phase", "") or "").strip().upper() + if phase == _phase_state.PHASE_PRELUDE: + return _phase_state.prelude_affordable_seconds(state) + outside = {"reason": "not_prelude", "phase": phase} + if session_deadline_sec is None: + return None, outside + remaining_sec = max(0.0, session_deadline_sec - time.monotonic()) + return remaining_sec, {**outside, "bound": "session_deadline", "affordable_sec": round(remaining_sec, 1)} + + +def warmup_pass_cap_sec( + cap: int, + *, + headroom_sec: float | None, + measured_expected_sec: float = 0.0, +) -> int | None: + """The hard cap for a discarded warmup pass, or ``None`` when it must not run. + + A round is two passes of the same workload -- a warmup whose throughput is + thrown away and the measured pass that re-attaches to the server it left + hot -- so the warmup may claim at most half of what the round has to spend. + That is not an estimate of anything: it is the pre-image of the gate that + runs after the warmup, which asks whether what is left still covers a pass + costing what the warmup cost. Both bounds behind ``headroom_sec`` fall by + exactly the wall-clock the warmup burns, so a warmup that survives this cap + is exactly a warmup that gate will pass, and one killed by it is exactly one + that gate would have refused after paying for it in full. + + The half is what makes the rule work on a session's first baseline, where + nothing has been measured and there is therefore nothing to predict a pass + from. Pricing the pair at the declared cap instead would refuse a baseline + in every default session: the caps are hang backstops of 7800s warm and + 9000s cold, each longer than the whole two-hour default the phase gets 40% + of. + + ``measured_expected_sec`` is what a later baseline knows a pass of this + workload costs. It only ever refuses the pass earlier than the cap would -- + a round already known not to fit is not worth half a round of GPU time to + re-discover -- and it is deliberately not consulted the other way: a + declared cap below one pass is a fact about the cap, not about the budget. + + Args: + cap: The timeout the caller would grant with an unbounded budget. + headroom_sec: Seconds this round may still spend, from + :func:`_round_headroom_sec`; ``None`` when it is under no budget. + measured_expected_sec: Seconds one pass of this workload is known to + take, or ``0`` when this session has not measured one. + + Returns: + int | None: The hard timeout for the warmup pass, or ``None`` when what + is left cannot fit both it and the measured round it warms for. + """ + if headroom_sec is None: + return int(cap) + share_sec = headroom_sec / 2.0 + if share_sec < 1.0 or (measured_expected_sec > 0.0 and share_sec < measured_expected_sec): + return None + return min(int(cap), int(share_sec)) def _disable_cuda_graph_flag(framework: str) -> str: @@ -1812,7 +1913,7 @@ def _session_capped_timeout( session deadline: the measured round is the last pass of the round, and a cap the session watchdog reaches first is a cap whose kill is attributed to the budget rather than to the model. The pass that does hold budget - back is the discarded warmup, in :meth:`_mn_warmup_capped_timeout`. + back is the discarded warmup, in :meth:`_warmup_pass_timeout`. Args: timeout_sec: The timeout this round would get on an unbounded budget. @@ -1830,47 +1931,50 @@ def _session_capped_timeout( ) @staticmethod - def _mn_warmup_capped_timeout( + def _warmup_pass_timeout( timeout_sec: int, - session_deadline_sec: float | None, *, output_dir: Path, + headroom_sec: float | None, measured_expected_sec: float | None, + evidence: dict[str, Any], ) -> int | None: - """The multi-node warmup pass's cap, or ``None`` when it should be skipped. + """The warmup pass's cap, or ``None`` when the round must not be started. - The decision itself is :func:`~._grid_runner.session_warmup_cap_sec`, so - both benching arms hold back the same seconds for the measured round they - discard a pass in front of; this adds the baseline's log lines. + The decision itself is :func:`warmup_pass_cap_sec`, shared by both + two-pass shapes a baseline has -- the single-node cold+hot double run and + the multi-node client warmup -- so a round is sized the same way whoever + launches it; this adds the log lines. Args: timeout_sec: The timeout this pass would get on an unbounded budget. - session_deadline_sec: Monotonic-clock session deadline, or ``None`` - when there is no budget to respect. output_dir: The warmup pass's workspace, for the log line. - measured_expected_sec: Seconds the measured round is expected to take, - or ``None`` when this session has not measured one yet. + headroom_sec: Seconds this round may still spend, or ``None`` when it + is under no budget. + measured_expected_sec: Seconds one pass of this workload is known to + take, or ``None`` when this session has not measured one. + evidence: The numbers behind ``headroom_sec``, for the log line. Returns: - int | None: The hard timeout for the warmup pass, or ``None`` when the - remaining budget cannot fit it and the measured round after it. + int | None: The hard timeout for the warmup pass, or ``None`` when + what is left cannot fit both passes of the round. """ - expected_sec = float(measured_expected_sec or 0.0) - clamped = session_warmup_cap_sec( + capped = warmup_pass_cap_sec( timeout_sec, - session_deadline_sec, - measured_expected_sec=expected_sec, + headroom_sec=headroom_sec, + measured_expected_sec=float(measured_expected_sec or 0.0), ) - if clamped is None: + if capped is None: log.warning( - "baseline_executor: skipping the MN warmup pass — what is left of the " - "session budget cannot fit both it and the measured round's expected " - "%.0fs, and a warmup capped under one pass warms nothing (round=%s)", - expected_sec, + "baseline_executor: not starting the round — a round is two passes " + "and %.0fs of budget is left for it (bound=%s), so the pass it could " + "pay for is the one whose number nothing may use (round=%s)", + float(headroom_sec or 0.0), + evidence.get("bound", ""), output_dir.name, ) return None - return _logged_session_clamp(timeout_sec, clamped, output_dir=output_dir) + return _logged_session_clamp(timeout_sec, capped, output_dir=output_dir) @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: @@ -3164,13 +3268,44 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: "baseline_executor: cold-start guard — warmup round (discarded, boots persistent server) in %s", warmup_dir, ) + # The warmup may claim at most half of what the round has, so a round + # that cannot fit both passes is refused before it spends the first + # one, and a warmup that survives its cap is one the gate below will + # pass. ``None`` is a budget under a whole round. + warmup_cap_sec, budget_evidence = self._double_run_warmup_budget( + ctx_extra=extra, + timeout_sec=timeout_sec, + warmup_dir=warmup_dir, + ) + round_meta = {"materialized_config": str(materialized_config_path)} + if warmup_cap_sec is None: + return _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], + round_label="cold+hot double round", + returncode=None, + runtime_sec=0.0, + output_dir=output_dir, + capture_meta=round_meta, + never_started=True, + evidence=budget_evidence, + ) warmup_result = await self._run_reported_round( label="warmup", config_path=warmup_cfg, output_dir=warmup_dir, - **common, + **{**common, "timeout_sec": warmup_cap_sec}, ) if warmup_result.get("status") != "succeeded": + if warmup_cap_sec < timeout_sec and warmup_result.get("error_class") == "timeout": + return _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], + round_label="cold+hot warmup round", + returncode=None, + runtime_sec=float(warmup_result.get("subprocess_runtime_sec") or 0.0), + output_dir=output_dir, + capture_meta=round_meta, + evidence={**budget_evidence, "warmup_cap_sec": warmup_cap_sec}, + ) # Warmup failure almost certainly recurs, so skip the # measured round. warmup_result.setdefault("nonfatal_warnings", []) @@ -3208,26 +3343,31 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: ) if not defer_accuracy_until_after_measure: - affordable, budget_evidence = self._measure_round_affordable( + affordable, gate_evidence = self._measure_round_affordable( warmup_runtime_sec=warmup_runtime, ctx_extra=extra, ) if not affordable: log.warning( - "baseline_executor: skipping the measured round — the warmup " - "took %.0fs and only %.0fs of preparation budget is left " - "(bound=%s). Keeping the warmup as the baseline; it is the " - "cold anchor a single-round baseline would have produced.", + "baseline_executor: the warmup took %.0fs and only %.0fs of " + "preparation budget is left (bound=%s), so the measured round " + "cannot follow it. The warmup is not kept: its throughput is " + "the cold-start number the round exists to discard, and an " + "anchor measured that way depresses every gain the session " + "goes on to report against it.", float(warmup_runtime or 0.0), - budget_evidence.get("affordable_sec", 0.0), - budget_evidence.get("bound", ""), + gate_evidence.get("affordable_sec", 0.0), + gate_evidence.get("bound", ""), ) - warmup_result.setdefault("nonfatal_warnings", []) - warmup_result["nonfatal_warnings"].append( - "baseline_measure_round_dropped_low_budget", + return _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], + round_label="cold+hot measured round", + returncode=None, + runtime_sec=float(warmup_runtime or 0.0), + output_dir=output_dir, + capture_meta=round_meta, + evidence=gate_evidence, ) - warmup_result["measure_round_dropped"] = budget_evidence - return warmup_result # Round 2 (measured): re-attach to the hot server (client only). # Warm re-attach is intentional — all comparison points (baseline, @@ -3395,6 +3535,40 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() + def _double_run_warmup_budget( + self, + *, + ctx_extra: dict[str, Any] | None, + timeout_sec: int, + warmup_dir: Path, + ) -> tuple[int | None, dict[str, Any]]: + """The cold round's cap and the numbers behind it, or ``None`` to not start. + + The single-node half of the same decision the multi-node warmup makes in + :meth:`_mn_warmup_pass`: both shapes discard a pass in front of the one + that is measured, so both size that pass out of the same headroom. + + Args: + ctx_extra: The runner context extras carrying ``shared_state``. + timeout_sec: The round's cap on an unbounded budget. + warmup_dir: The cold round's workspace, for the log line. + + Returns: + tuple[int | None, dict[str, Any]]: The cap, or ``None`` when what is + left cannot fit both passes, plus the evidence behind it. + """ + state = (ctx_extra or {}).get("shared_state") or self.shared_state + session_deadline_sec, measured_expected_sec = session_grid_bounds(state) + headroom_sec, evidence = _round_headroom_sec(state, session_deadline_sec) + cap_sec = self._warmup_pass_timeout( + timeout_sec, + output_dir=warmup_dir, + headroom_sec=headroom_sec, + measured_expected_sec=measured_expected_sec, + evidence=evidence, + ) + return cap_sec, evidence + def _measure_round_affordable( self, *, @@ -3403,17 +3577,15 @@ def _measure_round_affordable( ) -> tuple[bool, dict[str, Any]]: """Whether PRELUDE's remaining budget still covers the measured round. - The double-run exists to keep the baseline off cold-start numbers, and - on a normal model it costs minutes. On a large one it does not: two - field sessions spent 51 and 125 minutes on the pair, and in both the - cold and hot figures landed within 1% of each other — a correction the - session then had no time left to use. Dropping the second round when it - no longer fits leaves the single-round baseline the codebase already - supports and treats as valid, rather than a new half-measured state. - - The warmup's own runtime is the estimate: the measured round re-attaches - to the hot server, so it is an upper bound rather than a guess. Only - PRELUDE is guarded — a re-baseline in a later phase answers to that + The confirmation of what :func:`warmup_pass_cap_sec` sized the warmup + against, priced with the runtime the warmup actually had rather than the + share it was allowed: the round spends wall-clock either side of that + pass -- the server restart, the teardown -- which the cap does not bound. + The warmup's own runtime is the estimate for the round that follows it, + and an upper bound rather than a guess, because the measured round + re-attaches to a server the warmup has already booted. + + Only PRELUDE is guarded — a re-baseline in a later phase answers to that phase's budget. Args: @@ -3424,16 +3596,14 @@ def _measure_round_affordable( tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. """ state = (ctx_extra or {}).get("shared_state") or self.shared_state - if state is None: - return True, {"reason": "no_session_state"} - phase = str(getattr(state, "phase", "") or "").strip().upper() - if phase != _phase_state.PHASE_PRELUDE: - return True, {"reason": "not_prelude", "phase": phase} + headroom_sec, evidence = _round_headroom_sec(state, None) + if headroom_sec is None: + return True, evidence try: cost = float(warmup_runtime_sec or 0.0) except (TypeError, ValueError): cost = 0.0 - return _phase_state.prelude_can_afford(state, expected_cost_sec=cost) + return headroom_sec >= cost, {"expected_cost_sec": round(cost, 1), **evidence} def _double_run_enabled( self, @@ -3666,8 +3836,126 @@ async def _run_reported_round( config_path=config_path, output_dir=output_dir, **common, - ) + ) async def _mn_warmup_pass( + self, + *, + cmd: list[str], + env: dict[str, str], + output_dir: Path, + framework: str, + declared_timeout_sec: int, + timeout_sec: int, + session_deadline_sec: float | None, + measured_expected_sec: float | None, + state: Any, + capture_meta: dict[str, Any], + round_warnings: list[str], + ) -> dict[str, Any] | None: + """Run the discarded multi-node client warmup against the restarted server. + The pass exists because the restart just above it left a cold server and + this is the only thing that drives it before the measured pass. So the + two are one round: skipping the warmup does not save the round's cost, + it moves the round's measurement onto a cold server and anchors the + session's every later gain on it. A budget that cannot pay for both + therefore buys nothing here, and says so. + + Args: + cmd: The measured pass's command, re-pointed at the warmup slot. + env: The measured pass's environment, re-pointed the same way. + output_dir: The round's workspace; the warmup runs in a slot under it. + framework: Framework name, for the watchdog's server log. + declared_timeout_sec: The round's cap before any budget touched it, + which is what tells a hang apart from a budget-shortened pass. + timeout_sec: The round's cap after the session clamp. + session_deadline_sec: Monotonic-clock session deadline, or ``None``. + measured_expected_sec: Seconds one pass is known to take, or ``None``. + state: The session ``SharedState``, for the round's headroom. + capture_meta: Config/eval-contract facts every failure result carries. + round_warnings: Collected onto the round's result, for a warmup that + did not run and did not end the round. + + Returns: + dict[str, Any] | None: The round's result when the round is over, + else ``None`` to go on to the measured pass. + """ + warm_dir = output_dir / "mn_warmup" + headroom_sec, headroom_evidence = _round_headroom_sec(state, session_deadline_sec) + warm_timeout_sec = self._warmup_pass_timeout( + timeout_sec, + output_dir=warm_dir, + headroom_sec=headroom_sec, + measured_expected_sec=measured_expected_sec, + evidence=headroom_evidence, + ) + if warm_timeout_sec is None: + return _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], + round_label="multi-node round", + returncode=None, + runtime_sec=0.0, + output_dir=output_dir, + capture_meta=capture_meta, + never_started=True, + evidence=headroom_evidence, + ) + started_unix = time.time() + # The measurement is discarded, but the returncode is not: this pass is a + # full benchmark round, so a stop here ends the baseline round. Going on + # to the measured pass would spend a second round of GPU time the run has + # already been told to stop spending. + warm_rc: int | None = None + try: + warm_dir.mkdir(parents=True, exist_ok=True) + warm_cmd = [str(warm_dir) if c == str(output_dir) else c for c in cmd] + warm_env = dict(env) + warm_env["RESULT_DIR"] = str(warm_dir) + warm_env["EVAL_RESULT_DIR"] = str(warm_dir / "eval_output") + warm_env["SERVER_LOG"] = str(warm_dir / "server.log") + warm_env["GPU_METRICS_CSV"] = str(warm_dir / "gpu_metrics.csv") + async with heartbeat_while_output_flows( + unit="baseline_round", + label="mn_warmup", + ) as warm_activity: + warm_proc = await asyncio.to_thread( + run_with_session_kill, + warm_cmd, + env=warm_env, + cwd=str(warm_dir), + timeout=warm_timeout_sec, + server_log_path=_watchdog_server_log_path(warm_dir, framework), + on_output=warm_activity.note, + session_deadline_sec=session_deadline_sec, + ) + warm_rc = warm_proc.returncode + log.info("baseline_executor: MN warmup pass done (discarded) rc=%s", warm_rc) + except subprocess.TimeoutExpired as exc: + if warm_timeout_sec < declared_timeout_sec: + return _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], + round_label="multi-node warmup pass", + returncode=None, + runtime_sec=max(0.0, time.time() - started_unix), + output_dir=output_dir, + capture_meta=capture_meta, + evidence={**headroom_evidence, "warmup_cap_sec": warm_timeout_sec}, + ) + log.warning("baseline_executor: MN warmup pass hit its own hang backstop (ignored): %r", exc) + round_warnings.append(_MN_WARMUP_DID_NOT_WARM_WARNING) + except Exception as exc: # noqa: BLE001 - a warmup that fails on its own is best-effort + log.warning("baseline_executor: MN warmup pass failed (ignored): %r", exc) + round_warnings.append(_MN_WARMUP_DID_NOT_WARM_WARNING) + warm_stopped = stopped_by_the_run(warm_rc) + if warm_stopped is not None: + return _stopped_round_result( + warm_stopped, + round_label="multi-node warmup pass", + returncode=warm_rc, + runtime_sec=max(0.0, time.time() - started_unix), + output_dir=output_dir, + capture_meta=capture_meta, + ) + return None async def _run_single_benchmark( self, *, @@ -3788,9 +4076,9 @@ async def _run_single_benchmark( # The session's wall-clock budget, resolved per round rather than once # per task: a baseline runs up to three of them, and a warmup that # overran has already spent budget the ones after it were counting on. - session_deadline_sec, measured_expected_sec = session_grid_bounds( - ctx_extra.get("shared_state") or self.shared_state - ) + _session_state = ctx_extra.get("shared_state") or self.shared_state + session_deadline_sec, measured_expected_sec = session_grid_bounds(_session_state) + declared_timeout_sec = int(timeout_sec) timeout_sec = self._session_capped_timeout(timeout_sec, session_deadline_sec, output_dir=output_dir) if not ctx_extra.get("mn_round_restarted"): try: @@ -3855,68 +4143,23 @@ async def _run_single_benchmark( mn_bench_warmup_enabled as _mn_warm, ) - _mn_warm_dir = output_dir / "mn_warmup" - # What the measured round is expected to need is held back from the - # warmup, the same reserve the grid builds from the same number. Handed - # the whole cap, a slow warmup can spend the budget the measured round is - # then admitted without -- so the round that could never finish is the one - # launched and reaped, and the baseline is reported as - # ``session_time_exhausted`` after a warmup that actually succeeded. - # ``None`` means what is left cannot fit both passes, and the warmup is - # the one to drop. - _mn_warm_timeout_sec = ( - self._mn_warmup_capped_timeout( - timeout_sec, - session_deadline_sec, - output_dir=_mn_warm_dir, + round_warnings: list[str] = [] + if _mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted"): + _mn_warm_result = await self._mn_warmup_pass( + cmd=cmd, + env=env, + output_dir=output_dir, + framework=framework, + declared_timeout_sec=declared_timeout_sec, + timeout_sec=timeout_sec, + session_deadline_sec=session_deadline_sec, measured_expected_sec=measured_expected_sec, + state=_session_state, + capture_meta=capture_meta, + round_warnings=round_warnings, ) - if (_mn_imn() and _mn_warm() and not ctx_extra.get("mn_round_restarted")) - else None - ) - if _mn_warm_timeout_sec is not None: - _mn_warm_started_unix = time.time() - # The measurement is discarded, but the returncode is not: this pass - # is a full benchmark round, so a stop here ends the baseline round. - # Going on to the measured pass would spend a second round of GPU - # time the run has already been told to stop spending. - _mn_warm_rc: int | None = None - try: - _mn_warm_dir.mkdir(parents=True, exist_ok=True) - _mn_warm_cmd = [str(_mn_warm_dir) if c == str(output_dir) else c for c in cmd] - _mn_warm_env = dict(env) - _mn_warm_env["RESULT_DIR"] = str(_mn_warm_dir) - _mn_warm_env["EVAL_RESULT_DIR"] = str(_mn_warm_dir / "eval_output") - _mn_warm_env["SERVER_LOG"] = str(_mn_warm_dir / "server.log") - _mn_warm_env["GPU_METRICS_CSV"] = str(_mn_warm_dir / "gpu_metrics.csv") - async with heartbeat_while_output_flows( - unit="baseline_round", - label="mn_warmup", - ) as _mn_warm_activity: - _mn_warm_proc = await asyncio.to_thread( - run_with_session_kill, - _mn_warm_cmd, - env=_mn_warm_env, - cwd=str(_mn_warm_dir), - timeout=_mn_warm_timeout_sec, - server_log_path=_watchdog_server_log_path(_mn_warm_dir, framework), - on_output=_mn_warm_activity.note, - session_deadline_sec=session_deadline_sec, - ) - _mn_warm_rc = _mn_warm_proc.returncode - log.info("baseline_executor: MN warmup pass done (discarded) rc=%s", _mn_warm_rc) - except Exception as exc: # noqa: BLE001 - warmup is best-effort - log.warning("baseline_executor: MN warmup pass failed (ignored): %r", exc) - _mn_warm_stopped = stopped_by_the_run(_mn_warm_rc) - if _mn_warm_stopped is not None: - return _stopped_round_result( - _mn_warm_stopped, - round_label="multi-node warmup pass", - returncode=_mn_warm_rc, - runtime_sec=max(0.0, time.time() - _mn_warm_started_unix), - output_dir=output_dir, - capture_meta=capture_meta, - ) + if _mn_warm_result is not None: + return _mn_warm_result workspaces_before = snapshot_workspaces(output_dir) subprocess_started_unix = time.time() @@ -4172,7 +4415,7 @@ async def _run_single_benchmark( workspace=workspace, subprocess_started_unix=subprocess_started_unix, ) - warnings = list(measurement.pop("nonfatal_warnings", []) or []) + warnings = round_warnings + list(measurement.pop("nonfatal_warnings", []) or []) if proc_returncode != 0: warnings.append("magpie_nonzero_after_valid_measurement") for leak_src, _ in harvested: diff --git a/src/hyperloom/orchestrator/actions/executors/report.py b/src/hyperloom/orchestrator/actions/executors/report.py index 6259fe1b3c..3d94e6ef97 100644 --- a/src/hyperloom/orchestrator/actions/executors/report.py +++ b/src/hyperloom/orchestrator/actions/executors/report.py @@ -377,7 +377,11 @@ def _build_failure_summary( # PRELUDE-phase early exits (before optimization begins). "prelude_baseline_failed": "PRELUDE baseline failed before optimization could start; see the baseline failure summary.", "prelude_policy_loop": "The policy gate detected a decision loop during PRELUDE and stopped.", - "time_exhausted_during_prelude": "The wall-clock budget was exhausted while still in PRELUDE, before optimization began.", + "time_exhausted_during_prelude": ( + "The wall-clock budget PRELUDE had was exhausted before optimization began — either the session clock ran " + "out, or what was left of PRELUDE's share could no longer pay for a whole baseline round (a discarded " + "warmup pass and the measured pass it makes comparable)." + ), # Recipe KB knowledge-plane bootstrap failures. "recipe_kb_t0_failed": "Recipe KB knowledge-plane bootstrap (t0) failed; the run stopped early.", "recipe_kb_drain_failed": "Recipe KB knowledge-plane drain failed; the run stopped early.", diff --git a/src/hyperloom/orchestrator/actions/stop_attribution.py b/src/hyperloom/orchestrator/actions/stop_attribution.py index c207f63b33..78b65f145a 100644 --- a/src/hyperloom/orchestrator/actions/stop_attribution.py +++ b/src/hyperloom/orchestrator/actions/stop_attribution.py @@ -35,6 +35,7 @@ __all__ = [ "ORCHESTRATOR_CANCELLED_CLASS", + "SESSION_BUDGET_BELOW_ONE_ROUND_CLASS", "SESSION_TIME_EXHAUSTED_CLASS", "STOPPED_BY_THE_RUN", "StoppedByTheRun", @@ -52,6 +53,15 @@ # not face the shutdown. ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" +# Labels work refused, or cut short, because what is left of the budget cannot +# pay for a whole round of it. A benchmark round is two passes -- a discarded +# warmup and the measured pass it makes comparable -- so a budget that fits one +# fits none: the pass it could pay for is the one whose number nothing may use. +# Distinct from the class above because the deadline has not passed. Nothing was +# measured either way, but this one is also known to be permanent: the clock +# only shrinks, so the round that does not fit now never will. +SESSION_BUDGET_BELOW_ONE_ROUND_CLASS = "session_budget_below_one_round" + class StoppedByTheRun(NamedTuple): """How work that the run stopped from outside is recorded. @@ -87,6 +97,15 @@ class StoppedByTheRun(NamedTuple): never_started="the orchestrator cancelled this action before this round ran", ends_the_batch=True, ), + SESSION_BUDGET_BELOW_ONE_ROUND_CLASS: StoppedByTheRun( + error_class=SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, + interrupted=( + "the warmup pass was stopped at its share of the remaining budget, " + "which is therefore too small to fit both passes of this round" + ), + never_started="the remaining budget cannot fit both passes of this round", + ends_the_batch=True, + ), } diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 3e30d239da..62f407f796 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -25,7 +25,7 @@ summarize_change, ) from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON -from ..actions.stop_attribution import stopped_by_the_run_class +from ..actions.stop_attribution import SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, stopped_by_the_run_class from ..state.shared_state import SharedState, resolve_grading_anchor_tput from hyperloom.inference_optimizer.protocol.intent import Intent from ..bus.message_bus import Message @@ -1014,7 +1014,27 @@ async def _handle_unpromotable_result( ) if eval_failed: self._persist_eval_failure(result_payload) - if stopped_by_the_run: + if err_class == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS: + # Terminal on the first one. A baseline round is two passes and + # what is left cannot buy both, which is a fact about the clock + # rather than about this attempt: retrying re-derives the same + # answer from a budget that has only shrunk since. The two shapes + # already here both get it wrong -- a stop the run chose keeps the + # reactor proposing baselines until the session clock genuinely + # dies, which it need not do for hours after PRELUDE's own share + # is gone, and three counted failures close the run as + # ``baseline_failed``, blaming the model for the budget. The + # vocabulary already has the honest word for preparation running + # out of clock, and ``abort_prelude`` already routes it to CLOSE. + log.warning( + "baseline %s was refused by the budget (%s); closing as " + "time_exhausted_during_prelude rather than retrying a round the " + "clock can only afford less of", + task.task_id, + err_class, + ) + self.shared_state.set_stop_reason("time_exhausted_during_prelude") + elif stopped_by_the_run: log.warning( "baseline %s was stopped by the run (%s); the failure streak stays at %d " "because nothing about the baseline was measured", diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 002662c296..66bf1b3024 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -2078,44 +2078,42 @@ def _session_usable_seconds(state: Any) -> float | None: return session_remaining_seconds(state) -def prelude_can_afford( +def prelude_affordable_seconds( state: Any, *, - expected_cost_sec: float, now_unix: float | None = None, -) -> tuple[bool, dict[str, Any]]: - """Decide whether PRELUDE can still buy an optional arm costing ``expected_cost_sec``. +) -> tuple[float | None, dict[str, Any]]: + """Seconds PRELUDE may still spend, and the numbers the figure is built from. Two bounds apply and the tighter wins: what is left of PRELUDE's own share (:data:`PRELUDE_SPEND_CEILING_PCT`), and what is left of the session once the optimization phases' reserve (:data:`OPTIMIZATION_RESERVE_PCT`) is held - back. An arm that fits neither is not refused work the session needed — it - is refused work the session could not have used the result of. + back. Work that fits neither is not work the session needed — it is work + the session could not have used the result of. + + Read directly by callers that have to *size* a unit of work rather than + judge one they can already price, which is the position a first baseline is + in: it has no measured runtime to judge against, but the share it may spend + is known before anything runs. Args: state (Any): Frozen SharedState view. - expected_cost_sec (float): What the arm is expected to cost. Callers - should anchor this on something this session measured; the static - per-action estimates are calibrated on small models and understate - a large one by an order of magnitude. now_unix (float | None): Override for the current time. Returns: - tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. Evidence - carries the numbers behind the decision so a skip is legible in the log - and the phase record. + tuple[float | None, dict[str, Any]]: The affordable seconds — which may + be negative once the share is overspent — or ``None`` on an unbounded + budget, plus the evidence behind it. """ - cost = max(0.0, float(expected_cost_sec or 0.0)) max_sec = _max_minutes(state) * 60.0 usable = _session_usable_seconds(state) if max_sec <= 0.0 or usable is None: - return True, {"reason": "unbounded_budget", "expected_cost_sec": round(cost, 1)} + return None, {"reason": "unbounded_budget"} spent = phase_cumulative_seconds(state, phase=PHASE_PRELUDE, now_unix=now_unix) phase_headroom = max_sec * PRELUDE_SPEND_CEILING_PCT - spent session_headroom = usable - max_sec * OPTIMIZATION_RESERVE_PCT affordable_sec = min(phase_headroom, session_headroom) - evidence: dict[str, Any] = { - "expected_cost_sec": round(cost, 1), + return affordable_sec, { "prelude_spent_sec": round(spent, 1), "prelude_ceiling_sec": round(max_sec * PRELUDE_SPEND_CEILING_PCT, 1), "optimization_reserve_sec": round(max_sec * OPTIMIZATION_RESERVE_PCT, 1), @@ -2123,7 +2121,38 @@ def prelude_can_afford( "affordable_sec": round(affordable_sec, 1), "bound": "prelude_ceiling" if phase_headroom <= session_headroom else "optimization_reserve", } - return affordable_sec >= cost, evidence + + +def prelude_can_afford( + state: Any, + *, + expected_cost_sec: float, + now_unix: float | None = None, +) -> tuple[bool, dict[str, Any]]: + """Decide whether PRELUDE can still buy an optional arm costing ``expected_cost_sec``. + + The share itself is :func:`prelude_affordable_seconds`; this judges one cost + against it. + + Args: + state (Any): Frozen SharedState view. + expected_cost_sec (float): What the arm is expected to cost. Callers + should anchor this on something this session measured; the static + per-action estimates are calibrated on small models and understate + a large one by an order of magnitude. + now_unix (float | None): Override for the current time. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. Evidence + carries the numbers behind the decision so a skip is legible in the log + and the phase record. + """ + cost = max(0.0, float(expected_cost_sec or 0.0)) + affordable_sec, evidence = prelude_affordable_seconds(state, now_unix=now_unix) + priced = {"expected_cost_sec": round(cost, 1), **evidence} + if affordable_sec is None: + return True, priced + return affordable_sec >= cost, priced def exit_time_exhausted_prelude( @@ -3368,6 +3397,7 @@ def record_lifecycle_event( "exit_terminal_prelude", "exit_time_exhausted_prelude", "append_phase_evidence_row", + "prelude_affordable_seconds", "prelude_can_afford", "prelude_exit_viability", "is_action_allowed_in_phase", From db8e484fb71b91f23921c6d318ec96e633450d9b Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 08:43:24 +0000 Subject: [PATCH 44/65] grid_runner: the budget is re-checked after the uncapped server restart, not only at the top of the loop Co-authored-by: Cursor --- .../tests/test_grid_runner.py | 121 ++++++++-------- .../actions/executors/_grid_runner.py | 133 ++++++++---------- 2 files changed, 116 insertions(+), 138 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index cf79d9cf66..fbab4e05d7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio import inspect import json import os @@ -26,6 +27,7 @@ from hyperloom.orchestrator.actions.executors._grid_runner import ( _MN_BACKENDS_PRIORITY, _MN_PARAMS_PRIORITY, + SESSION_TIME_EXHAUSTED_CLASS, GridVariant, VariantResult, _build_variant_yaml, @@ -1518,69 +1520,6 @@ def test_state_without_the_deadline_accessor_is_tolerated(self): assert _grid_runner.session_grid_bounds(state) == (None, 600.0) -class TestTheWarmupHoldBack: - """The other half of ``session_grid_bounds``' contract: what a pass holds back. - - Both benching arms discard a warmup pass in front of a measured round, so both - have to hold back the same seconds for it. And because a reserve is the only - thing that can pull a cap before the session deadline, the same number decides - whether the warmup is worth launching at all. - """ - - def test_the_measured_rounds_expected_runtime_is_held_back(self): - cap = _grid_runner.session_warmup_cap_sec( - 7800, - time.monotonic() + 3600.0, - measured_expected_sec=600.0, - ) - assert cap == pytest.approx(3600 - 600 + _SESSION_KILL_GRACE_SEC, abs=2) - - def test_a_budget_that_fits_the_declared_cap_leaves_it_alone(self): - assert ( - _grid_runner.session_warmup_cap_sec( - 600, - time.monotonic() + 36000.0, - measured_expected_sec=600.0, - ) - == 600 - ) - - def test_a_warmup_that_cannot_fit_is_refused_rather_than_starved(self): - """Two passes do not fit in 800s, and the 1s floor is not a warmup.""" - assert ( - _grid_runner.session_warmup_cap_sec( - 7800, - time.monotonic() + 800.0, - measured_expected_sec=600.0, - ) - is None - ) - - def test_an_unmeasured_session_holds_nothing_back(self): - """Zero is "no round measured yet", not "the next round needs no time". - - With nothing known to reserve the cap stays past the deadline, which leaves - the session watchdog -- the only thing that attributes a budget kill - correctly -- as what stops the pass. - """ - cap = _grid_runner.session_warmup_cap_sec( - 7800, - time.monotonic() + 800.0, - measured_expected_sec=0.0, - ) - assert cap == pytest.approx(800 + _SESSION_KILL_GRACE_SEC, abs=2) - - def test_an_unbounded_budget_leaves_the_cap_alone(self): - """Not even a cap smaller than the pass is this function's business here. - - Nothing is being held back from anyone, so there is nothing for the warmup - to be short of; a declared cap under the measured runtime is a fact about - the cap, and refusing the pass over it would be this helper deciding - something the budget never asked it to. - """ - assert _grid_runner.session_warmup_cap_sec(600, None, measured_expected_sec=900.0) == 600 - - class TestSessionBudgetAdmission: """A variant is admitted on what it is expected to need, not on its backstop. @@ -2020,6 +1959,62 @@ async def test_admission_accounts_for_the_warmup_pass(self, tmp_path): assert recorded == [] assert [r.status for r in results] == ["skipped"] + @pytest.mark.asyncio + async def test_the_budget_is_re_checked_after_the_uncapped_server_restart( + self, + tmp_path, + monkeypatch, + ): + """The admission gate is taken before the one launch it does not cover. + + The per-variant multi-node server restart sits between the gate at the top + of the loop and the first pass, and it is under no cap of its own: booting + a large model across nodes can take longer than a benchmark pass. So a + variant can be admitted on a budget that fits both its passes and reach the + warmup with a budget that fits neither -- and the grid has no skip there, + so it launches the warmup anyway, watches it get killed, swallows that as + best-effort, and finds the measured round no longer fits. + + Scaled down by a thousand from the field shape (1300s left, 2x600s + admitted, a 300s restart) so the restart's cost is real elapsed time + rather than a clock the test pretends about. + """ + from hyperloom.orchestrator.actions.executors import _multi_node_env as mne + from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnsl + + base = tmp_path / "base.yaml" + _write_baseline_yaml_overrides(base) + monkeypatch.setattr(mne, "is_multi_node", lambda: True) + monkeypatch.setattr(mne, "mn_bench_warmup_enabled", lambda: True) + restarts: list[float] = [] + + async def slow_restart(**_kwargs): + restarts.append(time.monotonic()) + await asyncio.sleep(0.5) + + monkeypatch.setattr(mnsl, "restart_server_for_round", slow_restart) + recorded: list[dict] = [] + + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_capture_launches(recorded), + ): + results = await run_grid( + base_yaml_path=base, + base_extra_args="", + grid=[GridVariant("v0")], + output_root=tmp_path / "out", + variant_timeout_sec=600, + session_deadline_sec=time.monotonic() + 1.3, + variant_expected_sec=0.6, + ) + + assert restarts, "the restart never ran, so this is not the case under test" + launched = [c["round_slot"] for c in recorded] + assert launched == [], f"a pass was launched into a budget the restart had spent: {launched}" + assert [r.status for r in results] == ["skipped"] + assert [r.error_class for r in results] == [SESSION_TIME_EXHAUSTED_CLASS] + @pytest.mark.asyncio async def test_warmup_cap_reserves_budget_for_the_measured_round(self, tmp_path): base = tmp_path / "base.yaml" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 9dee2458b2..33fb71e070 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1301,10 +1301,9 @@ def session_clamped_timeout_sec( least ``_SESSION_KILL_GRACE_SEC`` past it, so the watchdog always gets there first. Every caller that reserves therefore re-opens the spurious-timeout window the grace exists to close, and owes a reason its round cannot be reaped - by its own cap -- either an admission gate built from the same number - (:func:`run_grid` refuses a variant whose rounds do not fit before it reserves - for them) or the decision to skip the round instead of starting one that will - be killed (:func:`session_warmup_cap_sec`). + by its own cap: :func:`run_grid` refuses a variant whose rounds do not fit, + both before the per-variant server restart and again after it, so what it + reserves for is a round it has just re-checked there is room for. Args: cap: The timeout the caller would grant with an unbounded budget. @@ -1322,52 +1321,6 @@ def session_clamped_timeout_sec( return int(cap) if usable >= int(cap) else max(1, usable) -def session_warmup_cap_sec( - cap: int, - session_deadline_sec: float | None, - *, - measured_expected_sec: float, -) -> int | None: - """The hard cap for a discarded warmup pass, or ``None`` when it must be skipped. - - A warmup is the pass whose measurement is thrown away, so it is the pass whose - budget is given up first: it holds back what the measured round after it is - expected to need. That reserve is also the only thing that can pull a cap - before the session deadline (see :func:`session_clamped_timeout_sec`), so the - same number decides whether to launch at all. A warmup granted less than one - full pass warms nothing: it is killed at its cap, both arms swallow that as - best-effort, and the measured round then runs against a server nothing ever - drove -- and its throughput is what the session anchors every later gain on. - A skipped warmup is a shape both arms already support; a warmup killed at its - cap is not. - - ``measured_expected_sec`` is the measured baseline runtime from - :func:`session_grid_bounds` -- what a normally-behaving pass of the same - workload needs, and what the admission gates upstream judge on. Zero means the - session has not measured a round yet, which leaves the warmup with no - hold-back: there is no number to reserve, and without a reserve the cap cannot - be pulled before the deadline, so the watchdog stays the thing that stops the - pass and attributes it to the budget. An unbounded budget is the same case for - the same reason -- nothing is being held back from anyone, so there is nothing - for the warmup to be short of, whatever its declared cap. - - Args: - cap: The timeout the caller would grant with an unbounded budget. - session_deadline_sec: Monotonic-clock session deadline, or ``None`` when - the budget is unbounded, which leaves ``cap`` untouched. - measured_expected_sec: Seconds one measured pass of this workload is - expected to take; ``0`` when the session has not measured one. - - Returns: - int | None: The hard timeout for the warmup pass, or ``None`` when what is - left cannot fit both it and the measured round it reserves for. - """ - if session_deadline_sec is None or measured_expected_sec <= 0: - return session_clamped_timeout_sec(cap, session_deadline_sec) - clamped = session_clamped_timeout_sec(cap, session_deadline_sec, reserve_sec=measured_expected_sec) - return clamped if clamped >= measured_expected_sec else None - - def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: """Resolve ``(session_deadline_sec, variant_expected_sec)`` for a :func:`run_grid` call. @@ -1736,36 +1689,60 @@ def _record_round_stop( _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds + def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: + """Skip variant ``idx`` and every one after it, or say the budget still fits. + + Checked twice per variant, because the two things that spend the budget + between the check and the launch are not under any cap: the per-variant + multi-node server restart, and the variant before this one overrunning. + Once a variant does not fit, none after it does either -- the clock only + shrinks -- so this ends the batch rather than trying the next name. + + Args: + idx: Zero-based index of the variant about to run. + spent_on: What has been spent since the last check, for the log line. + + Returns: + bool: ``True`` when the batch is over and nothing more was launched. + """ + if session_deadline_sec is None: + return False + remaining_sec = session_deadline_sec - time.monotonic() + # Falls back to a single ``variant_timeout_sec`` when no estimate was + # given, which is what callers that cannot estimate already got. + required_sec = ( + float(variant_expected_sec) * variant_rounds + if variant_expected_sec is not None + else float(variant_timeout_sec) + ) + if remaining_sec >= required_sec: + return False + log.warning( + "grid_runner: %.0fs left cannot fit this variant's %d round(s) of %.0fs " + "(spent on: %s); skipping %d remaining variant(s) rather than launching a " + "pass whose measured round cannot follow it", + max(0.0, remaining_sec), + variant_rounds, + required_sec, + spent_on, + len(grid) - idx, + ) + for skipped_variant in grid[idx:]: + results.append( + _not_run_skip_result( + skipped_variant, + _STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_RETURNCODE], + ) + ) + return True + for i, variant in enumerate(grid): # Session-budget stop: skip the remaining variants once the wall-clock # deadline is reached or the remaining budget cannot fit another variant, # so a timeout halts the grid instead of draining it (and the last variant # cannot overrun the close window). - if session_deadline_sec is not None: - remaining_sec = session_deadline_sec - time.monotonic() - # Falls back to a single ``variant_timeout_sec`` when no estimate was - # given, which is what callers that cannot estimate already got. - required_sec = ( - float(variant_expected_sec) * variant_rounds - if variant_expected_sec is not None - else float(variant_timeout_sec) - ) - if remaining_sec < required_sec: - log.warning( - "grid_runner: session budget exhausted (%.0fs left < %.0fs needed); " - "skipping %d remaining variant(s)", - max(0.0, remaining_sec), - required_sec, - len(grid) - i, - ) - for skipped_variant in grid[i:]: - results.append( - _not_run_skip_result( - skipped_variant, - _STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_RETURNCODE], - ) - ) - break + if _skip_rest_for_budget(i, spent_on="the variants before this one"): + break await _unit_started(i, "variant") slot = output_root / f"variant_{i:02d}_{_safe(variant.name)}" server_log = slot / "server.log" @@ -2191,6 +2168,12 @@ def _record_round_stop( break continue + # The restart above is a launch of its own and is under no cap, so the + # budget admitted at the top of the loop may no longer be there. Booting + # a large model can take longer than a benchmark pass. + if _skip_rest_for_budget(i, spent_on="this variant's server restart"): + break + # Multi-node client warmup: one discarded benchmark pass against the # just-restarted, persistent remote server to warm JIT / steady-state # before the measured pass. No lifecycle / no restart between; best- From 0b62de6a16676eef0666b766c7051bdfc9cc0632 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Fri, 14 Aug 2026 11:39:12 +0000 Subject: [PATCH 45/65] baseline: no pass of a round is shortened to pay for the next one A round's warmup was capped at a share of what the round had left, so the round could pay for the measured pass after it. Every way of sizing that share was wrong in the same direction, because the two passes do not cost the same thing: the warmup boots the server and pays weight load and graph capture -- which this file's own cold-start cap sizes at up to 9000s -- and the measured pass re-attaches to the server the warmup left hot. Pricing them as equal halves kills a first baseline that fits, on a default session, before it reaches the gate that would have passed it. Holding budget back has a second cost the file already documents in ``session_clamped_timeout_sec``: with nothing reserved, a round's cap sits past the session deadline so the watchdog reaches it first and the kill is recorded as the budget running out. A reserve moves the cap in front of the deadline and hands the kill back to the round's own timeout, which reads as the thing under test hanging. The discriminator written to undo that -- "the cap is below the declared one and the class is timeout" -- is true of every timeout on a default session, so a wedged server was filed as a budget shortfall, losing the retries that recover a transient boot wedge. So nothing is held back. Whether the measured round can follow its warmup is asked after the warmup, priced with the wall-clock that pass actually cost: an upper bound rather than a prediction, and available exactly when a first baseline needs it. A round that cannot afford the second pass keeps the first as its anchor and marks it cold, which is what this file did before the reserve and is strictly better than spending the GPU time and keeping nothing. The grid arm keeps its reserve -- it has the admission gate that ``session_clamped_timeout_sec`` requires of a caller that reserves -- but the gate's second check now charges only for the passes still ahead. It ran after the variant's own warmup and charged for it again, ending a batch over time that was already spent, and left without stopping the server that warmup had booted. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 246 +++----------- .../tests/test_coordinator_runtime.py | 44 --- .../actions/executors/_grid_runner.py | 86 +++-- .../actions/executors/baseline.py | 309 ++++-------------- .../orchestrator/actions/stop_attribution.py | 19 -- src/hyperloom/orchestrator/loop/writeback.py | 24 +- 6 files changed, 149 insertions(+), 579 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 50af5c8dc3..8990fb26cc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -28,11 +28,8 @@ from hyperloom.orchestrator.actions.executors.baseline import ( BASELINE_COLD_START_TIMEOUT_SEC, BASELINE_DEFAULT_TIMEOUT_SEC, + MEASURE_ROUND_DROPPED_WARNING, BaselineExecutor, - warmup_pass_cap_sec, -) -from hyperloom.orchestrator.actions.stop_attribution import ( - SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, ) from hyperloom.orchestrator.actions.executors.profile import ( PROFILE_DEFAULT_TIMEOUT_SEC, @@ -383,39 +380,37 @@ def _run_double_run_baseline(tmp_path, shared_state) -> dict: return result -def test_a_budget_below_one_round_produces_no_anchor_at_all(tmp_path): - """The MiniMax-M2 shape: preparation has spent the share the round needs. +def test_a_budget_that_cannot_pay_for_the_measured_round_keeps_the_cold_warmup(tmp_path): + """Preparation has spent the share the second pass needs; the first still ran. - The warmup carries a throughput figure and it is tempting to keep it -- it - is the number a single-round baseline would have produced. But the round runs - twice precisely because that number is cold-contaminated, and every later - comparison the session makes is computed against whatever is anchored here. - Promoting the discarded pass would depress the anchor and inflate every gain - reported against it, for the whole run, to save one round. So the round - produces nothing, and says why. + Nothing is predicted before the warmup, so the round starts and the warmup's + GPU time is spent before the shortfall is known. Refusing to keep its figure + would throw that away and leave the session with no anchor at all, which is + strictly worse than the cold anchor a single-round baseline would have + produced. So the warmup is promoted and marked: the number is depressed, and + the marker is what tells a reader of the session's later gains that their + denominator is. """ result = _run_double_run_baseline( tmp_path, _prelude_shared_state(spent_sec=10_000.0, usable_sec=500.0), ) - assert result["status"] == "failed" - assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS - assert result["_rounds_run"] == 0, "GPU time was spent on a round known not to fit" - assert result.get("output_throughput") is None, "the cold warmup was promoted as the anchor" - assert result["budget_shortfall"]["bound"] == "prelude_ceiling" + assert result["status"] == "succeeded" + assert result["_rounds_run"] == 1, "the measured round ran on a budget that cannot pay for it" + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] + assert result["measure_round_dropped"]["bound"] == "prelude_ceiling" def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_path): - """The cap bounds the warmup pass; the gate after it bounds the whole round. - - A cap of half the phase's headroom is exactly the pre-image of that gate -- - both bounds behind the headroom fall by the wall-clock the warmup burns, so a - pass that survives its cap is one the gate passes. What the cap does not - bound is the rest of the round: the server boot in front of the pass and the - teardown behind it are wall-clock too. Here the pass keeps well inside its - cap and the round still spends the share, which the gate sees because it - prices on what was actually spent rather than on what was allowed. + """The gate prices the round on what it spent, not on what its cap allowed. + + A warmup can keep well inside its own timeout and still leave the round + unable to pay for a second pass: the server boot in front of the pass and + the teardown behind it are wall-clock the cap never bounded. Asking after + the pass, against the clock rather than against the cap, is what catches + that -- and is why nothing is predicted before the pass instead. """ base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") @@ -437,9 +432,8 @@ def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_ assert [c["round_slot"] for c in calls] == ["warmup_round"], ( f"the measured round ran on a budget the round had already spent: {[c['round_slot'] for c in calls]}" ) - assert result["status"] == "failed" - assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS - assert result.get("output_throughput") is None + assert result["status"] == "succeeded" + assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): @@ -2127,38 +2121,6 @@ def _launch_one_grid_variant_under_budget( return calls -class TestHowMuchAWarmupPassMayClaim: - """The arithmetic behind the round-level behaviour, at its boundaries.""" - - def test_the_warmup_may_claim_half_of_what_the_round_has(self): - """Two passes of comparable cost, so the discarded one gets half.""" - assert warmup_pass_cap_sec(7800, headroom_sec=2880.0) == 1440 - - def test_a_declared_cap_under_the_share_is_left_alone(self): - """The share is a ceiling on the pass, not a grant to spend up to.""" - assert warmup_pass_cap_sec(600, headroom_sec=2880.0) == 600 - - def test_an_overspent_share_starts_nothing(self): - assert warmup_pass_cap_sec(7800, headroom_sec=-720.0) is None - - def test_a_known_pass_that_does_not_fit_the_share_is_refused_before_it_runs(self): - """History turns the cap into a prediction, which is cheaper than finding out.""" - assert warmup_pass_cap_sec(7800, headroom_sec=800.0, measured_expected_sec=600.0) is None - - def test_a_known_pass_that_fits_is_not_refused(self): - assert warmup_pass_cap_sec(7800, headroom_sec=2880.0, measured_expected_sec=600.0) == 1440 - - def test_a_declared_cap_under_a_known_pass_is_not_the_budgets_business(self): - """The first baseline runs under the 9000s cold cap and promotes its runtime; - every later one is given the 7800s warm cap, so an anchor in between sits - above the cap forever. Refusing the warmup over that would disable it for - the rest of the run and name the session budget as the reason.""" - assert warmup_pass_cap_sec(7800, headroom_sec=1_000_000.0, measured_expected_sec=8000.0) == 7800 - - def test_no_budget_at_all_leaves_the_cap_alone(self): - assert warmup_pass_cap_sec(600, headroom_sec=None, measured_expected_sec=900.0) == 600 - - class TestTheSessionBudgetReachesTheBaselineRound: """The arm #1146 names as the largest hole, and the one that motivated it. @@ -2299,18 +2261,17 @@ def test_a_first_baseline_on_a_default_session_runs_both_of_its_passes( tmp_path, monkeypatch, ): - """The regime three rounds of this mechanism have failed in, pinned directly. - - A session's first baseline has measured nothing, so there is no history to - predict a pass from -- and the only other number available, the round's - declared cap, is a hang backstop of 7800s warm and 9000s cold, each longer - than the whole two-hour default session PRELUDE gets 40% of. Pricing the - pair at that cap refuses a baseline in essentially every run, which turns - "produce nothing rather than a cold anchor" into "produce nothing". - - So the warmup is sized at half of what the phase can still spend, which - needs no history, and this ten-minute workload is nowhere near it: both - passes run and the round yields the warm anchor it exists to produce. + """The regime several rounds of this mechanism have failed in, pinned directly. + + A session's first baseline has measured nothing, so every rule that + shortens a pass by predicting the next one is guessing here. Each guess + tried so far killed a round that fit: the cold pass pays weight load and + graph capture, which the same file's cold-start cap sizes at up to 9000s, + so any share-of-the-budget cap lands under it on a default session. + + No pass is shortened now. The warmup is granted the round's own cap, and + this ten-minute workload runs both passes and yields the warm anchor the + round exists to produce. """ enable_multi_node(monkeypatch) pass_sec = 600.0 @@ -2326,141 +2287,14 @@ def test_a_first_baseline_on_a_default_session_runs_both_of_its_passes( assert result["status"] == "succeeded", f"a first baseline was refused a default session: {result}" assert result["output_throughput"] == pytest.approx(_HOT_TPUT) assert set(launches) >= set(_BASELINE_ROUND_SLOTS), f"the round did not run both passes: {list(launches)}" + remaining_sec = _DEFAULT_SESSION_MINUTES * 60.0 warmup = _mn_warmup_cap_sec(calls) - assert warmup is not None and warmup >= pass_sec, ( - f"the warmup was capped under the {pass_sec}s this pass takes, so it could only be killed: {warmup}" + assert warmup is not None and warmup >= remaining_sec, ( + f"the warmup's cap was moved in front of the session deadline, so the " + f"pass can now be killed by its own timeout before the watchdog reaches " + f"it: {warmup}s against {remaining_sec}s left" ) - def test_a_first_baseline_too_big_for_its_budget_costs_one_share_to_find_out( - self, - tmp_path, - monkeypatch, - ): - """With nothing measured, the shortfall is discovered by the cap, not predicted. - - Nothing this session measured says what a pass of this workload costs, and - the two numbers that could stand in for it both lie: the declared cap is a - hang backstop longer than the session, and the static per-action estimates - are calibrated on small models. So the warmup is launched with its share - and the cap is what finds out -- which bounds the cost of finding out to - that share. - - The alternative is what a warmup with no hold-back does: it is granted - more than the whole remaining budget, runs to completion, and leaves the - measured round to be admitted with nothing and reaped, so the session pays - for both passes and keeps neither. - """ - enable_multi_node(monkeypatch) - remaining_sec = 3600.0 - result, calls = _run_baseline_under_budget( - tmp_path, - remaining_sec=remaining_sec, - timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, - measured_expected_sec=0.0, - pass_duration_sec=2000.0, - ) - launches = launches_by_round_slot(calls) - - assert _MEASURED_ROUND_SLOT not in launches, "a measured round was launched into a budget that cannot pay" - warmup = _mn_warmup_cap_sec(calls) - assert warmup is not None and warmup <= remaining_sec / 2.0, ( - f"the warmup was allowed more than its share of what is left: {warmup}s of {remaining_sec}s" - ) - assert result["status"] == "failed" - assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, ( - f"the shortfall was reported as something else: {result.get('error_class')}" - ) - assert result["budget_shortfall"]["bound"] == "prelude_ceiling" - - def test_a_warmup_that_does_not_fit_leaves_no_anchor_rather_than_a_cold_one( - self, - tmp_path, - monkeypatch, - ): - """Dropping the warmup does not drop the harm the warmup exists to prevent. - - The per-round server restart runs under the same condition as the warmup, - so whenever the warmup would have run, the server was just restarted and - the warmup is the only thing that drives it. Running the measured pass - without it makes that pass the first traffic against a cold server, and - its throughput becomes the anchor every later gain is reported against -- - recorded as a plain success, with nothing saying so. - """ - enable_multi_node(monkeypatch) - pass_sec = 600.0 - result, calls = _run_baseline_under_budget( - tmp_path, - remaining_sec=800.0, - timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, - measured_expected_sec=pass_sec, - pass_duration_sec=pass_sec, - ) - - assert calls == [], ( - f"the round it already knows does not fit was launched anyway: {[c['round_slot'] for c in calls]}" - ) - assert result["status"] == "failed" - assert result["error_class"] == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS - assert result.get("output_throughput") is None, "a cold anchor was persisted anyway" - - def test_a_declared_cap_below_one_pass_is_not_blamed_on_the_budget( - self, - tmp_path, - monkeypatch, - ): - """A cap smaller than the workload is a fact about the cap, not the clock. - - The first baseline of a session runs under the 9000s cold-start cap and - promotes its runtime; every later one runs under the 7800s warm cap. An - anchor runtime in between therefore sits above the cap the next round is - given, permanently -- and a rule that refuses the warmup whenever its cap - is under one measured pass disables it for the rest of the run while - naming the session budget as the reason, with a week of clock left. - """ - enable_multi_node(monkeypatch) - _result, calls = _run_baseline_under_budget( - tmp_path, - remaining_sec=7.0 * 24 * 3600, - timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, - measured_expected_sec=BASELINE_COLD_START_TIMEOUT_SEC - 1000.0, - ) - - assert _mn_warmup_cap_sec(calls) == BASELINE_DEFAULT_TIMEOUT_SEC, ( - "the warmup was refused over its own declared cap while the session had a week left" - ) - - def test_neither_benching_arm_launches_a_pass_its_measured_round_cannot_follow( - self, - tmp_path, - monkeypatch, - ): - """The contract the two arms share, which is about outcomes, not seconds. - - They answer from different information -- the grid always knows what a - pass of this workload costs, a session's first baseline never does -- so - pinning them to the same number would pin one of them to a rule it has no - basis for. What must hold either way is that a discarded pass is never - started when the round it warms for cannot follow it: that is the only - shape in which the GPU time buys nothing at all. - """ - enable_multi_node(monkeypatch) - remaining_sec, declared_cap_sec, expected_sec = 800.0, 7200, 600.0 - _result, baseline_calls = _run_baseline_under_budget( - tmp_path, - remaining_sec=remaining_sec, - timeout_sec=declared_cap_sec, - measured_expected_sec=expected_sec, - ) - grid_calls = _launch_one_grid_variant_under_budget( - tmp_path / "grid_arm", - remaining_sec=remaining_sec, - variant_timeout_sec=declared_cap_sec, - variant_expected_sec=expected_sec, - ) - - assert _mn_warmup_cap_sec(baseline_calls) is None, "the baseline arm launched a doomed warmup" - assert _mn_warmup_cap_sec(grid_calls) is None, "the grid arm launched a doomed warmup" - def test_the_profile_arm_gets_all_of_it(self, tmp_path): """Profile is the same executor with a four-hour default -- longer than any session budget it could be given.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 56f480154a..4f88a60951 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1209,50 +1209,6 @@ async def test_baseline_rounds_the_run_stopped_do_not_charge_the_failure_streak( await c.stop() -@pytest.mark.asyncio -async def test_a_baseline_the_budget_cannot_fit_closes_the_run_naming_the_budget(session_dir): - """One shortfall is terminal, and it is not the model's fault. - - A baseline round is two passes and what is left cannot buy both. That is a - fact about the clock, not about this attempt, and the clock only shrinks -- - so a retry re-derives the same answer having spent more on the asking. The - two shapes already here both get it wrong. Left as a stop the run chose, the - streak stays at zero and the reactor keeps proposing baselines until the - session clock genuinely dies, which it need not do for hours after PRELUDE's - own share is gone. Counted as a failure, three of them close the run as - ``baseline_failed``, which reads as a model that cannot boot. - - So it closes on the first one, with the word the vocabulary already has for - preparation running out of clock -- and ``abort_prelude`` already routes that - word to CLOSE, so nothing new had to be taught how to end a run. - """ - from hyperloom.orchestrator.phases.machine_state import PHASE_CLOSE, compute_next_phase - - c = Coordinator(session_dir, backends=_silent_backends()) - _mute_action_scoring(c) - try: - await c._handle_unpromotable_result( - _mk_task("baseline", "t-below-one-round"), - { - "status": "failed", - "error_class": "session_budget_below_one_round", - "error": "the remaining budget cannot fit both passes of this round", - }, - ) - assert c.shared_state.stop_reason == "time_exhausted_during_prelude" - assert c.shared_state.baseline_failure_streak == 0, "the clock was charged to the model" - assert c.shared_state.baseline_total_failures == 0 - assert len(c.shared_state.last_action_failures) == 1, "the round was not recorded" - - c.shared_state.phase = "PRELUDE" - transition = compute_next_phase(c.shared_state) - assert transition is not None, "the run had no way to reach a terminal state" - assert transition[0] == PHASE_CLOSE - assert transition[1] == "time_exhausted_during_prelude" - finally: - await c.stop() - - @pytest.mark.asyncio async def test_a_stopped_baseline_round_does_not_clear_a_real_failure_streak(session_dir): """A stop the run chose neither charges the streak nor forgives what preceded it.""" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 33fb71e070..3f266763f6 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1689,7 +1689,7 @@ def _record_round_stop( _mn_warmup_rounds = 1 if (_mn_is_multi_node() and _mn_bench_warmup_enabled()) else 0 variant_rounds = 1 + (1 if auto_warmup_requested else 0) + _mn_warmup_rounds - def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: + def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant_rounds) -> bool: """Skip variant ``idx`` and every one after it, or say the budget still fits. Checked twice per variant, because the two things that spend the budget @@ -1698,9 +1698,16 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: Once a variant does not fit, none after it does either -- the clock only shrinks -- so this ends the batch rather than trying the next name. + ``rounds_left`` is what makes the second check a different question from + the first rather than the same one asked with less clock. A pass this + variant has already run is paid for, and charging for it again refuses a + variant that fits and throws away the GPU time already spent on it. + Args: idx: Zero-based index of the variant about to run. spent_on: What has been spent since the last check, for the log line. + rounds_left: Passes of this variant still to launch. Defaults to all + of them, which is right for the check before any has run. Returns: bool: ``True`` when the batch is over and nothing more was launched. @@ -1711,18 +1718,18 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: # Falls back to a single ``variant_timeout_sec`` when no estimate was # given, which is what callers that cannot estimate already got. required_sec = ( - float(variant_expected_sec) * variant_rounds + float(variant_expected_sec) * rounds_left if variant_expected_sec is not None else float(variant_timeout_sec) ) if remaining_sec >= required_sec: return False log.warning( - "grid_runner: %.0fs left cannot fit this variant's %d round(s) of %.0fs " - "(spent on: %s); skipping %d remaining variant(s) rather than launching a " - "pass whose measured round cannot follow it", + "grid_runner: %.0fs left cannot fit this variant's %d remaining round(s) " + "of %.0fs (spent on: %s); skipping %d remaining variant(s) rather than " + "launching a pass whose measured round cannot follow it", max(0.0, remaining_sec), - variant_rounds, + rounds_left, required_sec, spent_on, len(grid) - idx, @@ -1950,13 +1957,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: session_deadline_sec=session_deadline_sec, ) except subprocess.TimeoutExpired as exc: - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) log.warning( "grid_runner: variant %d/%d name=%s aborted: warmup timeout (timeout_sec=%d): %s", i + 1, @@ -1993,13 +1994,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: warmup_stopped = stopped_by_the_run(warmup_rc) if warmup_stopped is not None: - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) grid_is_over = _record_round_stop( warmup_stopped, idx=i, @@ -2034,13 +2029,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: subprocess_started_unix=warmup_started_unix, ) if warmup_rc != 0 or not warmup_measurement.get("valid_measurement"): - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) warmup_error = ( server_log_death_excerpt(str(warmup_server_log)) or redact_secret_values((warmup_stderr or warmup_stdout)[-2000:]) @@ -2170,8 +2159,17 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: # The restart above is a launch of its own and is under no cap, so the # budget admitted at the top of the loop may no longer be there. Booting - # a large model can take longer than a benchmark pass. - if _skip_rest_for_budget(i, spent_on="this variant's server restart"): + # a large model can take longer than a benchmark pass. Only the passes + # still ahead are charged for: the auto warmup above this point has + # already run, and re-charging it here would end the batch over time + # that is spent either way. + if _skip_rest_for_budget( + i, + spent_on="this variant's server restart", + rounds_left=1 + _mn_warmup_rounds, + ): + if auto_warmup: + _teardown_variant_server(slot, lifecycle) break # Multi-node client warmup: one discarded benchmark pass against the @@ -2306,13 +2304,7 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: continue finally: if auto_warmup: - from ._server_lifecycle import teardown_lifecycle_server - - teardown_lifecycle_server( - pid_dir=slot, - framework=str(lifecycle.get("framework") or ""), - port=int(lifecycle.get("port") or 0), - ) + _teardown_variant_server(slot, lifecycle) # Eval bounds could not be installed for a variant that runs eval, so # nothing launched. Labelled rather than left to the generic path, which @@ -2694,6 +2686,28 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str) -> bool: return results +def _teardown_variant_server(slot: Path, lifecycle: dict[str, Any]) -> None: + """Stop the server this variant booted for its own rounds. + + Every path out of a variant that booted one owes this call, including the + ones that leave over the budget rather than over a result: the process + outlives the loop iteration that started it, and the next variant boots its + own. Kept in one function so a new exit path is one line rather than a + four-line block someone can leave out. + + Args: + slot: The variant's workspace, which is also its pid directory. + lifecycle: The variant's resolved lifecycle, carrying framework and port. + """ + from ._server_lifecycle import teardown_lifecycle_server + + teardown_lifecycle_server( + pid_dir=slot, + framework=str(lifecycle.get("framework") or ""), + port=int(lifecycle.get("port") or 0), + ) + + def _not_run_skip_result(variant: GridVariant, stopped: StoppedByTheRun) -> VariantResult: """Build the ``skipped`` result for a variant the run never got to. diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 3bb692901d..2acd0a13a5 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -37,11 +37,7 @@ from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import heartbeat_while_output_flows, report_progress from ...phases import machine_state as _phase_state -from ..stop_attribution import ( - SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, - STOPPED_BY_THE_RUN, - StoppedByTheRun, -) +from ..stop_attribution import StoppedByTheRun from . import _server_lifecycle as _lifecycle from ._file_lock import best_effort_file_lock from ._aiter_jit import ( @@ -116,6 +112,11 @@ # because the warmup that exists to drive it did not. Its throughput is a cold # number, and the session anchors every later gain on it. _MN_WARMUP_DID_NOT_WARM_WARNING = "baseline_mn_warmup_did_not_run" +# The round kept its warmup pass as the baseline because the budget could not +# pay for the measured pass after it. Same consequence as the warning above -- +# a cold anchor -- reached from the other direction, and carried on the result +# so a reader of the session's gains knows the denominator is depressed. +MEASURE_ROUND_DROPPED_WARNING = "baseline_measure_round_dropped_low_budget" # The cold-start guard's two round directories. The warmup round is the only one # that measures accuracy (``RUN_EVAL=true``); the measured round is hot @@ -317,8 +318,6 @@ def _stopped_round_result( runtime_sec: float, output_dir: Path, capture_meta: dict[str, Any], - never_started: bool = False, - evidence: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build the result for a round the run itself stopped. @@ -341,26 +340,19 @@ def _stopped_round_result( runtime_sec: Wall-clock seconds the round had run for. output_dir: The task workspace, echoed onto the result. capture_meta: Config/eval-contract facts every failure result carries. - never_started: Whether the round was refused before it launched, which - is the other half of every cause here and reads differently in a - ledger: no GPU time was spent, so there is not even a partial round - to look for. - evidence: The numbers behind the stop, when the cause is one the run - computed rather than observed. Returns: dict[str, Any]: The failed result carrying the stop's own error class. """ - detail = stopped.never_started if never_started else stopped.interrupted + detail = stopped.interrupted log.warning( - "baseline_executor: %s %s after %.1fs: %s; error_class=%s.", + "baseline_executor: %s reaped after %.1fs: %s; error_class=%s.", round_label, - "refused" if never_started else "reaped", runtime_sec, detail, stopped.error_class, ) - result = { + return { "status": "failed", "error_class": stopped.error_class, "returncode": returncode, @@ -369,9 +361,6 @@ def _stopped_round_result( "output_dir": str(output_dir), **capture_meta, } - if evidence is not None: - result["budget_shortfall"] = evidence - return result def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple[float | None, dict[str, Any]]: @@ -405,56 +394,6 @@ def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple return remaining_sec, {**outside, "bound": "session_deadline", "affordable_sec": round(remaining_sec, 1)} -def warmup_pass_cap_sec( - cap: int, - *, - headroom_sec: float | None, - measured_expected_sec: float = 0.0, -) -> int | None: - """The hard cap for a discarded warmup pass, or ``None`` when it must not run. - - A round is two passes of the same workload -- a warmup whose throughput is - thrown away and the measured pass that re-attaches to the server it left - hot -- so the warmup may claim at most half of what the round has to spend. - That is not an estimate of anything: it is the pre-image of the gate that - runs after the warmup, which asks whether what is left still covers a pass - costing what the warmup cost. Both bounds behind ``headroom_sec`` fall by - exactly the wall-clock the warmup burns, so a warmup that survives this cap - is exactly a warmup that gate will pass, and one killed by it is exactly one - that gate would have refused after paying for it in full. - - The half is what makes the rule work on a session's first baseline, where - nothing has been measured and there is therefore nothing to predict a pass - from. Pricing the pair at the declared cap instead would refuse a baseline - in every default session: the caps are hang backstops of 7800s warm and - 9000s cold, each longer than the whole two-hour default the phase gets 40% - of. - - ``measured_expected_sec`` is what a later baseline knows a pass of this - workload costs. It only ever refuses the pass earlier than the cap would -- - a round already known not to fit is not worth half a round of GPU time to - re-discover -- and it is deliberately not consulted the other way: a - declared cap below one pass is a fact about the cap, not about the budget. - - Args: - cap: The timeout the caller would grant with an unbounded budget. - headroom_sec: Seconds this round may still spend, from - :func:`_round_headroom_sec`; ``None`` when it is under no budget. - measured_expected_sec: Seconds one pass of this workload is known to - take, or ``0`` when this session has not measured one. - - Returns: - int | None: The hard timeout for the warmup pass, or ``None`` when what - is left cannot fit both it and the measured round it warms for. - """ - if headroom_sec is None: - return int(cap) - share_sec = headroom_sec / 2.0 - if share_sec < 1.0 or (measured_expected_sec > 0.0 and share_sec < measured_expected_sec): - return None - return min(int(cap), int(share_sec)) - - def _disable_cuda_graph_flag(framework: str) -> str: """Return the framework-correct flag that disables cuda-graph capture. @@ -1909,11 +1848,13 @@ def _session_capped_timeout( of the session is left. A round granted more than the budget has runs past the end of the session and takes the closing phase with it. - Nothing is held back here, which is what keeps the cap sitting past the - session deadline: the measured round is the last pass of the round, and a - cap the session watchdog reaches first is a cap whose kill is attributed - to the budget rather than to the model. The pass that does hold budget - back is the discarded warmup, in :meth:`_warmup_pass_timeout`. + Nothing is held back here, and no pass of a baseline round holds anything + back either. That is what keeps every cap sitting past the session + deadline, so the watchdog reaches a round before the round's own timeout + does and the kill is attributed to the budget rather than to the model. + Whether a round should start, and whether its measured pass should follow + its warmup, are decided by the gates that price those questions -- not by + shortening a cap until the round dies of it. Args: timeout_sec: The timeout this round would get on an unbounded budget. @@ -1930,52 +1871,6 @@ def _session_capped_timeout( output_dir=output_dir, ) - @staticmethod - def _warmup_pass_timeout( - timeout_sec: int, - *, - output_dir: Path, - headroom_sec: float | None, - measured_expected_sec: float | None, - evidence: dict[str, Any], - ) -> int | None: - """The warmup pass's cap, or ``None`` when the round must not be started. - - The decision itself is :func:`warmup_pass_cap_sec`, shared by both - two-pass shapes a baseline has -- the single-node cold+hot double run and - the multi-node client warmup -- so a round is sized the same way whoever - launches it; this adds the log lines. - - Args: - timeout_sec: The timeout this pass would get on an unbounded budget. - output_dir: The warmup pass's workspace, for the log line. - headroom_sec: Seconds this round may still spend, or ``None`` when it - is under no budget. - measured_expected_sec: Seconds one pass of this workload is known to - take, or ``None`` when this session has not measured one. - evidence: The numbers behind ``headroom_sec``, for the log line. - - Returns: - int | None: The hard timeout for the warmup pass, or ``None`` when - what is left cannot fit both passes of the round. - """ - capped = warmup_pass_cap_sec( - timeout_sec, - headroom_sec=headroom_sec, - measured_expected_sec=float(measured_expected_sec or 0.0), - ) - if capped is None: - log.warning( - "baseline_executor: not starting the round — a round is two passes " - "and %.0fs of budget is left for it (bound=%s), so the pass it could " - "pay for is the one whose number nothing may use (round=%s)", - float(headroom_sec or 0.0), - evidence.get("bound", ""), - output_dir.name, - ) - return None - return _logged_session_clamp(timeout_sec, capped, output_dir=output_dir) - @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: """Resolve the InferenceX checkout the subprocess will ``cd`` into. @@ -3268,44 +3163,18 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: "baseline_executor: cold-start guard — warmup round (discarded, boots persistent server) in %s", warmup_dir, ) - # The warmup may claim at most half of what the round has, so a round - # that cannot fit both passes is refused before it spends the first - # one, and a warmup that survives its cap is one the gate below will - # pass. ``None`` is a budget under a whole round. - warmup_cap_sec, budget_evidence = self._double_run_warmup_budget( - ctx_extra=extra, - timeout_sec=timeout_sec, - warmup_dir=warmup_dir, - ) - round_meta = {"materialized_config": str(materialized_config_path)} - if warmup_cap_sec is None: - return _stopped_round_result( - STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], - round_label="cold+hot double round", - returncode=None, - runtime_sec=0.0, - output_dir=output_dir, - capture_meta=round_meta, - never_started=True, - evidence=budget_evidence, - ) + # The warmup runs under the round's own cap, which the session clamp + # leaves sitting past the session deadline so the watchdog reaches it + # first and a budget kill is recorded as one. Whether the measured + # round can follow is asked after this pass, priced with what it + # actually cost rather than a prediction of what it would. warmup_result = await self._run_reported_round( label="warmup", config_path=warmup_cfg, output_dir=warmup_dir, - **{**common, "timeout_sec": warmup_cap_sec}, + **common, ) if warmup_result.get("status") != "succeeded": - if warmup_cap_sec < timeout_sec and warmup_result.get("error_class") == "timeout": - return _stopped_round_result( - STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], - round_label="cold+hot warmup round", - returncode=None, - runtime_sec=float(warmup_result.get("subprocess_runtime_sec") or 0.0), - output_dir=output_dir, - capture_meta=round_meta, - evidence={**budget_evidence, "warmup_cap_sec": warmup_cap_sec}, - ) # Warmup failure almost certainly recurs, so skip the # measured round. warmup_result.setdefault("nonfatal_warnings", []) @@ -3349,25 +3218,23 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: ) if not affordable: log.warning( - "baseline_executor: the warmup took %.0fs and only %.0fs of " - "preparation budget is left (bound=%s), so the measured round " - "cannot follow it. The warmup is not kept: its throughput is " - "the cold-start number the round exists to discard, and an " - "anchor measured that way depresses every gain the session " - "goes on to report against it.", + "baseline_executor: the warmup took %.0fs and only %.0fs is " + "left (bound=%s), so the measured round cannot follow it. " + "Keeping the warmup as the baseline; it is the cold anchor a " + "single-round baseline would have produced, and the GPU time " + "it cost is already spent. The marker below says the figure " + "is cold so the session's later gains can be read against a " + "known-depressed denominator.", float(warmup_runtime or 0.0), gate_evidence.get("affordable_sec", 0.0), gate_evidence.get("bound", ""), ) - return _stopped_round_result( - STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], - round_label="cold+hot measured round", - returncode=None, - runtime_sec=float(warmup_runtime or 0.0), - output_dir=output_dir, - capture_meta=round_meta, - evidence=gate_evidence, + warmup_result.setdefault("nonfatal_warnings", []) + warmup_result["nonfatal_warnings"].append( + MEASURE_ROUND_DROPPED_WARNING, ) + warmup_result["measure_round_dropped"] = gate_evidence + return warmup_result # Round 2 (measured): re-attach to the hot server (client only). # Warm re-attach is intentional — all comparison points (baseline, @@ -3535,40 +3402,6 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() - def _double_run_warmup_budget( - self, - *, - ctx_extra: dict[str, Any] | None, - timeout_sec: int, - warmup_dir: Path, - ) -> tuple[int | None, dict[str, Any]]: - """The cold round's cap and the numbers behind it, or ``None`` to not start. - - The single-node half of the same decision the multi-node warmup makes in - :meth:`_mn_warmup_pass`: both shapes discard a pass in front of the one - that is measured, so both size that pass out of the same headroom. - - Args: - ctx_extra: The runner context extras carrying ``shared_state``. - timeout_sec: The round's cap on an unbounded budget. - warmup_dir: The cold round's workspace, for the log line. - - Returns: - tuple[int | None, dict[str, Any]]: The cap, or ``None`` when what is - left cannot fit both passes, plus the evidence behind it. - """ - state = (ctx_extra or {}).get("shared_state") or self.shared_state - session_deadline_sec, measured_expected_sec = session_grid_bounds(state) - headroom_sec, evidence = _round_headroom_sec(state, session_deadline_sec) - cap_sec = self._warmup_pass_timeout( - timeout_sec, - output_dir=warmup_dir, - headroom_sec=headroom_sec, - measured_expected_sec=measured_expected_sec, - evidence=evidence, - ) - return cap_sec, evidence - def _measure_round_affordable( self, *, @@ -3577,13 +3410,20 @@ def _measure_round_affordable( ) -> tuple[bool, dict[str, Any]]: """Whether PRELUDE's remaining budget still covers the measured round. - The confirmation of what :func:`warmup_pass_cap_sec` sized the warmup - against, priced with the runtime the warmup actually had rather than the - share it was allowed: the round spends wall-clock either side of that - pass -- the server restart, the teardown -- which the cap does not bound. - The warmup's own runtime is the estimate for the round that follows it, - and an upper bound rather than a guess, because the measured round - re-attaches to a server the warmup has already booted. + Asked after the warmup rather than before it, and priced with what that + pass actually cost rather than a prediction of what it would. A session's + first baseline has nothing to predict from -- the measured runtimes are + written only once an anchor lands -- so a gate that ran first would + either refuse every first baseline or wave every one through. The + warmup's own wall-clock is available exactly when this question is asked, + and it is an upper bound rather than an estimate: the measured round + re-attaches to a server this pass has already booted, so it cannot cost + more. + + A round that fails here has still produced a number. It is the cold one + the double run exists to discard, so the caller keeps it as the anchor + and marks it -- the GPU time is spent either way, and a marked cold + anchor beats no anchor. Only PRELUDE is guarded — a re-baseline in a later phase answers to that phase's budget. @@ -3843,11 +3683,8 @@ async def _run_reported_round( env: dict[str, str], output_dir: Path, framework: str, - declared_timeout_sec: int, timeout_sec: int, session_deadline_sec: float | None, - measured_expected_sec: float | None, - state: Any, capture_meta: dict[str, Any], round_warnings: list[str], ) -> dict[str, Any] | None: @@ -3857,20 +3694,21 @@ async def _run_reported_round( this is the only thing that drives it before the measured pass. So the two are one round: skipping the warmup does not save the round's cost, it moves the round's measurement onto a cold server and anchors the - session's every later gain on it. A budget that cannot pay for both - therefore buys nothing here, and says so. + session's every later gain on it. + + Both passes run under the round's own cap, which the session clamp leaves + past the session deadline so a budget kill arrives as the watchdog's + sentinel rather than as this pass timing out. Nothing is held back for + the measured pass: holding back moves the cap in front of the deadline + and puts the round back in reach of its own timeout. Args: cmd: The measured pass's command, re-pointed at the warmup slot. env: The measured pass's environment, re-pointed the same way. output_dir: The round's workspace; the warmup runs in a slot under it. framework: Framework name, for the watchdog's server log. - declared_timeout_sec: The round's cap before any budget touched it, - which is what tells a hang apart from a budget-shortened pass. timeout_sec: The round's cap after the session clamp. session_deadline_sec: Monotonic-clock session deadline, or ``None``. - measured_expected_sec: Seconds one pass is known to take, or ``None``. - state: The session ``SharedState``, for the round's headroom. capture_meta: Config/eval-contract facts every failure result carries. round_warnings: Collected onto the round's result, for a warmup that did not run and did not end the round. @@ -3880,25 +3718,6 @@ async def _run_reported_round( else ``None`` to go on to the measured pass. """ warm_dir = output_dir / "mn_warmup" - headroom_sec, headroom_evidence = _round_headroom_sec(state, session_deadline_sec) - warm_timeout_sec = self._warmup_pass_timeout( - timeout_sec, - output_dir=warm_dir, - headroom_sec=headroom_sec, - measured_expected_sec=measured_expected_sec, - evidence=headroom_evidence, - ) - if warm_timeout_sec is None: - return _stopped_round_result( - STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], - round_label="multi-node round", - returncode=None, - runtime_sec=0.0, - output_dir=output_dir, - capture_meta=capture_meta, - never_started=True, - evidence=headroom_evidence, - ) started_unix = time.time() # The measurement is discarded, but the returncode is not: this pass is a # full benchmark round, so a stop here ends the baseline round. Going on @@ -3922,7 +3741,7 @@ async def _run_reported_round( warm_cmd, env=warm_env, cwd=str(warm_dir), - timeout=warm_timeout_sec, + timeout=timeout_sec, server_log_path=_watchdog_server_log_path(warm_dir, framework), on_output=warm_activity.note, session_deadline_sec=session_deadline_sec, @@ -3930,16 +3749,6 @@ async def _run_reported_round( warm_rc = warm_proc.returncode log.info("baseline_executor: MN warmup pass done (discarded) rc=%s", warm_rc) except subprocess.TimeoutExpired as exc: - if warm_timeout_sec < declared_timeout_sec: - return _stopped_round_result( - STOPPED_BY_THE_RUN[SESSION_BUDGET_BELOW_ONE_ROUND_CLASS], - round_label="multi-node warmup pass", - returncode=None, - runtime_sec=max(0.0, time.time() - started_unix), - output_dir=output_dir, - capture_meta=capture_meta, - evidence={**headroom_evidence, "warmup_cap_sec": warm_timeout_sec}, - ) log.warning("baseline_executor: MN warmup pass hit its own hang backstop (ignored): %r", exc) round_warnings.append(_MN_WARMUP_DID_NOT_WARM_WARNING) except Exception as exc: # noqa: BLE001 - a warmup that fails on its own is best-effort @@ -4077,8 +3886,7 @@ async def _run_single_benchmark( # per task: a baseline runs up to three of them, and a warmup that # overran has already spent budget the ones after it were counting on. _session_state = ctx_extra.get("shared_state") or self.shared_state - session_deadline_sec, measured_expected_sec = session_grid_bounds(_session_state) - declared_timeout_sec = int(timeout_sec) + session_deadline_sec, _ = session_grid_bounds(_session_state) timeout_sec = self._session_capped_timeout(timeout_sec, session_deadline_sec, output_dir=output_dir) if not ctx_extra.get("mn_round_restarted"): try: @@ -4150,11 +3958,8 @@ async def _run_single_benchmark( env=env, output_dir=output_dir, framework=framework, - declared_timeout_sec=declared_timeout_sec, timeout_sec=timeout_sec, session_deadline_sec=session_deadline_sec, - measured_expected_sec=measured_expected_sec, - state=_session_state, capture_meta=capture_meta, round_warnings=round_warnings, ) diff --git a/src/hyperloom/orchestrator/actions/stop_attribution.py b/src/hyperloom/orchestrator/actions/stop_attribution.py index 78b65f145a..c207f63b33 100644 --- a/src/hyperloom/orchestrator/actions/stop_attribution.py +++ b/src/hyperloom/orchestrator/actions/stop_attribution.py @@ -35,7 +35,6 @@ __all__ = [ "ORCHESTRATOR_CANCELLED_CLASS", - "SESSION_BUDGET_BELOW_ONE_ROUND_CLASS", "SESSION_TIME_EXHAUSTED_CLASS", "STOPPED_BY_THE_RUN", "StoppedByTheRun", @@ -53,15 +52,6 @@ # not face the shutdown. ORCHESTRATOR_CANCELLED_CLASS = "orchestrator_cancelled" -# Labels work refused, or cut short, because what is left of the budget cannot -# pay for a whole round of it. A benchmark round is two passes -- a discarded -# warmup and the measured pass it makes comparable -- so a budget that fits one -# fits none: the pass it could pay for is the one whose number nothing may use. -# Distinct from the class above because the deadline has not passed. Nothing was -# measured either way, but this one is also known to be permanent: the clock -# only shrinks, so the round that does not fit now never will. -SESSION_BUDGET_BELOW_ONE_ROUND_CLASS = "session_budget_below_one_round" - class StoppedByTheRun(NamedTuple): """How work that the run stopped from outside is recorded. @@ -97,15 +87,6 @@ class StoppedByTheRun(NamedTuple): never_started="the orchestrator cancelled this action before this round ran", ends_the_batch=True, ), - SESSION_BUDGET_BELOW_ONE_ROUND_CLASS: StoppedByTheRun( - error_class=SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, - interrupted=( - "the warmup pass was stopped at its share of the remaining budget, " - "which is therefore too small to fit both passes of this round" - ), - never_started="the remaining budget cannot fit both passes of this round", - ends_the_batch=True, - ), } diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 62f407f796..3e30d239da 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -25,7 +25,7 @@ summarize_change, ) from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON -from ..actions.stop_attribution import SESSION_BUDGET_BELOW_ONE_ROUND_CLASS, stopped_by_the_run_class +from ..actions.stop_attribution import stopped_by_the_run_class from ..state.shared_state import SharedState, resolve_grading_anchor_tput from hyperloom.inference_optimizer.protocol.intent import Intent from ..bus.message_bus import Message @@ -1014,27 +1014,7 @@ async def _handle_unpromotable_result( ) if eval_failed: self._persist_eval_failure(result_payload) - if err_class == SESSION_BUDGET_BELOW_ONE_ROUND_CLASS: - # Terminal on the first one. A baseline round is two passes and - # what is left cannot buy both, which is a fact about the clock - # rather than about this attempt: retrying re-derives the same - # answer from a budget that has only shrunk since. The two shapes - # already here both get it wrong -- a stop the run chose keeps the - # reactor proposing baselines until the session clock genuinely - # dies, which it need not do for hours after PRELUDE's own share - # is gone, and three counted failures close the run as - # ``baseline_failed``, blaming the model for the budget. The - # vocabulary already has the honest word for preparation running - # out of clock, and ``abort_prelude`` already routes it to CLOSE. - log.warning( - "baseline %s was refused by the budget (%s); closing as " - "time_exhausted_during_prelude rather than retrying a round the " - "clock can only afford less of", - task.task_id, - err_class, - ) - self.shared_state.set_stop_reason("time_exhausted_during_prelude") - elif stopped_by_the_run: + if stopped_by_the_run: log.warning( "baseline %s was stopped by the run (%s); the failure streak stays at %d " "because nothing about the baseline was measured", From 603ebca11c0fb9d3e39cb91adb5f44a5fcda2b85 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 03:27:21 +0000 Subject: [PATCH 46/65] machine_state: preparation answers to the session clock, not a ledger the resume outlives PRELUDE's share was the tighter of two bounds: what was left of a 40% ceiling on its own banked spend, and what was left of the session once the optimization phases' 50% reserve was held back. The two read different clocks, and a resume that reanchors the budget makes them disagree -- the session clock restarts, the phase ledger carries every second the earlier leg banked -- so preparation was born over its ceiling with the whole budget ahead of it, and the measured half of the baseline was refused on every resumed run. The reserve alone answers the question that decides the matter: after this work, is enough left for the phases that produce the result? It reads the same usable remainder every other budget decision reads, so it survives the reanchor. The ceiling constant, the phase-spend lookup, and the ``now_unix`` override that existed only to date it are gone with it; no caller ever passed the override. One test that asserted the ceiling was passing on an artefact rather than on its subject. Its double set ``phase_started_unix`` to zero, which the policy read as a preparation phase running since the epoch, so a round meant to be refused for spending its share on server boot and teardown was refused for being 56 years old instead. The double now carries no phase clock at all, the round is placed where its overheads really do reach into the reserve, and the case the ceiling used to cover -- a long preparation followed by an expensive optional arm -- is still refused, by the reserve, on the same numbers. Also clamps the post-warmup gate's cost to non-negative, as its sibling ``prelude_can_afford`` already did: a negative runtime would otherwise satisfy a negative headroom and wave the measured round through on no budget. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 42 ++++++--------- .../tests/test_phase_state_machine.py | 30 +++++++---- .../actions/executors/baseline.py | 5 +- .../orchestrator/phases/machine_state.py | 54 +++++++++---------- 4 files changed, 64 insertions(+), 67 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 8990fb26cc..4f85806a82 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -381,7 +381,7 @@ def _run_double_run_baseline(tmp_path, shared_state) -> dict: def test_a_budget_that_cannot_pay_for_the_measured_round_keeps_the_cold_warmup(tmp_path): - """Preparation has spent the share the second pass needs; the first still ran. + """The session clock cannot pay for the second pass; the first still ran. Nothing is predicted before the warmup, so the round starts and the warmup's GPU time is spent before the shortfall is known. Refusing to keep its figure @@ -400,7 +400,7 @@ def test_a_budget_that_cannot_pay_for_the_measured_round_keeps_the_cold_warmup(t assert result["_rounds_run"] == 1, "the measured round ran on a budget that cannot pay for it" assert result["output_throughput"] == pytest.approx(_COLD_TPUT) assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] - assert result["measure_round_dropped"]["bound"] == "prelude_ceiling" + assert result["measure_round_dropped"]["bound"] == "optimization_reserve" def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_path): @@ -411,11 +411,18 @@ def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_ the teardown behind it are wall-clock the cap never bounded. Asking after the pass, against the clock rather than against the cap, is what catches that -- and is why nothing is predicted before the pass instead. + + The round enters with 7200s on the clock, of which 3600s is held for the + optimization phases, so a gate asked before the warmup would have found + 3600s of headroom and waved through a pass of any length. The round then + charges 4000s -- the pass itself returns at once, the overheads around it do + not -- and the same question asked afterwards finds the clock 400s into the + reserve. """ base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") state = _BudgetedState(remaining_sec=7200.0, double_run=True) - fake_run, calls = _capturing_fake_run(state=state, charge_sec=3000.0) + fake_run, calls = _capturing_fake_run(state=state, charge_sec=4000.0) executor = BaselineExecutor( magpie_python=sys.executable, default_config_path=base, @@ -1899,18 +1906,17 @@ class _BudgetedState: Production reads one session through two accessors -- the deadline the round reaper is handed and the usable-seconds figure the phase policy reads -- and a double that lets them drift cannot see the regime where they disagree. - Both are derived from one deadline here, and PRELUDE's spend is whatever the - session has already used, which is what a run that has not left PRELUDE has - spent it on. + Both are derived from one deadline here. A pass calls :meth:`charge` for the wall-clock it burned, which is the only way a test can reach the case this whole mechanism exists for: a round whose first pass leaves the second one nothing. - ``max_minutes`` defaults to a session that has just started with - ``remaining_sec`` on the clock, so a test that only cares how much is left - says only that. Give it explicitly to place the round part-way through a - session, which is what decides how much of PRELUDE's own share is gone. + It carries no phase clock on purpose. An earlier version set + ``phase_started_unix`` to zero, which the budget policy of the day read as a + preparation phase running since the epoch, and a test meant to show a + round's overheads exhausting the session passed on that arithmetic instead. + The policy answers to the session clock alone, so that is all this offers. """ def __init__( @@ -1918,18 +1924,13 @@ def __init__( *, remaining_sec: float | None, measured_expected_sec: float = 0.0, - max_minutes: float | None = None, phase: str = "PRELUDE", double_run: bool = False, ) -> None: self.baseline_double_run = double_run self.baseline_runtime_sec = measured_expected_sec self.phase = phase - if remaining_sec is None: - self.max_minutes = 0.0 - else: - self.max_minutes = remaining_sec / 60.0 if max_minutes is None else max_minutes - self.phase_started_unix = 0.0 + self.max_minutes = 0.0 if remaining_sec is None else remaining_sec / 60.0 self._deadline = None if remaining_sec is None else time.monotonic() + remaining_sec def charge(self, seconds: float) -> None: @@ -1943,13 +1944,6 @@ def grid_session_deadline_sec(self) -> float | None: def session_budget_usable_sec(self) -> float | None: return None if self._deadline is None else self._deadline - time.monotonic() - @property - def phase_elapsed_totals(self) -> dict[str, float]: - usable = self.session_budget_usable_sec() - if usable is None: - return {} - return {"PRELUDE": max(0.0, self.max_minutes * 60.0 - usable)} - def _capturing_fake_run( returncode: int = 0, @@ -2034,7 +2028,6 @@ def _run_baseline_under_budget( produces_workspace: bool = True, measured_expected_sec: float = 0.0, pass_duration_sec: float = 0.0, - max_minutes: float | None = None, phase: str = "PRELUDE", executor_cls=BaselineExecutor, ) -> tuple[dict, list[dict]]: @@ -2044,7 +2037,6 @@ def _run_baseline_under_budget( state = _BudgetedState( remaining_sec=remaining_sec, measured_expected_sec=measured_expected_sec, - max_minutes=max_minutes, phase=phase, ) fake_run, calls = _capturing_fake_run( diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py index d59a60f3e4..d555e047d9 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py @@ -245,25 +245,33 @@ def test_prelude_can_afford_an_arm_the_budget_still_covers(): state = _prelude_state(spent_sec=600.0, usable_sec=10_000.0) affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=300.0) assert affordable is True - assert evidence["prelude_spent_sec"] == 600.0 + # Half of 180 minutes is held for the optimization phases; the rest is PRELUDE's. + assert evidence["affordable_sec"] == pytest.approx(4600.0) -def test_prelude_refuses_an_arm_past_its_own_ceiling(): +def test_prelude_refuses_an_arm_that_would_eat_the_optimization_reserve(): """The Qwen3.5-397B shape: 51 minutes of baseline, then a roofline that costs another 45+.""" state = _prelude_state(spent_sec=3090.0, usable_sec=7700.0) affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=2706.0) assert affordable is False - assert evidence["bound"] == "prelude_ceiling" - # 40% of 180 minutes is 4320s; 3090 spent leaves 1230s, well under the arm. - assert evidence["affordable_sec"] == pytest.approx(1230.0) + assert evidence["bound"] == "optimization_reserve" + # 7700s left, 5400s of it spoken for, so the arm may cost at most 2300s. + assert evidence["affordable_sec"] == pytest.approx(2300.0) -def test_prelude_refuses_an_arm_that_would_eat_the_optimization_reserve(): - """Under the phase ceiling but over the session reserve: the tighter bound wins.""" - state = _prelude_state(spent_sec=60.0, usable_sec=6000.0) - affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=3000.0) - assert affordable is False - assert evidence["bound"] == "optimization_reserve" +def test_a_resumed_prelude_is_not_charged_for_what_the_earlier_leg_spent(): + """Banked phase spend and the session clock answer to different origins. + + A resume that reanchors the budget restarts the session clock while the + phase ledger keeps every second the earlier leg banked. A bound read off + the ledger therefore declared preparation overspent on a session that had + its whole budget ahead of it, and the measured half of the baseline was + refused on every resumed run. Only the clock decides. + """ + state = _prelude_state(spent_sec=10_000.0, usable_sec=10_000.0) + affordable, evidence = phase_state.prelude_can_afford(state, expected_cost_sec=2706.0) + assert affordable is True + assert evidence["affordable_sec"] == pytest.approx(4600.0) def test_prelude_budget_policy_is_inert_without_a_clock(): diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 2acd0a13a5..c386e9c3c9 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -366,7 +366,8 @@ def _stopped_round_result( def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple[float | None, dict[str, Any]]: """Seconds this round's budget may still spend, and the numbers behind it. - In PRELUDE that is the phase's own share + In PRELUDE that is what the session has left once the optimization phases' + reserve is held back (:func:`~...phases.machine_state.prelude_affordable_seconds`), which is the same figure the post-warmup gate judges the measured round against. Outside it a re-baseline answers to what is left before the session deadline, the @@ -3440,7 +3441,7 @@ def _measure_round_affordable( if headroom_sec is None: return True, evidence try: - cost = float(warmup_runtime_sec or 0.0) + cost = max(0.0, float(warmup_runtime_sec or 0.0)) except (TypeError, ValueError): cost = 0.0 return headroom_sec >= cost, {"expected_cost_sec": round(cost, 1), **evidence} diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 66bf1b3024..78f191b491 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -352,8 +352,7 @@ def is_valid_phase_exit_reason(value: str) -> bool: PHASE_CLOSE: 0.02, } -# Share of the session PRELUDE may spend before its optional arms are dropped, -# and the share held for the phases that actually produce a result. +# Share of the session held back for the phases that actually produce a result. # # PRELUDE's ``DEFAULT_PHASE_BUDGET_PCT`` entry is 3%, but nothing enforced it: # :func:`exit_normal_prelude` tests only whether a baseline landed, so the phase @@ -365,11 +364,20 @@ def is_valid_phase_exit_reason(value: str) -> bool: # nothing. Stopping the run on time is necessary and not sufficient -- the # phases that spend the budget have to be the ones that produce the result. # -# These bound the preparation rather than the session. They are deliberately +# This bounds the preparation rather than the session, and it is deliberately # looser than 3%: a baseline that legitimately takes an hour should still run, # it just cannot also buy an 80-minute roofline out of the optimization phases' # time. -PRELUDE_SPEND_CEILING_PCT: float = 0.40 +# +# A second bound used to sit beside it -- a ceiling on PRELUDE's own banked +# spend -- and it was removed because the two clocks it straddled can disagree. +# Banked phase spend accumulates against the phase ledger; the reserve is read +# off the session clock. A session resumed with a reanchored budget restarts the +# session clock and carries the ledger over, so preparation was born over its +# ceiling with hours genuinely left, and every resumed session was refused the +# measured half of its baseline. The reserve alone answers the question that +# decides the matter: after this work, is enough left for the phases that +# produce the result? OPTIMIZATION_RESERVE_PCT: float = 0.50 # Wall-clock ceiling for an unbounded run (``max_minutes`` == 0): the container @@ -2078,18 +2086,14 @@ def _session_usable_seconds(state: Any) -> float | None: return session_remaining_seconds(state) -def prelude_affordable_seconds( - state: Any, - *, - now_unix: float | None = None, -) -> tuple[float | None, dict[str, Any]]: +def prelude_affordable_seconds(state: Any) -> tuple[float | None, dict[str, Any]]: """Seconds PRELUDE may still spend, and the numbers the figure is built from. - Two bounds apply and the tighter wins: what is left of PRELUDE's own share - (:data:`PRELUDE_SPEND_CEILING_PCT`), and what is left of the session once - the optimization phases' reserve (:data:`OPTIMIZATION_RESERVE_PCT`) is held - back. Work that fits neither is not work the session needed — it is work - the session could not have used the result of. + What is left on the session clock once the optimization phases' reserve + (:data:`OPTIMIZATION_RESERVE_PCT`) is held back, measured against the same + usable remainder every other budget decision reads, so the figure survives + a resume that reanchors the budget. Work that does not fit is not work the + session needed — it is work the session could not have used the result of. Read directly by callers that have to *size* a unit of work rather than judge one they can already price, which is the position a first baseline is @@ -2098,28 +2102,23 @@ def prelude_affordable_seconds( Args: state (Any): Frozen SharedState view. - now_unix (float | None): Override for the current time. Returns: tuple[float | None, dict[str, Any]]: The affordable seconds — which may - be negative once the share is overspent — or ``None`` on an unbounded - budget, plus the evidence behind it. + be negative once the reserve is eaten into — or ``None`` on an + unbounded budget, plus the evidence behind it. """ max_sec = _max_minutes(state) * 60.0 usable = _session_usable_seconds(state) if max_sec <= 0.0 or usable is None: return None, {"reason": "unbounded_budget"} - spent = phase_cumulative_seconds(state, phase=PHASE_PRELUDE, now_unix=now_unix) - phase_headroom = max_sec * PRELUDE_SPEND_CEILING_PCT - spent - session_headroom = usable - max_sec * OPTIMIZATION_RESERVE_PCT - affordable_sec = min(phase_headroom, session_headroom) + reserve_sec = max_sec * OPTIMIZATION_RESERVE_PCT + affordable_sec = usable - reserve_sec return affordable_sec, { - "prelude_spent_sec": round(spent, 1), - "prelude_ceiling_sec": round(max_sec * PRELUDE_SPEND_CEILING_PCT, 1), - "optimization_reserve_sec": round(max_sec * OPTIMIZATION_RESERVE_PCT, 1), + "optimization_reserve_sec": round(reserve_sec, 1), "session_usable_sec": round(usable, 1), "affordable_sec": round(affordable_sec, 1), - "bound": "prelude_ceiling" if phase_headroom <= session_headroom else "optimization_reserve", + "bound": "optimization_reserve", } @@ -2127,7 +2126,6 @@ def prelude_can_afford( state: Any, *, expected_cost_sec: float, - now_unix: float | None = None, ) -> tuple[bool, dict[str, Any]]: """Decide whether PRELUDE can still buy an optional arm costing ``expected_cost_sec``. @@ -2140,7 +2138,6 @@ def prelude_can_afford( should anchor this on something this session measured; the static per-action estimates are calibrated on small models and understate a large one by an order of magnitude. - now_unix (float | None): Override for the current time. Returns: tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. Evidence @@ -2148,7 +2145,7 @@ def prelude_can_afford( and the phase record. """ cost = max(0.0, float(expected_cost_sec or 0.0)) - affordable_sec, evidence = prelude_affordable_seconds(state, now_unix=now_unix) + affordable_sec, evidence = prelude_affordable_seconds(state) priced = {"expected_cost_sec": round(cost, 1), **evidence} if affordable_sec is None: return True, priced @@ -3335,7 +3332,6 @@ def record_lifecycle_event( "DEFAULT_EXPLORE_FORCE_EXIT_HOURS_REMAINING", "DEFAULT_PHASE_BUDGET_PCT", "OPTIMIZATION_RESERVE_PCT", - "PRELUDE_SPEND_CEILING_PCT", "DEFAULT_PLATEAU_EXPLORE_EMPTY_STREAK", "DEFAULT_PLATEAU_EXPLORE_KEEP_GAIN_PCT", "DEFAULT_PLATEAU_EXPLORE_LOOKBACK", From 92951f0f06da1c348a13f52059a7257431656664 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 03:53:58 +0000 Subject: [PATCH 47/65] baseline: a round the session cannot finish does not boot a server to find out The post-warmup gate keeps the session honest but not thrifty: it can only ask its question once the warmup has already paid for the boot, the compile and the graph capture. On a first baseline that is unavoidable -- there is nothing to predict from, so a gate in front would either refuse every one of them or wave every one through. From the second round on the figures exist, and so does an anchor, which is what makes refusing free: the session keeps the number it already had instead of spending a cold pass to be told it cannot keep the new one. So the round now faces a gate before the lease is taken, judged on a lower bound built only from measurements rather than on an estimate, which is what makes refusing on it sound: if the part the session can prove does not fit, the whole round does not. The two passes are added separately because they buy different things, and a session can hold the cold figure without the hot one -- a round whose measured pass was dropped for budget promotes its cold number and has no hot number to write. That session is the one most in need of the gate, so its cold pass is still priced and only the unpriceable pass is left to the gate after the warmup. A refusal is the run stopping work, not the model failing a measurement, so it carries ``session_time_exhausted`` and no returncode, and it finally gives ``StoppedByTheRun.never_started`` the consumer it was written for -- until now every stop went through the ``interrupted`` wording, including the ones that had not started. Seven cases pin it, and each of five mutations -- the gate waved through, the double run priced on one pass, the gate going inert on a half-measured session, and the two result-shape slips -- is caught by exactly one of them. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 110 +++++++++++ .../actions/executors/baseline.py | 179 ++++++++++++++++-- 2 files changed, 274 insertions(+), 15 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 4f85806a82..621fa4ab7e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -46,6 +46,7 @@ ORCHESTRATOR_CANCELLED_RETURNCODE, SESSION_TIME_EXHAUSTED_RETURNCODE, ) +from hyperloom.orchestrator.actions.stop_attribution import STOPPED_BY_THE_RUN from hyperloom.orchestrator.state.shared_state import SharedState from hyperloom.orchestrator.trace.task_progress import progress_scope @@ -455,6 +456,109 @@ def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): assert "budget_shortfall" not in result +class TestARoundThatCannotFinishIsNotIgnited: + """The gate in front of a round, and the one thing it may be asked with. + + A round is refused before it boots only on what earlier rounds measured, so + the session's first one is never refused: it has nothing to be judged by, and + a gate that guessed would either refuse every first baseline or wave every + one through. From the second on, the figures exist and an anchor exists with + them, so refusing costs the session nothing it had while igniting costs it a + boot, a compile and a graph capture for a number it would then have to mark + as cold. + + Every session here has 3600s on its clock, half of which is held for the + phases that produce a result, so a round is judged against 1800s. + """ + + def test_a_first_round_is_not_judged_at_all(self, tmp_path): + """Nothing measured yet, so nothing to refuse it with.""" + result, calls = _run_baseline_under_budget(tmp_path, remaining_sec=3600.0) + + assert result["status"] == "succeeded" + assert calls, "a first baseline was refused on a prediction the session cannot have" + + def test_a_round_larger_than_what_is_left_boots_nothing(self, tmp_path): + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=1900.0, + ) + + assert calls == [], "GPU time was spent on a round the session cannot finish" + assert result["status"] == "failed" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert result["returncode"] is None, "a round that never launched reported a returncode" + assert result["error"] == STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS].never_started + assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(1900.0) + assert result["budget_shortfall"]["affordable_sec"] == pytest.approx(1800.0, abs=1.0) + + def test_a_round_that_still_fits_is_ignited(self, tmp_path): + """The gate must not turn a merely expensive round into a refused one.""" + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=1700.0, + ) + + assert result["status"] == "succeeded" + assert calls + + def test_a_double_run_pays_for_both_of_its_passes(self, tmp_path): + """Priced on the round, not on the pass: 1500s fits, 1500s plus 400s does not.""" + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=1500.0, + measured_warm_sec=400.0, + double_run=True, + ) + + assert calls == [], "the round was priced on one pass while planning to run two" + assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(1900.0) + + def test_a_single_round_pays_for_the_one_pass_it_runs(self, tmp_path): + """The same two figures, with no second pass to buy, still fit.""" + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=1500.0, + measured_warm_sec=400.0, + ) + + assert result["status"] == "succeeded" + assert calls + + def test_a_missing_hot_figure_still_pays_for_the_cold_pass(self, tmp_path): + """The state a previous cold-anchor drop leaves, and the one to catch. + + A round whose measured pass was dropped for budget promotes its cold + number and has no hot number to write. Going inert on a half-measured + session would exempt exactly the session that already ran out of budget + once. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=3000.0, + double_run=True, + ) + + assert calls == [] + assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(3000.0) + + def test_a_hot_pass_that_cannot_be_priced_does_not_refuse_a_cold_one_that_fits(self, tmp_path): + """Only the provable part refuses; the rest is the post-warmup gate's.""" + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=1500.0, + double_run=True, + ) + + assert calls, "a round was refused for a pass the session could not price" + + def test_deferred_accuracy_skips_eval_when_hot_throughput_regresses( tmp_path, ): @@ -1924,11 +2028,13 @@ def __init__( *, remaining_sec: float | None, measured_expected_sec: float = 0.0, + measured_warm_sec: float = 0.0, phase: str = "PRELUDE", double_run: bool = False, ) -> None: self.baseline_double_run = double_run self.baseline_runtime_sec = measured_expected_sec + self.baseline_warm_runtime_sec = measured_warm_sec self.phase = phase self.max_minutes = 0.0 if remaining_sec is None else remaining_sec / 60.0 self._deadline = None if remaining_sec is None else time.monotonic() + remaining_sec @@ -2027,8 +2133,10 @@ def _run_baseline_under_budget( returncode: int = 0, produces_workspace: bool = True, measured_expected_sec: float = 0.0, + measured_warm_sec: float = 0.0, pass_duration_sec: float = 0.0, phase: str = "PRELUDE", + double_run: bool = False, executor_cls=BaselineExecutor, ) -> tuple[dict, list[dict]]: """Run one baseline round against a session with ``remaining_sec`` left.""" @@ -2037,7 +2145,9 @@ def _run_baseline_under_budget( state = _BudgetedState( remaining_sec=remaining_sec, measured_expected_sec=measured_expected_sec, + measured_warm_sec=measured_warm_sec, phase=phase, + double_run=double_run, ) fake_run, calls = _capturing_fake_run( returncode, diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index c386e9c3c9..2d8cf238ff 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -37,7 +37,11 @@ from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import heartbeat_while_output_flows, report_progress from ...phases import machine_state as _phase_state -from ..stop_attribution import StoppedByTheRun +from ..stop_attribution import ( + SESSION_TIME_EXHAUSTED_CLASS, + STOPPED_BY_THE_RUN, + StoppedByTheRun, +) from . import _server_lifecycle as _lifecycle from ._file_lock import best_effort_file_lock from ._aiter_jit import ( @@ -318,6 +322,7 @@ def _stopped_round_result( runtime_sec: float, output_dir: Path, capture_meta: dict[str, Any], + started: bool = True, ) -> dict[str, Any]: """Build the result for a round the run itself stopped. @@ -333,25 +338,39 @@ def _stopped_round_result( Nothing here arms a retry either: the cause is the run, and a resume meets it again. + A round the budget stopped *before* it booted anything is the same cause and + carries the same class; only the wording and the absent returncode differ, so + a reader is told whether GPU time was spent. + Args: stopped: How to record the cause. round_label: Which round was stopped, for the log line. - returncode: The stopped round's returncode. + returncode: The stopped round's returncode, or ``None`` when nothing ran. runtime_sec: Wall-clock seconds the round had run for. output_dir: The task workspace, echoed onto the result. capture_meta: Config/eval-contract facts every failure result carries. + started: Whether the round had begun. ``False`` selects the wording for + work that never launched. Returns: dict[str, Any]: The failed result carrying the stop's own error class. """ - detail = stopped.interrupted - log.warning( - "baseline_executor: %s reaped after %.1fs: %s; error_class=%s.", - round_label, - runtime_sec, - detail, - stopped.error_class, - ) + detail = stopped.interrupted if started else stopped.never_started + if started: + log.warning( + "baseline_executor: %s reaped after %.1fs: %s; error_class=%s.", + round_label, + runtime_sec, + detail, + stopped.error_class, + ) + else: + log.warning( + "baseline_executor: %s not launched: %s; error_class=%s.", + round_label, + detail, + stopped.error_class, + ) return { "status": "failed", "error_class": stopped.error_class, @@ -395,6 +414,28 @@ def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple return remaining_sec, {**outside, "bound": "session_deadline", "affordable_sec": round(remaining_sec, 1)} +def _measured_runtime_sec(state: Any, field: str) -> float | None: + """Read a runtime some earlier round measured, or ``None`` when none did. + + Absent and zero are the same answer: both mean no round has landed an anchor + yet. Neither may read as a round that cost nothing, which is what a plain + ``float(... or 0.0)`` would make of them. + + Args: + state: The session ``SharedState``, or ``None``. + field: The SharedState attribute holding the runtime. + + Returns: + float | None: The measured seconds, or ``None`` when there is no + measurement to predict from. + """ + try: + value = float(getattr(state, field, 0.0) or 0.0) + except (TypeError, ValueError): + return None + return value if value > 0.0 else None + + def _disable_cuda_graph_flag(framework: str) -> str: """Return the framework-correct flag that disables cuda-graph capture. @@ -2935,6 +2976,36 @@ async def _run_once( materialized_config_path ) + # Asked before the lease, because a round that will not be run should not + # hold a GPU while being refused. + ignitable, ignition_evidence = self._round_affordable_before_ignition( + double_run=double_run, + ctx_extra=extra, + ) + if not ignitable: + stopped_result = _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + round_label="baseline round", + returncode=None, + runtime_sec=0.0, + output_dir=output_dir, + capture_meta={ + "materialized_config": str(materialized_config_path), + "run_eval_disabled": bool(run_eval_disabled), + }, + started=False, + ) + stopped_result["budget_shortfall"] = ignition_evidence + log.warning( + "baseline_executor: a whole round is expected to cost %.0fs and " + "only %.0fs is left (bound=%s), so nothing is booted. The anchor " + "this session already measured stands.", + ignition_evidence.get("expected_cost_sec", 0.0), + ignition_evidence.get("affordable_sec", 0.0), + ignition_evidence.get("bound", ""), + ) + return stopped_result + before_apply_sha = _git_head_sha(patch_target) def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if live_shared_state is None: @@ -3074,7 +3145,6 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: len(applied_patches), [p["patch_file"] for p in applied_patches], ) - # Ray-managed GPU execution (§12 T1): one held Ray lease (``num_gpus=TP``) # spans this baseline's benchmark rounds — a double-run's warmup + # measure reuse one persistent server, so both must run under the same @@ -3403,6 +3473,85 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() + def _round_cost_lower_bound_sec( + self, + *, + double_run: bool, + ctx_extra: dict[str, Any] | None = None, + ) -> float | None: + """What this round will cost at least, from what earlier rounds measured. + + A lower bound rather than an estimate, so that refusing on it is sound: + if the part the session can prove does not fit, the whole round does not. + The two passes are priced apart because they buy different things -- the + first boots the server and pays the one-time compile and graph capture, + the second re-attaches a client to a server already up -- and each is + added only once it has been measured. + + A session can hold the cold figure without the hot one, and that is not a + corner case: a round whose measured pass was dropped for budget promotes + its cold number as the anchor and has no hot number to write. Such a + session is the one most in need of this gate, so its cold pass is still + priced; only the pass it cannot price is left out, and whether that one + may follow is settled after the warmup as usual. + + Args: + double_run: Whether this round will run both passes. + ctx_extra: The runner context extras carrying ``shared_state``. + + Returns: + float | None: Seconds the round will cost at least, or ``None`` when + the session has measured nothing to predict from. + """ + state = (ctx_extra or {}).get("shared_state") or self.shared_state + cold_sec = _measured_runtime_sec(state, "baseline_runtime_sec") + if cold_sec is None or not double_run: + return cold_sec + warm_sec = _measured_runtime_sec(state, "baseline_warm_runtime_sec") + return cold_sec if warm_sec is None else cold_sec + warm_sec + + def _round_affordable_before_ignition( + self, + *, + double_run: bool, + ctx_extra: dict[str, Any] | None = None, + ) -> tuple[bool, dict[str, Any]]: + """Whether the budget still holds a whole round, asked before anything boots. + + The companion to :meth:`_measure_round_affordable`, and the two divide + the session's baselines between them. A session's first round is not + asked this question, because there is nothing to ask it with: the + measured runtimes are written only once an anchor lands, so a gate here + would either refuse every first baseline or wave every one through. That + round runs, and whether a second pass may follow is settled afterwards + against what the first actually cost. + + Every later round has at least the previous one's cold figure, and by then + an anchor already exists. So a session too poor to finish a round should + not spend a cold pass discovering it: refusing costs the session nothing + it had, while igniting costs it the boot, the compile, and the capture for + a number it will then have to mark as cold. + + Args: + double_run: Whether this round will run both passes. + ctx_extra: The runner context extras carrying ``shared_state``. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. + """ + state = (ctx_extra or {}).get("shared_state") or self.shared_state + cost = self._round_cost_lower_bound_sec( + double_run=double_run, + ctx_extra=ctx_extra, + ) + if cost is None: + return True, {"reason": "no_measured_round_to_predict_from"} + headroom_sec, evidence = _round_headroom_sec(state, None) + if headroom_sec is None: + return True, evidence + priced = {"expected_cost_sec": round(cost, 1), **evidence} + return headroom_sec >= cost, priced + def _measure_round_affordable( self, *, @@ -3412,15 +3561,15 @@ def _measure_round_affordable( """Whether PRELUDE's remaining budget still covers the measured round. Asked after the warmup rather than before it, and priced with what that - pass actually cost rather than a prediction of what it would. A session's - first baseline has nothing to predict from -- the measured runtimes are - written only once an anchor lands -- so a gate that ran first would - either refuse every first baseline or wave every one through. The + pass actually cost rather than a prediction of what it would. The warmup's own wall-clock is available exactly when this question is asked, and it is an upper bound rather than an estimate: the measured round re-attaches to a server this pass has already booted, so it cannot cost more. + This is the gate every round faces, including the first, which is the one + :meth:`_round_affordable_before_ignition` cannot judge. + A round that fails here has still produced a number. It is the cold one the double run exists to discard, so the caller keeps it as the anchor and marks it -- the GPU time is spent either way, and a marked cold From 85a08cffaff837effd0dc965639673940eaa8b56 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 04:45:06 +0000 Subject: [PATCH 48/65] baseline: say what the round's price really is, an over-prediction with one safe use The gate in front of a round called its figure a lower bound and justified refusing on it as sound. It is neither. The figure is an earlier round's full wall-clock, and that round paid a one-time JIT compile on a cold kernel cache which a later pass on the same signature does not pay again -- the executor's own cache probe is built on exactly that difference and picks a cap 20 minutes shorter when it finds the cache warm. So the figure over-predicts, and a gate built on it refuses rounds that would have fitted. Which is tolerable here, but only here, and for a reason worth writing down: the two outcomes are not symmetric. Igniting a round that cannot finish costs a boot, a compile and a capture for a number that must then be marked cold. Refusing one that would have fitted costs a fresh anchor the session did not need, because the anchor it measured earlier still stands and every later comparison is made against that one. A caller that would end a session on this figure would be trading the rest of the run against a systematic over-prediction, and the docstring now says so rather than leaving the next reader to borrow the justification. Renamed accordingly, and the survival property the argument rests on is now asserted instead of assumed: a refused round comes back with no throughput and no warning, so there is nothing on it that promotion could put over the anchor. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 18 ++++++++ .../actions/executors/baseline.py | 46 +++++++++++++------ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 621fa4ab7e..a4e3af61e1 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -547,6 +547,24 @@ def test_a_missing_hot_figure_still_pays_for_the_cold_pass(self, tmp_path): assert calls == [] assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(3000.0) + def test_a_refused_round_carries_nothing_that_could_replace_the_anchor(self, tmp_path): + """The property the whole gate rests on, asserted rather than assumed. + + The figure this gate judges by over-predicts: it comes from a pass that + paid a one-time compile on a cold kernel cache, which a later pass on a + warm one does not pay again. Refusing too readily is tolerable only + because the anchor the session already measured survives a refusal, so + the round must come back with no throughput to promote over it. + """ + result, _calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + measured_expected_sec=1900.0, + ) + + assert result.get("output_throughput") in (None, 0, 0.0) + assert not result.get("nonfatal_warnings"), "a refused round volunteered a warning to promote" + def test_a_hot_pass_that_cannot_be_priced_does_not_refuse_a_cold_one_that_fits(self, tmp_path): """Only the provable part refuses; the rest is the post-warmup gate's.""" result, calls = _run_baseline_under_budget( diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 2d8cf238ff..f0fb232807 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -3473,20 +3473,30 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() - def _round_cost_lower_bound_sec( + def _last_round_cost_sec( self, *, double_run: bool, ctx_extra: dict[str, Any] | None = None, ) -> float | None: - """What this round will cost at least, from what earlier rounds measured. + """What an earlier round of this session actually cost. + + An over-prediction of what the next one will cost, and knowingly so. The + figure comes from a pass that paid the one-time JIT compile on a fresh + kernel cache, which a later pass on the same signature does not pay + again; the executor's own cache probe is built on that difference, and + picks a cap 20 minutes shorter when it finds the cache warm. + + Over-predicting is the tolerable direction *for this caller only*, and + the reason is in :meth:`_round_affordable_before_ignition`: a refusal + here costs the session a fresh anchor it can do without, because the + anchor it already has still stands. No caller that would end a session on + this figure may use it. - A lower bound rather than an estimate, so that refusing on it is sound: - if the part the session can prove does not fit, the whole round does not. The two passes are priced apart because they buy different things -- the - first boots the server and pays the one-time compile and graph capture, - the second re-attaches a client to a server already up -- and each is - added only once it has been measured. + first boots the server and pays the compile and the graph capture, the + second re-attaches a client to a server already up -- and each is added + only once it has been measured. A session can hold the cold figure without the hot one, and that is not a corner case: a round whose measured pass was dropped for budget promotes @@ -3500,8 +3510,8 @@ def _round_cost_lower_bound_sec( ctx_extra: The runner context extras carrying ``shared_state``. Returns: - float | None: Seconds the round will cost at least, or ``None`` when - the session has measured nothing to predict from. + float | None: What an earlier round cost, or ``None`` when the + session has measured nothing to predict from. """ state = (ctx_extra or {}).get("shared_state") or self.shared_state cold_sec = _measured_runtime_sec(state, "baseline_runtime_sec") @@ -3527,10 +3537,18 @@ def _round_affordable_before_ignition( against what the first actually cost. Every later round has at least the previous one's cold figure, and by then - an anchor already exists. So a session too poor to finish a round should - not spend a cold pass discovering it: refusing costs the session nothing - it had, while igniting costs it the boot, the compile, and the capture for - a number it will then have to mark as cold. + an anchor already exists. That is what makes this gate safe to build on an + over-predicting figure: the two outcomes are not symmetric. Igniting a + round that cannot finish costs a boot, a compile and a graph capture for a + number that must then be marked cold. Refusing one that would have fitted + costs a fresh anchor the session did not need, because the anchor it + measured earlier still stands and every later comparison is made against + that one. + + The asymmetry is the whole justification, so it may not be borrowed. A + caller that would *end the session* on this figure would be trading the + rest of the run against a systematic over-prediction, and needs a bound + that errs the other way. Args: double_run: Whether this round will run both passes. @@ -3540,7 +3558,7 @@ def _round_affordable_before_ignition( tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. """ state = (ctx_extra or {}).get("shared_state") or self.shared_state - cost = self._round_cost_lower_bound_sec( + cost = self._last_round_cost_sec( double_run=double_run, ctx_extra=ctx_extra, ) From e5edb4eabee7a4de542fe9410585f672b10f9ca5 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 06:53:32 +0000 Subject: [PATCH 49/65] baseline: name the one thing that keeps a warmup's figure as a marked cold anchor Two paths end a round after its warmup has already paid for the boot, the compile and the capture, and both owe the same answer: keep the number, mark it cold. Only one of them exists today. Naming it now means the second arrives as a call rather than as a copy of these four lines. Co-authored-by: Cursor --- .../actions/executors/baseline.py | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index f0fb232807..f4f2d572b6 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -414,6 +414,35 @@ def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple return remaining_sec, {**outside, "bound": "session_deadline", "affordable_sec": round(remaining_sec, 1)} +def _cold_anchor_from_warmup( + warmup_result: dict[str, Any], + *, + dropped: dict[str, Any], +) -> dict[str, Any]: + """Keep the warmup's cold figure as the anchor, marked as the cold one. + + Two things end a round once its warmup has already paid for the boot, the + compile and the capture: the budget cannot cover a second pass, and the + session's budget reaped the second pass mid-flight. Either way a number + exists and the GPU time behind it is spent, so it is kept rather than + discarded -- a marked cold anchor beats no anchor. The marker is what tells a + reader of the session's later gains that their denominator is depressed. + + Args: + warmup_result: The succeeded warmup pass's result, mutated in place. + dropped: Why the measured round did not produce the figure, recorded so + the decision is legible in the result and the session record. + + Returns: + dict[str, Any]: ``warmup_result``, marked. + """ + warnings = warmup_result.setdefault("nonfatal_warnings", []) + if MEASURE_ROUND_DROPPED_WARNING not in warnings: + warnings.append(MEASURE_ROUND_DROPPED_WARNING) + warmup_result["measure_round_dropped"] = dropped + return warmup_result + + def _measured_runtime_sec(state: Any, field: str) -> float | None: """Read a runtime some earlier round measured, or ``None`` when none did. @@ -3300,12 +3329,7 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: gate_evidence.get("affordable_sec", 0.0), gate_evidence.get("bound", ""), ) - warmup_result.setdefault("nonfatal_warnings", []) - warmup_result["nonfatal_warnings"].append( - MEASURE_ROUND_DROPPED_WARNING, - ) - warmup_result["measure_round_dropped"] = gate_evidence - return warmup_result + return _cold_anchor_from_warmup(warmup_result, dropped=gate_evidence) # Round 2 (measured): re-attach to the hot server (client only). # Warm re-attach is intentional — all comparison points (baseline, From 7adef31ded1fc8916b51b28c67860ad665f63dcb Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 06:59:14 +0000 Subject: [PATCH 50/65] executors: a round reports what it spent booting apart from what it spent benchmarking A round's wall-clock is two things bought by different spenders. Every explore variant boots its own server, so a variant costs the sum; a pass that re-attaches to a server already up costs only the benchmark. Reported as one number, the total is the only price available, and pricing a re-attaching pass at it refuses work that fits comfortably. The instant is recorded where it is already known -- the gate loop that latches the ready marker for the from-ready soft deadline -- into a stamp beside the round's server.log. A file rather than a wider return value because the reader is not always the writer: on the Ray path the round runs inside an actor, whose monotonic clock has its own origin and whose return would have to be widened through every degraded branch to carry this. The caller already reads that same directory's server.log to classify server deaths, so the channel is not new. A stamp older than the round it is read for is reported as unknown rather than clamped, which would price a cold round as though it had never booted. Co-authored-by: Cursor --- .../tests/test_soft_deadline_from_ready.py | 132 +++++++++++++++++- .../actions/executors/_subprocess_kill.py | 100 +++++++++++++ .../actions/executors/baseline.py | 54 +++++++ src/hyperloom/orchestrator/loop/writeback.py | 11 ++ .../orchestrator/state/shared_state.py | 6 + 5 files changed, 302 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py index db92727dbe..f8fc414407 100644 --- a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py +++ b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py @@ -1,12 +1,17 @@ # SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Tests for the from-ready soft-deadline caliber. +"""Tests for the two things measured from the server-ready marker. When a ``server.log`` is available the explore overtime soft deadline measures only the post-ready phase: the clock starts at the server-ready marker, excluding pre-ready boot / weight load / first-request recompile. Opt out via ``INFERENCE_OPTIMIZER_SOFT_DEADLINE_FROM_READY=0``. + +The same marker is recorded so a round can be *priced* by its two parts rather +than its total, which is what lets later work be charged for what it will +actually spend: a variant boots its own server and pays both parts, a pass that +re-attaches pays only the second. """ from __future__ import annotations @@ -16,7 +21,10 @@ from hyperloom.orchestrator.actions.executors._subprocess_kill import ( OVERTIME_KILL_RETURNCODE, + clear_server_ready_stamp, + post_ready_runtime_sec, run_with_session_kill, + server_ready_unix, ) # Long stall grace so the detok-stall watchdog never interferes here. @@ -115,3 +123,125 @@ def test_no_server_log_uses_from_spawn(tmp_path): elapsed = time.monotonic() - start assert cp.returncode == OVERTIME_KILL_RETURNCODE assert elapsed < 15.0, f"from-spawn (no server.log) took {elapsed:.2f}s" + + +class TestARoundIsPricedByItsTwoParts: + """What a round spent booting and what it spent benchmarking, told apart. + + The whole point of separating them is that they are spent by different + things. Charging a re-attaching pass for a boot it never pays is what makes + a budget gate refuse work that fits. + """ + + def test_the_boot_is_not_charged_to_the_benchmark(self, tmp_path): + """A round that boots for 3s and benchmarks for 1s reports 1s, not 4s.""" + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(3)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + runtime_sec = time.time() - started_unix + assert cp.returncode == 0 + + post_ready = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + assert post_ready is not None, "the round reported ready but nothing recorded when" + # The benchmark's own second, found without the three the boot took. The + # windows are wide because the poll interval and process spawn are inside + # them; what is being pinned is that the two parts are told apart at all. + assert 0.5 <= post_ready <= 2.5, f"benchmark share read as {post_ready:.2f}s, expected ~1s" + boot_sec = runtime_sec - post_ready + assert 2.5 <= boot_sec <= 4.5, f"boot share read as {boot_sec:.2f}s, expected ~3s" + + def test_a_round_that_never_came_up_is_not_priced(self, tmp_path): + """No ready marker means no split, reported as unknown rather than guessed.""" + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(1)\n" + ) + started_unix = time.time() + run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + assert server_ready_unix(str(log_path)) is None + assert ( + post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=time.time() - started_unix, + ) + is None + ) + + def test_an_earlier_rounds_stamp_is_not_read_as_this_ones(self, tmp_path): + """A stamp predating the round is unknown, not "it never booted". + + The clamp alone would report such a stamp as a whole-round benchmark, + which is the reading that would price a cold round as a warm one. + """ + log_path = tmp_path / "server.log" + log_path.write_text("INFO loading weights\n", encoding="utf-8") + (tmp_path / "server_ready_at").write_text(f"{time.time() - 600.0:.3f}\n", encoding="utf-8") + + assert ( + post_ready_runtime_sec( + str(log_path), + started_unix=time.time(), + runtime_sec=90.0, + ) + is None + ) + + def test_a_stamp_is_cleared_so_the_next_round_starts_blind(self, tmp_path): + """Clearing is what keeps the case above from arising in a reused dir.""" + log_path = tmp_path / "server.log" + stamp = tmp_path / "server_ready_at" + stamp.write_text(f"{time.time():.3f}\n", encoding="utf-8") + assert server_ready_unix(str(log_path)) is not None + + clear_server_ready_stamp(str(log_path)) + + assert not stamp.exists() + assert server_ready_unix(str(log_path)) is None + # Idempotent: a round whose dir never had one must not fail to start. + clear_server_ready_stamp(str(log_path)) + + def test_a_clock_that_disagrees_cannot_produce_a_negative_price(self, tmp_path): + """The writer and the reader can be different hosts, so skew is possible. + + A stamp after the round's own end would price the benchmark at less than + nothing; it is floored instead, and the round's total caps the other end. + """ + log_path = tmp_path / "server.log" + started_unix = time.time() + (tmp_path / "server_ready_at").write_text(f"{started_unix + 500.0:.3f}\n", encoding="utf-8") + + priced = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=100.0, + ) + + assert priced == 0.0 diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 32a4b6338e..9bb2127a38 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -589,6 +589,99 @@ def server_log_death_excerpt(path: str, *, max_chars: int = 1200) -> str | None: return None +# Name of the stamp written beside the caller's ``server.log`` the moment the +# server first reports ready. +_READY_STAMP_NAME = "server_ready_at" + + +def _ready_stamp_path(server_log_path: str) -> Path: + """Return where a round's ready stamp lives, given its ``server.log`` path.""" + return Path(server_log_path).parent / _READY_STAMP_NAME + + +def _stamp_server_ready(server_log_path: str) -> None: + """Record, beside ``server_log_path``, that the server just reported ready. + + Written as a wall-clock (``time.time()``) instant rather than the + ``time.monotonic()`` one the gates run on, because the process that reads it + is not always the one that wrote it: on the Ray path the round runs inside an + actor, and a monotonic reading means nothing outside the process that took + it. A file is used for the same reason -- it crosses the actor boundary that + the round's return value would otherwise have to be widened to cross, and the + round's output directory is already how post-mortem evidence gets back (the + caller reads the same directory's ``server.log`` to classify server deaths). + + Best effort: a round whose stamp cannot be written loses a measurement, which + callers already have to handle, and must not lose the round. + + Args: + server_log_path: The ``/server.log`` path from the caller. + """ + try: + _ready_stamp_path(server_log_path).write_text(f"{time.time():.3f}\n", encoding="utf-8") + except OSError as exc: + log.warning("_subprocess_kill: could not stamp server-ready time (%s)", exc) + + +def clear_server_ready_stamp(server_log_path: str) -> None: + """Drop any ready stamp an earlier round left in this output directory. + + Args: + server_log_path: The ``/server.log`` path from the caller. + """ + try: + _ready_stamp_path(server_log_path).unlink(missing_ok=True) + except OSError as exc: + log.warning("_subprocess_kill: could not clear stale server-ready stamp (%s)", exc) + + +def server_ready_unix(server_log_path: str) -> float | None: + """Return when the server reported ready, or ``None`` when nothing recorded it. + + Args: + server_log_path: The ``/server.log`` path from the caller. + + Returns: + float | None: The wall-clock instant, or ``None`` when no stamp exists or + it is unreadable. + """ + try: + stamped = float(_ready_stamp_path(server_log_path).read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return None + return stamped if stamped > 0.0 else None + + +def post_ready_runtime_sec( + server_log_path: str, + *, + started_unix: float, + runtime_sec: float, +) -> float | None: + """Return how long a round ran *after* its server was ready. + + This is the part of a round's wall-clock that is the benchmark itself, with + boot, weight load, compile and graph capture excluded. It is what makes two + rounds comparable when one paid for a cold start and the other re-attached to + a server already up. + + Args: + server_log_path: The ``/server.log`` path from the caller. + started_unix: When the round was spawned. + runtime_sec: The round's full wall-clock. + + Returns: + float | None: Seconds after ready, bounded by the round's own runtime, or + ``None`` when the round never reported ready or the only stamp present + predates it (an earlier round's, left behind by a failed clear -- reading + it would report a cold round as though it had never booted). + """ + ready_unix = server_ready_unix(server_log_path) + if ready_unix is None or ready_unix < started_unix: + return None + return max(0.0, min(float(runtime_sec), started_unix + float(runtime_sec) - ready_unix)) + + def _resolve_scan_logs(server_log_path: str) -> list[str]: """Return the log files to scan for markers, newest-nesting first. @@ -1141,6 +1234,10 @@ def _communicate_with_soft_deadline( if scan.saw_ready and server_ready_since is None: server_ready_since = now last_activity_at = now # start the silence clock at ready + # Recorded for the caller, which prices later work off the + # post-ready segment rather than the whole round: a pass that + # re-attaches to this server pays none of the boot. + _stamp_server_ready(server_log_path) # type: ignore[arg-type] if scan.saw_eval_start and not soft_deadline_suspended: soft_deadline_suspended = True log.info( @@ -1238,10 +1335,13 @@ def _communicate_with_soft_deadline( "SESSION_TIME_EXHAUSTED_RETURNCODE", "STOP_GATE_POLL_SECONDS", "TERM_GRACE_SECONDS", + "clear_server_ready_stamp", "kill_my_spawned_server", "new_session_kwargs", + "post_ready_runtime_sec", "run_with_session_kill", "server_log_death_excerpt", + "server_ready_unix", "session_deadline_to_remaining_sec", "session_remaining_to_deadline_sec", ] diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index f4f2d572b6..bf47d08b58 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -68,6 +68,8 @@ from ._subprocess_kill import ( DETOKENIZER_STALL_RETURNCODE, SERVER_DEAD_RETURNCODE, + clear_server_ready_stamp, + post_ready_runtime_sec, run_with_session_kill, server_log_death_excerpt, session_deadline_to_remaining_sec, @@ -289,6 +291,37 @@ def _watchdog_server_log_path(output_dir: Path, framework: str) -> str | None: return str(output_dir / "server.log") +def _round_post_ready_sec( + server_log_path: str | None, + *, + started_unix: float, + runtime_sec: float, +) -> float | None: + """How much of a round's wall-clock was the benchmark rather than the boot. + + Splitting the two is what lets later work be priced on what it will actually + cost: an explore variant boots its own server and pays both, while a pass + that re-attaches to a server already up pays only the second. + + Args: + server_log_path: The round's ``server.log``, or ``None`` for a scriptable + framework, which runs no server and therefore has no split to make. + started_unix: When the round was spawned. + runtime_sec: The round's full wall-clock. + + Returns: + float | None: Seconds after the server reported ready, or ``None`` when + the round has no such boundary or nothing recorded one. + """ + if not server_log_path: + return None + return post_ready_runtime_sec( + server_log_path, + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + + def _logged_session_clamp(timeout_sec: int, clamped: int, *, output_dir: Path) -> int: """Announce a round's cap being cut by the session budget, and return the cut. @@ -3393,6 +3426,14 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: float(warmup_runtime), 2, ) + # The split belongs to round 1 for the same reason its + # wall-clock does: round 2 re-attached, so it has no boot to + # separate and its own reading says nothing about what booting + # this workload costs. Assigned with that wall-clock and not + # beside it, so the total and the part of it reported here can + # never come from different rounds -- their difference is + # published as this workload's boot. + result["post_ready_runtime_sec"] = warmup_result.get("post_ready_runtime_sec") _hot = result.get("output_throughput") or 0.0 _cold = warmup_tput or 0.0 log.info( @@ -4181,6 +4222,9 @@ async def _run_single_benchmark( stale_server_log, exc, ) + # And the ready stamp beside it, for the same reason: a prior attempt's + # would make this attempt's boot look like it never happened. + clear_server_ready_stamp(str(stale_server_log)) try: if serving_lease is not None: # Ray-managed GPU execution (§12 T1): run inside the lease's @@ -4472,6 +4516,16 @@ async def _run_single_benchmark( # promotes into ``SharedState.baseline_runtime_sec``, the explore # overtime-kill anchor. Omitted on failure paths. "subprocess_runtime_sec": round(subprocess_runtime_sec, 2), + # The benchmark's own share of that wall-clock, boot excluded. Kept + # separate rather than folded in because the two are spent by + # different things: every later variant boots again, so it is the + # sum that prices one, while only this part prices a pass that + # re-attaches. ``None`` when nothing recorded a ready boundary. + "post_ready_runtime_sec": _round_post_ready_sec( + watchdog_server_log, + started_unix=subprocess_started_unix, + runtime_sec=subprocess_runtime_sec, + ), # Authoritative (materialized-config) view of whether the serving # lm-eval ran this run. The accuracy-stop decision reads this rather # than re-deriving from params, so a YAML/reference-env RUN_EVAL=false diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 3e30d239da..72c1f0d881 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -3046,6 +3046,17 @@ async def _promote_baseline( elif float(getattr(self.shared_state, "baseline_warm_runtime_sec", 0.0) or 0.0) != 0.0: self.shared_state.baseline_warm_runtime_sec = 0.0 changed = True + # Promote the cold round's boot/benchmark split. Cleared the same way + # the warm figure is when a later baseline does not carry one, so a + # stale split can never be subtracted from a fresh total and reported + # as this workload's boot. + post_ready_raw = result.get("post_ready_runtime_sec") + if isinstance(post_ready_raw, (int, float)) and post_ready_raw > 0: + self.shared_state.baseline_post_ready_runtime_sec = float(post_ready_raw) + changed = True + elif float(getattr(self.shared_state, "baseline_post_ready_runtime_sec", 0.0) or 0.0) != 0.0: + self.shared_state.baseline_post_ready_runtime_sec = 0.0 + changed = True # current_best.tput follows the same hot baseline contract so the # gain numerator and denominator stay aligned. Once the stack carries a # validated layer, current_best belongs to the stack top and a baseline diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 6e7cc7b68a..21d5c23478 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -595,6 +595,12 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # Baseline WARM measure-round wall-clock (client-only, no boot); anchors the # explore overtime kill apples-to-apples. Zero => fall back to the cold anchor. baseline_warm_runtime_sec: float = 0.0 + # The benchmark's own share of the COLD round above, from the server-ready + # marker onward. Kept beside that total rather than replacing it because the + # difference between them is what booting this workload costs, and every + # later variant boots again: the pair prices one variant, while this figure + # alone prices a pass that re-attaches. Zero => never measured. + baseline_post_ready_runtime_sec: float = 0.0 current_best: dict[str, Any] = field(default_factory=dict) # Reference launch recipe from the operator's --reference-script: lowest-priority # base server args/envs seeding every baseline. Persisted. From d4f1062ceafbd08b9dbfdaf83d2f178c3f6ab189 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 17:35:47 +0000 Subject: [PATCH 51/65] state: a session carries what its baseline cost in two parts, and whether it is a cold figure The round's wall-clock alone cannot price later work. What a variant will spend is a boot and a benchmark, and only the difference between the round's total and the part of it that ran after the server was ready says what the boot costs. Both figures are kept, because subtracting a stale one from a fresh total would report a boot that never happened; the split is cleared the same way the warm figure is when a later baseline does not carry one. The marker is the other half. A baseline that had to keep its cold figure has a denominator inflated by the boot, the first request's kernel compile and the graph capture, and every variant read against it looks like an improvement over a baseline that was never the baseline. That is a decision for the session, not the round, so it is carried on the session -- and cleared by the next baseline that does land a hot figure, so a leg resumed with a fresh clock is not held to the earlier one's shortfall. Co-authored-by: Cursor --- ...coordinator_async_methods_coverage_unit.py | 75 +++++++++++++++++++ src/hyperloom/orchestrator/loop/writeback.py | 13 ++++ .../orchestrator/state/shared_state.py | 6 ++ 3 files changed, 94 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py index fff4705067..13ea21177a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py @@ -87,6 +87,81 @@ async def test_promote_single_round_baseline_clears_stale_warm_runtime(coord: Co assert coord.shared_state.baseline_warm_runtime_sec == 0.0 +@pytest.mark.asyncio +async def test_promote_baseline_carries_the_boot_and_benchmark_split(coord: Coordinator) -> None: + """The two figures that let later work be priced on what it will spend. + + The whole round and the part of it that ran after the server was ready; the + difference between them is what booting this workload costs, and every + variant boots again. + """ + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1000.0, + "subprocess_runtime_sec": 900.0, + "post_ready_runtime_sec": 550.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_runtime_sec == 900.0 + assert coord.shared_state.baseline_post_ready_runtime_sec == 550.0 + + +@pytest.mark.asyncio +async def test_promote_baseline_clears_a_split_a_later_round_did_not_report( + coord: Coordinator, +) -> None: + """A stale split would be subtracted from a fresh total and called the boot.""" + coord.shared_state.baseline_post_ready_runtime_sec = 550.0 + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1000.0, + "subprocess_runtime_sec": 900.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_post_ready_runtime_sec == 0.0 + + +@pytest.mark.asyncio +async def test_promote_baseline_carries_a_dropped_hot_pass_to_the_session( + coord: Coordinator, +) -> None: + """The marker drives a session-level decision, so it has to reach the session. + + PRELUDE routes to CLOSE on it rather than optimizing against a denominator + that was never the baseline, and it is cleared by the next baseline that does + land a hot figure -- otherwise a session resumed with a fresh clock stays + condemned by the earlier leg's shortfall. + """ + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1000.0, + "subprocess_runtime_sec": 900.0, + "measure_round_dropped": {"reason": "measure_round_reaped_by_the_run"}, + "workspace": "/tmp/ws", + }, + ) + assert coord.shared_state.baseline_measure_round_dropped is True + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 1200.0, + "subprocess_runtime_sec": 900.0, + "measure_round_runtime_sec": 400.0, + "workspace": "/tmp/ws", + }, + ) + assert coord.shared_state.baseline_measure_round_dropped is False + + @pytest.mark.asyncio async def test_promote_baseline_non_dict_is_noop(coord: Coordinator) -> None: await coord._promote_to_shared_state("baseline", "not-a-dict") # type: ignore[arg-type] diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 72c1f0d881..b211d0fc66 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -3046,6 +3046,19 @@ async def _promote_baseline( elif float(getattr(self.shared_state, "baseline_warm_runtime_sec", 0.0) or 0.0) != 0.0: self.shared_state.baseline_warm_runtime_sec = 0.0 changed = True + # Whether this baseline had to keep its cold figure because the budget + # could not fund a hot pass and a variant to use it. Carried on the + # session because the decision it drives is the session's: PRELUDE + # routes to CLOSE rather than optimizing against a denominator that + # was never the baseline. Cleared by a baseline that does land a hot + # figure, so a resumed session with a fresh clock is not held to the + # earlier leg's shortfall. + measure_round_dropped = bool(result.get("measure_round_dropped")) + if measure_round_dropped != bool( + getattr(self.shared_state, "baseline_measure_round_dropped", False) + ): + self.shared_state.baseline_measure_round_dropped = measure_round_dropped + changed = True # Promote the cold round's boot/benchmark split. Cleared the same way # the warm figure is when a later baseline does not carry one, so a # stale split can never be subtracted from a fresh total and reported diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 21d5c23478..d13d4fd39f 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -595,6 +595,12 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # Baseline WARM measure-round wall-clock (client-only, no boot); anchors the # explore overtime kill apples-to-apples. Zero => fall back to the cold anchor. baseline_warm_runtime_sec: float = 0.0 + # Whether the baseline's hot pass was dropped because the budget could not + # cover it plus one variant to read against it, leaving the cold pass's + # depressed figure as the anchor. Routes PRELUDE to CLOSE rather than letting + # later phases compare against a denominator that was never the baseline, and + # keeps a resumed session from treating preparation as finished. + baseline_measure_round_dropped: bool = False # The benchmark's own share of the COLD round above, from the server-ready # marker onward. Kept beside that total rather than replacing it because the # difference between them is what booting this workload costs, and every From 0e8d5723d8859430d1ebf163e1a5cb0733501b5f Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 17:36:05 +0000 Subject: [PATCH 52/65] phases, executors: price work by what it spends, and stop when nothing can use the result Every budget gate priced its work at the baseline's whole cold round, which is a boot plus a benchmark that also paid the first request's kernel compile. A variant does not pay that compile and a re-attaching pass does not pay the boot at all, so the session refused work that fitted: with a 900s cold round it declined variants it had 750s of work left for. The pricing now names the parts -- boot, benchmark, one further measured variant -- and every caller reads the same three functions, so the executor cannot refuse a round the phase machine had just called affordable. What the tighter numbers make safe is the harder rule they carry. A PRELUDE baseline is not a result; it is the denominator later results are read against and the anchor their overtime kill uses, so the clock must cover the round and one variant to follow it. Refusing before ignition costs nothing, since the anchor the session already has still stands. Refusing after the cold pass keeps that pass as a marked cold anchor -- its GPU time is spent either way -- and a run's clock that takes the hot pass mid-flight lands in the same place, rather than throwing away a pass that ran to completion. A cold anchor is then not a finished preparation: the session either measures a comparable baseline or stops with the figure marked, instead of optimizing against a denominator that was never the baseline. A re-baseline in a later phase is asked the narrower question. Its measurement is the deliverable, so requiring a successor would refuse the round that validates the run's own answer at the point in the budget where it is most likely to be the last thing left. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 618 +++++++++++++++--- .../tests/test_grid_runner.py | 43 ++ .../tests/test_phase_state_machine.py | 112 ++++ .../actions/executors/_grid_runner.py | 27 +- .../actions/executors/baseline.py | 334 ++++++---- .../orchestrator/actions/executors/report.py | 7 + .../orchestrator/phases/machine_state.py | 237 ++++++- 7 files changed, 1125 insertions(+), 253 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index a4e3af61e1..fb17e3624f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -18,6 +18,7 @@ import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -26,7 +27,6 @@ import yaml from hyperloom.orchestrator.actions.executors.baseline import ( - BASELINE_COLD_START_TIMEOUT_SEC, BASELINE_DEFAULT_TIMEOUT_SEC, MEASURE_ROUND_DROPPED_WARNING, BaselineExecutor, @@ -45,6 +45,7 @@ from hyperloom.orchestrator.actions.executors._subprocess_kill import ( ORCHESTRATOR_CANCELLED_RETURNCODE, SESSION_TIME_EXHAUSTED_RETURNCODE, + _stamp_server_ready, ) from hyperloom.orchestrator.actions.stop_attribution import STOPPED_BY_THE_RUN from hyperloom.orchestrator.state.shared_state import SharedState @@ -122,9 +123,20 @@ def _run(coro): _HOT_TPUT = 4701.6 -def _cold_then_hot_fake_run(captured: list | None = None): +def _cold_then_hot_fake_run( + captured: list | None = None, + *, + clock: _AClockOnlyThePassesMove | None = None, + boot_sec: float = 0.0, + benchmark_sec: float = 0.0, +): """Return a ``run_with_session_kill`` stand-in that emits a cold throughput - on its first call and a hot throughput thereafter.""" + on its first call and a hot throughput thereafter. + + Given a ``clock``, the first call spends it in the two parts a cold pass + spends it in and announces the server ready between them; later calls + re-attach, so they spend only the benchmark and announce nothing. + """ state = {"calls": 0} def fake_run(cmd, *args, **kwargs): @@ -135,6 +147,14 @@ def fake_run(cmd, *args, **kwargs): cfg = yaml.safe_load(Path(cmd[cfg_idx + 1]).read_text()) captured.append(cfg) tput = _COLD_TPUT if state["calls"] == 0 else _HOT_TPUT + if clock is not None: + server_log_path = kwargs.get("server_log_path") + if state["calls"] == 0: + clock.advance(boot_sec) + if server_log_path: + Path(server_log_path).parent.mkdir(parents=True, exist_ok=True) + _stamp_server_ready(server_log_path) + clock.advance(benchmark_sec) state["calls"] += 1 _fake_workspace(slot, tput=tput) return subprocess.CompletedProcess(cmd, 0, "ok", "") @@ -196,6 +216,7 @@ def test_baseline_discards_cold_first_round_via_lifecycle(tmp_path, monkeypatch) assert captured[0]["benchmark"]["benchmark_script"] == "vllm_mi300x.sh" +<<<<<<< HEAD def _run_capturing_rounds(executor, ctx, notes): """Run ``executor`` and record which round notes existed at each launch.""" at_launch: list[list[str]] = [] @@ -348,23 +369,41 @@ def test_a_failing_warmup_round_still_reported_that_it_started(tmp_path): assert [(n["label"], n["status"]) for n in notes] == [("warmup", "started")] -def _prelude_shared_state(*, spent_sec: float, usable_sec: float) -> SimpleNamespace: - """A PRELUDE session state with an explicit clock, as the budget policy reads it.""" +def _prelude_shared_state(*, usable_sec: float, phase: str = "PRELUDE") -> SimpleNamespace: + """A session state with an explicit clock, as the budget policy reads it. + + Only the usable remainder, because that is all the policy reads. An earlier + version also carried a phase ledger and a phase start, from when preparation + answered to a share of its own; a double that still offers them invites a + reader to believe they decide something. + + The phase is offered because the gates ask it one thing: whether the round's + worth depends on a variant following it. + """ return SimpleNamespace( baseline_double_run=True, - phase="PRELUDE", + phase=phase, max_minutes=180, - phase_elapsed_totals={"PRELUDE": spent_sec}, - phase_started_unix=0.0, session_budget_usable_sec=lambda: usable_sec, ) -def _run_double_run_baseline(tmp_path, shared_state) -> dict: +def _run_double_run_baseline( + tmp_path, + shared_state, + *, + clock: _AClockOnlyThePassesMove | None = None, + boot_sec: float = 0.0, + benchmark_sec: float = 0.0, +) -> dict: base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") output_dir = tmp_path / "ws" - fake_run, state = _cold_then_hot_fake_run() + fake_run, state = _cold_then_hot_fake_run( + clock=clock, + boot_sec=boot_sec, + benchmark_sec=benchmark_sec, + ) executor = BaselineExecutor( magpie_python=sys.executable, default_config_path=base, @@ -372,9 +411,12 @@ def _run_double_run_baseline(tmp_path, shared_state) -> dict: shared_state=shared_state, ) ctx = _make_ctx({"output_dir": str(output_dir), "timeout_sec": 10, "gpu_type": "mi300x"}) - with patch( - "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", - side_effect=fake_run, + with ( + _passes_time_the_executor_believes(clock or _AClockOnlyThePassesMove()), + patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ), ): result = _run(executor(ctx)) result["_rounds_run"] = state["calls"] @@ -382,48 +424,110 @@ def _run_double_run_baseline(tmp_path, shared_state) -> dict: def test_a_budget_that_cannot_pay_for_the_measured_round_keeps_the_cold_warmup(tmp_path): - """The session clock cannot pay for the second pass; the first still ran. - - Nothing is predicted before the warmup, so the round starts and the warmup's - GPU time is spent before the shortfall is known. Refusing to keep its figure - would throw that away and leave the session with no anchor at all, which is - strictly worse than the cold anchor a single-round baseline would have - produced. So the warmup is promoted and marked: the number is depressed, and - the marker is what tells a reader of the session's later gains that their - denominator is. + """The session clock cannot pay for the hot pass and a use for it; the cold one ran. + + Nothing is predicted before the warmup on a first baseline, so the round + starts and the warmup's GPU time is spent before the shortfall is known. + Refusing to keep its figure would throw that away and leave the session with + no anchor at all, which is strictly worse than the cold anchor a single-round + baseline would have produced. So the warmup is promoted and marked: the + number is depressed, and the marker is what tells a reader of the session's + later gains that their denominator is. + + The warmup boots for 350s and benchmarks for 550s, so the hot pass that would + follow costs 550s and a variant to read against it costs 900s. 1200s covers + the pass alone with room to spare and the pair not at all. """ + clock = _AClockOnlyThePassesMove() + result = _run_double_run_baseline( tmp_path, - _prelude_shared_state(spent_sec=10_000.0, usable_sec=500.0), + _prelude_shared_state(usable_sec=1200.0), + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, ) assert result["status"] == "succeeded" assert result["_rounds_run"] == 1, "the measured round ran on a budget that cannot pay for it" assert result["output_throughput"] == pytest.approx(_COLD_TPUT) assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] - assert result["measure_round_dropped"]["bound"] == "optimization_reserve" + dropped = result["measure_round_dropped"] + assert dropped["bound"] == "session_usable" + assert dropped["priced_by"] == "warmup_post_ready" + assert dropped["measure_round_sec"] == pytest.approx(550.0, abs=1.0) + assert dropped["one_more_measurement_sec"] == pytest.approx(900.0, abs=1.0) + assert dropped["measure_round_sec"] < 1200.0, ( + "the pass alone did not fit, so this case does not show what it claims to" + ) -def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_path): +def test_a_rebaselines_hot_pass_needs_only_its_own_wall_clock(tmp_path): + """The same 1200s that drops the hot pass in PRELUDE runs it here. + + In PRELUDE the hot pass buys a denominator, so a session that cannot follow + it with a variant gains nothing by running it. A re-baseline's hot pass is + the measurement the session came for, and covering it is the whole question. + """ + clock = _AClockOnlyThePassesMove() + + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=1200.0, phase="EXPLORE"), + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 2, "the measurement the round exists for was dropped" + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + assert MEASURE_ROUND_DROPPED_WARNING not in (result.get("nonfatal_warnings") or []) + + +def test_a_rebaseline_that_cannot_cover_its_hot_pass_keeps_the_cold_warmup(tmp_path): + """A later phase asks a narrower question, not no question. + + 550s of benchmarking does not fit in 400s, so the pass would be reaped + mid-flight and the warmup's figure lost with it. + """ + clock = _AClockOnlyThePassesMove() + + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=400.0, phase="EXPLORE"), + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 1 + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert result["measure_round_dropped"]["one_more_measurement_sec"] == pytest.approx(0.0) + + +def test_a_rounds_boot_is_priced_even_though_no_cap_bounded_it(tmp_path): """The gate prices the round on what it spent, not on what its cap allowed. - A warmup can keep well inside its own timeout and still leave the round - unable to pay for a second pass: the server boot in front of the pass and - the teardown behind it are wall-clock the cap never bounded. Asking after - the pass, against the clock rather than against the cap, is what catches - that -- and is why nothing is predicted before the pass instead. - - The round enters with 7200s on the clock, of which 3600s is held for the - optimization phases, so a gate asked before the warmup would have found - 3600s of headroom and waved through a pass of any length. The round then - charges 4000s -- the pass itself returns at once, the overheads around it do - not -- and the same question asked afterwards finds the clock 400s into the - reserve. + A warmup can keep well inside its own timeout and still leave the round unable + to pay for what should follow: the server boot in front of the pass is + wall-clock the cap never bounded. Asking after the pass, against the clock + rather than against the cap, is what catches that -- and is why nothing is + predicted before the pass on a first baseline instead. + + The cap here is 10 seconds and the pass spends 2600 of them, 1400 of which is + the boot. A gate priced on the cap would have waved through a round costing + three hundred times what it was allowed. """ base = tmp_path / "base.yaml" _write_yaml(base, framework="vllm") - state = _BudgetedState(remaining_sec=7200.0, double_run=True) - fake_run, calls = _capturing_fake_run(state=state, charge_sec=4000.0) + clock = _AClockOnlyThePassesMove() + state = _BudgetedState(remaining_sec=3600.0, double_run=True) + fake_run, calls = _capturing_fake_run( + state=state, + clock=clock, + boot_sec=1400.0, + benchmark_sec=1200.0, + ) executor = BaselineExecutor( magpie_python=sys.executable, default_config_path=base, @@ -431,24 +535,128 @@ def test_a_round_whose_overheads_spend_the_share_is_caught_after_the_warmup(tmp_ shared_state=state, ) ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) - with patch( - "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", - side_effect=fake_run, + with ( + _passes_time_the_executor_believes(clock), + patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ), ): result = _run(executor(ctx)) assert [c["round_slot"] for c in calls] == ["warmup_round"], ( f"the measured round ran on a budget the round had already spent: {[c['round_slot'] for c in calls]}" ) + assert float(calls[0]["timeout"]) <= 10.0, "the cap was not the small one this case rests on" + assert result["status"] == "succeeded" + assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] + dropped = result["measure_round_dropped"] + assert dropped["one_more_measurement_sec"] == pytest.approx(2600.0, abs=1.0) + assert dropped["expected_cost_sec"] == pytest.approx(3800.0, abs=1.0) + + +def _warmup_then_reaped_fake_run(tmp_path): + """A double run whose warmup lands and whose measured pass the clock takes. + + The regime a prediction cannot rule out: the gate before the measured pass + admitted it on what the warmup had just cost, and the pass overran that. + """ + state = {"calls": 0} + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + state["calls"] += 1 + if state["calls"] == 1: + _fake_workspace(slot, tput=_COLD_TPUT) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + return subprocess.CompletedProcess(cmd, SESSION_TIME_EXHAUSTED_RETURNCODE, "", "") + + return fake_run, state + + +def test_a_measured_round_the_clock_takes_mid_flight_keeps_the_cold_warmup(tmp_path): + """The GPU time behind the warmup's figure is spent either way. + + Reporting the round as failed would discard it and leave the session with + nothing to show for a pass that ran to completion, so the warmup is kept and + marked exactly as a refused measured round keeps it. A session that reaches + this state has already paid for a cold anchor; what it must not do is pay + again for nothing. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + fake_run, state = _warmup_then_reaped_fake_run(tmp_path) + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=_prelude_shared_state(usable_sec=10_000.0), + ) + ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) + + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert state["calls"] == 2, "the measured round did not run, so it cannot have been reaped" assert result["status"] == "succeeded" + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) assert MEASURE_ROUND_DROPPED_WARNING in result["nonfatal_warnings"] + assert result["measure_round_dropped"]["reason"] == "measure_round_reaped_by_the_run" + + +def test_a_measured_round_that_fails_on_its_own_is_still_a_failure(tmp_path): + """Only the run's clock earns the fallback. + + A pass that broke for a reason of its own is a fact about the configuration, + and the warmup having succeeded does not make the round's figure comparable. + Promoting a cold anchor here would bury a real failure under a warning. + """ + base = tmp_path / "base.yaml" + _write_yaml(base, framework="vllm") + calls = {"n": 0} + + def fake_run(cmd, *args, **kwargs): + if "--output-dir" not in cmd: + return subprocess.CompletedProcess(cmd, 0, "ok", "") + slot = Path(cmd[cmd.index("--output-dir") + 1]) + calls["n"] += 1 + if calls["n"] == 1: + _fake_workspace(slot, tput=_COLD_TPUT) + return subprocess.CompletedProcess(cmd, 0, "ok", "") + return subprocess.CompletedProcess(cmd, 1, "", "CUDA error") + + executor = BaselineExecutor( + magpie_python=sys.executable, + default_config_path=base, + session_dir=tmp_path, + shared_state=_prelude_shared_state(usable_sec=10_000.0), + ) + ctx = _make_ctx({"output_dir": str(tmp_path / "ws"), "timeout_sec": 10, "gpu_type": "mi300x"}) + + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + assert result["status"] == "failed" + assert result["error_class"] != SESSION_TIME_EXHAUSTED_CLASS + assert MEASURE_ROUND_DROPPED_WARNING not in (result.get("nonfatal_warnings") or []) def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): """The guard must not turn every double-run into a single one.""" result = _run_double_run_baseline( tmp_path, - _prelude_shared_state(spent_sec=300.0, usable_sec=10_000.0), + _prelude_shared_state(usable_sec=10_000.0), + clock=_AClockOnlyThePassesMove(), + boot_sec=350.0, + benchmark_sec=550.0, ) assert result["_rounds_run"] == 2 @@ -456,19 +664,64 @@ def test_measured_round_survives_a_budget_that_still_covers_it(tmp_path): assert "budget_shortfall" not in result +def test_a_double_run_reports_the_boot_split_of_the_pass_that_paid_it(tmp_path): + """The round's total and the part of it that was the benchmark, from one pass. + + Round 1 boots for 350s and benchmarks for 550s; round 2 re-attaches and + benchmarks for 550s more. The split belongs to round 1, whose 900s total is + also what the round reports: their difference is published as what booting + this workload costs, so two rounds cannot each supply one of them. Round 2's + own split says nothing, having never booted. + """ + result = _run_double_run_baseline( + tmp_path, + _prelude_shared_state(usable_sec=10_000.0), + clock=_AClockOnlyThePassesMove(), + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["subprocess_runtime_sec"] == pytest.approx(900.0, abs=1.0) + assert result["post_ready_runtime_sec"] == pytest.approx(550.0, abs=1.0) + assert result["measure_round_runtime_sec"] == pytest.approx(550.0, abs=1.0) + boot_sec = result["subprocess_runtime_sec"] - result["post_ready_runtime_sec"] + assert boot_sec == pytest.approx(350.0, abs=1.0) + + +# The workload every case in the gate class below is priced against, and the +# figures the pricing derives from it. A round that boots for 350s and then +# benchmarks is what a variant costs too, because a variant's config differs in +# the knobs that decide how a server comes up and it has to bring up its own. +_COLD_ROUND_SEC = 900.0 +_COLD_POST_READY_SEC = 550.0 +_HOT_ROUND_SEC = 400.0 +# 900 - 550: the part of the cold round that was not the benchmark. +_BOOT_SEC = 350.0 +# One further measured variant: its own boot, then its own benchmark. +_ONE_MORE_SEC = _BOOT_SEC + _HOT_ROUND_SEC +# A single-pass round is a boot and a benchmark; a double-run round adds a second +# benchmark, which re-attaches and so buys no second boot. +_SINGLE_ROUND_SEC = _ONE_MORE_SEC +_DOUBLE_ROUND_SEC = _ONE_MORE_SEC + _HOT_ROUND_SEC + + class TestARoundThatCannotFinishIsNotIgnited: - """The gate in front of a round, and the one thing it may be asked with. + """The gate in front of a round, and the two things it must be asked with. A round is refused before it boots only on what earlier rounds measured, so the session's first one is never refused: it has nothing to be judged by, and a gate that guessed would either refuse every first baseline or wave every - one through. From the second on, the figures exist and an anchor exists with - them, so refusing costs the session nothing it had while igniting costs it a - boot, a compile and a graph capture for a number it would then have to mark - as cold. + one through. From the second on -- and on a resumed session's first, which + carries the earlier leg's figures -- the answer is available before a second + of GPU time is spent. + + What must fit is the round *and one further measured variant*. A baseline is + not a result; it is the denominator results are read against and the anchor + their overtime kill uses. A round no variant can follow buys neither, so the + wall-clock it would spend produces nothing. - Every session here has 3600s on its clock, half of which is held for the - phases that produce a result, so a round is judged against 1800s. + Priced on this workload: boot 350s, benchmark 400s, so one variant costs 750s, + a single-pass round costs 750s and a double-run round 1150s. """ def test_a_first_round_is_not_judged_at_all(self, tmp_path): @@ -479,10 +732,13 @@ def test_a_first_round_is_not_judged_at_all(self, tmp_path): assert calls, "a first baseline was refused on a prediction the session cannot have" def test_a_round_larger_than_what_is_left_boots_nothing(self, tmp_path): + """750s for the round and 750s for a variant to use it: 1400s cannot.""" result, calls = _run_baseline_under_budget( tmp_path, - remaining_sec=3600.0, - measured_expected_sec=1900.0, + remaining_sec=1400.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, ) assert calls == [], "GPU time was spent on a round the session cannot finish" @@ -490,92 +746,189 @@ def test_a_round_larger_than_what_is_left_boots_nothing(self, tmp_path): assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS assert result["returncode"] is None, "a round that never launched reported a returncode" assert result["error"] == STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS].never_started - assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(1900.0) - assert result["budget_shortfall"]["affordable_sec"] == pytest.approx(1800.0, abs=1.0) - - def test_a_round_that_still_fits_is_ignited(self, tmp_path): - """The gate must not turn a merely expensive round into a refused one.""" + shortfall = result["budget_shortfall"] + assert shortfall["expected_cost_sec"] == pytest.approx(_SINGLE_ROUND_SEC + _ONE_MORE_SEC) + assert shortfall["round_sec"] == pytest.approx(_SINGLE_ROUND_SEC) + assert shortfall["one_more_measurement_sec"] == pytest.approx(_ONE_MORE_SEC) + assert shortfall["affordable_sec"] == pytest.approx(1400.0, abs=1.0) + + def test_a_round_the_budget_covers_is_ignited(self, tmp_path): + """The gate must not turn a merely expensive round into a refused one. + + Given a margin over the 1500s the round and its use need, rather than + exactly that: the headroom is read from a live clock, so a case pinned to + the boundary would decide on how long the test itself took to get there. + """ result, calls = _run_baseline_under_budget( tmp_path, - remaining_sec=3600.0, - measured_expected_sec=1700.0, + remaining_sec=_SINGLE_ROUND_SEC + _ONE_MORE_SEC + 60.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, ) assert result["status"] == "succeeded" assert calls - def test_a_double_run_pays_for_both_of_its_passes(self, tmp_path): - """Priced on the round, not on the pass: 1500s fits, 1500s plus 400s does not.""" + def test_a_round_the_budget_covers_with_nothing_left_to_use_it_is_refused(self, tmp_path): + """The requirement that is not about finishing the round. + + 1000s covers the 750s round with room to spare, and the round would run + to completion. It is still refused, because what it produces is a + denominator, and 250s buys no variant to read against it. Wall-clock + spent on a number nothing is ever compared to is wall-clock wasted, and + a session that stops here keeps the anchor it already had. + """ result, calls = _run_baseline_under_budget( tmp_path, - remaining_sec=3600.0, - measured_expected_sec=1500.0, - measured_warm_sec=400.0, - double_run=True, + remaining_sec=1000.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert calls == [], "a round ran that nothing could be measured against" + assert result["budget_shortfall"]["round_sec"] == pytest.approx(_SINGLE_ROUND_SEC) + assert result["budget_shortfall"]["round_sec"] < 1000.0, ( + "the round itself did not fit, so this case is not the one it claims to be" ) - assert calls == [], "the round was priced on one pass while planning to run two" - assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(1900.0) + def test_a_rebaseline_in_a_later_phase_needs_no_successor(self, tmp_path): + """The same 1000s that refuses a PRELUDE round admits this one. - def test_a_single_round_pays_for_the_one_pass_it_runs(self, tmp_path): - """The same two figures, with no second pass to buy, still fit.""" + A re-baseline re-measures the stack the session has assembled, and that + measurement is what the session is for. Requiring a variant after it + would refuse the round that validates the run's own answer, at the point + in the budget where it is most likely to be the last thing left. + """ result, calls = _run_baseline_under_budget( tmp_path, - remaining_sec=3600.0, - measured_expected_sec=1500.0, - measured_warm_sec=400.0, + remaining_sec=1000.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + phase="EXPLORE", ) + assert calls, "the round that validates the stack was refused for lack of a successor" assert result["status"] == "succeeded" - assert calls - def test_a_missing_hot_figure_still_pays_for_the_cold_pass(self, tmp_path): + def test_a_rebaseline_larger_than_what_is_left_is_still_refused(self, tmp_path): + """Dropping the successor does not drop the round's own cost. + + A later phase is a narrower question, not an absent one: a round that + cannot finish inside the budget burns a boot and a compile for a number + the reaper takes away before it lands. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=_SINGLE_ROUND_SEC - 100.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + phase="EXPLORE", + ) + + assert calls == [], "GPU time was spent on a round the session cannot finish" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + shortfall = result["budget_shortfall"] + assert shortfall["one_more_measurement_sec"] == pytest.approx(0.0) + assert shortfall["expected_cost_sec"] == pytest.approx(_SINGLE_ROUND_SEC) + + def test_a_double_run_pays_for_the_second_pass_but_not_a_second_boot(self, tmp_path): + """One budget, two answers: it covers a single-pass round, not a double one. + + The difference between them is one benchmark and no second boot, because + the second pass re-attaches to the server the first left running. Both + sides run against the same figure so the refusal can only come from that. + """ + budget_sec = _SINGLE_ROUND_SEC + _ONE_MORE_SEC + 60.0 + for side in ("single", "double"): + (tmp_path / side).mkdir() + + fits, fitting_calls = _run_baseline_under_budget( + tmp_path / "single", + remaining_sec=budget_sec, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + refused, refused_calls = _run_baseline_under_budget( + tmp_path / "double", + remaining_sec=budget_sec, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, + double_run=True, + ) + + assert fits["status"] == "succeeded" and fitting_calls + assert refused_calls == [], "the round was priced on one pass while planning to run two" + shortfall = refused["budget_shortfall"] + assert shortfall["round_sec"] == pytest.approx(_DOUBLE_ROUND_SEC) + assert shortfall["expected_cost_sec"] == pytest.approx(_DOUBLE_ROUND_SEC + _ONE_MORE_SEC) + + def test_a_session_with_no_hot_figure_prices_the_benchmark_from_the_cold_pass(self, tmp_path): """The state a previous cold-anchor drop leaves, and the one to catch. A round whose measured pass was dropped for budget promotes its cold - number and has no hot number to write. Going inert on a half-measured - session would exempt exactly the session that already ran out of budget - once. + number and has no hot number to write. Going inert on such a session would + exempt exactly the one that already ran out of budget once, so the cold + round's post-ready segment stands in for the benchmark. It over-predicts, + having also paid the first request's compile, which is why the hot figure + wins whenever one exists. """ + one_more_sec = _BOOT_SEC + _COLD_POST_READY_SEC + result, calls = _run_baseline_under_budget( tmp_path, - remaining_sec=3600.0, - measured_expected_sec=3000.0, - double_run=True, + remaining_sec=one_more_sec * 2 - 1.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, ) assert calls == [] - assert result["budget_shortfall"]["expected_cost_sec"] == pytest.approx(3000.0) + assert result["budget_shortfall"]["round_sec"] == pytest.approx(one_more_sec) + assert result["budget_shortfall"]["round_sec"] > _SINGLE_ROUND_SEC, ( + "the fallback did not over-predict, so it cannot be the post-ready segment" + ) + + def test_a_round_whose_boot_was_never_measured_is_not_judged(self, tmp_path): + """A cold wall-clock alone cannot say what a variant costs. + + Without the split, the boot is unknown, and a boot is most of what a + variant pays. Refusing on a figure that cannot be built would refuse + sessions arbitrarily, so the question is left to the post-warmup gate, + which prices the pass from what the warmup just spent. + """ + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=100.0, + cold_round_sec=_COLD_ROUND_SEC, + hot_round_sec=_HOT_ROUND_SEC, + ) + + assert calls, "a round was refused on a price the session cannot compute" + assert result["status"] == "succeeded" def test_a_refused_round_carries_nothing_that_could_replace_the_anchor(self, tmp_path): """The property the whole gate rests on, asserted rather than assumed. - The figure this gate judges by over-predicts: it comes from a pass that - paid a one-time compile on a cold kernel cache, which a later pass on a - warm one does not pay again. Refusing too readily is tolerable only - because the anchor the session already measured survives a refusal, so - the round must come back with no throughput to promote over it. + A refusal is only cheap if what the session already measured survives it. + The anchor is held in session state, not in this result, so the refused + round must come back with no throughput of its own to promote over it. """ result, _calls = _run_baseline_under_budget( tmp_path, - remaining_sec=3600.0, - measured_expected_sec=1900.0, + remaining_sec=1400.0, + cold_round_sec=_COLD_ROUND_SEC, + cold_post_ready_sec=_COLD_POST_READY_SEC, + hot_round_sec=_HOT_ROUND_SEC, ) assert result.get("output_throughput") in (None, 0, 0.0) assert not result.get("nonfatal_warnings"), "a refused round volunteered a warning to promote" - def test_a_hot_pass_that_cannot_be_priced_does_not_refuse_a_cold_one_that_fits(self, tmp_path): - """Only the provable part refuses; the rest is the post-warmup gate's.""" - result, calls = _run_baseline_under_budget( - tmp_path, - remaining_sec=3600.0, - measured_expected_sec=1500.0, - double_run=True, - ) - - assert calls, "a round was refused for a pass the session could not price" - def test_deferred_accuracy_skips_eval_when_hot_throughput_regresses( tmp_path, @@ -2045,14 +2398,16 @@ def __init__( self, *, remaining_sec: float | None, - measured_expected_sec: float = 0.0, - measured_warm_sec: float = 0.0, + cold_round_sec: float = 0.0, + cold_post_ready_sec: float = 0.0, + hot_round_sec: float = 0.0, phase: str = "PRELUDE", double_run: bool = False, ) -> None: self.baseline_double_run = double_run - self.baseline_runtime_sec = measured_expected_sec - self.baseline_warm_runtime_sec = measured_warm_sec + self.baseline_runtime_sec = cold_round_sec + self.baseline_post_ready_runtime_sec = cold_post_ready_sec + self.baseline_warm_runtime_sec = hot_round_sec self.phase = phase self.max_minutes = 0.0 if remaining_sec is None else remaining_sec / 60.0 self._deadline = None if remaining_sec is None else time.monotonic() + remaining_sec @@ -2069,6 +2424,37 @@ def session_budget_usable_sec(self) -> float | None: return None if self._deadline is None else self._deadline - time.monotonic() +class _AClockOnlyThePassesMove: + """The wall clock the executor prices rounds by, moved by hand. + + A round's price is wall-clock, so a test that needs a pass to cost twenty + minutes can sleep for them, assert on nothing, or hand the executor a clock it + can move. Only the third is both quick and about the thing under test. + + Installed over ``time.time``, which is what the executor times rounds with; + ``time.monotonic``, which the budget deadline runs on, is left alone so the + two cannot be confused for each other. + """ + + def __init__(self) -> None: + self._now = time.time() + + def __call__(self) -> float: + """Answer as ``time.time`` does.""" + return self._now + + def advance(self, seconds: float) -> None: + """Spend ``seconds``, as a pass that ran that long does.""" + self._now += float(seconds) + + +@contextmanager +def _passes_time_the_executor_believes(clock: _AClockOnlyThePassesMove): + """Run the block with ``clock`` standing in for the wall clock.""" + with patch("time.time", clock): + yield + + def _capturing_fake_run( returncode: int = 0, *, @@ -2076,6 +2462,9 @@ def _capturing_fake_run( pass_duration_sec: float = 0.0, state: _BudgetedState | None = None, charge_sec: float | None = None, + clock: _AClockOnlyThePassesMove | None = None, + boot_sec: float = 0.0, + benchmark_sec: float = 0.0, ): """A ``run_with_session_kill`` stand-in that records how each round was launched. @@ -2091,6 +2480,11 @@ def _capturing_fake_run( ``charge_sec`` charges the session more than the pass itself ran, which is what a round with a server restart and a teardown around the pass costs. + + ``boot_sec`` and ``benchmark_sec`` spend ``clock`` in the two parts a real + pass spends it in, announcing the server ready between them exactly as the + gate loop does. A pass that models one duration cannot reach the pricing at + all, which is built on telling the two apart. """ calls: list[dict] = [] @@ -2108,6 +2502,13 @@ def fake_run(cmd, *args, **kwargs): until_deadline = max(0.0, deadline - time.monotonic()) reaped_by_the_session = until_deadline < ran_sec ran_sec = min(ran_sec, until_deadline) + if clock is not None: + clock.advance(boot_sec) + server_log_path = kwargs.get("server_log_path") + if server_log_path: + Path(server_log_path).parent.mkdir(parents=True, exist_ok=True) + _stamp_server_ready(server_log_path) + clock.advance(benchmark_sec) if state is not None: state.charge(charge_sec if charge_sec is not None else ran_sec) if pass_duration_sec and ran_sec < pass_duration_sec: @@ -2150,8 +2551,9 @@ def _run_baseline_under_budget( timeout_sec: int = 7200, returncode: int = 0, produces_workspace: bool = True, - measured_expected_sec: float = 0.0, - measured_warm_sec: float = 0.0, + cold_round_sec: float = 0.0, + cold_post_ready_sec: float = 0.0, + hot_round_sec: float = 0.0, pass_duration_sec: float = 0.0, phase: str = "PRELUDE", double_run: bool = False, @@ -2162,8 +2564,9 @@ def _run_baseline_under_budget( _write_yaml(base, framework="vllm") state = _BudgetedState( remaining_sec=remaining_sec, - measured_expected_sec=measured_expected_sec, - measured_warm_sec=measured_warm_sec, + cold_round_sec=cold_round_sec, + cold_post_ready_sec=cold_post_ready_sec, + hot_round_sec=hot_round_sec, phase=phase, double_run=double_run, ) @@ -2399,7 +2802,6 @@ def test_a_first_baseline_on_a_default_session_runs_both_of_its_passes( tmp_path, remaining_sec=_DEFAULT_SESSION_MINUTES * 60.0, timeout_sec=BASELINE_DEFAULT_TIMEOUT_SEC, - measured_expected_sec=0.0, pass_duration_sec=pass_sec, ) launches = launches_by_round_slot(calls) diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py index fbab4e05d7..0df03a1044 100644 --- a/src/hyperloom/inference_optimizer/tests/test_grid_runner.py +++ b/src/hyperloom/inference_optimizer/tests/test_grid_runner.py @@ -1519,6 +1519,49 @@ def test_state_without_the_deadline_accessor_is_tolerated(self): state = SimpleNamespace(baseline_runtime_sec=600.0) assert _grid_runner.session_grid_bounds(state) == (None, 600.0) + def test_a_variant_is_priced_as_a_boot_and_a_benchmark(self): + """What a variant actually spends, rather than what the baseline spent. + + A variant cannot re-attach to anyone else's server -- its config differs + in the very knobs that decide how one comes up -- so it pays a boot and + then a benchmark. The baseline's 900s cold round is 350s of boot and 550s + of benchmarking that also paid the first request's kernel compile; the + variant pays that boot and the 400s a benchmark costs once the compile is + cached. Admitting on the 900s abandons 150s of every variant's worth of + tail budget. + """ + state = SimpleNamespace( + grid_session_deadline_sec=lambda: 4242.0, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + baseline_warm_runtime_sec=400.0, + ) + + deadline, variant_sec = _grid_runner.session_grid_bounds(state) + + assert deadline == 4242.0 + assert variant_sec == pytest.approx(750.0) + + def test_a_baseline_with_no_hot_pass_prices_the_benchmark_from_the_cold_one(self): + """The post-ready segment stands in, over-predicting by the compile.""" + state = SimpleNamespace( + grid_session_deadline_sec=lambda: None, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + ) + + assert _grid_runner.session_grid_bounds(state)[1] == pytest.approx(900.0) + + def test_a_baseline_that_never_reported_its_boot_falls_back_to_the_whole_round(self): + """A scriptable workload runs no server, so there is no split to read.""" + state = SimpleNamespace( + grid_session_deadline_sec=lambda: None, + baseline_runtime_sec=900.0, + baseline_warm_runtime_sec=400.0, + ) + + assert _grid_runner.session_grid_bounds(state)[1] == pytest.approx(900.0) + class TestSessionBudgetAdmission: """A variant is admitted on what it is expected to need, not on its backstop. diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py index d555e047d9..38c0e0f3ea 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py @@ -227,6 +227,10 @@ def _prelude_state( usable_sec: float | None = None, baseline_tput: float = 0.0, baseline_runtime_sec: float = 0.0, + baseline_post_ready_runtime_sec: float = 0.0, + baseline_warm_runtime_sec: float = 0.0, + baseline_measure_round_dropped: bool = False, + baseline_double_run: bool = False, ) -> SimpleNamespace: """A PRELUDE-phase state with an explicit clock, as the budget policy reads it.""" return SimpleNamespace( @@ -236,6 +240,10 @@ def _prelude_state( phase_started_unix=0.0, baseline_tput=baseline_tput, baseline_runtime_sec=baseline_runtime_sec, + baseline_post_ready_runtime_sec=baseline_post_ready_runtime_sec, + baseline_warm_runtime_sec=baseline_warm_runtime_sec, + baseline_measure_round_dropped=baseline_measure_round_dropped, + baseline_double_run=baseline_double_run, session_budget_usable_sec=lambda: usable_sec, ) @@ -315,6 +323,110 @@ def test_prelude_exit_states_whether_one_optimization_round_still_fits(): assert evidence["fits_one_optimization_round"] is False +# The workload the cold-anchor cases below are priced against: a 900s cold round +# whose last 550s was the benchmark, so the boot took 350s, and a 400s hot pass. +# One further measured variant therefore costs 750s and a double-run baseline +# round 1150s, which is what a session must be able to afford before it is worth +# measuring another baseline. +_COLD_ANCHOR_WORKLOAD = { + "baseline_tput": 1074.7, + "baseline_runtime_sec": 900.0, + "baseline_post_ready_runtime_sec": 550.0, + "baseline_warm_runtime_sec": 400.0, + "baseline_double_run": True, +} +_RETRY_COST_SEC = 1150.0 + 750.0 + + +class TestAColdAnchorIsNotAFinishedPrelude: + """What happens to a session whose baseline could only keep its cold figure. + + The figure exists, so every rule that asks only whether a baseline landed + reads preparation as done. It is not: the number carries the boot, the first + request's compile and the graph capture, so every variant measured against it + reads as an improvement over a baseline that was never the baseline. + + Two outcomes are correct and the budget picks between them -- measure another + baseline, or stop and say why -- and neither is "optimize against it". + """ + + def test_a_dropped_hot_pass_does_not_finish_the_phase(self): + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=_RETRY_COST_SEC + 60.0, + ) + + assert phase_state.exit_normal_prelude(state) is None + + state.baseline_measure_round_dropped = False + assert phase_state.exit_normal_prelude(state)[0] == "prelude_done" + + def test_a_session_that_cannot_afford_another_baseline_closes(self): + """1900s buys a round and a variant to read against it; 1200s buys neither.""" + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=1200.0, + ) + + out = phase_state.compute_next_phase(state, kernel_enabled=True) + + assert out is not None + next_phase, reason, evidence = out + assert (next_phase, reason) == ("CLOSE", "prelude_cold_anchor_low_budget") + assert evidence["terminal"] is True + assert evidence["baseline_anchor"] == "cold" + assert evidence["retry_round_sec"] == pytest.approx(1150.0) + assert phase_state.is_valid_stop_reason(reason) + assert phase_state.is_valid_phase_exit_reason(reason) + + def test_a_session_resumed_with_a_fresh_clock_measures_another_baseline(self): + """The marker outlives the shortfall, so it must not decide on its own. + + A resume reanchors the session clock while the marker from the earlier leg + persists. Closing on the marker would end every resumed run before it + began, and no later baseline could clear the marker because none would + run. So the phase stays open with nothing to advance it but a new + baseline. + """ + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=_RETRY_COST_SEC + 60.0, + ) + + assert phase_state.exit_cold_anchor_prelude(state) is None + assert phase_state.compute_next_phase(state, kernel_enabled=True) is None + + def test_a_single_round_baseline_is_not_mistaken_for_a_dropped_one(self): + """A cold figure by configuration is consistent with what follows it. + + A session that never asked for a hot pass measures everything the same + way, so its comparisons hold. Only a pass that was *dropped* leaves a + denominator out of step with the numerators. + """ + state = _prelude_state( + baseline_tput=1074.7, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + usable_sec=1200.0, + ) + + assert phase_state.exit_cold_anchor_prelude(state) is None + assert phase_state.exit_normal_prelude(state)[0] == "prelude_done" + + def test_a_session_with_no_clock_is_not_closed_for_a_budget_it_does_not_have(self): + """An unbounded run cannot fail an affordability test, so it retries.""" + state = _prelude_state( + **_COLD_ANCHOR_WORKLOAD, + baseline_measure_round_dropped=True, + usable_sec=None, + ) + + assert phase_state.exit_cold_anchor_prelude(state) is None + + def test_exit_terminal_prelude_after_three_baseline_failures(): state = SimpleNamespace(baseline_failure_streak=2) assert phase_state.exit_terminal_prelude(state) is None diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index 3f266763f6..aadbb0bd2b 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -32,6 +32,7 @@ scrub_benchmark_process_env, ) +from ...phases import machine_state as _phase_state from ...roles.robustness_pulse import pulse as _robustness_pulse from ...trace.task_progress import heartbeat_while_output_flows, report_progress from ..cancel_channel import stop_was_asked_for @@ -1329,11 +1330,18 @@ def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: next produces arms that abandon different amounts of the tail budget. Both are read here so there is one definition. - ``variant_expected_sec`` is the measured baseline runtime -- what a - normally-behaving variant needs -- and is deliberately not the declared - timeout, which is a catastrophic-hang backstop roughly twice as large. - Admitting on the backstop would refuse to start a 20-minute round with 30 - minutes left. + ``variant_expected_sec`` is what a normally-behaving variant needs -- its own + server boot and then its benchmark + (:func:`~...phases.machine_state.one_more_measurement_sec`) -- and is + deliberately not the declared timeout, which is a catastrophic-hang backstop + roughly twice as large. Admitting on the backstop would refuse to start a + 20-minute round with 30 minutes left. + + Nor is it the baseline's cold wall-clock, which is the fallback only for a + session whose baseline reported no boot/benchmark split. That figure also + carries the first request's kernel compile on a cold JIT cache, which a + variant on the now-populated cache does not pay again, so admitting on it + abandons the tail of the budget to variants that would have finished. Args: shared_state: The session ``SharedState``, or ``None`` when the caller @@ -1349,11 +1357,10 @@ def session_grid_bounds(shared_state: Any) -> tuple[float | None, float | None]: return (None, None) deadline_fn = getattr(shared_state, "grid_session_deadline_sec", None) deadline = deadline_fn() if callable(deadline_fn) else None - try: - baseline_sec = float(getattr(shared_state, "baseline_runtime_sec", 0.0) or 0.0) - except (TypeError, ValueError): - baseline_sec = 0.0 - return (deadline, baseline_sec if baseline_sec > 0 else None) + variant_sec = _phase_state.one_more_measurement_sec(shared_state) + if variant_sec is None: + variant_sec = _phase_state.measured_seconds(shared_state, "baseline_runtime_sec") + return (deadline, variant_sec) async def run_grid( diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index bf47d08b58..184ac1ac00 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -418,12 +418,18 @@ def _stopped_round_result( def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple[float | None, dict[str, Any]]: """Seconds this round's budget may still spend, and the numbers behind it. - In PRELUDE that is what the session has left once the optimization phases' - reserve is held back - (:func:`~...phases.machine_state.prelude_affordable_seconds`), which is the - same figure the post-warmup gate judges the measured round against. Outside - it a re-baseline answers to what is left before the session deadline, the - only bound the round is under there. + The session's own usable remainder, which is what every other admission + decision reads, so a round cannot be judged against a figure the rest of the + run disagrees with. Not a share of it: the share held back for the + optimization phases + (:func:`~...phases.machine_state.prelude_affordable_seconds`) sizes the + *optional* arms of preparation, where the question is proportion. A round is + a feasibility question, and answering it with a percentage refuses rounds + that fit -- a session with ninety minutes left and a twelve-minute pass to + run is told it has six. + + Falls back to the session deadline for a caller with no session state at all, + and to no bound when there is neither. Args: state: The session ``SharedState``, or ``None`` when the caller has no @@ -434,13 +440,13 @@ def _round_headroom_sec(state: Any, session_deadline_sec: float | None) -> tuple tuple[float | None, dict[str, Any]]: The headroom, or ``None`` when the round is under no budget at all, plus the evidence behind it. """ - if state is None: - outside: dict[str, Any] = {"reason": "no_session_state"} + if state is not None: + usable_sec = _phase_state.session_usable_seconds(state) + if usable_sec is not None: + return usable_sec, {"bound": "session_usable", "affordable_sec": round(usable_sec, 1)} + outside: dict[str, Any] = {"reason": "unbounded_session_budget"} else: - phase = str(getattr(state, "phase", "") or "").strip().upper() - if phase == _phase_state.PHASE_PRELUDE: - return _phase_state.prelude_affordable_seconds(state) - outside = {"reason": "not_prelude", "phase": phase} + outside = {"reason": "no_session_state"} if session_deadline_sec is None: return None, outside remaining_sec = max(0.0, session_deadline_sec - time.monotonic()) @@ -476,26 +482,47 @@ def _cold_anchor_from_warmup( return warmup_result -def _measured_runtime_sec(state: Any, field: str) -> float | None: - """Read a runtime some earlier round measured, or ``None`` when none did. +def _a_use_must_follow_the_round(state: Any) -> bool: + """Whether this round is only worth running if something can be measured after it. - Absent and zero are the same answer: both mean no round has landed an anchor - yet. Neither may read as a round that cost nothing, which is what a plain - ``float(... or 0.0)`` would make of them. + A PRELUDE baseline is not a result. It is the denominator later results are + read against and the anchor their overtime kill uses, so a session that + cannot afford one variant after it would spend the wall-clock on a number + nothing ever reads. + + A re-baseline in a later phase is the opposite: it re-measures the stack the + session has assembled, and that measurement is the deliverable. Requiring a + successor would refuse exactly the round that validates the run's own answer, + at the point in the budget where it is most likely to be the last thing left. Args: state: The session ``SharedState``, or ``None``. - field: The SharedState attribute holding the runtime. Returns: - float | None: The measured seconds, or ``None`` when there is no - measurement to predict from. + bool: ``True`` while the round's worth depends on a successor. + """ + phase = str(getattr(state, "phase", "") or "").strip().upper() + return phase == _phase_state.PHASE_PRELUDE + + +def _positive_seconds(value: Any) -> float | None: + """Coerce a duration a round reported to seconds, or ``None`` when it did not. + + Absent, unparseable and zero are one answer: nothing was measured. None of + them may read as work that took no time, which is what a plain + ``float(... or 0.0)`` would make of them. + + Args: + value: The reported duration, from a round's result. + + Returns: + float | None: The seconds, or ``None`` when there is no measurement. """ try: - value = float(getattr(state, field, 0.0) or 0.0) + seconds = float(value or 0.0) except (TypeError, ValueError): return None - return value if value > 0.0 else None + return seconds if seconds > 0.0 else None def _disable_cuda_graph_flag(framework: str) -> str: @@ -3058,14 +3085,28 @@ async def _run_once( started=False, ) stopped_result["budget_shortfall"] = ignition_evidence - log.warning( - "baseline_executor: a whole round is expected to cost %.0fs and " - "only %.0fs is left (bound=%s), so nothing is booted. The anchor " - "this session already measured stands.", - ignition_evidence.get("expected_cost_sec", 0.0), - ignition_evidence.get("affordable_sec", 0.0), - ignition_evidence.get("bound", ""), - ) + if ignition_evidence.get("one_more_measurement_sec"): + log.warning( + "baseline_executor: this round (%.0fs) and one variant to read " + "against it (%.0fs) need %.0fs, and %.0fs is left (bound=%s), so " + "nothing is booted. A baseline no variant can follow is a " + "denominator with no numerator; the anchor this session already " + "measured stands.", + ignition_evidence.get("round_sec", 0.0), + ignition_evidence.get("one_more_measurement_sec", 0.0), + ignition_evidence.get("expected_cost_sec", 0.0), + ignition_evidence.get("affordable_sec", 0.0), + ignition_evidence.get("bound", ""), + ) + else: + log.warning( + "baseline_executor: this round needs %.0fs and only %.0fs is left " + "(bound=%s), so nothing is booted. The anchor this session already " + "measured stands.", + ignition_evidence.get("expected_cost_sec", 0.0), + ignition_evidence.get("affordable_sec", 0.0), + ignition_evidence.get("bound", ""), + ) return stopped_result before_apply_sha = _git_head_sha(patch_target) @@ -3334,6 +3375,7 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: return warmup_result warmup_tput = warmup_result.get("output_throughput") warmup_runtime = warmup_result.get("subprocess_runtime_sec") + warmup_post_ready = warmup_result.get("post_ready_runtime_sec") await report_progress( unit="baseline_round", label="warmup", @@ -3347,18 +3389,33 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if not defer_accuracy_until_after_measure: affordable, gate_evidence = self._measure_round_affordable( warmup_runtime_sec=warmup_runtime, + warmup_post_ready_sec=warmup_post_ready, ctx_extra=extra, ) if not affordable: + if gate_evidence.get("one_more_measurement_sec"): + why = ( + "a hot pass (%.0fs) and one variant to read against it " + "(%.0fs) need %.0fs" % ( + gate_evidence.get("measure_round_sec", 0.0), + gate_evidence.get("one_more_measurement_sec", 0.0), + gate_evidence.get("expected_cost_sec", 0.0), + ) + ) + else: + why = "a hot pass needs %.0fs" % ( + gate_evidence.get("expected_cost_sec", 0.0), + ) log.warning( - "baseline_executor: the warmup took %.0fs and only %.0fs is " - "left (bound=%s), so the measured round cannot follow it. " - "Keeping the warmup as the baseline; it is the cold anchor a " - "single-round baseline would have produced, and the GPU time " - "it cost is already spent. The marker below says the figure " - "is cold so the session's later gains can be read against a " - "known-depressed denominator.", - float(warmup_runtime or 0.0), + "baseline_executor: %s, and %.0fs is left (bound=%s), so the hot " + "pass is not run. It would have bought a denominator nothing " + "could then be compared to, and its own overtime anchor would " + "have gone unused. Keeping the warmup as the baseline; it is the " + "cold anchor a single-round baseline would have produced, and " + "the GPU time it cost is already spent. The marker below says " + "the figure is cold so the session's later gains can be read " + "against a known-depressed denominator.", + why, gate_evidence.get("affordable_sec", 0.0), gate_evidence.get("bound", ""), ) @@ -3408,6 +3465,31 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: result["warm_kernel_apply_results"] = list( params.get("warm_kernel_apply_results") or [] ) + if ( + result.get("status") != "succeeded" + and result.get("error_class") == SESSION_TIME_EXHAUSTED_CLASS + ): + # The gate before this pass admitted it and the run's clock took + # it anyway -- the pass overran what it was priced at. Reporting + # the round as failed would throw away the warmup's figure too, + # leaving the session with nothing from GPU time it has already + # spent, so the warmup is kept and marked exactly as a refusal + # keeps it. Only a reap is handled this way: a pass that failed + # for a reason of its own is a failure worth surfacing, and the + # warmup having succeeded does not make it comparable. + log.warning( + "baseline_executor: the run's clock stopped the measured " + "round mid-flight, so the warmup stands as the baseline. It " + "is the cold anchor a single-round baseline would have " + "produced, and the marker below says so.", + ) + return _cold_anchor_from_warmup( + warmup_result, + dropped={ + "reason": "measure_round_reaped_by_the_run", + "measure_round_error": result.get("error"), + }, + ) if result.get("status") == "succeeded": result.setdefault("nonfatal_warnings", []) result["nonfatal_warnings"].append( @@ -3538,82 +3620,41 @@ def _persist_recipe_snapshot(manifest: dict[str, Any]) -> bool: if bench_lease is not None: bench_lease.close() - def _last_round_cost_sec( - self, - *, - double_run: bool, - ctx_extra: dict[str, Any] | None = None, - ) -> float | None: - """What an earlier round of this session actually cost. - - An over-prediction of what the next one will cost, and knowingly so. The - figure comes from a pass that paid the one-time JIT compile on a fresh - kernel cache, which a later pass on the same signature does not pay - again; the executor's own cache probe is built on that difference, and - picks a cap 20 minutes shorter when it finds the cache warm. - - Over-predicting is the tolerable direction *for this caller only*, and - the reason is in :meth:`_round_affordable_before_ignition`: a refusal - here costs the session a fresh anchor it can do without, because the - anchor it already has still stands. No caller that would end a session on - this figure may use it. - - The two passes are priced apart because they buy different things -- the - first boots the server and pays the compile and the graph capture, the - second re-attaches a client to a server already up -- and each is added - only once it has been measured. - - A session can hold the cold figure without the hot one, and that is not a - corner case: a round whose measured pass was dropped for budget promotes - its cold number as the anchor and has no hot number to write. Such a - session is the one most in need of this gate, so its cold pass is still - priced; only the pass it cannot price is left out, and whether that one - may follow is settled after the warmup as usual. - - Args: - double_run: Whether this round will run both passes. - ctx_extra: The runner context extras carrying ``shared_state``. - - Returns: - float | None: What an earlier round cost, or ``None`` when the - session has measured nothing to predict from. - """ - state = (ctx_extra or {}).get("shared_state") or self.shared_state - cold_sec = _measured_runtime_sec(state, "baseline_runtime_sec") - if cold_sec is None or not double_run: - return cold_sec - warm_sec = _measured_runtime_sec(state, "baseline_warm_runtime_sec") - return cold_sec if warm_sec is None else cold_sec + warm_sec - def _round_affordable_before_ignition( self, *, double_run: bool, ctx_extra: dict[str, Any] | None = None, ) -> tuple[bool, dict[str, Any]]: - """Whether the budget still holds a whole round, asked before anything boots. + """Whether the budget holds a whole round *and a use for it*, before anything boots. The companion to :meth:`_measure_round_affordable`, and the two divide the session's baselines between them. A session's first round is not asked this question, because there is nothing to ask it with: the measured runtimes are written only once an anchor lands, so a gate here would either refuse every first baseline or wave every one through. That - round runs, and whether a second pass may follow is settled afterwards + round runs, and whether its second pass may follow is settled afterwards against what the first actually cost. - Every later round has at least the previous one's cold figure, and by then - an anchor already exists. That is what makes this gate safe to build on an - over-predicting figure: the two outcomes are not symmetric. Igniting a - round that cannot finish costs a boot, a compile and a graph capture for a - number that must then be marked cold. Refusing one that would have fitted - costs a fresh anchor the session did not need, because the anchor it - measured earlier still stands and every later comparison is made against - that one. - - The asymmetry is the whole justification, so it may not be borrowed. A - caller that would *end the session* on this figure would be trading the - rest of the run against a systematic over-prediction, and needs a bound - that errs the other way. + Every later round is asked, and a resumed session's first round is the + case that most needs it: it holds the earlier leg's measurements, so a + round that cannot lead anywhere can be refused before a single second of + GPU time is spent on it, and the refusal names a number an operator can + act on. + + What is required in PRELUDE is the round *plus one further measured + variant*, not the round alone. A baseline is not a result there; it is the + denominator later results are read against, and one that nothing is ever + compared to is wall-clock spent on a number no one uses. A variant's own + measurement is a result, which is why the variant gates ask only whether + the variant itself fits -- and why a re-baseline in a later phase is asked + the same narrower question by :func:`_a_use_must_follow_the_round`. + + The round is priced by + :func:`~...phases.machine_state.baseline_round_cost_sec`, which the phase + machine also reads to decide whether a session stopped for this reason may + try again on a fresh clock. Two definitions would let the executor refuse + rounds the phase machine had just decided were affordable. Args: double_run: Whether this round will run both passes. @@ -3623,46 +3664,71 @@ def _round_affordable_before_ignition( tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. """ state = (ctx_extra or {}).get("shared_state") or self.shared_state - cost = self._last_round_cost_sec( - double_run=double_run, - ctx_extra=ctx_extra, - ) - if cost is None: + round_sec = _phase_state.baseline_round_cost_sec(state, double_run=double_run) + if round_sec is None: return True, {"reason": "no_measured_round_to_predict_from"} + use_sec = 0.0 + if _a_use_must_follow_the_round(state): + use_sec = _phase_state.one_more_measurement_sec(state) or 0.0 headroom_sec, evidence = _round_headroom_sec(state, None) if headroom_sec is None: return True, evidence - priced = {"expected_cost_sec": round(cost, 1), **evidence} + cost = round_sec + use_sec + priced = { + "expected_cost_sec": round(cost, 1), + "round_sec": round(round_sec, 1), + "one_more_measurement_sec": round(use_sec, 1), + **evidence, + } return headroom_sec >= cost, priced def _measure_round_affordable( self, *, warmup_runtime_sec: Any, + warmup_post_ready_sec: Any = None, ctx_extra: dict[str, Any] | None = None, ) -> tuple[bool, dict[str, Any]]: - """Whether PRELUDE's remaining budget still covers the measured round. - - Asked after the warmup rather than before it, and priced with what that - pass actually cost rather than a prediction of what it would. The - warmup's own wall-clock is available exactly when this question is asked, - and it is an upper bound rather than an estimate: the measured round - re-attaches to a server this pass has already booted, so it cannot cost - more. + """Whether the budget covers the measured round *and a use for it*. - This is the gate every round faces, including the first, which is the one + Asked after the warmup rather than before it, and priced from what that + pass just measured rather than from a prediction. This is the gate every + round faces, including the first, which is the one :meth:`_round_affordable_before_ignition` cannot judge. - A round that fails here has still produced a number. It is the cold one - the double run exists to discard, so the caller keeps it as the anchor - and marks it -- the GPU time is spent either way, and a marked cold - anchor beats no anchor. + The measured round re-attaches to the server the warmup left running, so + it costs a benchmark and no boot. The warmup's post-ready segment is what + prices it: the same pass, with its boot taken off. That segment still + over-predicts, since it also paid the first request's kernel compile, but + it is the tightest honest figure a session has before its hot pass has + ever run -- and far tighter than the warmup's whole wall-clock, which + prices a client-only pass as though it booted a server. + + In PRELUDE, covering the pass is not enough to justify running it. A hot + baseline is an input there, not a result: it is the denominator later + variants are read against, and it is what anchors their overtime kill. + Neither buys anything if no variant can follow, so what is required is the + pass plus one further measured variant -- a boot and a benchmark, because + a variant's config differs in the very knobs that decide how a server + comes up and it cannot re-attach to anyone else's. A re-baseline in a + later phase is its own deliverable and is asked only to cover its pass; + :func:`_a_use_must_follow_the_round` draws that line. - Only PRELUDE is guarded — a re-baseline in a later phase answers to that - phase's budget. + A round that fails here has still produced a number. It is the cold one + the double run exists to discard, so the caller keeps it as the anchor and + marks it -- the GPU time is spent either way, and a marked cold anchor + beats no anchor. Args: warmup_runtime_sec: Wall-clock the warmup round took. + warmup_post_ready_sec: The part of it that ran after the server was + ready, or ``None`` when nothing recorded the boundary -- a + scriptable workload, which runs no server, or a stamp that could + not be written. Both terms then fall back to the warmup's whole + wall-clock, which remains an upper bound on each: the measured + round re-attaches to a server this pass booted, and a variant is a + boot plus a benchmark where this figure is a boot plus a benchmark + that also paid the first request's compile. ctx_extra: The runner context extras carrying ``shared_state``. Returns: @@ -3672,11 +3738,27 @@ def _measure_round_affordable( headroom_sec, evidence = _round_headroom_sec(state, None) if headroom_sec is None: return True, evidence - try: - cost = max(0.0, float(warmup_runtime_sec or 0.0)) - except (TypeError, ValueError): - cost = 0.0 - return headroom_sec >= cost, {"expected_cost_sec": round(cost, 1), **evidence} + warmup_sec = _positive_seconds(warmup_runtime_sec) or 0.0 + benchmark_sec = _positive_seconds(warmup_post_ready_sec) + if benchmark_sec is None: + priced_by = "warmup_wall_clock" + benchmark_sec = warmup_sec + boot_sec = 0.0 + else: + priced_by = "warmup_post_ready" + boot_sec = max(0.0, warmup_sec - benchmark_sec) + use_sec = 0.0 + if _a_use_must_follow_the_round(state): + use_sec = boot_sec + benchmark_sec + cost = benchmark_sec + use_sec + priced = { + "expected_cost_sec": round(cost, 1), + "priced_by": priced_by, + "measure_round_sec": round(benchmark_sec, 1), + "one_more_measurement_sec": round(use_sec, 1), + **evidence, + } + return headroom_sec >= cost, priced def _double_run_enabled( self, diff --git a/src/hyperloom/orchestrator/actions/executors/report.py b/src/hyperloom/orchestrator/actions/executors/report.py index 3d94e6ef97..7136591938 100644 --- a/src/hyperloom/orchestrator/actions/executors/report.py +++ b/src/hyperloom/orchestrator/actions/executors/report.py @@ -382,6 +382,13 @@ def _build_failure_summary( "out, or what was left of PRELUDE's share could no longer pay for a whole baseline round (a discarded " "warmup pass and the measured pass it makes comparable)." ), + "prelude_cold_anchor_low_budget": ( + "The baseline's hot pass was skipped because the clock could not cover it together with one variant to " + "measure against it, so the only figure available is the cold pass's — depressed by the server boot, the " + "first request's kernel compile and the graph capture. Optimizing against it would report every variant as " + "an improvement over a baseline that was never the baseline, so the run stopped with the figure kept and " + "marked. Resume with more budget to measure a comparable baseline." + ), # Recipe KB knowledge-plane bootstrap failures. "recipe_kb_t0_failed": "Recipe KB knowledge-plane bootstrap (t0) failed; the run stopped early.", "recipe_kb_drain_failed": "Recipe KB knowledge-plane drain failed; the run stopped early.", diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 78f191b491..130c383796 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -235,6 +235,7 @@ def render_phase_proposable_bullets( "recipe_kb_drain_failed", "recipe_kb_commit_failed", "prelude_baseline_failed", + "prelude_cold_anchor_low_budget", # PRELUDE → CLOSE; only a cold anchor, nothing comparable to it affordable "prelude_policy_loop", "policy_loop", "crash_threshold_exceeded", @@ -267,6 +268,7 @@ def render_phase_proposable_bullets( "robustness_escalated", "user_stop_requested", "prelude_baseline_failed", + "prelude_cold_anchor_low_budget", "prelude_policy_loop", "time_exhausted_during_prelude", "recipe_kb_t0_failed", @@ -1974,6 +1976,14 @@ def enablement_engaged(state: Any) -> bool: def exit_normal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """``baseline_tput > 0`` and warm-replay settled → ``prelude_done`` (else ``None``). + A figure whose hot pass was dropped for budget does not finish preparation, + even though it is a figure. It is depressed by the boot, the compile and the + graph capture it could not discard, so declaring PRELUDE done on it would + hand the optimization phases a denominator that was never the baseline. The + phase stays open instead, and what happens next is decided by the budget: + :func:`exit_cold_anchor_prelude` closes a session that still cannot afford a + comparable baseline, and one resumed with a fresh clock measures another. + Args: state (Any): Frozen SharedState view exposing ``baseline_tput`` and the warm-replay outcome. @@ -1984,6 +1994,8 @@ def exit_normal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """ if warm_replay_in_flight(state): return None + if bool(getattr(state, "baseline_measure_round_dropped", False)): + return None try: tput = float(getattr(state, "baseline_tput", 0.0) or 0.0) except (TypeError, ValueError): @@ -1993,6 +2005,127 @@ def exit_normal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: return None +def measured_seconds(state: Any, field: str) -> float | None: + """Read a duration an earlier round measured, or ``None`` when none did. + + Every budget decision that prices work off a measurement has to tell "no + round has run" apart from "a round ran and took no time", because the first + means the decision cannot be made and the second would make everything look + free. + + Args: + state (Any): Frozen SharedState view. + field (str): The ``SharedState`` attribute holding the duration. + + Returns: + float | None: The duration when one was measured, else ``None``. + """ + try: + value = float(getattr(state, field, 0.0) or 0.0) + except (TypeError, ValueError): + return None + return value if value > 0.0 else None + + +def boot_cost_sec(state: Any) -> float | None: + """What bringing this workload's server up costs, or ``None`` when unmeasured. + + The baseline's cold round is the one round that pays this in the open, so it + is where the figure comes from: its wall-clock less the part of it that ran + after the server reported ready. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Seconds spent before the server was ready, or ``None`` when + no round has reported the split (no baseline yet, or a scriptable + workload, which runs no server and so has no boot to separate). + """ + total_sec = measured_seconds(state, "baseline_runtime_sec") + post_ready_sec = measured_seconds(state, "baseline_post_ready_runtime_sec") + if total_sec is None or post_ready_sec is None: + return None + return max(0.0, total_sec - post_ready_sec) + + +def benchmark_cost_sec(state: Any) -> float | None: + """What one benchmark pass costs on a server already up, or ``None``. + + Two figures can answer this and they are not equally good, so the better one + wins when it exists: + + * The measured hot pass (``baseline_warm_runtime_sec``) is the answer, being + exactly a benchmark against a server someone else booted. + * The cold round's post-ready segment is the fallback, and it over-predicts: + that segment also pays the first request's kernel compile, which a pass on + a now-populated JIT cache does not. It is what a session has before its hot + pass runs, and over-predicting a benchmark is a smaller error than pricing + one at a whole cold round. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Seconds one benchmark pass costs, or ``None`` when neither + figure has been measured. + """ + hot_sec = measured_seconds(state, "baseline_warm_runtime_sec") + if hot_sec is not None: + return hot_sec + return measured_seconds(state, "baseline_post_ready_runtime_sec") + + +def baseline_round_cost_sec(state: Any, *, double_run: bool) -> float | None: + """What a baseline round costs, or ``None`` when unmeasured. + + A round is one or two passes and they are priced apart, because they buy + different things. The first brings a server up and then benchmarks it, so it + costs what any measured variant costs. The second re-attaches to the server + the first left running, so it costs a benchmark and no second boot. + + Args: + state (Any): Frozen SharedState view. + double_run (bool): Whether the round runs a warmup pass and a measured + one, or a single pass. + + Returns: + float | None: Seconds the round costs, or ``None`` when the session has + measured nothing to price it from. + """ + first_pass_sec = one_more_measurement_sec(state) + if first_pass_sec is None or not double_run: + return first_pass_sec + second_pass_sec = benchmark_cost_sec(state) + return first_pass_sec if second_pass_sec is None else first_pass_sec + second_pass_sec + + +def one_more_measurement_sec(state: Any) -> float | None: + """What the next measured variant will cost, or ``None`` when unmeasured. + + A variant is not a benchmark; it is a boot and then a benchmark. Its config + differs from the baseline's in the very knobs that decide how a server comes + up -- parallelism, quantization, kernel backends -- so it cannot re-attach to + a server already running and has to bring up its own. + + This is the unit every "is there time left to use a result" question is asked + in, because a result nothing gets measured against is a result the session + could not have used. + + Args: + state (Any): Frozen SharedState view. + + Returns: + float | None: Seconds one further measured variant costs, or ``None`` + when either half of it is unmeasured. + """ + boot_sec = boot_cost_sec(state) + benchmark_sec = benchmark_cost_sec(state) + if boot_sec is None or benchmark_sec is None: + return None + return boot_sec + benchmark_sec + + def prelude_exit_viability(state: Any) -> dict[str, Any]: """Report whether the budget PRELUDE leaves behind can still fund one optimization round. @@ -2005,6 +2138,14 @@ def prelude_exit_viability(state: Any) -> dict[str, Any]: the answer is actionable and the first moment it is knowable: the baseline is measured, so one benchmark round has a price rather than an estimate. + Priced as what an optimization round actually is -- one boot and one + benchmark (:func:`one_more_measurement_sec`) -- rather than as the baseline's + whole cold wall-clock, which also carries the first request's compile and so + over-reports a prepared session as a spent one. The cold figure remains the + fallback for a session whose baseline reported no split, with ``priced_by`` + naming which ruler answered so two runs' evidence cannot be compared as + though they used the same one. + Args: state (Any): Frozen SharedState view. @@ -2012,16 +2153,18 @@ def prelude_exit_viability(state: Any) -> dict[str, Any]: dict[str, Any]: Evidence for the phase record; empty when the budget is unbounded or no round has been measured. """ - usable = _session_usable_seconds(state) - try: - round_sec = float(getattr(state, "baseline_runtime_sec", 0.0) or 0.0) - except (TypeError, ValueError): - round_sec = 0.0 - if usable is None or round_sec <= 0.0: + usable = session_usable_seconds(state) + round_sec = one_more_measurement_sec(state) + priced_by = "boot_plus_benchmark" + if round_sec is None: + round_sec = measured_seconds(state, "baseline_runtime_sec") + priced_by = "cold_round" + if usable is None or round_sec is None: return {} return { "session_usable_sec": round(usable, 1), "measured_round_sec": round(round_sec, 1), + "priced_by": priced_by, "affordable_rounds": round(usable / round_sec, 2), "fits_one_optimization_round": usable >= round_sec, } @@ -2061,7 +2204,7 @@ def append_phase_evidence_row(history: Any, *, key: str, row: dict[str, Any]) -> return True -def _session_usable_seconds(state: Any) -> float | None: +def session_usable_seconds(state: Any) -> float | None: """Seconds a unit of work may still claim, from the session's own accounting. Prefers ``SharedState.session_budget_usable_sec`` — the single number @@ -2109,7 +2252,7 @@ def prelude_affordable_seconds(state: Any) -> tuple[float | None, dict[str, Any] unbounded budget, plus the evidence behind it. """ max_sec = _max_minutes(state) * 60.0 - usable = _session_usable_seconds(state) + usable = session_usable_seconds(state) if max_sec <= 0.0 or usable is None: return None, {"reason": "unbounded_budget"} reserve_sec = max_sec * OPTIMIZATION_RESERVE_PCT @@ -2176,7 +2319,7 @@ def exit_time_exhausted_prelude( tuple[str, dict[str, Any]] | None: ``("time_exhausted_during_prelude", evidence)`` when the budget is gone, else ``None``. """ - usable = _session_usable_seconds(state) + usable = session_usable_seconds(state) if usable is None or usable > 0.0: return None return "time_exhausted_during_prelude", { @@ -2188,6 +2331,70 @@ def exit_time_exhausted_prelude( } +def exit_cold_anchor_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: + """Route to CLOSE when PRELUDE could only produce a cold anchor. + + A baseline's hot pass is dropped when the budget cannot cover it together with + one variant to read against it. What survives is the cold pass's figure, and + it is depressed: it carries the server boot, the first request's kernel + compile and the graph capture in its throughput denominator. + + Continuing on it is worse than stopping. Every variant measured against a + depressed denominator reads as an improvement over a baseline that was never + the baseline, so the session would spend the rest of its clock producing + findings that a later run cannot reproduce. Stopping keeps the number and the + marker that says what it is. + + Fires only on the dropped-pass marker, not on a cold figure as such: a session + configured for a single-round baseline reports a cold figure by design, and + its comparisons are consistent because everything downstream is measured the + same way. + + And only while the budget still cannot buy a comparable baseline. A session + resumed with a fresh clock carries the earlier leg's marker but not its + shortfall, and it can now do the thing it was stopped for: measure a hot + baseline. Firing on the marker alone would send it straight back to CLOSE on + the strength of a constraint that no longer holds, and no later baseline could + ever clear the marker because none would run. + + Args: + state (Any): Frozen SharedState view. + + Returns: + tuple[str, dict[str, Any]] | None: ``("prelude_cold_anchor_low_budget", + evidence)`` when the hot pass was dropped and still cannot be afforded, + else ``None``. + """ + if not bool(getattr(state, "baseline_measure_round_dropped", False)): + return None + usable = session_usable_seconds(state) + if usable is None: + return None + # Without the boot/benchmark split -- a scriptable workload runs no server, so + # it has no ready boundary to split on -- the cold round's whole wall-clock + # stands in for each half, the same upper bound the round's own gate falls + # back to. Undecidable rather than assumed only when nothing was measured at + # all, which cannot happen with the marker set but is not worth closing on. + cold_sec = measured_seconds(state, "baseline_runtime_sec") + round_sec = ( + baseline_round_cost_sec( + state, + double_run=bool(getattr(state, "baseline_double_run", False)), + ) + or cold_sec + ) + use_sec = one_more_measurement_sec(state) or cold_sec + if round_sec is None or use_sec is None: + return None + if usable >= round_sec + use_sec: + return None + return "prelude_cold_anchor_low_budget", { + "baseline_anchor": "cold", + "retry_round_sec": round(round_sec, 1), + **prelude_exit_viability(state), + } + + def exit_terminal_prelude(state: Any) -> tuple[str, dict[str, Any]] | None: """Decide the PRELUDE terminal exit on repeated baseline failures. @@ -2815,6 +3022,11 @@ def compute_next_phase( term = exit_terminal_prelude(state) if term is not None: return PHASE_CLOSE, term[0], {"terminal": True, **term[1]} + # Asked before the normal exit, which sees only that a figure exists: a + # cold anchor is a figure the later phases cannot honestly compare to. + cold = exit_cold_anchor_prelude(state) + if cold is not None: + return PHASE_CLOSE, cold[0], {"terminal": True, **cold[1]} norm = exit_normal_prelude(state) if norm is None: # No baseline and no clock left: name the failure instead of @@ -3388,14 +3600,21 @@ def record_lifecycle_event( "exit_normal_explore", "exit_normal_framework_agent", "exit_normal_kernel", + "exit_cold_anchor_prelude", "exit_normal_prelude", "exit_normal_sweep", "exit_terminal_prelude", "exit_time_exhausted_prelude", "append_phase_evidence_row", + "baseline_round_cost_sec", + "benchmark_cost_sec", + "boot_cost_sec", + "measured_seconds", + "one_more_measurement_sec", "prelude_affordable_seconds", "prelude_can_afford", "prelude_exit_viability", + "session_usable_seconds", "is_action_allowed_in_phase", "is_action_llm_proposable_in_phase", "llm_proposable_actions_for", From 60a700c308cc38cc8c35ab33bf057e50383e8f16 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 17:36:17 +0000 Subject: [PATCH 53/65] phases: a session that stopped in CLOSE reopens where a fresh clock can be spent CLOSE is the machine's only terminal phase, and a resumed leg loaded whatever phase was persisted. Loading CLOSE meant staying there: the machine has no transition out, and the run loop stops on a stop_reason rather than on the phase, so the leg ticked to the end in a phase admitting only report, session_breakdown and recover. Every stop taken on the promise of "resume with more budget" was answered by a leg that spent the budget on none of the work it was resumed for. Reopened at PRELUDE, the phase that works out where a session belongs: it exits on its first evaluation when the anchor the run needs is already measured, and measures one when it is not. A session that still cannot fund the work is not held open by this -- the PRELUDE exits price the new clock and route it back, this time against the budget it actually has. The earlier leg's close_sequence_done goes with it. The flag means "the sequencer already wrote the breakdown", and carried into a leg that then never reaches CLOSE it silences the safety net that would have written one. Co-authored-by: Cursor --- .../inference_optimizer/tests/test_resume.py | 74 ++++++++++++++++++- src/hyperloom/orchestrator/phases/machine.py | 44 +++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_resume.py b/src/hyperloom/inference_optimizer/tests/test_resume.py index 7739648240..8349a943e1 100644 --- a/src/hyperloom/inference_optimizer/tests/test_resume.py +++ b/src/hyperloom/inference_optimizer/tests/test_resume.py @@ -4,8 +4,8 @@ """Coordinator resume tests. Covers resume detection, ``replay_for_resume`` rebuilding undecided -pending_proposals, pruned_families preservation, and lazy replay on the first -``tick()``. +pending_proposals, pruned_families preservation, lazy replay on the first +``tick()``, and reopening the phase machine for a session that stopped in CLOSE. """ from __future__ import annotations @@ -62,6 +62,76 @@ async def test_existing_state_json_triggers_resume(session_dir): await c.stop() +class TestAClosedSessionIsReopenedOnResume: + """CLOSE has no way out, so a leg that loads it would tick in it to the end. + + The machine's only terminal phase, and the run loop stops on ``stop_reason`` + rather than on the phase. A resumed leg that keeps CLOSE therefore spends its + whole new clock in a phase admitting nothing but ``report``, + ``session_breakdown`` and ``recover``. Every design that stops early on the + promise of "resume with more budget" rests on this being reopened. + """ + + @pytest.mark.asyncio + async def test_a_session_stopped_in_close_starts_the_next_leg_at_the_entrance( + self, + session_dir, + ): + SharedState(session_id="closed", phase="CLOSE").save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.phase == "PRELUDE" + finally: + await coordinator.stop() + + @pytest.mark.asyncio + async def test_the_reopening_is_recorded_as_the_transition_it_is(self, session_dir): + """A phase the run did not reach by working its way there needs saying so.""" + SharedState(session_id="closed", phase="CLOSE").save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + latest = coordinator.shared_state.phase_history[-1] + finally: + await coordinator.stop() + + assert latest["from_phase"] == "CLOSE" + assert latest["to_phase"] == "PRELUDE" + assert latest["evidence"]["trigger"] == "resumed_from_close" + + @pytest.mark.asyncio + async def test_the_earlier_legs_close_sequence_does_not_count_for_this_one( + self, + session_dir, + ): + """The flag means "the sequencer already wrote the breakdown". + + Carried into a leg that then never reaches CLOSE, it silences the + end-of-run safety net that would have written one, and the leg finishes + with no breakdown at all. + """ + SharedState(session_id="closed", phase="CLOSE", close_sequence_done=True).save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.close_sequence_done is False + finally: + await coordinator.stop() + + @pytest.mark.asyncio + async def test_a_session_stopped_anywhere_else_resumes_where_it_stopped(self, session_dir): + """Only the phase with no exit is reopened; the rest can still make progress.""" + SharedState(session_id="mid", phase="EXPLORE").save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.phase == "EXPLORE" + assert coordinator.shared_state.phase_history == [] + finally: + await coordinator.stop() + + @pytest.mark.asyncio async def test_existing_events_triggers_resume(session_dir): c1 = Coordinator(session_dir, backends=_backends_full()) diff --git a/src/hyperloom/orchestrator/phases/machine.py b/src/hyperloom/orchestrator/phases/machine.py index 0922c91e23..5a495c4222 100644 --- a/src/hyperloom/orchestrator/phases/machine.py +++ b/src/hyperloom/orchestrator/phases/machine.py @@ -35,6 +35,9 @@ def _ensure_phase_initialised(self) -> None: if not state.phase_budget_pct: state.phase_budget_pct = dict(self._phase_budget_pct) current = (state.phase or "").strip().upper() + if current == _phase_state.PHASE_CLOSE: + self._reopen_a_session_that_was_left_closed() + current = _phase_state.PHASE_PRELUDE if current in _phase_state.PHASE_NAMES: # Already initialised; keep the CLI-side budget override authoritative. state.phase_budget_pct = dict(self._phase_budget_pct) @@ -54,6 +57,47 @@ def _ensure_phase_initialised(self) -> None: except Exception: # noqa: BLE001 — defensive log.exception("Coordinator: save after phase init failed") + def _reopen_a_session_that_was_left_closed(self) -> None: + """Put a session persisted in CLOSE back at the phase machine's entrance. + + CLOSE is terminal -- the machine has no transition out of it -- so a + resumed session that loads it stays there for the whole leg. The run loop + does not stop on CLOSE either, so what such a leg actually does is tick + in a phase that admits only ``report``, ``session_breakdown`` and + ``recover``: it spends its new clock on none of the work it was resumed + for. + + Reopened at PRELUDE rather than at the phase CLOSE was entered from, + because PRELUDE is the phase that works out where a session belongs. It + exits on its first evaluation when the anchor the run needs is already + measured, and it measures one when it is not. A session stopped for a + cold anchor is the second case, and re-measuring is the whole reason its + stop was worth resuming from. + + A session that still cannot fund the work is not kept open by this: the + PRELUDE exits price the new clock and route it back to CLOSE, this time + against the budget it actually has. + + Reached only from the constructor, so a session cannot reopen itself + mid-run -- within one leg, CLOSE is entered long after this has run. + """ + state = self.shared_state + log.info( + "Coordinator: session resumed in CLOSE, a phase with no way out; " + "reopening at PRELUDE so the new budget can be spent on the work " + "the earlier leg stopped short of." + ) + state.record_phase_transition( + to_phase=_phase_state.PHASE_PRELUDE, + reason="phase_entered", + evidence={"trigger": "resumed_from_close"}, + ) + # Locked True by the CLOSE sequencer and read by the end-of-run safety + # nets as "the sequencer already wrote the breakdown". Carried into a leg + # that then never reaches CLOSE, it suppresses the write that would have + # stood in for it, and the leg produces no breakdown at all. + state.close_sequence_done = False + def _ensure_recipe_kb_t0_anchored(self) -> None: """Defensive T0 anchor for SDK callers constructed without cli plumbing. Skips when recipe_kb is None or recipe_kb_session_id set.""" client = self.recipe_kb From 5b3c7230303e477696bcd6bf564980d8389b0dd5 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 19:01:30 +0000 Subject: [PATCH 54/65] executors, phases: measure a round's price rather than rebuild it, and let a hot pass correct a cold anchor Findings from an adversarial read of the cost-based gates. A cold anchor holds PRELUDE open until a hot pass replaces it, and the rule that keeps a later, lower re-baseline from displacing the anchor rejected exactly that replacement -- so a retry that did land a hot pass, but measured lower because the "cold" warmup was not really cold, left the marker set and the session re-measured whole baseline rounds until the clock killed it. A hot figure over a marked cold one is a correction, not a regression: the two are not comparable, which is what the marker says. The round price was rebuilt as boot-plus-hot-benchmark, dropping the compile the cold pass pays and under-pricing the round by that much, while the gate after the cold pass priced the same second pass at the post-ready segment. A band of budgets was therefore admitted before ignition and refused for certain afterwards, at the cost of a whole cold pass. Both now read what the session measured: the round's first pass whole, and the hot pass from a hot pass. Reading the measured total also gives a price to rounds with no boot boundary to reconstruct from. Multi-node had lost both gates -- the shared pre-warmup refusal was deleted and neither replacement reached it -- so a pair of passes launched on a budget for one and the measured pass was reaped, leaving no anchor. It is gated again, priced as the two passes it is. Also: a prior attempt's nested workspace could latch this round's ready marker and report a fifteen-minute boot as zero; the ready timestamp was tied to an unrelated stall watchdog whose knob silently withdrew it; a missing timestamp made the post-warmup gate demand two whole cold rounds at the one site where refusing ends the session; and the exit glossary still described a share mechanism this branch removed. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 193 +++++++++++++++--- ...coordinator_async_methods_coverage_unit.py | 76 +++++++ .../tests/test_phase_state_machine.py | 13 +- .../tests/test_soft_deadline_from_ready.py | 75 +++++++ .../actions/executors/_subprocess_kill.py | 54 ++++- .../actions/executors/baseline.py | 114 ++++++++--- .../orchestrator/actions/executors/report.py | 6 +- src/hyperloom/orchestrator/loop/writeback.py | 23 ++- .../orchestrator/phases/machine_state.py | 20 +- 9 files changed, 507 insertions(+), 67 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index fb17e3624f..c2e6f87f84 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -51,9 +51,12 @@ from hyperloom.orchestrator.state.shared_state import SharedState from hyperloom.orchestrator.trace.task_progress import progress_scope -from .conftest import chatty_child, enable_multi_node, suppression_window_s - -from .conftest import enable_multi_node, launches_by_round_slot +from .conftest import ( + chatty_child, + enable_multi_node, + launches_by_round_slot, + suppression_window_s, +) @pytest.fixture(autouse=True) @@ -216,7 +219,6 @@ def test_baseline_discards_cold_first_round_via_lifecycle(tmp_path, monkeypatch) assert captured[0]["benchmark"]["benchmark_script"] == "vllm_mi300x.sh" -<<<<<<< HEAD def _run_capturing_rounds(executor, ctx, notes): """Run ``executor`` and record which round notes existed at each launch.""" at_launch: list[list[str]] = [] @@ -388,6 +390,28 @@ def _prelude_shared_state(*, usable_sec: float, phase: str = "PRELUDE") -> Simpl ) +def _a_session_the_passes_spend( + *, + usable_sec: float, + clock: _AClockOnlyThePassesMove, + **measured: float, +) -> SimpleNamespace: + """A PRELUDE session whose remaining budget falls as the passes spend it. + + The fixed-remainder double cannot show the two gates disagreeing, because the + second one is asked after the warmup has spent its share and a budget that + never moves hides exactly that. This reads the same clock the passes move. + """ + started = clock() + return SimpleNamespace( + baseline_double_run=True, + phase="PRELUDE", + max_minutes=180, + session_budget_usable_sec=lambda: usable_sec - (clock() - started), + **measured, + ) + + def _run_double_run_baseline( tmp_path, shared_state, @@ -462,6 +486,74 @@ def test_a_budget_that_cannot_pay_for_the_measured_round_keeps_the_cold_warmup(t ) +def test_a_round_admitted_before_ignition_is_not_refused_after_its_cold_pass(tmp_path): + """The two gates price the same second pass, so they must reach the same answer. + + A gate before ignition that admits what the gate after the cold pass will + certainly refuse spends a whole cold pass to learn something it already knew. + The disagreement is in the ruler: this session has measured a 400s hot pass, + and pricing the pass to come at the warmup's 550s post-ready segment instead + -- a segment that also paid the first request's compile -- demands 300s more + than ignition was allowed to require. + + 2100s is inside that band. Ignition needs the 1300s round and a 750s variant; + after the warmup spends 900s, the hot pass and its variant need 1150s of the + 1200s left, while the post-ready ruler would have called for 1450s. + """ + clock = _AClockOnlyThePassesMove() + state = _a_session_the_passes_spend( + usable_sec=2100.0, + clock=clock, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + baseline_warm_runtime_sec=400.0, + ) + + result = _run_double_run_baseline( + tmp_path, + state, + clock=clock, + boot_sec=350.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 2, ( + "a cold pass was spent on a round the gate after it was always going to refuse" + ) + assert result["output_throughput"] == pytest.approx(_HOT_TPUT) + + +def test_a_warmup_that_overran_its_prediction_still_drops_the_hot_pass(tmp_path): + """Agreeing with the earlier gate is not the same as admitting everything. + + Once the two price the same work, the only rounds left for this gate to + refuse are the ones that cost more than they were admitted on -- which is + precisely what a gate asked after the pass, against the clock rather than + against a prediction, exists for. This round was admitted at 2050s and its + warmup then took 1250s instead of 900s, leaving 850s where 1150s is needed. + """ + clock = _AClockOnlyThePassesMove() + state = _a_session_the_passes_spend( + usable_sec=2100.0, + clock=clock, + baseline_runtime_sec=900.0, + baseline_post_ready_runtime_sec=550.0, + baseline_warm_runtime_sec=400.0, + ) + + result = _run_double_run_baseline( + tmp_path, + state, + clock=clock, + boot_sec=700.0, + benchmark_sec=550.0, + ) + + assert result["_rounds_run"] == 1 + assert result["output_throughput"] == pytest.approx(_COLD_TPUT) + assert result["measure_round_dropped"]["priced_by"] == "session_hot_pass" + + def test_a_rebaselines_hot_pass_needs_only_its_own_wall_clock(tmp_path): """The same 1200s that drops the hot pass in PRELUDE runs it here. @@ -697,12 +789,16 @@ def test_a_double_run_reports_the_boot_split_of_the_pass_that_paid_it(tmp_path): _HOT_ROUND_SEC = 400.0 # 900 - 550: the part of the cold round that was not the benchmark. _BOOT_SEC = 350.0 -# One further measured variant: its own boot, then its own benchmark. +# One further measured variant: its own boot, then its own benchmark. The +# benchmark is priced hot because the variant runs on a JIT cache this session +# has already populated. _ONE_MORE_SEC = _BOOT_SEC + _HOT_ROUND_SEC -# A single-pass round is a boot and a benchmark; a double-run round adds a second -# benchmark, which re-attaches and so buys no second boot. -_SINGLE_ROUND_SEC = _ONE_MORE_SEC -_DOUBLE_ROUND_SEC = _ONE_MORE_SEC + _HOT_ROUND_SEC +# A round's first pass is the cold one, measured whole rather than rebuilt from +# its halves -- rebuilding it as boot-plus-hot would drop the compile it paid and +# under-price the round by 150s. A double run adds a second benchmark, which +# re-attaches and so buys no second boot. +_SINGLE_ROUND_SEC = _COLD_ROUND_SEC +_DOUBLE_ROUND_SEC = _COLD_ROUND_SEC + _HOT_ROUND_SEC class TestARoundThatCannotFinishIsNotIgnited: @@ -868,38 +964,41 @@ def test_a_double_run_pays_for_the_second_pass_but_not_a_second_boot(self, tmp_p assert shortfall["round_sec"] == pytest.approx(_DOUBLE_ROUND_SEC) assert shortfall["expected_cost_sec"] == pytest.approx(_DOUBLE_ROUND_SEC + _ONE_MORE_SEC) - def test_a_session_with_no_hot_figure_prices_the_benchmark_from_the_cold_pass(self, tmp_path): + def test_a_session_with_no_hot_figure_prices_the_variant_from_the_cold_pass(self, tmp_path): """The state a previous cold-anchor drop leaves, and the one to catch. A round whose measured pass was dropped for budget promotes its cold number and has no hot number to write. Going inert on such a session would exempt exactly the one that already ran out of budget once, so the cold - round's post-ready segment stands in for the benchmark. It over-predicts, - having also paid the first request's compile, which is why the hot figure - wins whenever one exists. + round's post-ready segment stands in for the variant's benchmark. It + over-predicts, having also paid the first request's compile, which is why + the hot figure wins whenever one exists. """ one_more_sec = _BOOT_SEC + _COLD_POST_READY_SEC result, calls = _run_baseline_under_budget( tmp_path, - remaining_sec=one_more_sec * 2 - 1.0, + remaining_sec=_SINGLE_ROUND_SEC + one_more_sec - 1.0, cold_round_sec=_COLD_ROUND_SEC, cold_post_ready_sec=_COLD_POST_READY_SEC, ) assert calls == [] - assert result["budget_shortfall"]["round_sec"] == pytest.approx(one_more_sec) - assert result["budget_shortfall"]["round_sec"] > _SINGLE_ROUND_SEC, ( + shortfall = result["budget_shortfall"] + assert shortfall["one_more_measurement_sec"] == pytest.approx(one_more_sec) + assert shortfall["one_more_measurement_sec"] > _ONE_MORE_SEC, ( "the fallback did not over-predict, so it cannot be the post-ready segment" ) - def test_a_round_whose_boot_was_never_measured_is_not_judged(self, tmp_path): - """A cold wall-clock alone cannot say what a variant costs. + def test_a_round_whose_boot_was_never_measured_is_still_judged(self, tmp_path): + """A round with no split is priced at whole cold rounds, not waved through. - Without the split, the boot is unknown, and a boot is most of what a - variant pays. Refusing on a figure that cannot be built would refuse - sessions arbitrarily, so the question is left to the post-warmup gate, - which prices the pass from what the warmup just spent. + Multi-node and scriptable workloads never report a boot boundary -- one + brings its server up outside the round, the other runs no server at all -- + so a gate that goes inert without the split exempts them permanently. Both + terms fall back to what the session did measure: the cold round's own + wall-clock, which is a boot and a benchmark, and so is what a variant + costs too. """ result, calls = _run_baseline_under_budget( tmp_path, @@ -908,8 +1007,10 @@ def test_a_round_whose_boot_was_never_measured_is_not_judged(self, tmp_path): hot_round_sec=_HOT_ROUND_SEC, ) - assert calls, "a round was refused on a price the session cannot compute" - assert result["status"] == "succeeded" + assert calls == [], "a workload with no boot boundary was exempted from the gate" + shortfall = result["budget_shortfall"] + assert shortfall["round_sec"] == pytest.approx(_COLD_ROUND_SEC) + assert shortfall["one_more_measurement_sec"] == pytest.approx(_COLD_ROUND_SEC) def test_a_refused_round_carries_nothing_that_could_replace_the_anchor(self, tmp_path): """The property the whole gate rests on, asserted rather than assumed. @@ -2817,6 +2918,50 @@ def test_a_first_baseline_on_a_default_session_runs_both_of_its_passes( f"it: {warmup}s against {remaining_sec}s left" ) + def test_a_multi_node_round_that_cannot_pay_for_both_passes_launches_neither( + self, + tmp_path, + monkeypatch, + ): + """A multi-node round is two client passes, and the figure covers one. + + The server comes up outside the round, so both passes are the same shape + and the recorded wall-clock -- taken after the warmup -- is one of them. + Pricing the round at that one figure admits a pair that cannot fit: the + warmup spends its half and the measured pass meets the deadline, leaving + the round with no anchor and the GPU time gone. + + 1500s is the band only this gate catches: the generic gate before it + prices the round at one 600s pass and a variant at another, admits at + 1200s, and has no way to know a second pass is coming. The pair plus a + variant needs 1800s. + """ + enable_multi_node(monkeypatch) + + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=1500.0, + cold_round_sec=600.0, + ) + + assert calls == [], "a multi-node pair ran on a budget that covers one pass" + assert result["error_class"] == SESSION_TIME_EXHAUSTED_CLASS + assert result["returncode"] is None + assert result["budget_shortfall"]["round_sec"] == pytest.approx(1200.0) + + def test_a_multi_node_round_the_budget_covers_runs_both_passes(self, tmp_path, monkeypatch): + """The gate must not refuse the pairs that fit, only the ones that cannot.""" + enable_multi_node(monkeypatch) + + result, calls = _run_baseline_under_budget( + tmp_path, + remaining_sec=3600.0, + cold_round_sec=600.0, + ) + + assert set(launches_by_round_slot(calls)) >= set(_BASELINE_ROUND_SLOTS) + assert result["status"] == "succeeded" + def test_the_profile_arm_gets_all_of_it(self, tmp_path): """Profile is the same executor with a four-hour default -- longer than any session budget it could be given.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py index 13ea21177a..88514af235 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py @@ -162,6 +162,82 @@ async def test_promote_baseline_carries_a_dropped_hot_pass_to_the_session( assert coord.shared_state.baseline_measure_round_dropped is False +class TestAHotPassCorrectsAColdAnchor: + """The escape from the marker, without which PRELUDE cannot finish. + + A cold anchor holds the phase open until a hot pass replaces it. The rule + that keeps a later, lower re-baseline from displacing the anchor would reject + that replacement whenever the cold figure reads higher -- which it does + whenever the "cold" pass was not really cold, its weights already in page + cache and its kernels already compiled by an earlier run. The session would + then re-measure whole baseline rounds until the clock killed it, each one + landing the very measurement that was supposed to release it. + """ + + @pytest.mark.asyncio + async def test_a_lower_hot_figure_replaces_a_marked_cold_one(self, coord: Coordinator) -> None: + coord.shared_state.baseline_tput = 1000.0 + coord.shared_state.baseline_measure_round_dropped = True + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 980.0, + "subprocess_runtime_sec": 900.0, + "measure_round_runtime_sec": 400.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_tput == 980.0 + assert coord.shared_state.baseline_measure_round_dropped is False + assert coord.shared_state.baseline_warm_runtime_sec == 400.0 + + @pytest.mark.asyncio + async def test_a_lower_cold_figure_does_not_replace_a_marked_cold_one( + self, + coord: Coordinator, + ) -> None: + """Only a hot pass corrects the anchor; another cold one is just noisier. + + Two cold figures are comparable to each other, so the ordinary rule + applies and the better one stands. Nothing has been corrected, so the + marker stays and the phase stays open. + """ + coord.shared_state.baseline_tput = 1000.0 + coord.shared_state.baseline_measure_round_dropped = True + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 980.0, + "subprocess_runtime_sec": 900.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_tput == 1000.0 + assert coord.shared_state.baseline_measure_round_dropped is True + + @pytest.mark.asyncio + async def test_a_lower_hot_figure_still_loses_to_a_hot_anchor(self, coord: Coordinator) -> None: + """With no marker there is nothing to correct, so drift is refused again.""" + coord.shared_state.baseline_tput = 1000.0 + coord.shared_state.baseline_measure_round_dropped = False + + await coord._promote_to_shared_state( + "baseline", + { + "output_throughput": 980.0, + "subprocess_runtime_sec": 900.0, + "measure_round_runtime_sec": 400.0, + "workspace": "/tmp/ws", + }, + ) + + assert coord.shared_state.baseline_tput == 1000.0 + + @pytest.mark.asyncio async def test_promote_baseline_non_dict_is_noop(coord: Coordinator) -> None: await coord._promote_to_shared_state("baseline", "not-a-dict") # type: ignore[arg-type] diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py index 38c0e0f3ea..364976e7e6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_state_machine.py @@ -325,9 +325,10 @@ def test_prelude_exit_states_whether_one_optimization_round_still_fits(): # The workload the cold-anchor cases below are priced against: a 900s cold round # whose last 550s was the benchmark, so the boot took 350s, and a 400s hot pass. -# One further measured variant therefore costs 750s and a double-run baseline -# round 1150s, which is what a session must be able to afford before it is worth -# measuring another baseline. +# One further measured variant therefore costs 750s -- its own boot and a +# benchmark on a populated JIT cache -- while a double-run round costs the whole +# measured cold pass plus a second benchmark, 1300s. Together they are what a +# session must afford before measuring another baseline is worth doing. _COLD_ANCHOR_WORKLOAD = { "baseline_tput": 1074.7, "baseline_runtime_sec": 900.0, @@ -335,7 +336,7 @@ def test_prelude_exit_states_whether_one_optimization_round_still_fits(): "baseline_warm_runtime_sec": 400.0, "baseline_double_run": True, } -_RETRY_COST_SEC = 1150.0 + 750.0 +_RETRY_COST_SEC = 1300.0 + 750.0 class TestAColdAnchorIsNotAFinishedPrelude: @@ -363,7 +364,7 @@ def test_a_dropped_hot_pass_does_not_finish_the_phase(self): assert phase_state.exit_normal_prelude(state)[0] == "prelude_done" def test_a_session_that_cannot_afford_another_baseline_closes(self): - """1900s buys a round and a variant to read against it; 1200s buys neither.""" + """2050s buys a round and a variant to read against it; 1200s buys neither.""" state = _prelude_state( **_COLD_ANCHOR_WORKLOAD, baseline_measure_round_dropped=True, @@ -377,7 +378,7 @@ def test_a_session_that_cannot_afford_another_baseline_closes(self): assert (next_phase, reason) == ("CLOSE", "prelude_cold_anchor_low_budget") assert evidence["terminal"] is True assert evidence["baseline_anchor"] == "cold" - assert evidence["retry_round_sec"] == pytest.approx(1150.0) + assert evidence["retry_round_sec"] == pytest.approx(1300.0) assert phase_state.is_valid_stop_reason(reason) assert phase_state.is_valid_phase_exit_reason(reason) diff --git a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py index f8fc414407..15a6b0047a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py +++ b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py @@ -168,6 +168,81 @@ def test_the_boot_is_not_charged_to_the_benchmark(self, tmp_path): boot_sec = runtime_sec - post_ready assert 2.5 <= boot_sec <= 4.5, f"boot share read as {boot_sec:.2f}s, expected ~3s" + def test_a_previous_attempts_log_does_not_time_this_rounds_boot(self, tmp_path): + """Only bytes this round writes may say when its server came up. + + Magpie writes into a ``benchmark_*/`` workspace under the round's output + dir, and a reused dir can still hold one from an earlier attempt. Scanned + from byte zero, its ready line latches on the first poll -- seconds after + spawn -- and the round reports a boot that took minutes as one that took + none. Every variant is then admitted at a benchmark's price and reaped. + """ + stale = tmp_path / "benchmark_vllm_20200101" / "server.log" + stale.parent.mkdir(parents=True) + stale.write_text("Application startup complete\n", encoding="utf-8") + + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(3)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + runtime_sec = time.time() - started_unix + assert cp.returncode == 0 + + post_ready = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + assert post_ready is not None + boot_sec = runtime_sec - post_ready + assert boot_sec >= 2.5, ( + f"the stale log timed the boot: read {boot_sec:.2f}s of boot for a round " + f"that spent 3s on it" + ) + + def test_the_split_does_not_depend_on_an_unrelated_watchdog(self, tmp_path): + """The stall grace is a hang backstop, not a switch for the cost model. + + Turning it off used to withdraw the ready timestamp with it, and a session + run that way prices every round at its whole cold wall-clock without + anything saying so. + """ + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(2)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=0.0, + session_deadline_sec=time.monotonic() + 30.0, + ) + assert cp.returncode == 0 + assert server_ready_unix(str(log_path)) is not None, ( + "the boot/benchmark split was withdrawn along with the stall watchdog" + ) + def test_a_round_that_never_came_up_is_not_priced(self, tmp_path): """No ready marker means no split, reported as unknown rather than guessed.""" log_path = tmp_path / "server.log" diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 9bb2127a38..f06dbd4806 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -735,6 +735,40 @@ class _LogScan(NamedTuple): child_spoke: bool +def _stale_scan_log_sizes(server_log_path: str) -> dict[str, int]: + """Current byte length of each nested log that already exists at spawn. + + Seeded as starting offsets so such a log contributes only what it grows by, + never what a previous attempt left in it. + + Only the nested ``benchmark_*/`` workspaces, not the path the caller named. + That one the caller owns: it clears it when it means this round to start + clean, and a caller that means to attach to a server already up says so with + ``server_already_ready``. The nested ones are found by globbing a directory + the caller reuses, so nothing in them was asserted by anyone -- and a stale + ready line there would otherwise latch on the first poll and report a boot + that took minutes as one that took none. + + Args: + server_log_path: The ``/server.log`` path from the caller. + + Returns: + dict[str, int]: Per-path byte lengths; a path whose size cannot be read + is left out, so it is scanned from the start as an absent one would be. + """ + owned_dir = Path(server_log_path).parent + sizes: dict[str, int] = {} + for path in _resolve_scan_logs(server_log_path): + candidate = Path(path) + if candidate.parent == owned_dir: + continue + try: + sizes[path] = candidate.stat().st_size + except OSError: + continue + return sizes + + def _scan_logs_increment(server_log_path: str, offsets: dict[str, int]) -> _LogScan: """Scan every resolved log for markers, advancing ``offsets`` in place. @@ -1183,12 +1217,14 @@ def _communicate_with_soft_deadline( not in {"0", "false", "no", "off"} ) # The log increment scan feeds the stall watchdog, the from-ready - # soft-deadline anchor and the eval-start boundary; run it once per slice - # when any of them needs it. Broader than ``soft_from_ready``: a warm-reuse - # round (``server_already_ready``) still needs the eval-start boundary, so - # any soft deadline with a log present scans. - soft_watches_log = soft_active and bool(server_log_path) - scan_active = stall_active or soft_watches_log + # soft-deadline anchor, the eval-start boundary and the ready timestamp the + # caller prices later work off; run it once per slice whenever a log is + # present. Not narrowed to the watchdogs that consume it: tying it to + # ``stall_active`` let an unrelated knob + # (``INFERENCE_OPTIMIZER_DETOK_STALL_GRACE_SEC=0``) silently withdraw the + # boot/benchmark split for a whole session. It costs nothing where it did not + # already run, since without a gate this call never enters the loop at all. + scan_active = bool(server_log_path) and gated poll_interval = STOP_GATE_POLL_SECONDS start = time.monotonic() dead_marker_since: float | None = None @@ -1197,6 +1233,12 @@ def _communicate_with_soft_deadline( # new output (seeded to the ready time). The gate only arms once # ``server_ready_since`` is set. scan_offsets: dict[str, int] = {} + if scan_active: + # A reused output_dir can still hold a prior attempt's nested workspace, + # whose markers are not this round's. The workspace this round creates + # does not exist yet, so it is discovered later at offset zero, which is + # correct: all of its bytes are this round's. + scan_offsets.update(_stale_scan_log_sizes(server_log_path)) # type: ignore[arg-type] server_ready_since: float | None = None last_activity_at: float | None = None # Latched once the accuracy eval starts: the soft deadline bounds the diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 184ac1ac00..2db5518eca 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -3664,12 +3664,38 @@ def _round_affordable_before_ignition( tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. """ state = (ctx_extra or {}).get("shared_state") or self.shared_state - round_sec = _phase_state.baseline_round_cost_sec(state, double_run=double_run) - if round_sec is None: + return self._round_affordable( + state, + round_sec=_phase_state.baseline_round_cost_sec(state, double_run=double_run), + ) + + @staticmethod + def _round_affordable(state: Any, *, round_sec: float | None) -> tuple[bool, dict[str, Any]]: + """Whether the budget holds a round costing ``round_sec`` and a use for it. + + The shared half of every before-ignition gate, so the two round shapes a + baseline has -- the single-node cold-then-hot double run and the + multi-node pair of client passes -- differ only in what they cost, never + in what they are asked. Each supplies its own price and this decides. + + Args: + state: The session ``SharedState``, or ``None``. + round_sec: What this round is expected to cost, or ``None`` when the + session has measured nothing to price it from. + + Returns: + tuple[bool, dict[str, Any]]: ``(affordable, evidence)``. + """ + cold_sec = _phase_state.measured_seconds(state, "baseline_runtime_sec") + if cold_sec is None or round_sec is None: return True, {"reason": "no_measured_round_to_predict_from"} use_sec = 0.0 if _a_use_must_follow_the_round(state): - use_sec = _phase_state.one_more_measurement_sec(state) or 0.0 + # Without the split, a variant is priced at a whole cold round, which + # is what it is: a boot and a benchmark that pays the compile. The + # same fallback the phase machine uses, so the two agree on when a + # stopped session may try again. + use_sec = _phase_state.one_more_measurement_sec(state) or cold_sec headroom_sec, evidence = _round_headroom_sec(state, None) if headroom_sec is None: return True, evidence @@ -3697,13 +3723,19 @@ def _measure_round_affordable( :meth:`_round_affordable_before_ignition` cannot judge. The measured round re-attaches to the server the warmup left running, so - it costs a benchmark and no boot. The warmup's post-ready segment is what - prices it: the same pass, with its boot taken off. That segment still - over-predicts, since it also paid the first request's kernel compile, but - it is the tightest honest figure a session has before its hot pass has - ever run -- and far tighter than the warmup's whole wall-clock, which + it costs a benchmark and no boot. A hot pass this session already measured + prices it best, being exactly that; before one has ever run, the warmup's + post-ready segment stands in -- the same pass, with its boot taken off. The + segment over-predicts, since it also paid the first request's kernel + compile, but it is far tighter than the warmup's whole wall-clock, which prices a client-only pass as though it booted a server. + Reading the session's hot figure first is also what keeps this gate and + :meth:`_round_affordable_before_ignition` on one ruler. Both price the + same second pass, so a round admitted before ignition would otherwise meet + a stricter question here and be refused for certain, having spent a whole + cold pass to find out. + In PRELUDE, covering the pass is not enough to justify running it. A hot baseline is an input there, not a result: it is the denominator later variants are read against, and it is what anchors their overtime kill. @@ -3722,13 +3754,13 @@ def _measure_round_affordable( Args: warmup_runtime_sec: Wall-clock the warmup round took. warmup_post_ready_sec: The part of it that ran after the server was - ready, or ``None`` when nothing recorded the boundary -- a - scriptable workload, which runs no server, or a stamp that could - not be written. Both terms then fall back to the warmup's whole - wall-clock, which remains an upper bound on each: the measured - round re-attaches to a server this pass booted, and a variant is a - boot plus a benchmark where this figure is a boot plus a benchmark - that also paid the first request's compile. + ready. Only a stamp that could not be written leaves this unset, + since this path runs a server by definition -- the workloads with + no ready boundary to record never reach a double run at all -- so + it is a defect rather than a shape, and the gate waves the round + through rather than guess. Guessing high is what a gate before + ignition may safely do; here a refusal ends the session, and a + session ended by a missing timestamp is the worse error. ctx_extra: The runner context extras carrying ``shared_state``. Returns: @@ -3738,18 +3770,17 @@ def _measure_round_affordable( headroom_sec, evidence = _round_headroom_sec(state, None) if headroom_sec is None: return True, evidence - warmup_sec = _positive_seconds(warmup_runtime_sec) or 0.0 - benchmark_sec = _positive_seconds(warmup_post_ready_sec) + warmup_sec = _positive_seconds(warmup_runtime_sec) + priced_by = "session_hot_pass" + benchmark_sec = _phase_state.measured_seconds(state, "baseline_warm_runtime_sec") if benchmark_sec is None: - priced_by = "warmup_wall_clock" - benchmark_sec = warmup_sec - boot_sec = 0.0 - else: priced_by = "warmup_post_ready" - boot_sec = max(0.0, warmup_sec - benchmark_sec) + benchmark_sec = _positive_seconds(warmup_post_ready_sec) + if benchmark_sec is None or warmup_sec is None: + return True, {"reason": "no_measured_benchmark_to_predict_from", **evidence} use_sec = 0.0 if _a_use_must_follow_the_round(state): - use_sec = boot_sec + benchmark_sec + use_sec = _phase_state.one_more_measurement_sec(state) or warmup_sec cost = benchmark_sec + use_sec priced = { "expected_cost_sec": round(cost, 1), @@ -4002,6 +4033,7 @@ async def _run_reported_round( session_deadline_sec: float | None, capture_meta: dict[str, Any], round_warnings: list[str], + ctx_extra: dict[str, Any] | None = None, ) -> dict[str, Any] | None: """Run the discarded multi-node client warmup against the restarted server. @@ -4027,11 +4059,46 @@ async def _run_reported_round( capture_meta: Config/eval-contract facts every failure result carries. round_warnings: Collected onto the round's result, for a warmup that did not run and did not end the round. + ctx_extra: The runner context extras carrying ``shared_state``, read + to price the pair of passes against what is left of the budget. Returns: dict[str, Any] | None: The round's result when the round is over, else ``None`` to go on to the measured pass. """ + # A multi-node round is two client passes of the same shape against a + # server the round did not boot, and the session's measured figure is one + # of them -- the round's wall-clock is taken after this pass -- so the pair + # costs twice it. Asked before the warmup because that is the pass whose + # number nothing may use: spending it and then meeting the deadline in the + # measured pass leaves the round with no anchor and the session with the + # GPU time gone. + state = (ctx_extra or {}).get("shared_state") or self.shared_state + one_pass_sec = _phase_state.measured_seconds(state, "baseline_runtime_sec") + affordable, evidence = self._round_affordable( + state, + round_sec=None if one_pass_sec is None else one_pass_sec * 2.0, + ) + if not affordable: + log.warning( + "baseline_executor: a multi-node round is two passes needing %.0fs " + "and %.0fs is left (bound=%s), so neither is launched. The anchor " + "this session already measured stands.", + evidence.get("expected_cost_sec", 0.0), + evidence.get("affordable_sec", 0.0), + evidence.get("bound", ""), + ) + refused = _stopped_round_result( + STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + round_label="multi-node round", + returncode=None, + runtime_sec=0.0, + output_dir=output_dir, + capture_meta=capture_meta, + started=False, + ) + refused["budget_shortfall"] = evidence + return refused warm_dir = output_dir / "mn_warmup" started_unix = time.time() # The measurement is discarded, but the returncode is not: this pass is a @@ -4277,6 +4344,7 @@ async def _run_single_benchmark( session_deadline_sec=session_deadline_sec, capture_meta=capture_meta, round_warnings=round_warnings, + ctx_extra=ctx_extra, ) if _mn_warm_result is not None: return _mn_warm_result diff --git a/src/hyperloom/orchestrator/actions/executors/report.py b/src/hyperloom/orchestrator/actions/executors/report.py index 7136591938..5888d56492 100644 --- a/src/hyperloom/orchestrator/actions/executors/report.py +++ b/src/hyperloom/orchestrator/actions/executors/report.py @@ -378,9 +378,9 @@ def _build_failure_summary( "prelude_baseline_failed": "PRELUDE baseline failed before optimization could start; see the baseline failure summary.", "prelude_policy_loop": "The policy gate detected a decision loop during PRELUDE and stopped.", "time_exhausted_during_prelude": ( - "The wall-clock budget PRELUDE had was exhausted before optimization began — either the session clock ran " - "out, or what was left of PRELUDE's share could no longer pay for a whole baseline round (a discarded " - "warmup pass and the measured pass it makes comparable)." + "The session's wall-clock budget ran out during preparation, before optimization began. Whatever PRELUDE " + "was doing when the clock reached zero — measuring the baseline, bringing up the framework agent, taking " + "the roofline — is where the time went; the phase record shows which arms ran and what each cost." ), "prelude_cold_anchor_low_budget": ( "The baseline's hot pass was skipped because the clock could not cover it together with one variant to " diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index b211d0fc66..1759b4011e 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -2938,10 +2938,31 @@ async def _promote_baseline( # revalidation is exempt: the enablement patch changed the stack, so the # prior anchor no longer describes anything reproducible. prior_anchor = float(self.shared_state.baseline_tput or 0.0) + # A hot pass measured against a recorded COLD anchor is a correction, not + # a regression, so it lands even when the number is lower. The two are not + # comparable -- that is what the cold marker says -- and a cold figure can + # read higher than the hot one that replaces it whenever the "cold" pass + # was not really cold: weights in page cache and a JIT cache a prior run + # populated leave it paying none of the startup its depressed reputation + # assumes. Without this the session cannot escape the marker. PRELUDE + # refuses to finish while it is set, the retry that would clear it is + # rejected for measuring lower, and the run re-measures whole baseline + # rounds until the clock kills it. + hot_pass_ran = result.get("measure_round_runtime_sec") + corrects_a_cold_anchor = ( + bool(getattr(self.shared_state, "baseline_measure_round_dropped", False)) + and isinstance(hot_pass_ran, (int, float)) + and hot_pass_ran > 0 + ) anchor_accepted = bool( isinstance(tput, (int, float)) and tput > 0 - and (prior_anchor <= 0.0 or float(tput) > prior_anchor or is_revalidation) + and ( + prior_anchor <= 0.0 + or float(tput) > prior_anchor + or is_revalidation + or corrects_a_cold_anchor + ) ) if isinstance(tput, (int, float)) and tput > 0: if anchor_accepted: diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 130c383796..58ec61ccdf 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -2080,9 +2080,21 @@ def baseline_round_cost_sec(state: Any, *, double_run: bool) -> float | None: """What a baseline round costs, or ``None`` when unmeasured. A round is one or two passes and they are priced apart, because they buy - different things. The first brings a server up and then benchmarks it, so it - costs what any measured variant costs. The second re-attaches to the server - the first left running, so it costs a benchmark and no second boot. + different things. The first brings a server up and benchmarks it cold, paying + the first request's kernel compile on the way; the session has measured that + whole pass directly, so it is read rather than reconstructed. The second + re-attaches to the server the first left running, so it costs a benchmark and + no second boot. + + Reconstructing the first pass as boot-plus-hot-benchmark would drop the + compile and under-price the round, which matters because this figure guards + ignition while the post-warmup gate that follows prices the same work from + the pass it just watched. A round admitted here and then certainly refused + there costs a whole cold pass to learn nothing. + + Reading the measured total also gives a price to rounds with no boot/benchmark + split to reconstruct from -- multi-node and scriptable workloads, which never + report one -- so those are gated rather than waved through. Args: state (Any): Frozen SharedState view. @@ -2093,7 +2105,7 @@ def baseline_round_cost_sec(state: Any, *, double_run: bool) -> float | None: float | None: Seconds the round costs, or ``None`` when the session has measured nothing to price it from. """ - first_pass_sec = one_more_measurement_sec(state) + first_pass_sec = measured_seconds(state, "baseline_runtime_sec") if first_pass_sec is None or not double_run: return first_pass_sec second_pass_sec = benchmark_cost_sec(state) From 225e307a4376297968995ccd3e8561668906d837 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 20:33:20 +0000 Subject: [PATCH 55/65] executors, policy: measure the boot on one clock, and let a cold anchor be re-measured Two holes in the cold-anchor path, both on the resume leg it exists for. The boot was derived by subtracting the ready stamp's wall-clock instant from the caller's own. On the Ray path those are two clocks, possibly on two hosts, and the difference between them was charged to the boot -- inflating it, and making the budget gates refuse rounds that fit. The stamp now carries the boot as a duration measured end to end inside the process that spawned the child, and keeps the instant only to tell this round's stamp from an earlier one's. A stamp missing the duration reads as no stamp at all: zero is a legitimate boot, so it cannot also stand for "not recorded". The singleton rule refuses a repeat baseline because it "re-measures a reference the run already has". A marked cold anchor is the case where the run does not have one, and it is a positive tput, so the rule refused the single round that clears the mark. A session resumed on a fresh clock therefore reopened at PRELUDE, declined to finish while the mark was set, declined to close while the clock was healthy, and had no admissible way forward. The rule now exempts a marked anchor -- after the authoring-round check, which is a reason to wait whatever the anchor says. Co-authored-by: Cursor --- .../tests/test_baseline_warmup_double_run.py | 4 +- .../tests/test_dispatched_task_policy.py | 58 ++++++++++++ .../inference_optimizer/tests/test_resume.py | 34 +++++++ .../tests/test_soft_deadline_from_ready.py | 88 +++++++++++++++++-- .../actions/executors/_subprocess_kill.py | 84 +++++++++++++----- src/hyperloom/orchestrator/policy/gate.py | 20 ++++- 6 files changed, 258 insertions(+), 30 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index c2e6f87f84..428d668c8e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -156,7 +156,7 @@ def fake_run(cmd, *args, **kwargs): clock.advance(boot_sec) if server_log_path: Path(server_log_path).parent.mkdir(parents=True, exist_ok=True) - _stamp_server_ready(server_log_path) + _stamp_server_ready(server_log_path, boot_sec) clock.advance(benchmark_sec) state["calls"] += 1 _fake_workspace(slot, tput=tput) @@ -2608,7 +2608,7 @@ def fake_run(cmd, *args, **kwargs): server_log_path = kwargs.get("server_log_path") if server_log_path: Path(server_log_path).parent.mkdir(parents=True, exist_ok=True) - _stamp_server_ready(server_log_path) + _stamp_server_ready(server_log_path, boot_sec) clock.advance(benchmark_sec) if state is not None: state.charge(charge_sec if charge_sec is not None else ran_sec) diff --git a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py index ef8f9c7410..fbed1ab62d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py @@ -818,3 +818,61 @@ def test_enablement_round_in_flight_allows_after_cleared(tmp_path, monkeypatch): payload={"action_name": "baseline", "params": {}}, ) gate.validate_intent("orchestration", intent) + + +class TestAColdAnchorIsNotAnEstablishedOne: + """The rule refuses a reference the run already has, and this is not one. + + A session that could not afford its hot pass keeps the warmup's cold figure + and marks it. PRELUDE will not finish while the mark is set, so the only way + on is another baseline -- and that is the round this rule would refuse, + leaving the phase with no way forward and no way out. The state arises on + resume, which is the whole point of keeping the figure recoverable. + """ + + def _a_baseline_is_proposed(self, gate): + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.DELEGATE, + payload={"action_name": "baseline", "params": {}}, + ), + ) + + def test_a_marked_cold_anchor_does_not_refuse_the_round_that_would_fix_it( + self, + tmp_path, + monkeypatch, + ): + gate, _ = _gate(tmp_path, monkeypatch) + gate.shared_state.baseline_tput = 1000.0 + gate.shared_state.baseline_measure_round_dropped = True + + self._a_baseline_is_proposed(gate) + + def test_an_established_anchor_still_refuses_a_repeat(self, tmp_path, monkeypatch): + gate, _ = _gate(tmp_path, monkeypatch) + gate.shared_state.baseline_tput = 1000.0 + gate.shared_state.baseline_measure_round_dropped = False + + with pytest.raises(PolicyDenied) as exc_info: + self._a_baseline_is_proposed(gate) + + assert exc_info.value.rule == "baseline_phase_singleton" + + def test_an_authoring_round_in_flight_still_wins(self, tmp_path, monkeypatch): + """The exemption is about which reference exists, not about what may run. + + A specialist rewriting the framework underneath a baseline is a reason to + wait whatever the anchor says; letting the cold mark through here would + launch a round against a stack that is being changed as it runs. + """ + gate, _ = _gate(tmp_path, monkeypatch) + gate.shared_state.baseline_tput = 1000.0 + gate.shared_state.baseline_measure_round_dropped = True + gate.shared_state.enablement.inflight_task_id = "spec-abc" + + with pytest.raises(PolicyDenied) as exc_info: + self._a_baseline_is_proposed(gate) + + assert exc_info.value.rule == "enablement_round_in_flight" diff --git a/src/hyperloom/inference_optimizer/tests/test_resume.py b/src/hyperloom/inference_optimizer/tests/test_resume.py index 8349a943e1..1a5c06c43c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_resume.py +++ b/src/hyperloom/inference_optimizer/tests/test_resume.py @@ -131,6 +131,40 @@ async def test_a_session_stopped_anywhere_else_resumes_where_it_stopped(self, se finally: await coordinator.stop() + @pytest.mark.asyncio + async def test_the_reopened_leg_may_actually_measure_the_baseline_it_reopened_for( + self, + session_dir, + ): + """Reopening the phase is only half of it; the round has to be admissible. + + A cold anchor is a positive ``baseline_tput``, which is what the singleton + rule refuses repeats on -- so the leg would reopen at PRELUDE, decline to + finish while the mark is set, decline to close while the clock is healthy, + and have the one round that clears the mark denied on its way in. This is + the last link in the chain the whole cold-anchor design rests on, and + nothing above it can tell whether it holds. + """ + SharedState( + session_id="cold", + phase="CLOSE", + baseline_tput=1000.0, + baseline_measure_round_dropped=True, + ).save(session_dir) + + coordinator = Coordinator(session_dir, backends=_backends_full()) + try: + assert coordinator.shared_state.phase == "PRELUDE" + coordinator.policy.validate_intent( + "orchestration", + Intent( + type=IntentType.DELEGATE, + payload={"action_name": "baseline", "params": {}}, + ), + ) + finally: + await coordinator.stop() + @pytest.mark.asyncio async def test_existing_events_triggers_resume(session_dir): diff --git a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py index 15a6b0047a..329ddf9f1a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py +++ b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py @@ -19,6 +19,8 @@ import sys import time +import pytest + from hyperloom.orchestrator.actions.executors._subprocess_kill import ( OVERTIME_KILL_RETURNCODE, clear_server_ready_stamp, @@ -243,6 +245,53 @@ def test_the_split_does_not_depend_on_an_unrelated_watchdog(self, tmp_path): "the boot/benchmark split was withdrawn along with the stall watchdog" ) + def test_a_reader_on_another_clock_gets_the_same_boot(self, tmp_path): + """The split survives being read where it was not written. + + On the Ray path the round runs inside an actor, possibly on another host, + and the caller subtracting its own clock from the actor's would charge the + boot for whatever the two disagree by -- inflating it, and making the + budget gates refuse rounds that fit. The boot is a duration taken on one + clock, so a reader whose clock is five seconds off reads the same figure. + """ + log_path = tmp_path / "server.log" + script = ( + "import sys, time\n" + "f = open(sys.argv[1], 'w')\n" + "f.write('INFO loading weights\\n'); f.flush()\n" + "time.sleep(3)\n" + "f.write('Application startup complete\\n'); f.flush()\n" + "time.sleep(1)\n" + "raise SystemExit(0)\n" + ) + started_unix = time.time() + cp = run_with_session_kill( + [sys.executable, "-c", script, str(log_path)], + timeout=30, + server_log_path=str(log_path), + detok_stall_grace_sec=_LONG_STALL_GRACE, + ) + runtime_sec = time.time() - started_unix + assert cp.returncode == 0 + + agreed = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=runtime_sec, + ) + # A caller whose clock runs five seconds behind the writer's. The instant + # is still this round's, so the stamp is accepted, and the boot it + # reports does not move. + skewed = post_ready_runtime_sec( + str(log_path), + started_unix=started_unix - 5.0, + runtime_sec=runtime_sec, + ) + assert agreed is not None + assert skewed == pytest.approx(agreed), ( + f"a five-second clock disagreement moved the boot: {skewed} vs {agreed}" + ) + def test_a_round_that_never_came_up_is_not_priced(self, tmp_path): """No ready marker means no split, reported as unknown rather than guessed.""" log_path = tmp_path / "server.log" @@ -278,7 +327,8 @@ def test_an_earlier_rounds_stamp_is_not_read_as_this_ones(self, tmp_path): """ log_path = tmp_path / "server.log" log_path.write_text("INFO loading weights\n", encoding="utf-8") - (tmp_path / "server_ready_at").write_text(f"{time.time() - 600.0:.3f}\n", encoding="utf-8") + # A complete stamp, so what rejects it can only be its instant. + (tmp_path / "server_ready_at").write_text(f"{time.time() - 600.0:.3f} 30.000\n", encoding="utf-8") assert ( post_ready_runtime_sec( @@ -293,7 +343,7 @@ def test_a_stamp_is_cleared_so_the_next_round_starts_blind(self, tmp_path): """Clearing is what keeps the case above from arising in a reused dir.""" log_path = tmp_path / "server.log" stamp = tmp_path / "server_ready_at" - stamp.write_text(f"{time.time():.3f}\n", encoding="utf-8") + stamp.write_text(f"{time.time():.3f} 30.000\n", encoding="utf-8") assert server_ready_unix(str(log_path)) is not None clear_server_ready_stamp(str(log_path)) @@ -303,15 +353,17 @@ def test_a_stamp_is_cleared_so_the_next_round_starts_blind(self, tmp_path): # Idempotent: a round whose dir never had one must not fail to start. clear_server_ready_stamp(str(log_path)) - def test_a_clock_that_disagrees_cannot_produce_a_negative_price(self, tmp_path): - """The writer and the reader can be different hosts, so skew is possible. + def test_a_boot_longer_than_the_round_cannot_produce_a_negative_price(self, tmp_path): + """A corrupt stamp must cost a measurement, never produce a nonsense one. - A stamp after the round's own end would price the benchmark at less than - nothing; it is floored instead, and the round's total caps the other end. + The two figures come from two places -- the boot from inside the round, + the total from the caller around it -- so nothing structurally forbids a + boot larger than the total. It is floored, and the total caps the other + end. """ log_path = tmp_path / "server.log" started_unix = time.time() - (tmp_path / "server_ready_at").write_text(f"{started_unix + 500.0:.3f}\n", encoding="utf-8") + (tmp_path / "server_ready_at").write_text(f"{started_unix + 10.0:.3f} 500.000\n", encoding="utf-8") priced = post_ready_runtime_sec( str(log_path), @@ -320,3 +372,25 @@ def test_a_clock_that_disagrees_cannot_produce_a_negative_price(self, tmp_path): ) assert priced == 0.0 + + def test_a_stamp_with_no_boot_recorded_is_no_stamp_at_all(self, tmp_path): + """A missing boot must not read as a round that never booted. + + Zero is a legitimate boot, so it cannot also mean "not recorded" -- + reading it that way hands the whole round to the benchmark, which is the + figure every later variant is then admitted on, and every one of them is + reaped. Unmeasured is reported as unmeasured; the gates know what to do + with that. + """ + log_path = tmp_path / "server.log" + started_unix = time.time() + (tmp_path / "server_ready_at").write_text(f"{started_unix + 10.0:.3f}\n", encoding="utf-8") + + assert ( + post_ready_runtime_sec( + str(log_path), + started_unix=started_unix, + runtime_sec=100.0, + ) + is None + ) diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index f06dbd4806..3a221f9e1e 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -599,26 +599,42 @@ def _ready_stamp_path(server_log_path: str) -> Path: return Path(server_log_path).parent / _READY_STAMP_NAME -def _stamp_server_ready(server_log_path: str) -> None: +def _stamp_server_ready(server_log_path: str, boot_sec: float = 0.0) -> None: """Record, beside ``server_log_path``, that the server just reported ready. - Written as a wall-clock (``time.time()``) instant rather than the - ``time.monotonic()`` one the gates run on, because the process that reads it - is not always the one that wrote it: on the Ray path the round runs inside an - actor, and a monotonic reading means nothing outside the process that took - it. A file is used for the same reason -- it crosses the actor boundary that - the round's return value would otherwise have to be widened to cross, and the - round's output directory is already how post-mortem evidence gets back (the - caller reads the same directory's ``server.log`` to classify server deaths). + Two numbers, because they answer two questions and one clock cannot answer + both. ``boot_sec`` is how long the round took to come up, measured from spawn + to this moment on one ``time.monotonic()`` reading in the process that + spawned the child. The wall-clock instant beside it only ever says *which + round* the stamp belongs to. + + Keeping the boot a duration is what makes it safe to read across a process + boundary. On the Ray path the round runs inside an actor, possibly on another + host; subtracting the actor's wall-clock from the driver's would charge the + boot for whatever the two clocks disagree by, and a positive disagreement + inflates the boot and makes the budget gates refuse rounds that fit. A + duration crosses the boundary meaning the same thing on both sides -- the + same reason ``session_remaining_sec`` is passed to the actor as a duration + rather than as a deadline. + + A file is used because it crosses that boundary without widening the round's + return value, and the round's output directory is already how post-mortem + evidence gets back (the caller reads the same directory's ``server.log`` to + classify server deaths). Best effort: a round whose stamp cannot be written loses a measurement, which callers already have to handle, and must not lose the round. Args: server_log_path: The ``/server.log`` path from the caller. + boot_sec: Seconds from spawn to this moment, on the spawning process's + monotonic clock. """ try: - _ready_stamp_path(server_log_path).write_text(f"{time.time():.3f}\n", encoding="utf-8") + _ready_stamp_path(server_log_path).write_text( + f"{time.time():.3f} {max(0.0, float(boot_sec)):.3f}\n", + encoding="utf-8", + ) except OSError as exc: log.warning("_subprocess_kill: could not stamp server-ready time (%s)", exc) @@ -635,6 +651,30 @@ def clear_server_ready_stamp(server_log_path: str) -> None: log.warning("_subprocess_kill: could not clear stale server-ready stamp (%s)", exc) +def _read_ready_stamp(server_log_path: str) -> tuple[float, float] | None: + """Return a round's ``(ready_unix, boot_sec)``, or ``None`` when unrecorded. + + Args: + server_log_path: The ``/server.log`` path from the caller. + + Both fields are required. A stamp missing its boot is reported as no stamp at + all rather than as a boot of zero: zero is a legitimate boot, so it cannot + also stand for "not recorded", and reading it that way would hand the whole + round to the benchmark -- the figure every later variant is then admitted on. + + Returns: + tuple[float, float] | None: The wall-clock instant the stamp was written + and the boot it measured, or ``None`` when no readable stamp exists. + """ + try: + fields = _ready_stamp_path(server_log_path).read_text(encoding="utf-8").split() + ready_unix = float(fields[0]) + boot_sec = float(fields[1]) + except (OSError, ValueError, IndexError): + return None + return (ready_unix, max(0.0, boot_sec)) if ready_unix > 0.0 else None + + def server_ready_unix(server_log_path: str) -> float | None: """Return when the server reported ready, or ``None`` when nothing recorded it. @@ -645,11 +685,8 @@ def server_ready_unix(server_log_path: str) -> float | None: float | None: The wall-clock instant, or ``None`` when no stamp exists or it is unreadable. """ - try: - stamped = float(_ready_stamp_path(server_log_path).read_text(encoding="utf-8").strip()) - except (OSError, ValueError): - return None - return stamped if stamped > 0.0 else None + stamp = _read_ready_stamp(server_log_path) + return None if stamp is None else stamp[0] def post_ready_runtime_sec( @@ -665,6 +702,11 @@ def post_ready_runtime_sec( rounds comparable when one paid for a cold start and the other re-attached to a server already up. + The round's wall-clock less the boot the stamp measured. The boot is a + duration taken on one clock, so this holds however far apart the writer and + the reader are; ``started_unix`` is only compared against the stamp's own + instant, to tell this round's stamp from one an earlier round left behind. + Args: server_log_path: The ``/server.log`` path from the caller. started_unix: When the round was spawned. @@ -676,10 +718,10 @@ def post_ready_runtime_sec( predates it (an earlier round's, left behind by a failed clear -- reading it would report a cold round as though it had never booted). """ - ready_unix = server_ready_unix(server_log_path) - if ready_unix is None or ready_unix < started_unix: + stamp = _read_ready_stamp(server_log_path) + if stamp is None or stamp[0] < started_unix: return None - return max(0.0, min(float(runtime_sec), started_unix + float(runtime_sec) - ready_unix)) + return max(0.0, min(float(runtime_sec), float(runtime_sec) - stamp[1])) def _resolve_scan_logs(server_log_path: str) -> list[str]: @@ -1278,8 +1320,10 @@ def _communicate_with_soft_deadline( last_activity_at = now # start the silence clock at ready # Recorded for the caller, which prices later work off the # post-ready segment rather than the whole round: a pass that - # re-attaches to this server pays none of the boot. - _stamp_server_ready(server_log_path) # type: ignore[arg-type] + # re-attaches to this server pays none of the boot. Taken as + # ``now - start`` so the boot is measured end to end on this + # process's own clock, whatever host the caller reads it on. + _stamp_server_ready(server_log_path, now - start) # type: ignore[arg-type] if scan.saw_eval_start and not soft_deadline_suspended: soft_deadline_suspended = True log.info( diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index de11f5cdb7..db505c9250 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -1543,7 +1543,19 @@ def _validate_baseline_singleton( self, payload: dict[str, Any], ) -> None: - """Deny a baseline proposal when an enablement round is in flight or the anchor is established.""" + """Deny a baseline proposal when an enablement round is in flight or the anchor is established. + + "Established" is the whole rule: a repeat baseline is refused because it + re-measures a reference the run already has. A cold anchor is the case + where it does not. The session kept a warmup's figure because the clock + could not fund the hot pass that would have made it comparable, and + marked it as such; PRELUDE will not finish while the mark is set, because + every variant read against a depressed denominator reads as an + improvement over a baseline that never existed. Refusing the round that + would clear the mark leaves the phase with no way forward and no way out, + which is the state a session resumed on a fresh clock arrives in -- + exactly the one the mark exists to make recoverable. + """ ss = getattr(self, "shared_state", None) if ss is None: return @@ -1558,6 +1570,12 @@ def _validate_baseline_singleton( "before re-running baseline." ), ) + # Checked after the authoring round, which is a reason to wait whatever + # the anchor says: a specialist rewriting the framework underneath a + # baseline would have this round measuring a stack that changes as it + # runs. + if bool(getattr(ss, "baseline_measure_round_dropped", False)): + return anchor = getattr(ss, "baseline_tput", 0.0) if not isinstance(anchor, (int, float)) or anchor <= 0: return From 8c265536bb3ee2a30a3622c1c0e8c9d4c13bd076 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sat, 15 Aug 2026 20:43:21 +0000 Subject: [PATCH 56/65] Require the boot duration when stamping server-ready The stamp's two fields answer two questions, and only one of them has a safe default. A caller that omitted the boot would write a well-formed stamp claiming the round came up instantly, which prices the whole round as benchmark -- the one wrong reading the two-field format was added to make impossible. Dropping the default turns that into a call-site error. Co-authored-by: Cursor --- .../actions/executors/_subprocess_kill.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py index 3a221f9e1e..9f80e573d1 100644 --- a/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py +++ b/src/hyperloom/orchestrator/actions/executors/_subprocess_kill.py @@ -599,7 +599,7 @@ def _ready_stamp_path(server_log_path: str) -> Path: return Path(server_log_path).parent / _READY_STAMP_NAME -def _stamp_server_ready(server_log_path: str, boot_sec: float = 0.0) -> None: +def _stamp_server_ready(server_log_path: str, boot_sec: float) -> None: """Record, beside ``server_log_path``, that the server just reported ready. Two numbers, because they answer two questions and one clock cannot answer @@ -628,7 +628,10 @@ def _stamp_server_ready(server_log_path: str, boot_sec: float = 0.0) -> None: Args: server_log_path: The ``/server.log`` path from the caller. boot_sec: Seconds from spawn to this moment, on the spawning process's - monotonic clock. + monotonic clock. Required rather than defaulted: a caller that + omitted it would write a well-formed stamp claiming the round booted + instantly, which reads as a whole round of benchmark and is the one + wrong answer the two-field format exists to make impossible. """ try: _ready_stamp_path(server_log_path).write_text( @@ -654,14 +657,14 @@ def clear_server_ready_stamp(server_log_path: str) -> None: def _read_ready_stamp(server_log_path: str) -> tuple[float, float] | None: """Return a round's ``(ready_unix, boot_sec)``, or ``None`` when unrecorded. - Args: - server_log_path: The ``/server.log`` path from the caller. - Both fields are required. A stamp missing its boot is reported as no stamp at all rather than as a boot of zero: zero is a legitimate boot, so it cannot also stand for "not recorded", and reading it that way would hand the whole round to the benchmark -- the figure every later variant is then admitted on. + Args: + server_log_path: The ``/server.log`` path from the caller. + Returns: tuple[float, float] | None: The wall-clock instant the stamp was written and the boot it measured, or ``None`` when no readable stamp exists. From 66a00ae98b7b5473a676591d78005003e8b7d91b Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sun, 16 Aug 2026 11:49:45 +0000 Subject: [PATCH 57/65] Record the multi-node successor as a known gap in the PRELUDE gate A multi-node variant runs two client passes, so reserving one leaves a pass unfunded. Teaching the phase machine's pricing what shape the cluster is costs more than the gap, which needs an enablement round holding PRELUDE open past an anchor to reach at all. Noted where the fallback is, so it reads as a bound rather than an oversight. Co-authored-by: Cursor --- selfcheck_gates.py | 124 ++++++++++++++++++ .../actions/executors/baseline.py | 8 ++ 2 files changed, 132 insertions(+) create mode 100644 selfcheck_gates.py diff --git a/selfcheck_gates.py b/selfcheck_gates.py new file mode 100644 index 0000000000..c5fcce8f98 --- /dev/null +++ b/selfcheck_gates.py @@ -0,0 +1,124 @@ +"""Throwaway self-review: sweep the real pricing functions for disagreements.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from hyperloom.orchestrator.phases import machine_state as ms + +BOOT, COLD_BENCH, HOT = 350.0, 550.0, 400.0 +COLD_ROUND = BOOT + COLD_BENCH + + +def state(usable, *, phase="PRELUDE", warm=HOT, post_ready=COLD_BENCH, marked=False, double=True): + return SimpleNamespace( + phase=phase, + max_minutes=180, + baseline_tput=1000.0, + baseline_runtime_sec=COLD_ROUND, + baseline_post_ready_runtime_sec=post_ready, + baseline_warm_runtime_sec=warm, + baseline_measure_round_dropped=marked, + baseline_double_run=double, + session_budget_usable_sec=lambda: usable, + ) + + +def gate1_need(s): + """What the pre-ignition gate demands, mirroring _round_affordable.""" + cold = ms.measured_seconds(s, "baseline_runtime_sec") + rnd = ms.baseline_round_cost_sec(s, double_run=bool(s.baseline_double_run)) + if cold is None or rnd is None: + return None + use = (ms.one_more_measurement_sec(s) or cold) if s.phase == "PRELUDE" else 0.0 + return rnd + use + + +def gate2_need(s, *, warmup_sec, warmup_post_ready): + """What the post-warmup gate demands, mirroring _measure_round_affordable.""" + bench = ms.measured_seconds(s, "baseline_warm_runtime_sec") + if bench is None: + bench = warmup_post_ready + if bench is None or warmup_sec is None: + return None + use = (ms.one_more_measurement_sec(s) or warmup_sec) if s.phase == "PRELUDE" else 0.0 + return bench + use + + +def main() -> int: + bad = 0 + + # 1. The band: is there a budget gate 1 admits and gate 2 then certainly refuses? + # Gate 2 is asked after the warmup has spent a cold pass. + band = [] + for usable in range(0, 6001, 10): + s = state(float(usable)) + need1 = gate1_need(s) + admitted = need1 is not None and usable >= need1 + if not admitted: + continue + after = state(float(usable) - COLD_ROUND) + need2 = gate2_need(after, warmup_sec=COLD_ROUND, warmup_post_ready=COLD_BENCH) + if need2 is not None and (usable - COLD_ROUND) < need2: + band.append(usable) + if band: + bad += 1 + print(f"BAND gate 1 admits and gate 2 refuses for usable in {band[0]}..{band[-1]}") + else: + print("ok no budget is admitted before ignition only to be refused after the cold pass") + + # 2. Livelock: with the mark set, does every budget either close or admit a retry? + stuck = [] + for usable in range(0, 8001, 10): + s = state(float(usable), marked=True) + closes = ms.exit_cold_anchor_prelude(s) is not None + need1 = gate1_need(s) + admits = need1 is not None and usable >= need1 + if not closes and not admits: + stuck.append(usable) + if stuck: + bad += 1 + print(f"LIVELOCK neither closes nor admits for usable in {stuck[0]}..{stuck[-1]}") + else: + print("ok a marked session always either closes or may retry") + + # 3. A session with no split measured (multi-node / scriptable shape). + s = state(3000.0, warm=0.0, post_ready=0.0) + need = gate1_need(s) + if need is None: + bad += 1 + print("UNGATED a round with no boot boundary is waved through") + else: + print(f"ok a round with no split is priced at {need:.0f}s (whole cold rounds)") + + # 4. A first baseline must never be judged. + first = SimpleNamespace( + phase="PRELUDE", + max_minutes=180, + baseline_tput=0.0, + baseline_runtime_sec=0.0, + baseline_post_ready_runtime_sec=0.0, + baseline_warm_runtime_sec=0.0, + baseline_measure_round_dropped=False, + baseline_double_run=True, + session_budget_usable_sec=lambda: 60.0, + ) + if gate1_need(first) is not None: + bad += 1 + print("PREDICTED a first baseline was priced from measurements it cannot have") + else: + print("ok a first baseline is not judged") + + # 5. Later phases ask only whether the round fits. + later = state(2000.0, phase="EXPLORE") + if gate1_need(later) != ms.baseline_round_cost_sec(later, double_run=True): + bad += 1 + print("SCOPED a re-baseline outside PRELUDE was charged for a successor") + else: + print("ok a re-baseline outside PRELUDE pays only for itself") + + return bad + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 2db5518eca..1454bb98d8 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -3695,6 +3695,14 @@ def _round_affordable(state: Any, *, round_sec: float | None) -> tuple[bool, dic # is what it is: a boot and a benchmark that pays the compile. The # same fallback the phase machine uses, so the two agree on when a # stopped session may try again. + # + # Known gap: a multi-node variant runs two client passes, not one + # (``_grid_runner`` reserves for it as ``x (1 + _mn_warmup_rounds)``), + # so one pass is left unreserved here. It needs a multi-node round to + # reach PRELUDE with an earlier one already measured, which takes an + # enablement round holding the phase open past an anchor that would + # otherwise finish it -- narrow enough not to be worth teaching the + # phase machine's pricing what shape the cluster is. use_sec = _phase_state.one_more_measurement_sec(state) or cold_sec headroom_sec, evidence = _round_headroom_sec(state, None) if headroom_sec is None: From 06705931aa23fd9dc86229bffae984f2b82dbaa1 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sun, 16 Aug 2026 15:27:05 +0000 Subject: [PATCH 58/65] Stop the usable-budget lock test from racing the clock Two live session_budget_usable_sec reads differ by microseconds, which failed exact equality on py3.11 shard 4. Freeze elapsed time instead. Also drop an unused started_unix and close the ServingLease fixture with `with`, which is how the class is meant to be released. Co-authored-by: Cursor --- src/hyperloom/inference_optimizer/tests/conftest.py | 5 +---- .../tests/test_agent_roles_and_policy.py | 7 +++---- .../tests/test_soft_deadline_from_ready.py | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/conftest.py b/src/hyperloom/inference_optimizer/tests/conftest.py index a894bf54c9..43abbb1229 100644 --- a/src/hyperloom/inference_optimizer/tests/conftest.py +++ b/src/hyperloom/inference_optimizer/tests/conftest.py @@ -464,8 +464,5 @@ def serving_lease_on_a_ray_double(monkeypatch): monkeypatch.setitem(sys.modules, "ray", RayDouble()) monkeypatch.setattr(rb, "get_ray_backend", lambda: SimpleNamespace(ensure=lambda **_kw: None)) - lease = rs.ServingLease(num_gpus=1) - try: + with rs.ServingLease(num_gpus=1) as lease: yield lease - finally: - lease.close() diff --git a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py index cd64866643..a5bf9a5a40 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py @@ -5,11 +5,8 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone - import pytest -from hyperloom.common.timeutil import iso_z from hyperloom.orchestrator.roles.agent_role import ( BackendType, DEFAULT_CLAUDE_MODEL, @@ -742,7 +739,9 @@ def test_gate_update_state_cannot_move_a_session_end_time(gate): def test_a_forged_closing_reserve_would_have_spent_the_session_outright(): """Names what the lock prevents: one field, and the run has no usable time left.""" state = SharedState(session_id="s", max_minutes=100) - state.start_ts = iso_z(datetime.now(timezone.utc) - timedelta(minutes=90)) + # Freeze elapsed time: two live ``session_budget_usable_sec`` reads race + # the clock by tens of microseconds, which is enough for ``==`` to fail. + state.elapsed_minutes = lambda **_kw: 90.0 # type: ignore[method-assign] honest = state.session_budget_usable_sec() applied = state.apply_changes({"closing_grace_sec": 1e9}, allow_core=False) diff --git a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py index 329ddf9f1a..64185af8a1 100644 --- a/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py +++ b/src/hyperloom/inference_optimizer/tests/test_soft_deadline_from_ready.py @@ -232,7 +232,6 @@ def test_the_split_does_not_depend_on_an_unrelated_watchdog(self, tmp_path): "time.sleep(1)\n" "raise SystemExit(0)\n" ) - started_unix = time.time() cp = run_with_session_kill( [sys.executable, "-c", script, str(log_path)], timeout=30, From c86cd693d516daee9370027e0a66e27746748b3a Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Sun, 16 Aug 2026 16:47:11 +0000 Subject: [PATCH 59/65] Stop recover from running after the session budget is spent. Recover is not a closing action: it takes the server-lifecycle lane and can hold CLOSE open past the wall clock, which is the overrun this branch exists to stop. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 30 +++++++++++++++++++ .../orchestrator/loop/coordinator_helpers.py | 10 ++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 7919d8ad77..53ec177c5f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -318,6 +318,10 @@ def test_an_action_with_no_registry_entry_is_admitted(self, coord: Coordinator): _set_budget(coord, minutes=1) assert coord._time_budget_denial_for_action("frobnicate") is None + def test_only_the_closing_actions_are_exempt_from_the_budget(self): + """Recover restarts the server; it is not how a session ends.""" + assert TIME_BUDGET_EXEMPT_ACTIONS == frozenset({"report", "session_breakdown"}) + def test_the_closing_actions_stay_startable_on_an_empty_budget(self, coord: Coordinator): """Refusing these would strand the session with nothing to show.""" _set_budget(coord, minutes=60, elapsed_min=60.0) @@ -325,6 +329,14 @@ def test_the_closing_actions_stay_startable_on_an_empty_budget(self, coord: Coor for action in TIME_BUDGET_EXEMPT_ACTIONS: assert coord._time_budget_denial_for_action(action) is None, action + def test_recover_is_refused_on_an_empty_budget(self, coord: Coordinator): + """A spent session that still starts recover cannot close.""" + _set_budget(coord, minutes=60, elapsed_min=60.0) + assert coord.shared_state.session_budget_usable_sec() == 0.0 + denied = coord._time_budget_denial_for_action("recover") + assert isinstance(denied, PolicyDenied) + assert denied.rule == "time_budget" + def test_a_stopping_session_leaves_the_gate_to_the_stop_path(self, coord: Coordinator): _set_budget(coord, minutes=1) coord.shared_state.stop_reason = "time_exhausted" @@ -492,6 +504,24 @@ async def test_a_queued_task_that_still_fits_is_left_alone(self, coord: Coordina assert await coord.dispatcher._cancel_queued_task_over_budget(task) is False assert (await coord.tasks.get(task.task_id)).state == "queued" + @pytest.mark.asyncio + async def test_a_queued_recover_is_dropped_when_the_budget_is_spent( + self, + coord: Coordinator, + ): + _set_budget(coord, minutes=600) + task, _ = await coord.tasks.create_or_return_existing( + kind="recover", + params={}, + idempotency_key="q-recover", + ) + _set_budget(coord, minutes=600, elapsed_min=600.0) + + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert [t.task_id for t, _, _ in spawned] == [] + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + # One of the closing actions, exempt from the budget because the closing reserve # is held back so it can run. diff --git a/src/hyperloom/orchestrator/loop/coordinator_helpers.py b/src/hyperloom/orchestrator/loop/coordinator_helpers.py index 6061ab8e62..a332f1339b 100644 --- a/src/hyperloom/orchestrator/loop/coordinator_helpers.py +++ b/src/hyperloom/orchestrator/loop/coordinator_helpers.py @@ -220,14 +220,16 @@ def _positive_int(*keys: str) -> bool: _MAX_ROOFLINE_FAILURE_RETRIES: int = 3 -# Actions that must stay startable no matter how little budget is left: they are -# how a session ends cleanly (report/breakdown) or unsticks itself (recover), so -# a time gate that refused them would strand the run with nothing to show. +# Actions that must stay startable no matter how little budget is left: they +# are how a session ends cleanly, so a time gate that refused them would +# strand the run with nothing to show. ``recover`` is not among them — it +# takes the server-lifecycle lane, prices at five catalogue minutes, and +# holds a twenty-minute lease; treating it as a closing action is what let a +# spent session keep working past the wall clock. TIME_BUDGET_EXEMPT_ACTIONS: frozenset[str] = frozenset( { "report", "session_breakdown", - "recover", } ) From 07e23b944c099a7688ced7b3b6116ec60c2c4dd6 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Mon, 17 Aug 2026 03:33:15 +0000 Subject: [PATCH 60/65] Ignore the EXPLORE hours leave-behind when it covers the whole session. Co-authored-by: Cursor --- .../tests/test_phase_force_exit.py | 56 +++++++++++++++++++ .../orchestrator/phases/machine_state.py | 42 +++++++++++++- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py b/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py index c789b8d6eb..1b9048b435 100644 --- a/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py +++ b/src/hyperloom/inference_optimizer/tests/test_phase_force_exit.py @@ -153,6 +153,62 @@ def test_force_exit_unlimited_run_never_fires(): assert fired is False # Without max_minutes nothing is computable. assert "session_remaining_seconds" not in evidence + assert "hours_remaining_gate" not in evidence + + +def test_force_exit_hours_leavebehind_cannot_cover_the_session(): + """IR-6's 3h default on a 3h session must not skip EXPLORE at first tick. + + Remaining starts at max_hours, so remaining <= 3h is true as soon as any + time has been spent. CI's 3h smoke (and the 3h example) would otherwise + leave EXPLORE with 0 grids. + """ + state = _make_explore_state( + max_minutes=180, + started_hours_ago=0.0, + phase_started_hours_ago=0.0, + ) + fired, evidence = phase_state.should_force_exit_explore( + state, + hours_remaining_threshold=3.0, + budget_pct_threshold=0.20, + ) + assert fired is False + assert "session_remaining" not in evidence["fired_reasons"] + assert evidence["hours_remaining_gate"] == "disabled_leavebehind_covers_session" + + +def test_force_exit_hours_leavebehind_disabled_after_prelude_on_three_hour_session(): + """CI shape: ~67 min spent before EXPLORE on a 3h budget.""" + state = _make_explore_state( + max_minutes=180, + started_hours_ago=1.12, + phase_started_hours_ago=0.0, + ) + fired, evidence = phase_state.should_force_exit_explore( + state, + hours_remaining_threshold=3.0, + budget_pct_threshold=0.20, + ) + assert fired is False + assert "session_remaining" not in evidence["fired_reasons"] + assert evidence["hours_remaining_gate"] == "disabled_leavebehind_covers_session" + + +def test_force_exit_explicit_hours_leavebehind_still_fires_on_short_session(): + """An operator-set leave-behind smaller than the session still fires.""" + state = _make_explore_state( + max_minutes=180, + started_hours_ago=2.96, + phase_started_hours_ago=0.01, + ) + fired, evidence = phase_state.should_force_exit_explore( + state, + hours_remaining_threshold=0.05, + budget_pct_threshold=0.0, + ) + assert fired is True + assert "session_remaining" in evidence["fired_reasons"] def test_exit_normal_explore_force_exit_takes_priority_over_plateau(): diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index 58ec61ccdf..c23e0dbb39 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -402,6 +402,9 @@ def is_valid_phase_exit_reason(value: str) -> bool: # EXPLORE hard force-exit thresholds (IR-6 HARD time gate; overrides plateau). # Fires when remaining wall-clock < HOURS_REMAINING OR EXPLORE budget fraction < BUDGET_PCT. +# HOURS_REMAINING is a leave-behind for later phases on long runs. When it is +# not strictly smaller than the session, the hours gate is ignored so a 3h +# smoke does not skip EXPLORE the moment the phase starts. DEFAULT_EXPLORE_FORCE_EXIT_HOURS_REMAINING: float = 3.0 DEFAULT_EXPLORE_FORCE_EXIT_BUDGET_PCT: float = 0.20 @@ -1292,6 +1295,29 @@ def phase_cap_exceeded( # EXPLORE hard force-exit (HARD time gate) +def _explore_hours_leavebehind_applies(state: Any, *, threshold_hours: float) -> bool: + """Whether IR-6's hours-remaining gate can fire for this session. + + Remaining starts at ``max_hours``, so a leave-behind that is not strictly + smaller than the session fires the moment EXPLORE starts. The 3h default + is for long runs; a 3h session (the CI smoke, the 3h example) must not + inherit it. + + Args: + state (Any): Frozen SharedState view. + threshold_hours (float): Configured hours-remaining leave-behind. + + Returns: + bool: ``True`` when the hours gate may fire. + """ + if float(threshold_hours) <= 0.0: + return False + session_hours = _max_minutes(state) / 60.0 + if session_hours <= 0.0: + return False + return float(threshold_hours) < session_hours + + def session_remaining_seconds( state: Any, *, @@ -1344,12 +1370,14 @@ def should_force_exit_explore( """Return ``(True, evidence)`` when HARD EXPLORE force-exit fires (IR-6). Fires when session remaining ≤ hours_threshold*3600 OR phase remaining - pct ≤ budget_pct_threshold; ``evidence`` records which fired. + pct ≤ budget_pct_threshold; ``evidence`` records which fired. The hours + gate is ignored when the leave-behind is not strictly smaller than the + session (it would otherwise fire the moment EXPLORE starts). Args: state (Any): Frozen SharedState view. hours_remaining_threshold (float): Session-hours-remaining gate; - non-positive disables it. + non-positive disables it. Also ignored when it covers the session. budget_pct_threshold (float): Phase-budget-fraction gate; non-positive disables it. budget_pct (dict[str, float] | None): Phase-budget overrides; defaults @@ -1368,8 +1396,16 @@ def should_force_exit_explore( fired_reasons: list[str] = [] # Non-positive threshold = disabled; both disabled turns force-exit off. - hours_threshold_enabled = float(hours_remaining_threshold) > 0.0 + # A leave-behind that covers the whole session is also disabled: remaining + # starts at max_hours, so it would fire the moment EXPLORE starts. + hours_threshold_enabled = _explore_hours_leavebehind_applies( + state, threshold_hours=hours_remaining_threshold + ) pct_threshold_enabled = float(budget_pct_threshold) > 0.0 + if float(hours_remaining_threshold) > 0.0 and not hours_threshold_enabled: + session_hours = _max_minutes(state) / 60.0 + if session_hours > 0.0: + evidence["hours_remaining_gate"] = "disabled_leavebehind_covers_session" session_remaining = session_remaining_seconds(state, now_unix=now_unix) if session_remaining is not None and hours_threshold_enabled: From b2a43f0972f82e9f7f4d00629ae93ca081c8b656 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Mon, 17 Aug 2026 10:17:17 +0000 Subject: [PATCH 61/65] Cancel a tick step that outlives the session bound so the run can still close. Co-authored-by: Cursor --- .../tests/test_session_time_budget.py | 91 +++++++++++++++++- .../orchestrator/loop/coordinator.py | 94 +++++++++++++++++-- 2 files changed, 176 insertions(+), 9 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 53ec177c5f..9607d829b4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -3,7 +3,7 @@ """The session wall-clock budget defences that live in the orchestrator loop. -Two of the four layers are here, the ones outside the executors: +Three of the defences are here, the ones outside the executors: * Admission -- an action whose expected cost cannot fit the budget that is left never starts. Covers the pure fit decision, the SharedState accessor both it @@ -16,6 +16,9 @@ landing terminal instead of stranding at ``running``, which of those handles the pump owns on its way out, and the pump and ``Coordinator.stop`` paths that trigger it. +* Tick bound -- a reactor turn or phase-enter await that never returns is + cancelled when the session (or closing) bound elapses, so the tick can still + reach the wall-clock stop. The remaining two layers are enforced inside the executors and tested next to them: the timeout clamp in ``test_explore_executor``, and the subprocess session @@ -1220,3 +1223,89 @@ async def test_stop_cancels_the_actions_still_running(self, coord: Coordinator): assert atask.cancelled() assert coord.dispatcher._inflight_actions == {} + + +async def _idle(*_args, **_kwargs) -> None: + return None + + +async def _hang_forever(*_args, **_kwargs) -> None: + await asyncio.sleep(3600) + + +class TestATickCannotOutliveTheSessionBound: + """A step that never returns used to skip the wall-clock stop at tick end.""" + + @pytest.mark.asyncio + async def test_a_hanging_reactor_still_stops_when_the_budget_ends( + self, + coord: Coordinator, + monkeypatch, + ): + monkeypatch.setattr(coord, "_advance_phase_if_needed", _idle) + monkeypatch.setattr(coord, "_reactor_pass", _hang_forever) + monkeypatch.setattr(coord, "_pump_dispatcher_once", _idle) + started = time.monotonic() + try: + reason = await asyncio.wait_for( + coord.run(max_minutes=0.05, closing_grace_sec=0.0, tick_interval_sec=0.0), + timeout=15.0, + ) + finally: + await coord.stop() + assert reason == "time_exhausted" + assert time.monotonic() - started < 10.0 + + @pytest.mark.asyncio + async def test_a_hanging_phase_enter_still_stops_when_the_budget_ends( + self, + coord: Coordinator, + monkeypatch, + ): + monkeypatch.setattr(coord, "_advance_phase_if_needed", _hang_forever) + monkeypatch.setattr(coord, "_reactor_pass", _idle) + monkeypatch.setattr(coord, "_pump_dispatcher_once", _idle) + started = time.monotonic() + try: + reason = await asyncio.wait_for( + coord.run(max_minutes=0.05, closing_grace_sec=0.0, tick_interval_sec=0.0), + timeout=15.0, + ) + finally: + await coord.stop() + assert reason == "time_exhausted" + assert time.monotonic() - started < 10.0 + + @pytest.mark.asyncio + async def test_a_spent_bound_does_not_start_the_next_step(self, coord: Coordinator): + coord._run_deadline = time.monotonic() - 1.0 + started: list[bool] = [] + + async def _must_not_run() -> None: + started.append(True) + + await coord._await_within_session_bound(_must_not_run, stage="test") + assert started == [] + + @pytest.mark.asyncio + async def test_no_deadline_still_runs_the_step(self, coord: Coordinator): + started: list[bool] = [] + + async def _ok() -> None: + started.append(True) + + await coord._await_within_session_bound(_ok, stage="test") + assert started == [True] + + @pytest.mark.asyncio + async def test_closing_uses_the_grace_bound_not_the_session_deadline(self, coord: Coordinator): + coord._run_deadline = time.monotonic() - 10.0 + coord._closing_deadline = time.monotonic() + 60.0 + coord.shared_state.closing_phase = True + started: list[bool] = [] + + async def _ok() -> None: + started.append(True) + + await coord._await_within_session_bound(_ok, stage="close") + assert started == [True] diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 89844563e3..9f6f81f51b 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -864,6 +864,9 @@ def _ckpt_fraction(env_key: str, default: float) -> float: # Wall-clock budget tracking for per-tick Time-budget prompt injection. self._run_deadline: float | None = None self._run_started_monotonic: float | None = None + # Closing-grace bound; used only while ``closing_phase`` is set so CLOSE + # work is not skipped just because the session deadline has passed. + self._closing_deadline: float | None = None # Latest objective wired by run(); refreshes target_gap_pct each tick. None outside a run. self._current_objective: Objective | None = None @@ -1519,18 +1522,30 @@ async def tick(self, n: int = 1) -> None: # hint (for example current GEAK returning no_gain -> skip_to_sweep). # Consume that before prompting agents again so stale phase prompts # cannot enqueue legacy work. - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_pre_reactor", + ) if str(getattr(self.shared_state, "pending_escalate_hint", "") or "").strip(): - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_hint", + ) for name in self._tick_roles: - await self._reactor_pass(name) + await self._await_within_session_bound( + lambda n=name: self._reactor_pass(n), + stage=f"reactor:{name}", + ) await self._pump_dispatcher_once() # FRAMEWORK_AGENT phase pump: enqueue next candidate / fetch next batch. await self._pump_framework_agent_phase_safely(caller="tick") # Phase-independent enablement pump: repair a non-runnable combo. await self._pump_enablement_safely(caller="tick") # phase machine advance at tick boundary. - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase", + ) def _record_coordinator_exception( self, @@ -1563,6 +1578,56 @@ def _record_coordinator_exception( except Exception: # noqa: BLE001 log.exception("failed to persist Coordinator exception metadata") + def _seconds_until_session_bound(self) -> float | None: + """Seconds left on the active run or closing bound; ``None`` if unbounded. + + During CLOSE the session deadline has already passed, so the bound + switches to ``_closing_deadline`` and CLOSE work is not skipped. + + Returns: + Remaining seconds, or ``None`` when no bound is armed. + """ + if bool(getattr(self.shared_state, "closing_phase", False)): + bound = self._closing_deadline + else: + bound = self._run_deadline + if bound is None: + return None + return float(bound) - time.monotonic() + + async def _await_within_session_bound( + self, + factory: Callable[[], Awaitable[Any]], + *, + stage: str, + ) -> None: + """Run one tick step, cancelling it when the session/closing bound elapses. + + The wall-clock stop lives at the end of each tick. A conversational + reactor turn or a long phase-enter await that never returns would skip + that stop. Cancelling the step lets the tick finish and enter CLOSE. + + Args: + factory: Builds the awaitable so a skipped step is never started. + stage: Label for the warning log. + """ + remaining = self._seconds_until_session_bound() + if remaining is not None and remaining <= 0.0: + log.warning("Coordinator: skipping %s; session bound already elapsed", stage) + return + awaitable = factory() + if remaining is None: + await awaitable + return + try: + await asyncio.wait_for(awaitable, timeout=remaining) + except asyncio.TimeoutError: + log.warning( + "Coordinator: %s hit the session bound after %.1fs; cancelled so the tick can close", + stage, + remaining, + ) + # Long-run interface async def run( self, @@ -1649,9 +1714,15 @@ async def run( # Bump the persistent tick counter — drives phase/plateau math. self.shared_state.increment_tick() try: - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_pre_reactor", + ) if str(getattr(self.shared_state, "pending_escalate_hint", "") or "").strip(): - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase_hint", + ) except Exception as exc: # noqa: BLE001 log.exception("phase advance before reactors (run) failed") self._record_coordinator_exception( @@ -1665,7 +1736,10 @@ async def run( for name in self._tick_roles: if self._stop.is_set(): break - await self._reactor_pass(name) + await self._await_within_session_bound( + lambda n=name: self._reactor_pass(n), + stage=f"reactor:{name}", + ) # Orchestration checkpoint/compaction; cadence-based. if not self._stop.is_set(): try: @@ -1690,7 +1764,10 @@ async def run( log.exception("targeted-build tick raised") # phase machine advance; runs even in_closing so CLOSE is recorded. try: - await self._advance_phase_if_needed() + await self._await_within_session_bound( + self._advance_phase_if_needed, + stage="advance_phase", + ) except Exception as exc: # noqa: BLE001 log.exception("phase advance (run) failed") self._record_coordinator_exception( @@ -1731,6 +1808,7 @@ async def run( closing_deadline = await self._enter_closing_phase( grace_sec=grace_sec, ) + self._closing_deadline = closing_deadline continue if in_closing: report_terminal = await self._closing_report_terminal() From c218d3fc7ea0cdae0f549524831ad54b2decc044 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Mon, 17 Aug 2026 10:36:57 +0000 Subject: [PATCH 62/65] Drop the unused await path that CodeQL flagged, and stop requiring a reactor turn on a spent budget. Co-authored-by: Cursor --- src/hyperloom/inference_optimizer/tests/test_objective.py | 3 ++- src/hyperloom/orchestrator/loop/coordinator.py | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_objective.py b/src/hyperloom/inference_optimizer/tests/test_objective.py index 35a730b21c..8b2815c453 100644 --- a/src/hyperloom/inference_optimizer/tests/test_objective.py +++ b/src/hyperloom/inference_optimizer/tests/test_objective.py @@ -342,8 +342,9 @@ async def _enter_and_record(*, grace_sec: float) -> float: closing_grace_sec=5.0, tick_interval_sec=0.0, ) - assert spy.calls >= 1 assert calls_at_closing, "expected closing phase to be entered" + # A spent bound cancels phase-enter and skips reactors on the tick + # that trips CLOSE. CLOSE itself must still not add LLM turns. assert spy.calls == calls_at_closing[0] finally: await c.stop() diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 9f6f81f51b..db15f3dbf0 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1615,12 +1615,9 @@ async def _await_within_session_bound( if remaining is not None and remaining <= 0.0: log.warning("Coordinator: skipping %s; session bound already elapsed", stage) return - awaitable = factory() - if remaining is None: - await awaitable - return try: - await asyncio.wait_for(awaitable, timeout=remaining) + # ``timeout=None`` waits until the step finishes (unbounded run). + await asyncio.wait_for(factory(), timeout=remaining) except asyncio.TimeoutError: log.warning( "Coordinator: %s hit the session bound after %.1fs; cancelled so the tick can close", From 6f23e0adbee3c5b54e2fb012e641cc2a394af093 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Tue, 18 Aug 2026 03:53:16 +0000 Subject: [PATCH 63/65] Fix a rebase merge that concatenated two BaselineExecutor methods. _run_reported_round from main and _mn_warmup_pass from this branch were left on one line, which is a SyntaxError. --- src/hyperloom/orchestrator/actions/executors/baseline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 1454bb98d8..ba469f752d 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -4030,7 +4030,9 @@ async def _run_reported_round( config_path=config_path, output_dir=output_dir, **common, - ) async def _mn_warmup_pass( + ) + + async def _mn_warmup_pass( self, *, cmd: list[str], From 57c7b662954d80e1da2fb75184b2489b598e9942 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Tue, 18 Aug 2026 06:20:03 +0000 Subject: [PATCH 64/65] Close SWEEP honestly when conc_sweep cannot fit the session budget. Admission already refused the sweep, but last_conc_sweep stayed empty, so the LLM's skip_to_close was labelled robustness_escalated and CI exited 1 on a successful run. --- .../tests/test_longrun_phase1.py | 81 +++++++++++++++++++ .../inference_optimizer/tests/test_report.py | 9 +++ .../tests/test_session_time_budget.py | 52 ++++++++++++ .../tests/test_sweep_phase_auto.py | 33 ++++++++ .../orchestrator/loop/coordinator.py | 2 + src/hyperloom/orchestrator/loop/dispatcher.py | 12 +++ src/hyperloom/orchestrator/phases/machine.py | 9 +++ .../orchestrator/phases/machine_state.py | 30 ++++++- src/hyperloom/orchestrator/phases/sweep.py | 25 ++++++ 9 files changed, 252 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py b/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py index a0a5026c0e..46c5ccaa89 100644 --- a/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py +++ b/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py @@ -91,6 +91,55 @@ def test_sweep_closes_when_insufficient_remaining(): assert evidence["reloop_blocked"] == "insufficient_remaining" +def test_sweep_skip_to_close_does_not_override_a_settled_conc_sweep(): + """LLM skip_to_close after a refused conc_sweep must not become robustness_escalated.""" + st = _sweep_state(max_minutes=180, started_hours_ago=166 / 60.0) + st.last_sweep = {} + st.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "skip_reason": "session_time_budget", + } + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + nxt = ps.compute_next_phase(st, max_hours=3.0) + assert nxt is not None + target, reason, evidence = nxt + assert target == ps.PHASE_CLOSE + assert reason == "conc_sweep_done" + assert evidence.get("conc_sweep_status") == "skipped" + + +def test_sweep_skip_to_close_still_escalates_when_conc_sweep_never_settled(): + """skip_to_close remains a robustness abort when SWEEP has nothing to close on.""" + st = _sweep_state(max_minutes=180, started_hours_ago=1.0) + st.last_sweep = {} + st.last_conc_sweep = {} + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + nxt = ps.compute_next_phase(st, max_hours=3.0) + assert nxt is not None + target, reason, _evidence = nxt + assert target == ps.PHASE_CLOSE + assert reason == "robustness_escalated" + + +def test_sweep_skip_to_close_yields_to_reloop_when_conc_sweep_was_skipped(): + """A skipped conc_sweep with budget left must not be aborted by skip_to_close.""" + st = _sweep_state(macro_cycle=0, cumulative_gain=5.0, gain_at_cycle_start=0.0) + st.last_sweep = {} + st.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "skip_reason": "session_time_budget", + } + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + nxt = ps.compute_next_phase(st, max_hours=96.0) + assert nxt is not None + target, reason, evidence = nxt + assert target == ps.PHASE_EXPLORE + assert reason == "cycle_reloop" + assert evidence["loopback"] is True + + def test_short_bounded_run_reloops_when_budget_and_leverage_remain(): # 12h bounded run: macro-loop is available even though budget accounting # stays in short-run charge-back mode. @@ -342,6 +391,38 @@ async def test_coordinator_applies_loopback(cyclic_coordinator): assert loopback_row["cycle"] == 1 +@pytest.mark.asyncio +async def test_skip_to_close_is_consumed_when_sweep_already_settled( + cyclic_coordinator, + monkeypatch, +): + """A suppressed skip_to_close must not leak into the next phase.""" + c = cyclic_coordinator + st = c.shared_state + now = datetime.now(timezone.utc) + st.phase = ps.PHASE_SWEEP + st.start_ts = (now - timedelta(minutes=166)).isoformat() + st.max_minutes = 180 + st.macro_cycle = 0 + st.last_sweep = {} + st.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "skip_reason": "session_time_budget", + } + st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) + + async def _entered(*, from_phase, to_phase): + return None + + monkeypatch.setattr(c.phase_machine, "_on_phase_entered", _entered) + await c._advance_phase_if_needed() + + assert st.phase == ps.PHASE_CLOSE + assert st.pending_escalate_hint == "" + assert st.last_consumed_escalate_hint == ps.ESCALATE_HINT_SKIP_TO_CLOSE + + @pytest.mark.asyncio async def test_coordinator_converged_close_sets_stop_reason(cyclic_coordinator): c = cyclic_coordinator diff --git a/src/hyperloom/inference_optimizer/tests/test_report.py b/src/hyperloom/inference_optimizer/tests/test_report.py index 0e5e3f3b24..d65fd9db09 100644 --- a/src/hyperloom/inference_optimizer/tests/test_report.py +++ b/src/hyperloom/inference_optimizer/tests/test_report.py @@ -274,6 +274,15 @@ def test_a_skip_with_no_recorded_reason_still_says_it_was_skipped(): assert "did not run" in rp._explain_stop_reason("conc_sweep_done", state) +def test_a_session_budget_skip_is_described_as_a_sweep_that_did_not_run(): + state = _SweepState( + {"status": "skipped", "was_skipped": True, "skip_reason": "session_time_budget"} + ) + msg = rp._explain_stop_reason("conc_sweep_done", state) + assert "did not run" in msg + assert "session_time_budget" in msg + + def test_a_sweep_that_spent_its_budget_is_not_reported_as_one_that_never_ran(tmp_path): """The budget path records was_skipped for a sweep that ran its whole ladder.""" from hyperloom.orchestrator.state.shared_state import SharedState diff --git a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py index 9607d829b4..94133eec74 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_time_budget.py @@ -526,6 +526,58 @@ async def test_a_queued_recover_is_dropped_when_the_budget_is_spent( assert (await coord.tasks.get(task.task_id)).state == "cancelled" + @pytest.mark.asyncio + async def test_a_queued_conc_sweep_the_budget_outlived_is_recorded_as_skipped( + self, + coord: Coordinator, + ): + """Cancelling conc_sweep at dispatch must stamp last_conc_sweep so SWEEP can close.""" + from hyperloom.orchestrator.phases.machine_state import exit_normal_sweep + + _set_budget(coord, minutes=180) + task, _ = await coord.tasks.create_or_return_existing( + kind="conc_sweep", + params={}, + idempotency_key="q-conc-sweep", + ) + _set_budget(coord, minutes=180, elapsed_min=166.0) + + spawned = await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert [t.task_id for t, _, _ in spawned] == [] + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + assert coord.shared_state.last_conc_sweep["status"] == "skipped" + assert coord.shared_state.last_conc_sweep["skip_reason"] == "session_time_budget" + assert coord.shared_state.last_conc_sweep["was_skipped"] is True + result = exit_normal_sweep(coord.shared_state) + assert result is not None + reason, evidence = result + assert reason == "conc_sweep_done" + assert evidence["conc_sweep_status"] == "skipped" + + @pytest.mark.asyncio + async def test_dropping_an_over_budget_conc_sweep_does_not_erase_a_prior_result( + self, + coord: Coordinator, + ): + """A later cancel must not overwrite a conc_sweep the session already measured.""" + _set_budget(coord, minutes=180) + coord.shared_state.record_conc_sweep( + {"status": "succeeded", "was_skipped": False, "summary": {"successful_pairs": 3}} + ) + task, _ = await coord.tasks.create_or_return_existing( + kind="conc_sweep", + params={}, + idempotency_key="q-conc-sweep-prior", + ) + _set_budget(coord, minutes=180, elapsed_min=166.0) + + await coord.dispatcher._spawn_fitting_queued(exclude_ids=set()) + + assert (await coord.tasks.get(task.task_id)).state == "cancelled" + assert coord.shared_state.last_conc_sweep["status"] == "succeeded" + + # One of the closing actions, exempt from the budget because the closing reserve # is held back so it can run. _CLOSING_ACTION = "report" diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py index 38a92680a0..0266178e15 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py @@ -48,6 +48,11 @@ class _BareState: conc_sweep_total_budget_sec: int = 60 conc_sweep_variant_timeout_sec: int = 30 save_count: int = 0 + stop_reason: str = "" + usable_sec: float | None = None + + def session_budget_usable_sec(self, *, reserve_sec=None) -> float | None: + return self.usable_sec def save(self, _session_dir: Path | None) -> None: self.save_count += 1 @@ -947,6 +952,34 @@ async def test_on_enter_sweep_skips_when_no_validated_gain_since_last_conc_sweep assert coord.shared_state.save_count >= 1 +@pytest.mark.asyncio +async def test_on_enter_sweep_skips_when_the_session_budget_cannot_fit_conc_sweep(coord): + """A conc_sweep the clock cannot pay for must not be enqueued, or SWEEP idles.""" + coord.shared_state.usable_sec = 14 * 60.0 + coord.shared_state.phase_history = [ + {"to_phase": "SWEEP", "reason": "plateau_kernel", "evidence": {}}, + ] + await coord._on_enter_sweep(from_phase="KERNEL") + assert coord.tasks._tasks == {} + evidence = coord.shared_state.phase_history[-1]["evidence"] + assert evidence["auto_conc_sweep_skipped"] == "session_time_budget" + assert coord.shared_state.last_conc_sweep["status"] == "skipped" + assert coord.shared_state.last_conc_sweep["skip_reason"] == "session_time_budget" + assert coord.shared_state.last_conc_sweep["was_skipped"] is True + + +@pytest.mark.asyncio +async def test_on_enter_sweep_still_enqueues_when_the_session_budget_fits(coord): + """The session-budget skip must not fire when the catalogue cost still fits.""" + coord.shared_state.usable_sec = 60 * 60.0 + coord.shared_state.phase_history = [ + {"to_phase": "SWEEP", "reason": "plateau_kernel", "evidence": {}}, + ] + await coord._on_enter_sweep(from_phase="KERNEL") + assert "internal-conc_sweep-phase_entry" in coord.tasks._tasks + assert coord.shared_state.last_conc_sweep == {} + + @pytest.mark.asyncio async def test_on_enter_sweep_runs_when_validated_gain_improved(coord): """A new validated gain after the last conc_sweep watermark dispatches conc_sweep.""" diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index db15f3dbf0..6b55f540ca 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -943,6 +943,8 @@ def router(self) -> IntentRouter: "_enqueue_internal_conc_sweep_task": "phase_sweep", "_enqueue_internal_sweep_task": "phase_sweep", "_build_sweep_params_from_recipe": "phase_sweep", + "_record_session_budget_conc_sweep_skip": "phase_sweep", + "_record_terminal_conc_sweep_skip": "phase_sweep", "_derive_close_stop_reason": "phase_close", "_session_integrated_kernel_patch": "phase_close", "_maybe_run_close_post_opt_roofline": "phase_close", diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index a6b99bb493..3cd71f870a 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -1548,6 +1548,18 @@ async def _cancel_queued_task_over_budget(self, task: Task) -> bool: "dispatcher: could not record time-budget denial for task=%s", task.task_id, ) + # A cancelled conc_sweep never writes last_conc_sweep on its own, so + # SWEEP would idle until the LLM emits skip_to_close and CI would read + # that as robustness_escalated. Stamp the skip here so the phase + # machine can close on conc_sweep_done. + if str(task.kind or "") == "conc_sweep": + try: + self._record_session_budget_conc_sweep_skip(denied=denied) + except Exception: # noqa: BLE001 — a stamp miss must not abort the pump + log.exception( + "dispatcher: could not record conc_sweep time-budget skip for task=%s", + task.task_id, + ) return True def _sequence_denial_for_request( diff --git a/src/hyperloom/orchestrator/phases/machine.py b/src/hyperloom/orchestrator/phases/machine.py index 5a495c4222..7a7661bb96 100644 --- a/src/hyperloom/orchestrator/phases/machine.py +++ b/src/hyperloom/orchestrator/phases/machine.py @@ -268,6 +268,15 @@ async def _advance_phase_if_needed(self) -> None: # Consume escalate hint after a hint-driven transition. if isinstance(evidence, dict) and (evidence.get("evidence") == "llm_escalation" or "hint" in evidence): state.consume_pending_escalate_hint() + elif ( + str(prior or "").strip().upper() == _phase_state.PHASE_SWEEP + and str(getattr(state, "pending_escalate_hint", "") or "").strip() + == _phase_state.ESCALATE_HINT_SKIP_TO_CLOSE + ): + # SWEEP already had an honest closeout, so skip_to_close was + # suppressed. Drop it here or the next phase inherits it and + # becomes robustness_escalated. + state.consume_pending_escalate_hint() # Terminal transition (target=CLOSE): mirror the stop_reason onto state. if ( target == _phase_state.PHASE_CLOSE diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index c23e0dbb39..39a511e6f6 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -1661,11 +1661,36 @@ def compute_plateau_kernel( } +# Statuses on last_sweep / last_conc_sweep that exit_normal_sweep already +# treats as SWEEP closeout. skip_to_close must not override those: the LLM +# emits it when conc_sweep was refused, and mapping that to +# robustness_escalated turns a successful run into a CI failure. +_SWEEP_DONE_STATUSES: frozenset[str] = frozenset({"succeeded", "partial", "completed"}) +_CONC_SWEEP_CLOSEOUT_STATUSES: frozenset[str] = frozenset( + {"succeeded", "partial", "completed", "skipped", "failed"} +) + + +def _sweep_has_recorded_closeout(state: Any) -> bool: + """Whether SWEEP already recorded a result the phase machine can close on.""" + last_sweep = getattr(state, "last_sweep", None) or {} + if isinstance(last_sweep, dict): + if str(last_sweep.get("status") or "").lower() in _SWEEP_DONE_STATUSES: + return True + last_conc = getattr(state, "last_conc_sweep", None) or {} + if isinstance(last_conc, dict): + if str(last_conc.get("status") or "").lower() in _CONC_SWEEP_CLOSEOUT_STATUSES: + return True + return False + + # terminal / abort (global) def _global_terminal(state: Any) -> tuple[str, dict[str, Any]] | None: """Return ``(stop_reason, evidence)`` for a phase-orthogonal stop. - Priority: 1. ``skip_to_close`` → ``robustness_escalated``; 2. Coordinator ``stop_reason``. + Priority: 1. ``skip_to_close`` → ``robustness_escalated``, except in SWEEP + when a sweep/conc_sweep closeout is already recorded (the honest SWEEP + exit wins); 2. Coordinator ``stop_reason``. Args: state (Any): Frozen SharedState view exposing ``stop_reason`` and any @@ -1677,6 +1702,9 @@ def _global_terminal(state: Any) -> tuple[str, dict[str, Any]] | None: """ hint = _pending_escalate_hint(state) if hint == ESCALATE_HINT_SKIP_TO_CLOSE: + current = (getattr(state, "phase", "") or "").strip().upper() + if current == PHASE_SWEEP and _sweep_has_recorded_closeout(state): + return None return "robustness_escalated", { "evidence": "llm_escalation", "hint": hint, diff --git a/src/hyperloom/orchestrator/phases/sweep.py b/src/hyperloom/orchestrator/phases/sweep.py index a4872d2958..cffe471790 100644 --- a/src/hyperloom/orchestrator/phases/sweep.py +++ b/src/hyperloom/orchestrator/phases/sweep.py @@ -59,6 +59,16 @@ async def _on_enter_sweep(self, *, from_phase: str) -> None: auto_conc_sweep_skipped_validated_gain=cur_validated, ) return + denied = self._time_budget_denial_for_action("conc_sweep") + if denied is not None: + log.info( + "SWEEP entry (from=%s): conc_sweep cannot fit the session budget " + "(%s); recording terminal skip.", + from_phase or "", + denied, + ) + self._record_session_budget_conc_sweep_skip(denied=denied) + return try: task = await self._enqueue_internal_conc_sweep_task( reason="phase_entry", @@ -167,6 +177,21 @@ async def _enqueue_internal_conc_sweep_task( self._record_phase_entry_evidence(auto_conc_sweep_task_id=task.task_id) return task + def _record_session_budget_conc_sweep_skip(self, *, denied: object) -> None: + """Stamp last_conc_sweep skipped when the session clock refused conc_sweep. + + No-op when a conc_sweep result is already on the session: a later + over-budget cancel must not erase a measurement the phase can close on. + """ + last = getattr(self.shared_state, "last_conc_sweep", None) or {} + if str(last.get("status") or "").strip(): + return + self._record_terminal_conc_sweep_skip( + skip_reason="session_time_budget", + auto_conc_sweep_skipped="session_time_budget", + auto_conc_sweep_denied=str(denied), + ) + def _record_terminal_conc_sweep_skip( self, *, From f2c241cbc32ee9f020858c118de289ebe0ea29d5 Mon Sep 17 00:00:00 2001 From: zoroyihan7 Date: Tue, 18 Aug 2026 06:47:15 +0000 Subject: [PATCH 65/65] Remove the leftover rebase duplicate of enable_multi_node and an unused NamedTuple import. --- src/hyperloom/inference_optimizer/tests/conftest.py | 11 ----------- .../orchestrator/actions/executors/_grid_runner.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/conftest.py b/src/hyperloom/inference_optimizer/tests/conftest.py index 43abbb1229..ccfb24d0a3 100644 --- a/src/hyperloom/inference_optimizer/tests/conftest.py +++ b/src/hyperloom/inference_optimizer/tests/conftest.py @@ -361,17 +361,6 @@ def suppression_window_s() -> float: return StallConfig().stall_timeout_s -def enable_multi_node(monkeypatch, nodes: int = 2) -> None: - """Put the executors in multi-node mode with a no-op per-round server restart.""" - from hyperloom.orchestrator.actions.executors import _multi_node_server_lifecycle as mnl - - async def _no_restart(*_args, **_kwargs) -> None: - return None - - monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) - monkeypatch.setattr(mnl, "restart_server_for_round", _no_restart) - - class _RayDoubleActorClass: """The ``@ray.remote`` class: ``.options(...)`` then ``.remote()`` for a handle.""" diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index aadbb0bd2b..491a15c916 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -19,7 +19,7 @@ import subprocess import time from pathlib import Path -from typing import Any, Callable, NamedTuple +from typing import Any, Callable import yaml