diff --git a/docs/conf.py b/docs/conf.py index eb9ac8992..4f8ffe0a0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,4 +85,4 @@ html_title = f"{project} {version_number} documentation" -external_projects_current_project = "Hyperloom" \ No newline at end of file +external_projects_current_project = "Hyperloom" diff --git a/selfcheck_gates.py b/selfcheck_gates.py deleted file mode 100644 index c5fcce8f9..000000000 --- a/selfcheck_gates.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Throwaway self-review: sweep the real pricing functions for disagreements.""" - -from __future__ import annotations - -from types import SimpleNamespace - -from hyperloom.orchestrator.phases import machine_state as ms - -BOOT, COLD_BENCH, HOT = 350.0, 550.0, 400.0 -COLD_ROUND = BOOT + COLD_BENCH - - -def state(usable, *, phase="PRELUDE", warm=HOT, post_ready=COLD_BENCH, marked=False, double=True): - return SimpleNamespace( - phase=phase, - max_minutes=180, - baseline_tput=1000.0, - baseline_runtime_sec=COLD_ROUND, - baseline_post_ready_runtime_sec=post_ready, - baseline_warm_runtime_sec=warm, - baseline_measure_round_dropped=marked, - baseline_double_run=double, - session_budget_usable_sec=lambda: usable, - ) - - -def gate1_need(s): - """What the pre-ignition gate demands, mirroring _round_affordable.""" - cold = ms.measured_seconds(s, "baseline_runtime_sec") - rnd = ms.baseline_round_cost_sec(s, double_run=bool(s.baseline_double_run)) - if cold is None or rnd is None: - return None - use = (ms.one_more_measurement_sec(s) or cold) if s.phase == "PRELUDE" else 0.0 - return rnd + use - - -def gate2_need(s, *, warmup_sec, warmup_post_ready): - """What the post-warmup gate demands, mirroring _measure_round_affordable.""" - bench = ms.measured_seconds(s, "baseline_warm_runtime_sec") - if bench is None: - bench = warmup_post_ready - if bench is None or warmup_sec is None: - return None - use = (ms.one_more_measurement_sec(s) or warmup_sec) if s.phase == "PRELUDE" else 0.0 - return bench + use - - -def main() -> int: - bad = 0 - - # 1. The band: is there a budget gate 1 admits and gate 2 then certainly refuses? - # Gate 2 is asked after the warmup has spent a cold pass. - band = [] - for usable in range(0, 6001, 10): - s = state(float(usable)) - need1 = gate1_need(s) - admitted = need1 is not None and usable >= need1 - if not admitted: - continue - after = state(float(usable) - COLD_ROUND) - need2 = gate2_need(after, warmup_sec=COLD_ROUND, warmup_post_ready=COLD_BENCH) - if need2 is not None and (usable - COLD_ROUND) < need2: - band.append(usable) - if band: - bad += 1 - print(f"BAND gate 1 admits and gate 2 refuses for usable in {band[0]}..{band[-1]}") - else: - print("ok no budget is admitted before ignition only to be refused after the cold pass") - - # 2. Livelock: with the mark set, does every budget either close or admit a retry? - stuck = [] - for usable in range(0, 8001, 10): - s = state(float(usable), marked=True) - closes = ms.exit_cold_anchor_prelude(s) is not None - need1 = gate1_need(s) - admits = need1 is not None and usable >= need1 - if not closes and not admits: - stuck.append(usable) - if stuck: - bad += 1 - print(f"LIVELOCK neither closes nor admits for usable in {stuck[0]}..{stuck[-1]}") - else: - print("ok a marked session always either closes or may retry") - - # 3. A session with no split measured (multi-node / scriptable shape). - s = state(3000.0, warm=0.0, post_ready=0.0) - need = gate1_need(s) - if need is None: - bad += 1 - print("UNGATED a round with no boot boundary is waved through") - else: - print(f"ok a round with no split is priced at {need:.0f}s (whole cold rounds)") - - # 4. A first baseline must never be judged. - first = SimpleNamespace( - phase="PRELUDE", - max_minutes=180, - baseline_tput=0.0, - baseline_runtime_sec=0.0, - baseline_post_ready_runtime_sec=0.0, - baseline_warm_runtime_sec=0.0, - baseline_measure_round_dropped=False, - baseline_double_run=True, - session_budget_usable_sec=lambda: 60.0, - ) - if gate1_need(first) is not None: - bad += 1 - print("PREDICTED a first baseline was priced from measurements it cannot have") - else: - print("ok a first baseline is not judged") - - # 5. Later phases ask only whether the round fits. - later = state(2000.0, phase="EXPLORE") - if gate1_need(later) != ms.baseline_round_cost_sec(later, double_run=True): - bad += 1 - print("SCOPED a re-baseline outside PRELUDE was charged for a successor") - else: - print("ok a re-baseline outside PRELUDE pays only for itself") - - return bad - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py b/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py index d0279da02..d8ce9165d 100644 --- a/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py +++ b/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py @@ -19,12 +19,6 @@ from pathlib import Path from typing import Any -# Sibling import: kernel-agent tools cannot rely on the ``hyperloom`` import root. -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _io_utils import truthy # noqa: E402 - -sys.path.pop(0) - def _load_input_json(path: str) -> dict[str, Any]: if not path: diff --git a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py index e74a41faa..fe6a9df5a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py @@ -317,7 +317,6 @@ def test_crash_and_success_produce_same_fingerprint(self, tmp_path): def test_workload_change_changes_fingerprint(self, tmp_path): """A change to workload shape (ISL) changes the fingerprint.""" - import yaml as _yaml cfg1 = tmp_path / "c1.yaml" cfg2 = tmp_path / "c2.yaml" diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 69c127b0e..05c1a6eea 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -1366,7 +1366,7 @@ async def test_compose_prompt_orchestration_all_advisory_blocks( assert token in out -def test_research_scout_seed_block_keeps_all_rounds(coord: Coordinator) -> None: +def test_research_scout_seed_block_keeps_findings_and_questions_only(coord: Coordinator) -> None: from hyperloom.orchestrator.knowledge import research_hints research_hints.append_hints( @@ -1410,11 +1410,14 @@ def test_research_scout_seed_block_keeps_all_rounds(coord: Coordinator) -> None: assert "hint one" in block assert "hint two" in block - assert '"name": "first"' in block - assert '"name": "second"' in block assert "question one" in block assert "question two" in block assert "ignore-me" not in block + # Proposals moved to the shared untested-proposal queue, which also drops + # the ones already benched; rendering them here as well would double them. + assert "Untested executable proposals" not in block + assert '"name": "first"' not in block + assert '"name": "second"' not in block @pytest.mark.asyncio diff --git a/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py index 141f3b578..78252a18c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py @@ -6,7 +6,6 @@ from __future__ import annotations import json -from types import SimpleNamespace # --------------------------------------------------------------------------- # diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index 6946d1319..12cf15538 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -2073,10 +2073,16 @@ def _build_specialist_prompt_text() -> str: return system_prompt + "\n" + user_prompt -def test_specialist_prompt_renders_top_6_target(): +def test_specialist_prompt_renders_proposal_target_and_ceiling(): text = _build_specialist_prompt_text() - assert "AT MOST **6** entries" in text - assert "top-6" in text + # Section 8 states both numbers; Section 1 repeats them so the target is + # visible before the specialist starts working, not only at exit time. + assert "**2 entries is the norm, 4 the hard cap.**" in text + assert "**2 proposals is the norm, 4 the hard" in text + # Both the padding and the keep-going pressures need a stated counterweight. + assert "``empty=true`` is better than one" in text + assert "stop once" in text and "not the only stop" in text + assert "a coin-flip proposal is worse than none" in text assert "reviews each surviving variant" in text diff --git a/src/hyperloom/inference_optimizer/tests/test_custom_framework.py b/src/hyperloom/inference_optimizer/tests/test_custom_framework.py index 30072ea8e..f52f91860 100644 --- a/src/hyperloom/inference_optimizer/tests/test_custom_framework.py +++ b/src/hyperloom/inference_optimizer/tests/test_custom_framework.py @@ -13,7 +13,6 @@ from __future__ import annotations import os -from pathlib import Path import pytest import yaml diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py b/src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py new file mode 100644 index 000000000..e00148438 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The grid author's two prompt surfaces agree on where a grid comes from and how big it gets.""" + +from __future__ import annotations + + +def _grid_prompt_surfaces() -> list[str]: + from hyperloom.orchestrator.prompts.prompt_builder import ( + _idea_generation_lines, + _format_grid_injection_hint, + ) + + return [_format_grid_injection_hint("explore") or "", "\n".join(_idea_generation_lines())] + + +def test_both_grid_surfaces_state_the_same_target_and_ceiling(): + for surface in _grid_prompt_surfaces(): + assert "4" in surface + assert "maximum 6" in surface + assert all("Untested proposals (current cycle)" in s for s in _grid_prompt_surfaces()[1:]) + + +def test_the_stale_proposal_set_wording_is_gone(): + from pathlib import Path + + from hyperloom.orchestrator.prompts import prompt_builder + + stale = "proposal_set drives the next" + assert stale not in Path(prompt_builder.__file__).read_text(encoding="utf-8") + md = Path(prompt_builder.__file__).parent / "orchestration.md" + assert "specialist proposal_set" not in md.read_text(encoding="utf-8") diff --git a/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py b/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py index 2c8669e72..3d8f7aa97 100644 --- a/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py +++ b/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py @@ -465,7 +465,6 @@ async def test_runner_does_not_log_generic_template_for_any_domain(tmp_path): PolicyDenied, PolicyGate, SPECIALIST_ACTION_NAME, - SPECIALIST_FROM_AGENT_PREFIX, ) from hyperloom.orchestrator.bus.resource_lock import ( KNOWN_LANES, diff --git a/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py b/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py new file mode 100644 index 000000000..840db5f32 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Shared proposal/variant identity. + +The fingerprints below were captured from the explore executor's own identity +block before it was refactored onto this helper. They are pinned rather than +recomputed: a change here re-keys ``explore_search["tested"]``, so every +resumed session would re-bench its whole history. +""" + +from __future__ import annotations + +import pytest + +from hyperloom.orchestrator.actions.executors._proposal_identity import ( + controls_of, + effective_fingerprint, + is_executable, + normalize_proposal, +) + + +# (label, extra_args, extra_envs, variant controls, base controls, expected fingerprint) +_GOLDEN = [ + ("plain", "--max-num-seqs 128", {"A": "1"}, ([], [], "append"), ([], [], ""), "caacd6bf1da76201"), + ("variant-only", "--max-num-seqs 128", {"A": "1"}, (["--x"], ["E"], "append"), ([], [], ""), "39cfaf303d04885d"), + ("base-only", "--max-num-seqs 128", {"A": "1"}, ([], [], "append"), (["--b"], ["BE"], ""), "e85521a43890c743"), + ("both", "--max-num-seqs 128", {"A": "1"}, (["--x"], ["E"], "append"), (["--b"], ["BE"], ""), "1e630973fad523ab"), + ("overlap", "--max-num-seqs 128", {}, (["--b"], [], "append"), (["--b"], [], ""), "9cb8463804e7a11c"), + ("variant-replace", "", {"A": "1"}, ([], [], "replace"), ([], [], ""), "7f4d4d32a84df525"), + ("base-replace", "", {"A": "1"}, ([], [], "append"), ([], [], "replace"), "7f4d4d32a84df525"), + ("removal-only", "", {}, (["--enable-prefix-caching"], [], "append"), ([], [], ""), "ca4d2e9e9760543a"), + ("empty", "", {}, ([], [], "append"), ([], [], ""), "164374825086dc65"), +] + + +def _controls(remove_args, unset_envs, args_mode) -> dict: + return controls_of( + normalize_proposal({"remove_args": remove_args, "unset_envs": unset_envs, "args_mode": args_mode}) + ) + + +@pytest.mark.parametrize( + "args,envs,variant,base,expected", + [row[1:] for row in _GOLDEN], + ids=[row[0] for row in _GOLDEN], +) +def test_effective_fingerprint_matches_the_pinned_executor_values(args, envs, variant, base, expected): + b_remove, b_unset, b_mode = base + assert ( + effective_fingerprint( + args, + envs, + controls=_controls(*variant), + base_remove_args=b_remove, + base_unset_envs=b_unset, + base_args_mode=b_mode, + ) + == expected + ) + + +def test_base_controls_change_the_fingerprint(): + controls = _controls(["--x"], [], "append") + assert effective_fingerprint("--a 1", {}, controls=controls) != effective_fingerprint( + "--a 1", {}, controls=controls, base_remove_args=["--b"] + ) + + +def test_removal_union_is_base_first_and_deduped(): + both = effective_fingerprint("", {}, controls=_controls(["--b", "--v"], [], "append"), base_remove_args=["--b"]) + assert both == effective_fingerprint("", {}, controls=_controls(["--b", "--v"], [], "append")) + + +@pytest.mark.parametrize( + "proposal,expected", + [ + ({"extra_args": "--a 1"}, True), + ({"extra_server_args": "--a 1"}, True), + ({"extra_envs": {"A": "1"}}, True), + ({"remove_args": ["--a"]}, True), + ({"unset_envs": ["A"]}, True), + ({"args_mode": "replace"}, True), + ({"name": "research-only", "reason": "read the scheduler"}, False), + ({"extra_args": " ", "extra_envs": {}}, False), + ], +) +def test_is_executable(proposal, expected): + assert is_executable(normalize_proposal(proposal)) is expected + + +def test_normalize_resolves_the_args_alias_and_keeps_atomic(): + fields = normalize_proposal( + { + "name": " coupled ", + "extra_server_args": " --a 1 ", + "extra_envs": {"A": 1}, + "remove_args": "--drop", + "args_mode": "REPLACE", + "atomic": True, + "reason": "needs the paired headroom", + } + ) + assert fields == { + "name": "coupled", + "extra_args": "--a 1", + "extra_envs": {"A": "1"}, + "remove_args": ["--drop"], + "unset_envs": [], + "args_mode": "replace", + "atomic": True, + "reason": "needs the paired headroom", + } + + +def test_controls_of_drops_defaults(): + assert controls_of(normalize_proposal({"extra_args": "--a 1"})) == {} + assert controls_of(normalize_proposal({"remove_args": ["--x"], "args_mode": "replace"})) == { + "remove_args": ["--x"], + "args_mode": "replace", + } + + +@pytest.mark.parametrize( + "value,expected", + [ + (["--max-num-seqs", "128"], "--max-num-seqs 128"), + (("--a", "1"), "--a 1"), + (["--a", "", " ", "1"], "--a 1"), + ("--a 1", "--a 1"), + (None, ""), + ], +) +def test_list_form_args_are_space_joined_not_repr(value, expected): + assert normalize_proposal({"extra_args": value})["extra_args"] == expected + + +def test_the_queue_and_the_grid_parser_agree_on_a_list_form_variant(): + from hyperloom.orchestrator.actions.executors.explore import _grid_variants_from_payload + + payload = {"name": "v", "extra_args": ["--max-num-seqs", "128"], "extra_envs": {"E": "1"}} + parsed = _grid_variants_from_payload([payload])[0] + fields = normalize_proposal(payload) + assert fields["extra_args"] == parsed.extra_server_args + assert effective_fingerprint( + fields["extra_args"], fields["extra_envs"], controls=controls_of(fields) + ) == effective_fingerprint(parsed.extra_server_args, parsed.extra_envs, controls={}) diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py index 3905170ee..1635d7c91 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py @@ -19,7 +19,6 @@ from hyperloom.inference_optimizer.protocol.intent import ( Intent, - IntentType, ) from hyperloom.orchestrator.policy.gate import SPECIALIST_FROM_AGENT_PREFIX diff --git a/src/hyperloom/inference_optimizer/tests/test_ssh_client.py b/src/hyperloom/inference_optimizer/tests/test_ssh_client.py index 41f385ce3..f6be0d756 100644 --- a/src/hyperloom/inference_optimizer/tests/test_ssh_client.py +++ b/src/hyperloom/inference_optimizer/tests/test_ssh_client.py @@ -12,7 +12,6 @@ from __future__ import annotations import base64 -import os import shlex import pytest diff --git a/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py b/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py new file mode 100644 index 000000000..50e4bd404 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The untested-proposal queue renderer and its injection into the prompt.""" + +from __future__ import annotations + +import pytest + +from hyperloom.orchestrator.actions.executors._proposal_identity import ( + controls_of, + effective_fingerprint, + normalize_proposal, +) +from hyperloom.orchestrator.phases.machine_state import ( + PHASE_CLOSE, + PHASE_EXPLORE, + PHASE_FRAMEWORK_AGENT, + PHASE_KERNEL_AGENT, + PHASE_PRELUDE, + PHASE_SWEEP, +) +from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan +from hyperloom.orchestrator.state.shared_state import SharedState +from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + +@pytest.fixture +def coord(session_dir) -> Coordinator: + plan = ScriptedPlan( + turns=[], + default_intent=Intent(type=IntentType.SEND_MESSAGE, payload={"topic": "heartbeat", "body_md": "ok"}), + ) + backends: dict[str, Backend] = { + name: MockBackend(plan, name=name) for name in ("orchestration", "critic", "robustness") + } + return Coordinator(session_dir, backends=backends) + + +def _state(rounds, *, cycle: int = 0, tested=None, gaps=None, current_best=None) -> SharedState: + state = SharedState() + state.macro_cycle = cycle + state.specialist_rounds = list(rounds) + state.explore_search = {"tested": dict(tested or {})} + state.gaps = list(gaps or []) + state.current_best = dict(current_best or {}) + return state + + +def _ledger_row(proposal) -> dict: + """A tested row as the executor writes it: variant-own fields, server-args key.""" + fields = normalize_proposal(proposal) + return { + "extra_server_args": fields["extra_args"], + "extra_envs": fields["extra_envs"], + **controls_of(fields), + } + + +def _fingerprint(proposal, **base) -> str: + fields = normalize_proposal(proposal) + return effective_fingerprint(fields["extra_args"], fields["extra_envs"], controls=controls_of(fields), **base) + + +def _round(proposals, *, cycle: int = 0, domain: str = "serving_specialist", gap: str = "") -> dict: + return { + "cycle": cycle, + "domain": domain, + "gap_canonical_id": gap, + "proposal_set": list(proposals), + } + + +def test_empty_queue_renders_nothing(): + assert _state([]).to_untested_proposals_summary() == "" + assert _state([_round([{"name": "research-only", "reason": "read it"}])]).to_untested_proposals_summary() == "" + + +def test_only_the_current_cycle_is_rendered(): + state = _state( + [ + _round([{"name": "old", "extra_args": "--old"}], cycle=0), + _round([{"name": "new", "extra_args": "--new"}], cycle=1), + ], + cycle=1, + ) + out = state.to_untested_proposals_summary() + assert "new" in out + assert "old" not in out + + +def test_a_round_missing_its_cycle_field_reads_as_zero(): + entry = _round([{"name": "legacy", "extra_args": "--legacy"}]) + entry.pop("cycle") + assert "legacy" in _state([entry], cycle=0).to_untested_proposals_summary() + assert _state([entry], cycle=1).to_untested_proposals_summary() == "" + + +def test_benched_proposals_are_dropped(): + proposal = {"name": "benched", "extra_args": "--a 1"} + tested = {"whatever-key": dict(_ledger_row(proposal), outcome="REVERT")} + assert _state([_round([proposal])], tested=tested).to_untested_proposals_summary() == "" + + +def test_a_keep_that_changes_the_stack_does_not_resurrect_benched_proposals(): + """The ledger key is stack-relative; matching on it would miss after a KEEP.""" + proposal = {"name": "with-removal", "extra_args": "--a 1", "remove_args": ["--v"]} + round_start = _fingerprint(proposal) + after_keep = _fingerprint(proposal, base_remove_args=["--b"], base_args_mode="replace") + assert round_start != after_keep + + state = _state( + [_round([proposal])], + tested={round_start: dict(_ledger_row(proposal), outcome="REVERT")}, + current_best={"remove_args": ["--b"], "args_mode": "replace"}, + ) + assert state.to_untested_proposals_summary() == "" + + +def test_a_nameless_proposal_gets_a_stable_name_the_grid_parser_accepts(): + from hyperloom.orchestrator.actions.executors.explore import _grid_variants_from_payload + + entry = _round([{"extra_args": "--z"}], domain="comm_specialist") + entry["task_id"] = "deadbeef99" + line = _state([entry]).to_untested_proposals_summary() + assert "comm-deadbeef-0" in line + assert _grid_variants_from_payload([{"name": "comm-deadbeef-0", "extra_args": "--z"}]) + + +def test_a_non_numeric_cycle_does_not_break_prompt_assembly(): + entry = _round([{"name": "v", "extra_args": "--a"}]) + entry["cycle"] = "bad" + assert "v" in _state([entry], cycle=0).to_untested_proposals_summary() + + +def test_duplicate_fingerprints_collapse_across_rounds(): + proposal = {"name": "dup", "extra_args": "--a 1"} + out = _state([_round([proposal]), _round([dict(proposal, name="dup-again")])]).to_untested_proposals_summary() + assert out.count("•") == 1 + + +def test_every_control_field_reaches_the_line(): + proposal = { + "name": "coupled", + "extra_args": "--a 1", + "extra_envs": {"E": "2"}, + "remove_args": ["--drop"], + "unset_envs": ["DROP_ENV"], + "args_mode": "replace", + "atomic": True, + "reason": "splitting it OOMs", + } + line = _state([_round([proposal])]).to_untested_proposals_summary() + assert "ATOMIC" in line + assert "+args=--a 1" in line + assert "+envs=E=2" in line + assert "-args=--drop" in line + assert "-envs=DROP_ENV" in line + assert "mode=replace" in line + assert "why=splitting it OOMs" in line + + +def test_a_removal_only_proposal_does_not_render_as_a_no_op(): + proposal = {"name": "drop-prefix-caching", "remove_args": ["--enable-prefix-caching"]} + line = _state([_round([proposal])]).to_untested_proposals_summary() + assert "-args=--enable-prefix-caching" in line + assert "+args=" not in line + + +def test_ranking_is_gap_severity_then_recency(): + state = _state( + [ + _round([{"name": "low-old", "extra_args": "--1"}], gap="gap.low"), + _round([{"name": "high", "extra_args": "--2"}], gap="gap.high"), + _round([{"name": "low-new", "extra_args": "--3"}], gap="gap.low"), + ], + gaps=[ + {"canonical_id": "gap.high", "severity": "high"}, + {"canonical_id": "gap.low", "severity": "low"}, + ], + ) + names = [line.split()[1] for line in state.to_untested_proposals_summary().splitlines() if line.startswith("•")] + assert names == ["high", "low-new", "low-old"] + + +def test_a_pruned_gap_sorts_last_but_is_not_dropped(): + state = _state( + [ + _round([{"name": "orphan", "extra_args": "--1"}], gap="gap.gone"), + _round([{"name": "known", "extra_args": "--2"}], gap="gap.low"), + ], + gaps=[{"canonical_id": "gap.low", "severity": "low"}], + ) + out = state.to_untested_proposals_summary() + names = [line.split()[1] for line in out.splitlines() if line.startswith("•")] + assert names == ["known", "orphan"] + assert "sev?" in out + + +def test_overflow_is_truncated_and_counted(): + proposals = [{"name": f"v{i}", "extra_args": f"--flag {i}"} for i in range(20)] + out = _state([_round(proposals)]).to_untested_proposals_summary(max_entries=12) + assert out.count("•") == 12 + assert "(+8 more not shown)" in out + + +@pytest.mark.parametrize( + "phase,expected", + [ + (PHASE_PRELUDE, False), + (PHASE_FRAMEWORK_AGENT, False), + (PHASE_EXPLORE, True), + (PHASE_KERNEL_AGENT, False), + (PHASE_SWEEP, False), + (PHASE_CLOSE, False), + ], +) +@pytest.mark.asyncio +async def test_the_block_is_injected_only_in_explore(coord, phase, expected): + coord.shared_state.phase = phase + coord.shared_state.macro_cycle = 0 + coord.shared_state.specialist_rounds = [_round([{"name": "queued", "extra_args": "--a 1"}])] + out = await coord._compose_prompt("orchestration") + assert ("=== Untested proposals (current cycle) ===" in out) is expected + + +@pytest.mark.asyncio +async def test_the_block_survives_a_delta_turn(coord, monkeypatch): + coord.shared_state.phase = PHASE_EXPLORE + coord.shared_state.macro_cycle = 0 + coord.shared_state.specialist_rounds = [_round([{"name": "queued", "extra_args": "--a 1"}])] + seed = await coord._compose_prompt("orchestration") + + monkeypatch.setattr(type(coord.conversation), "_orchestration_conversational", lambda self: True) + coord._orchestration_seeded = True + delta = await coord._compose_prompt("orchestration") + + assert "=== Shared session state ===" in seed + assert "=== Shared session state ===" not in delta + assert "queued" in seed + assert "queued" in delta diff --git a/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py b/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py new file mode 100644 index 000000000..97e27b4fa --- /dev/null +++ b/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""One identity for a specialist proposal and the explore variant it becomes. + +The proposal is a mapping keyed ``extra_args``; the variant is a ``GridVariant`` +keyed ``extra_server_args``. Both fingerprint into ``explore_search["tested"]`` +under the variant's own args and envs folded together with the union of its +removal controls and the current stack's, so the key is stack-relative and a +proposal hashed on its own would not match the ledger. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from hyperloom.common.coerce import to_str_list + +from ._canonical_fingerprint import canonical_fingerprint + + +__all__ = [ + "coerce_args", + "controls_of", + "effective_fingerprint", + "is_executable", + "normalize_proposal", +] + + +def coerce_args(value: Any) -> str: + """Coerce a payload ``extra_args`` / ``extra_server_args`` value to a shell-arg string. + + The LLM sometimes emits the flags as a JSON list; ``str(list)`` would yield + a Python repr the server rejects, so lists are space-joined into tokens. + + Args: + value: The raw payload value (string, list/tuple, or ``None``). + + Returns: + The coerced shell-arg string. + """ + if isinstance(value, (list, tuple)): + return " ".join(str(v).strip() for v in value if str(v).strip()) + return str(value or "").strip() + + +def _args_mode_of(value: Any) -> str: + """Coerce an args-mode to ``"replace"`` or ``"append"``.""" + return "replace" if str(value or "").strip().lower() == "replace" else "append" + + +def normalize_proposal(proposal: Mapping[str, Any]) -> dict[str, Any]: + """Project a ``proposal_set`` entry onto the variant field set. + + Args: + proposal: One ``specialist_done.proposal_set`` entry. + + Returns: + ``name`` / ``extra_args`` / ``extra_envs`` / ``remove_args`` / + ``unset_envs`` / ``args_mode`` / ``atomic`` / ``reason``, with the + ``extra_args`` / ``extra_server_args`` alias resolved. + """ + envs = proposal.get("extra_envs") + return { + "name": str(proposal.get("name") or "").strip(), + "extra_args": coerce_args(proposal.get("extra_args") or proposal.get("extra_server_args")), + "extra_envs": {str(k): str(v) for k, v in envs.items()} if isinstance(envs, Mapping) else {}, + "remove_args": to_str_list(proposal.get("remove_args")), + "unset_envs": to_str_list(proposal.get("unset_envs")), + "args_mode": _args_mode_of(proposal.get("args_mode")), + "atomic": bool(proposal.get("atomic")), + "reason": str(proposal.get("reason") or "").strip(), + } + + +def is_executable(fields: Mapping[str, Any]) -> bool: + """Whether a server restart could apply these fields. + + A removal-only entry qualifies; a research-only entry does not. + + Args: + fields: A :func:`normalize_proposal` result. + + Returns: + ``True`` when the entry carries args, envs, or a removal/replacement + control. + """ + return bool( + fields["extra_args"] + or fields["extra_envs"] + or fields["remove_args"] + or fields["unset_envs"] + or fields["args_mode"] == "replace" + ) + + +def controls_of(fields: Mapping[str, Any]) -> dict[str, Any]: + """Return only the non-default removal/replacement controls. + + Args: + fields: A :func:`normalize_proposal` result. + + Returns: + The controls that differ from the default; all-default yields ``{}`` so + a plain variant fingerprints unchanged. + """ + out: dict[str, Any] = {} + if fields["remove_args"]: + out["remove_args"] = list(fields["remove_args"]) + if fields["unset_envs"]: + out["unset_envs"] = list(fields["unset_envs"]) + if fields["args_mode"] == "replace": + out["args_mode"] = "replace" + return out + + +def effective_fingerprint( + extra_args: Any, + extra_envs: Any, + *, + controls: Mapping[str, Any] | None = None, + base_remove_args: Any = None, + base_unset_envs: Any = None, + base_args_mode: Any = None, +) -> str: + """Fingerprint a variant against the stack it will be launched on. + + Removals union base-first with order preserved; a ``replace`` base + args-mode wins over the variant's, since the base is what it launches on. + + Args: + extra_args: The variant's own server-args string. + extra_envs: The variant's own env mapping. + controls: The variant's own non-default controls. + base_remove_args: ``base_remove_args`` from the current stack. + base_unset_envs: ``base_unset_envs`` from the current stack. + base_args_mode: ``base_args_mode`` from the current stack. + + Returns: + The 16-char fingerprint ``explore_search["tested"]`` is keyed on. + """ + identity = dict(controls or {}) + remove_args = list(dict.fromkeys(to_str_list(base_remove_args) + to_str_list(identity.get("remove_args")))) + unset_envs = list(dict.fromkeys(to_str_list(base_unset_envs) + to_str_list(identity.get("unset_envs")))) + if remove_args: + identity["remove_args"] = remove_args + if unset_envs: + identity["unset_envs"] = unset_envs + if _args_mode_of(base_args_mode) == "replace": + identity["args_mode"] = "replace" + return canonical_fingerprint(extra_args, extra_envs, **identity) diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 1f2b08ff7..e58a21237 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -59,7 +59,8 @@ parse_eval_results, ) from . import _framework_switch_manifest as _switch_manifest -from ._canonical_fingerprint import canonical_fingerprint, workload_signature +from ._canonical_fingerprint import workload_signature +from ._proposal_identity import effective_fingerprint, normalize_proposal from ._grid_runner import ( _MN_BACKENDS_PRIORITY, _MN_PARAMS_PRIORITY, @@ -123,27 +124,6 @@ def _initial_explore_search_state() -> dict[str, Any]: } -def _coerce_args_str(value: Any) -> str: - """Coerce a payload ``extra_args`` / ``extra_server_args`` value into a - shell-arg string. - - The LLM sometimes emits the server flags as a JSON list instead of a single - string; a naive ``str(list)`` yields the Python repr which the server - rejects. Lists/tuples are space-joined into individual tokens. - - Args: - value: The raw payload value (string, list/tuple, or None). - - Returns: - The coerced shell-arg string ("" when ``value`` is None). - """ - if value is None: - return "" - if isinstance(value, (list, tuple)): - return " ".join(str(v).strip() for v in value if str(v).strip()) - return str(value) - - # Audit/provenance metadata stashed on a GridVariant that must survive being # rebuilt into a derived variant. _CARRIED_VARIANT_ATTRS: tuple[str, ...] = ( @@ -173,7 +153,7 @@ def _carry_variant_metadata(src: Any, dst: Any) -> Any: def _variant_control_fields(variant: Any) -> dict[str, Any]: - """Return non-default remove/unset/replace controls for ledger rows.""" + """Return non-default remove/unset/replace controls for identity and ledger rows.""" remove_args = to_str_list(getattr(variant, "remove_args", [])) unset_envs = to_str_list(getattr(variant, "unset_envs", [])) args_mode = str(getattr(variant, "args_mode", "append") or "append").strip().lower() @@ -220,17 +200,15 @@ def _grid_variants_from_payload(payload: list[Any]) -> list[GridVariant]: for raw in payload or []: if not isinstance(raw, dict) or not raw.get("name"): continue - args = _coerce_args_str(raw.get("extra_args") or raw.get("extra_server_args") or "").strip() - envs_raw = raw.get("extra_envs") or {} - envs = {str(k): str(v) for k, v in envs_raw.items()} if isinstance(envs_raw, dict) else {} + fields = normalize_proposal(raw) gv = GridVariant( - name=str(raw["name"]), - extra_server_args=args, - extra_envs=envs, + name=fields["name"], + extra_server_args=fields["extra_args"], + extra_envs=fields["extra_envs"], note=str(raw.get("note") or raw.get("provenance") or ""), - remove_args=to_str_list(raw.get("remove_args")), - unset_envs=to_str_list(raw.get("unset_envs")), - args_mode=str(raw.get("args_mode") or "append"), + remove_args=fields["remove_args"], + unset_envs=fields["unset_envs"], + args_mode=fields["args_mode"], ) # Stash extra metadata on the GridVariant so the ledger writer can # pull provenance/evidence. @@ -994,23 +972,13 @@ async def __call__(self, ctx) -> dict[str, Any]: unique_in_round: dict[str, GridVariant] = {} skipped_dup: list[dict[str, Any]] = [] for gv in grid: - identity_controls = _variant_control_fields(gv) - identity_remove_args = list( - dict.fromkeys(base_remove_args + to_str_list(identity_controls.get("remove_args"))) - ) - identity_unset_envs = list( - dict.fromkeys(base_unset_envs + to_str_list(identity_controls.get("unset_envs"))) - ) - if identity_remove_args: - identity_controls["remove_args"] = identity_remove_args - if identity_unset_envs: - identity_controls["unset_envs"] = identity_unset_envs - if base_args_mode == "replace": - identity_controls["args_mode"] = "replace" - fp = canonical_fingerprint( + fp = effective_fingerprint( gv.extra_server_args, gv.extra_envs, - **identity_controls, + controls=_variant_control_fields(gv), + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + base_args_mode=base_args_mode, ) gv.canonical_fp = fp # type: ignore[attr-defined] if fp in unique_in_round: diff --git a/src/hyperloom/orchestrator/framework/build_actions.py b/src/hyperloom/orchestrator/framework/build_actions.py index 30063a36e..dd2c86d23 100644 --- a/src/hyperloom/orchestrator/framework/build_actions.py +++ b/src/hyperloom/orchestrator/framework/build_actions.py @@ -17,6 +17,7 @@ from __future__ import annotations +import re as _re from dataclasses import dataclass, field from typing import Any, Literal, Mapping @@ -252,8 +253,6 @@ def build_novelty_key( ) -import re as _re - _GITHUB_PR_RE = _re.compile(r"https?://github\.com/([^/]+/[^/]+)/pull/(\d+)", _re.IGNORECASE) _PR_REF_RE = _re.compile(r"^PR:(\d+)$") # An issue is a discussion thread, not a branch: GitHub publishes diff --git a/src/hyperloom/orchestrator/framework/targeted_build.py b/src/hyperloom/orchestrator/framework/targeted_build.py index baa0e7b5b..40528a6ad 100644 --- a/src/hyperloom/orchestrator/framework/targeted_build.py +++ b/src/hyperloom/orchestrator/framework/targeted_build.py @@ -337,9 +337,6 @@ def run_aiter_build( so the coordinator tick loop is never blocked). All subprocess calls go through the injectable ``run`` shim for testability. """ - import json - import shutil - import subprocess as _subprocess import time as _time from .build_utils import ( @@ -1058,7 +1055,6 @@ def _driver_main(argv: list[str] | None = None) -> int: """Driver subprocess entry: load plan.json, call run_aiter_build, write result.json.""" import argparse import json - import sys as _sys parser = argparse.ArgumentParser(description="Off-loop targeted-build driver") parser.add_argument("--attempt-root", required=True, help="Attempt directory") diff --git a/src/hyperloom/orchestrator/loop/conversation.py b/src/hyperloom/orchestrator/loop/conversation.py index e50947134..043bb158c 100644 --- a/src/hyperloom/orchestrator/loop/conversation.py +++ b/src/hyperloom/orchestrator/loop/conversation.py @@ -547,6 +547,12 @@ async def _compose_prompt(self, agent_name: str, *, system_prompt: str | None = denial_summary = self.shared_state.to_policy_denial_summary(top_k=6) if denial_summary: sections.append(denial_summary) + # Outside the SEED gate: a queue seen once is the amnesia it fixes. + if (self.shared_state.phase or "").strip().upper() == _phase_state.PHASE_EXPLORE: + untested_block = self.shared_state.to_untested_proposals_summary() + if untested_block: + sections.append("=== Untested proposals (current cycle) ===") + sections.append(untested_block) # Recipe KB T0 warm-start snapshot + structured gaps[] ledger. if agent_name == "orchestration" and push_full: @@ -1184,8 +1190,12 @@ def _recent_proposed_variants( return out def _research_scout_seed_block(self) -> str: - """Render all persisted research-scout findings for an Orchestration SEED.""" - from ..actions.executors._canonical_fingerprint import canonical_fingerprint + """Render the persisted research-scout findings for an Orchestration SEED. + + The scout's executable proposals are not rendered here: they go through + ``=== Untested proposals (current cycle) ===`` alongside every other + domain's, which also drops the ones already benched. + """ from ..knowledge import research_hints as _research_hints hints = _research_hints.load_hints(self.session_dir) @@ -1203,38 +1213,15 @@ def _research_scout_seed_block(self) -> str: for hint in hints: lines.append(json.dumps(hint, sort_keys=True)) - proposals: list[dict[str, Any]] = [] - proposal_names: set[str] = set() - proposal_fingerprints: set[str] = set() questions: list[str] = [] seen_questions: set[str] = set() for row in rounds: - for proposal in row.get("proposal_set") or []: - if not isinstance(proposal, dict): - continue - name = str(proposal.get("name") or "").strip() - fingerprint = canonical_fingerprint( - str(proposal.get("extra_args") or proposal.get("extra_server_args") or ""), - proposal.get("extra_envs") if isinstance(proposal.get("extra_envs"), dict) else {}, - remove_args=proposal.get("remove_args"), - unset_envs=proposal.get("unset_envs"), - args_mode=str(proposal.get("args_mode") or "append"), - ) - if (name and name in proposal_names) or fingerprint in proposal_fingerprints: - continue - if name: - proposal_names.add(name) - proposal_fingerprints.add(fingerprint) - proposals.append(proposal) for question in row.get("residual_questions") or []: text = str(question).strip() if text and text not in seen_questions: seen_questions.add(text) questions.append(text) - if proposals: - lines.append("Untested executable proposals:") - lines.extend(json.dumps(proposal, sort_keys=True) for proposal in proposals) if questions: lines.append("Residual questions:") lines.extend(f"- {question}" for question in questions) diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 9223d7adf..1221462e3 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -1438,6 +1438,7 @@ async def _maybe_materialize_mn_explore( if bool((getattr(task, "params", None) or {}).get("framework_config_generation")): return from ..actions.executors._multi_node_env import is_multi_node + from ..actions.executors._proposal_identity import controls_of, is_executable, normalize_proposal if not is_multi_node() or not proposals: return @@ -1445,37 +1446,17 @@ async def _maybe_materialize_mn_explore( for i, p in enumerate(proposals[: self._MN_AUTO_EXPLORE_GRID_CAP]): if not isinstance(p, dict): continue - args = str(p.get("extra_args") or p.get("extra_server_args") or "").strip() - envs_raw = p.get("extra_envs") - envs = {str(k): str(v) for k, v in envs_raw.items()} if isinstance(envs_raw, dict) else {} - controls: dict[str, Any] = {} - for key in ("remove_args", "unset_envs"): - raw = p.get(key) - if isinstance(raw, str): - vals = [raw.strip()] if raw.strip() else [] - elif isinstance(raw, (list, tuple, set)): - vals = [str(v).strip() for v in raw if str(v).strip()] - else: - vals = [] - if vals: - controls[key] = vals - mode = str(p.get("args_mode") or "append").strip().lower() - if mode == "replace": - controls["args_mode"] = "replace" - # Drop entries with neither a server-arg nor an env override — - # nothing for the restart to apply (e.g. research-only items) - # unless the entry removes inherited args/envs. - if not args and not envs and not controls: + fields = normalize_proposal(p) + if not is_executable(fields): continue - name = str(p.get("name") or "").strip() or (f"{domain or 'specialist'}-{task.task_id[:8]}-{i}") grid.append( { - "name": name, - "extra_args": args, - "extra_envs": envs, - **controls, + "name": fields["name"] or f"{domain or 'specialist'}-{task.task_id[:8]}-{i}", + "extra_args": fields["extra_args"], + "extra_envs": fields["extra_envs"], + **controls_of(fields), "provenance": f"specialist:{domain}" if domain else "specialist", - "note": str(p.get("reason") or "")[:200], + "note": fields["reason"][:200], } ) if not grid: diff --git a/src/hyperloom/orchestrator/prompts/orchestration.md b/src/hyperloom/orchestrator/prompts/orchestration.md index 822089099..eaf34669b 100644 --- a/src/hyperloom/orchestrator/prompts/orchestration.md +++ b/src/hyperloom/orchestrator/prompts/orchestration.md @@ -185,9 +185,10 @@ phases' goals are omitted because you cannot act on them from here. **Decision priority**: pick the next action by reading facts in this order: (a) current phase + `allowed_actions`, (b) gaps / KB sub-graph / recent -winners / specialist proposal_set, (c) mandatory ordering (baseline first; -`explore` revalidates the stack inline — no separate rebench step), -(d) `phase_budget_remaining_pct` as the urgency signal. +winners / `=== Untested proposals (current cycle) ===`, (c) mandatory +ordering (baseline first; `explore` revalidates the stack inline — no +separate rebench step), (d) `phase_budget_remaining_pct` as the urgency +signal. ### PRELUDE — phase goal @@ -213,6 +214,16 @@ provide KB/PR/source evidence for `explore` grids and may produce patches for `integrate_patch`. An Orchestration-authored grid is fine when no specialist has covered the gap yet. +**Where a grid comes from.** `=== Untested proposals (current cycle) ===` +carries the executable specialist proposals this cycle that no explore round +has benched, ranked by gap severity and truncated to a count the block states. +Draw from it first and copy an entry's fields verbatim — an entry marked +ATOMIC is a coupled set that must go in as one variant, never split or +re-authored. Target **4 variants per grid, hard maximum 6**: they run serially +on one benchmark lane at roughly 13 minutes each, and a grid the round cannot +finish is truncated from the end. Top up from the idea-generation moves only +after the queue holds nothing else worth running. + **GPU specialists** hold the same cards as the serving stack and acquire `gpu_research_lane` (mutually exclusive with benchmark/profile/serving lanes). Use them opportunistically in the idle research window — while diff --git a/src/hyperloom/orchestrator/prompts/prompt_builder.py b/src/hyperloom/orchestrator/prompts/prompt_builder.py index b5e47cd18..15b7c21a9 100644 --- a/src/hyperloom/orchestrator/prompts/prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/prompt_builder.py @@ -464,7 +464,13 @@ def _format_grid_injection_hint(name: str) -> str | None: "Use remove_args/unset_envs to ablate harmful base flags; " "args_mode='replace' to drop inherited server args. " "provenance values: 'llm_direct', 'default_grid', " - "'specialist:' (audit/advisory, not a gate)." + "'specialist:' (audit/advisory, not a gate). " + "SIZE: target 4 variants, hard maximum 6. Variants run serially " + "on a single benchmark lane at ~13min each, so a 4-variant round " + "is about an hour of GPU. Submit a 5th or 6th only when it still " + "beats the median of the four you already have; a grid the round " + "cannot finish is truncated from the end, dropping whatever you " + "ranked last rather than whatever is worth least." ) if name == "sweep": return ( @@ -589,8 +595,11 @@ def _section_decision_framework(*, kernel_enabled: bool, phase: str = "", transp " `last_action_failures` + `explore_search.winners_history`.", " c. **KB sub-graphs + warm-start recipe** when present —", " cross-session priors carry " + "*qualitative* hints (what worked / what failed last time).", - " d. **specialist proposal_set** — when an explore round just", - " finished, the proposal_set drives the next `explore` grid.", + " d. **`=== Untested proposals (current cycle) ===`** — the", + " executable specialist proposals this cycle that no explore", + " round has benched, ranked by gap severity and truncated to", + " a count the block states. This is the grid's primary", + " source; an entry marked ATOMIC goes in verbatim.", " e. **Ordering facts**: baseline runs before anything else", " (invariant). ``analysis.md`` / ``last_profile_trace`` arrive", " automatically from the Coordinator-owned analysis task at", @@ -711,6 +720,10 @@ def _idea_generation_lines() -> list[str]: "`extra_server_args` is framework-neutral (routed to EXTRA_SGLANG_ARGS", "/ EXTRA_VLLM_ARGS / EXTRA_ATOM_ARGS by `--framework`).", "", + "Draw first from `=== Untested proposals (current cycle) ===`; the", + "five moves above are for topping the grid up to its target of 4", + "(hard maximum 6) once the queue is drained of anything worth running.", + "", "An explore round that produces zero new ideas is a bug — heartbeat", "with body_md='idea-pipeline-empty' so Robustness can intervene.", ] diff --git a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py index 805440de0..eef69f7af 100644 --- a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py @@ -1029,9 +1029,10 @@ def _section_identity(inp: SpecialistPromptInputs) -> list[str]: "source roots (Section 7), search any public GitHub repo or NVIDIA PR,", capability_line, "to be thorough. Be creative. Investigate deeply. One-turn shortcuts", - "are discouraged when a real bottleneck is on the table. Quality is", - "scored over quantity: cap your final ``proposal_set`` at the", - "**top-6** ranked picks (see Section 8).", + "are discouraged when a real bottleneck is on the table — but stop once", + "rounds stop yielding new findings; the wall clock is not the only stop", + "signal. Quality over quantity: **2 proposals is the norm, 4 the hard", + "cap**. One real beats two padded; ``empty=true`` beats one padded.", "", "Division of labour: the Coordinator owns the serving GPU, runs the E2E", "benchmark, and decides KEEP/REVERT — you do not have to validate final", @@ -1447,9 +1448,8 @@ def _section_gap(inp: SpecialistPromptInputs) -> list[str]: # Section 4 — optional KB context def _is_cold_start(inp: SpecialistPromptInputs) -> bool: - """Return True when every prior KB/PR/research source is empty, so a - cold-start directive is injected instead of letting specialists return - an empty proposal_set. + """Return True when every prior KB/PR/research source is empty, so the + cold-start directive is injected in place of the KB block. Args: inp: The specialist prompt inputs. @@ -1495,7 +1495,7 @@ def _section_kb_subgraph(inp: SpecialistPromptInputs) -> list[str]: ) return rows if cold: - # Cold-start directive: propose domain-focus defaults, not an empty set. + # Cold-start directive: fall back to the Section 1 defaults, or exit empty with a rationale. rows.extend( [ "**COLD-START MODE — no priors available.**", @@ -1508,27 +1508,22 @@ def _section_kb_subgraph(inp: SpecialistPromptInputs) -> list[str]: "- Warm-start recipe: ``(none)`` (Section 5).", "- Use ``mcp__pr_monitor__*`` tools (Section 6) to query PRs on demand.", "", - "**Directive — DO NOT return an empty proposal_set.** " - + "Treat the *Winning techniques* + *Pitfalls* in your " - + "**domain focus** block (Section 1) as your fallback " - + "prior. Pick the **1–2 most conservative, " - + "well-attested defaults** from those bullets that are " - + "compatible with the hardware (Section 2) and the " - + "gap symptom (Section 3); flag each with " - + "``provenance: domain_focus_default`` in the proposal " - + "and say it is an unvalidated fallback prior in the " - + "proposal's ``reason``. Do NOT add a ``confidence`` " - + "field: self-reported confidence / gain fields are " - + "stripped from your output before review. Use the " - + "``residual_questions`` field to record what RecipeKB, " - + "research, or ``mcp__pr_monitor__*`` query a future round should pursue.", - "", - "If the *Winning techniques* block is generic enough " - + "that no proposal is safer than a coin-flip, you may " - + "still emit ``empty=true`` — but you MUST cite which " - + "bullets you considered and why each was rejected " - + "(in ``summary``). A bare empty exit with no rationale " - + "will be treated as a tool failure by the Coordinator.", + "**Directive — a coin-flip proposal is worse than none.** " + + "Treat the *Winning techniques* + *Pitfalls* bullets in " + + "Section 1 as your fallback prior and take the **1–2 " + + "most conservative, well-attested defaults** that fit " + + "the hardware (Section 2) and the gap symptom " + + "(Section 3); flag each ``provenance: " + + "domain_focus_default`` and call it an unvalidated " + + "fallback in the proposal's ``reason``. If none clears " + + "that bar, emit ``empty=true`` and cite in ``summary`` " + + "which you considered and why each was rejected — a " + + "bare empty exit with no rationale reads as a tool " + + "failure. Do NOT add a ``confidence`` field: " + + "self-reported confidence / gain fields are stripped " + + "before review. Record in ``residual_questions`` what " + + "RecipeKB, research, or ``mcp__pr_monitor__*`` query a " + + "future round should pursue.", ] ) else: @@ -2227,11 +2222,14 @@ def _section_output_protocol(inp: SpecialistPromptInputs) -> list[str]: "coupling across several proposals." ), ( - "- ``proposal_set`` MUST contain AT MOST **6** entries. You are a " - "curator, not a brainstormer: rank candidates by expected gain x " - "your confidence, drop everything that contradicts ``kb_subgraph`` " - "/ ``pr_evidence`` already in your prompt, and only emit the " - "surviving top 6. Fewer is better than padding." + "- ``proposal_set``: **2 entries is the norm, 4 the hard cap.** You " + "are a curator, not a brainstormer: rank by expected gain x " + "confidence, drop anything contradicting ``kb_subgraph`` / " + "``pr_evidence``, and stop at 2. A 3rd or 4th must beat the median " + "of the first two. Padding is a failure, not thoroughness: each " + "weak entry costs a Critic reject and a slot on the serial " + "benchmark queue. One real proposal is a better round than two " + "padded ones, and ``empty=true`` is better than one." ), ( "- The Critic reviews each surviving variant against the KB " diff --git a/src/hyperloom/orchestrator/state/_shared_state/render.py b/src/hyperloom/orchestrator/state/_shared_state/render.py index 2b7071b1c..7caac94f3 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/render.py +++ b/src/hyperloom/orchestrator/state/_shared_state/render.py @@ -443,6 +443,120 @@ def to_gaps_summary(self, *, max_entries: int = 10, max_attempts: int = 0) -> st rows.append(f" · (+{len(ordered) - max_entries} older gaps elided; see state.json `gaps[]`)") return "\n".join(rows) + def _untested_proposal_rows(self) -> list[dict[str, Any]]: + """Executable proposals from this cycle that no explore round has benched. + + Matched on the proposal's own content. The ledger is keyed on the + variant folded together with whatever removal controls the stack + carried at the time, so a KEEP that changes those controls mid-round + would otherwise make everything benched before it look untried. + + Returns: + Rows ranked by gap severity then recency, each carrying the + normalized proposal fields plus ``domain`` / ``severity``. + """ + from hyperloom.common.coerce import to_int + + from ...actions.executors._proposal_identity import ( + controls_of, + effective_fingerprint, + is_executable, + normalize_proposal, + ) + + def content_fingerprint(fields: dict[str, Any]) -> str: + return effective_fingerprint(fields["extra_args"], fields["extra_envs"], controls=controls_of(fields)) + + cycle = to_int(self.macro_cycle, default=0) + benched = { + content_fingerprint(normalize_proposal(row)) + for row in ((self.explore_search or {}).get("tested") or {}).values() + if isinstance(row, dict) + } + severity_of = { + str(g.get("canonical_id") or ""): str(g.get("severity") or "").strip().lower() + for g in (self.gaps or []) + if isinstance(g, dict) + } + rank = {"high": 3, "medium": 2, "low": 1} + + ranked: list[tuple[int, int, dict[str, Any]]] = [] + seen: set[str] = set() + for order, entry in enumerate(self.specialist_rounds or []): + if not isinstance(entry, dict) or to_int(entry.get("cycle"), default=0) != cycle: + continue + domain = str(entry.get("domain") or "?").removesuffix("_specialist") + severity = severity_of.get(str(entry.get("gap_canonical_id") or ""), "") + task_id = str(entry.get("task_id") or "")[:8] + for index, proposal in enumerate(entry.get("proposal_set") or []): + if not isinstance(proposal, dict): + continue + row = normalize_proposal(proposal) + if not is_executable(row): + continue + fingerprint = content_fingerprint(row) + if fingerprint in benched or fingerprint in seen: + continue + seen.add(fingerprint) + row["name"] = row["name"] or f"{domain or 'specialist'}-{task_id}-{index}" + row["domain"] = domain + row["severity"] = severity + ranked.append((rank.get(severity, 0), order, row)) + ranked.sort(key=lambda r: (-r[0], -r[1])) + return [row for _, _, row in ranked] + + @staticmethod + def _untested_proposal_line(row: dict[str, Any]) -> str: + """Render one queue row, marking each field it carries. + + Args: + row: One row from :meth:`_untested_proposal_rows`. + + Returns: + A single ``•``-prefixed line. + """ + parts = [f"• {row['name']} [{row['domain']}·{row['severity'] or 'sev?'}]"] + if row["atomic"]: + parts.append("ATOMIC") + if row["extra_args"]: + parts.append(f"+args={row['extra_args']}") + if row["extra_envs"]: + parts.append("+envs=" + ",".join(f"{k}={v}" for k, v in sorted(row["extra_envs"].items()))) + if row["remove_args"]: + parts.append("-args=" + " ".join(row["remove_args"])) + if row["unset_envs"]: + parts.append("-envs=" + ",".join(row["unset_envs"])) + if row["args_mode"] == "replace": + parts.append("mode=replace") + reason = row["reason"].replace("\n", " ").strip()[:80].rstrip() + if reason: + parts.append(f"why={reason}") + return _flatten_for_prompt(" ".join(parts)) + + def to_untested_proposals_summary(self, *, max_entries: int = 12) -> str: + """Render the specialist proposals still waiting for a benchmark slot. + + Args: + max_entries (int): Rows to render before collapsing the rest into a + count. + + Returns: + str: The rendered queue, or ``""`` when nothing is waiting. + """ + rows = self._untested_proposal_rows() + if not rows: + return "" + out = [ + "Executable specialist proposals from this cycle that no explore round has benched.", + "Ranked by gap severity, then most recent. Compose the next `explore` grid from these;", + "dispatch an ATOMIC entry verbatim as one variant — never split or re-derive its flags.", + "", + ] + out.extend(self._untested_proposal_line(row) for row in rows[:max_entries]) + if len(rows) > max_entries: + out.append(f"(+{len(rows) - max_entries} more not shown)") + return "\n".join(out) + def to_proposal_scores_summary(self, *, max_rounds: int = 2) -> str: """Render advisory multi-model proposal scores for Orchestration; no mean/sorting, rater identities anonymized. Empty when no recent round carries scores.