From 9364b45deefbc86ff163825c85b09dc6d32010cd Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 19 Aug 2026 06:44:57 +0000 Subject: [PATCH 1/9] fix(specialist): tighten the proposal-set target to 2 and the ceiling to 4 The ceiling was the only number the prompt gave, and it became the target: after the Section 1 reminder landed, no run exceeded 6 but a quarter of them returned exactly 6. "Fewer is better than padding" did not carry, because it named no alternative. State a target in both places the specialist reads before it starts working, and give it a test for going past the target rather than a preference. Every extra entry costs a Critic review and a slot on the serial benchmark queue, so say that too. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_critic_verdict_map.py | 9 ++++++--- .../prompts/specialist_prompt_builder.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index 6946d1319..add175fd6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -2073,10 +2073,13 @@ def _build_specialist_prompt_text() -> str: return system_prompt + "\n" + user_prompt -def test_specialist_prompt_renders_top_6_target(): +def test_specialist_prompt_renders_proposal_target_and_ceiling(): text = _build_specialist_prompt_text() - assert "AT MOST **6** entries" in text - assert "top-6" in text + # Section 8 states both numbers; Section 1 repeats them so the target is + # visible before the specialist starts working, not only at exit time. + assert "**target 4 entries, hard maximum 6.**" in text + assert "**aim for 4** ranked proposals" in text + assert "**6 is a hard ceiling, never a target**" in text assert "reviews each surviving variant" in text diff --git a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py index 805440de0..cabe978b4 100644 --- a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py @@ -1030,8 +1030,8 @@ def _section_identity(inp: SpecialistPromptInputs) -> list[str]: capability_line, "to be thorough. Be creative. Investigate deeply. One-turn shortcuts", "are discouraged when a real bottleneck is on the table. Quality is", - "scored over quantity: cap your final ``proposal_set`` at the", - "**top-6** ranked picks (see Section 8).", + "scored over quantity: **aim for 4** ranked proposals in your final", + "``proposal_set``. **6 is a hard ceiling, never a target** (Section 8).", "", "Division of labour: the Coordinator owns the serving GPU, runs the E2E", "benchmark, and decides KEEP/REVERT — you do not have to validate final", @@ -2227,11 +2227,14 @@ def _section_output_protocol(inp: SpecialistPromptInputs) -> list[str]: "coupling across several proposals." ), ( - "- ``proposal_set`` MUST contain AT MOST **6** entries. You are a " - "curator, not a brainstormer: rank candidates by expected gain x " + "- ``proposal_set``: **target 4 entries, hard maximum 6.** You are " + "a curator, not a brainstormer: rank candidates by expected gain x " "your confidence, drop everything that contradicts ``kb_subgraph`` " - "/ ``pr_evidence`` already in your prompt, and only emit the " - "surviving top 6. Fewer is better than padding." + "/ ``pr_evidence`` already in your prompt, and cut at 4. Emit a 5th " + "or 6th ONLY if it still beats the median of the four you already " + "have. Filling the ceiling is a failure, not thoroughness: every " + "weak entry costs a Critic reject and a slot on the serial " + "benchmark queue." ), ( "- The Critic reviews each surviving variant against the KB " From b3c4e96bdff925ab87ef40885177448ed8c4bd74 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 19 Aug 2026 06:58:24 +0000 Subject: [PATCH 2/9] feat(state): surface the untested specialist proposals as a ranked queue A specialist's config proposals reached the grid author exactly once, inside a delegated_result the inbox renders as a single status line, and the inbox tail is lossy. Only the research scout's proposals were ever rendered in full, and they are the only ones that got benched at any rate: 45% against 4% for everything else, while converting at a lower KEEP rate once benched. The selection was following the rendering channel, not the evidence. Derive the queue instead of storing it. specialist_rounds already persists the proposals and explore_search already records what was benched, so the block is a projection of two ledgers that were both already there. Identity is the subtle half. The executor keys explore_search on the variant's args and envs folded together with the union of its own removal controls and the stack's, so a proposal carrying remove_args hashes differently once the stack contributes its own. Hashing the proposal alone would silently never match the ledger, and one in six ledger rows carries such a control. Both sides now compute it through one helper, with the executor's pre-existing values pinned in a test so the shared path cannot re-key a resumed session. The line carries remove_args, unset_envs, args_mode and atomic rather than just args and envs: a removal-only proposal would otherwise render as two empty fields and read as a no-op, and two in five proposals are atomic, which the grid author is required to dispatch without re-deriving. Rendered on every turn rather than only on a seed, since a queue seen once is the amnesia this replaces, but only in EXPLORE, the sole phase where explore is proposable. Scoped to the current macro-cycle. The research-scout block gives up its proposal half to avoid rendering them twice. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_coordinator_async_batch2_unit.py | 9 +- .../tests/test_proposal_identity_unit.py | 122 ++++++++++ .../test_untested_proposal_queue_unit.py | 217 ++++++++++++++++++ .../actions/executors/_proposal_identity.py | 135 +++++++++++ .../orchestrator/actions/executors/explore.py | 25 +- .../orchestrator/loop/conversation.py | 37 +-- src/hyperloom/orchestrator/phases/explore.py | 35 +-- .../state/_shared_state/render.py | 106 +++++++++ 8 files changed, 614 insertions(+), 72 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py create mode 100644 src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py create mode 100644 src/hyperloom/orchestrator/actions/executors/_proposal_identity.py diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 407b497fc..bcda8bc8e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -1288,7 +1288,7 @@ async def test_compose_prompt_orchestration_all_advisory_blocks( assert token in out -def test_research_scout_seed_block_keeps_all_rounds(coord: Coordinator) -> None: +def test_research_scout_seed_block_keeps_findings_and_questions_only(coord: Coordinator) -> None: from hyperloom.orchestrator.knowledge import research_hints research_hints.append_hints( @@ -1332,11 +1332,14 @@ def test_research_scout_seed_block_keeps_all_rounds(coord: Coordinator) -> None: assert "hint one" in block assert "hint two" in block - assert '"name": "first"' in block - assert '"name": "second"' in block assert "question one" in block assert "question two" in block assert "ignore-me" not in block + # Proposals moved to the shared untested-proposal queue, which also drops + # the ones already benched; rendering them here as well would double them. + assert "Untested executable proposals" not in block + assert '"name": "first"' not in block + assert '"name": "second"' not in block @pytest.mark.asyncio diff --git a/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py b/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py new file mode 100644 index 000000000..179f244cc --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Shared proposal/variant identity. + +The fingerprints below were captured from the explore executor's own identity +block before it was refactored onto this helper. They are pinned rather than +recomputed: a change here re-keys ``explore_search["tested"]``, so every +resumed session would re-bench its whole history. +""" + +from __future__ import annotations + +import pytest + +from hyperloom.orchestrator.actions.executors._proposal_identity import ( + controls_of, + effective_fingerprint, + is_executable, + normalize_proposal, +) + + +# (label, extra_args, extra_envs, variant controls, base controls, expected fingerprint) +_GOLDEN = [ + ("plain", "--max-num-seqs 128", {"A": "1"}, ([], [], "append"), ([], [], ""), "caacd6bf1da76201"), + ("variant-only", "--max-num-seqs 128", {"A": "1"}, (["--x"], ["E"], "append"), ([], [], ""), "39cfaf303d04885d"), + ("base-only", "--max-num-seqs 128", {"A": "1"}, ([], [], "append"), (["--b"], ["BE"], ""), "e85521a43890c743"), + ("both", "--max-num-seqs 128", {"A": "1"}, (["--x"], ["E"], "append"), (["--b"], ["BE"], ""), "1e630973fad523ab"), + ("overlap", "--max-num-seqs 128", {}, (["--b"], [], "append"), (["--b"], [], ""), "9cb8463804e7a11c"), + ("variant-replace", "", {"A": "1"}, ([], [], "replace"), ([], [], ""), "7f4d4d32a84df525"), + ("base-replace", "", {"A": "1"}, ([], [], "append"), ([], [], "replace"), "7f4d4d32a84df525"), + ("removal-only", "", {}, (["--enable-prefix-caching"], [], "append"), ([], [], ""), "ca4d2e9e9760543a"), + ("empty", "", {}, ([], [], "append"), ([], [], ""), "164374825086dc65"), +] + + +def _controls(remove_args, unset_envs, args_mode) -> dict: + return controls_of( + normalize_proposal({"remove_args": remove_args, "unset_envs": unset_envs, "args_mode": args_mode}) + ) + + +@pytest.mark.parametrize( + "args,envs,variant,base,expected", + [row[1:] for row in _GOLDEN], + ids=[row[0] for row in _GOLDEN], +) +def test_effective_fingerprint_matches_the_pinned_executor_values(args, envs, variant, base, expected): + b_remove, b_unset, b_mode = base + assert ( + effective_fingerprint( + args, + envs, + controls=_controls(*variant), + base_remove_args=b_remove, + base_unset_envs=b_unset, + base_args_mode=b_mode, + ) + == expected + ) + + +def test_base_controls_change_the_fingerprint(): + controls = _controls(["--x"], [], "append") + assert effective_fingerprint("--a 1", {}, controls=controls) != effective_fingerprint( + "--a 1", {}, controls=controls, base_remove_args=["--b"] + ) + + +def test_removal_union_is_base_first_and_deduped(): + both = effective_fingerprint("", {}, controls=_controls(["--b", "--v"], [], "append"), base_remove_args=["--b"]) + assert both == effective_fingerprint("", {}, controls=_controls(["--b", "--v"], [], "append")) + + +@pytest.mark.parametrize( + "proposal,expected", + [ + ({"extra_args": "--a 1"}, True), + ({"extra_server_args": "--a 1"}, True), + ({"extra_envs": {"A": "1"}}, True), + ({"remove_args": ["--a"]}, True), + ({"unset_envs": ["A"]}, True), + ({"args_mode": "replace"}, True), + ({"name": "research-only", "reason": "read the scheduler"}, False), + ({"extra_args": " ", "extra_envs": {}}, False), + ], +) +def test_is_executable(proposal, expected): + assert is_executable(normalize_proposal(proposal)) is expected + + +def test_normalize_resolves_the_args_alias_and_keeps_atomic(): + fields = normalize_proposal( + { + "name": " coupled ", + "extra_server_args": " --a 1 ", + "extra_envs": {"A": 1}, + "remove_args": "--drop", + "args_mode": "REPLACE", + "atomic": True, + "reason": "needs the paired headroom", + } + ) + assert fields == { + "name": "coupled", + "extra_args": "--a 1", + "extra_envs": {"A": "1"}, + "remove_args": ["--drop"], + "unset_envs": [], + "args_mode": "replace", + "atomic": True, + "reason": "needs the paired headroom", + } + + +def test_controls_of_drops_defaults(): + assert controls_of(normalize_proposal({"extra_args": "--a 1"})) == {} + assert controls_of(normalize_proposal({"remove_args": ["--x"], "args_mode": "replace"})) == { + "remove_args": ["--x"], + "args_mode": "replace", + } diff --git a/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py b/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py new file mode 100644 index 000000000..70dbc2062 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The untested-proposal queue renderer and its injection into the prompt.""" + +from __future__ import annotations + +import pytest + +from hyperloom.orchestrator.actions.executors._canonical_fingerprint import canonical_fingerprint +from hyperloom.orchestrator.actions.executors._proposal_identity import ( + controls_of, + effective_fingerprint, + normalize_proposal, +) +from hyperloom.orchestrator.phases.machine_state import ( + PHASE_CLOSE, + PHASE_EXPLORE, + PHASE_FRAMEWORK_AGENT, + PHASE_KERNEL_AGENT, + PHASE_PRELUDE, + PHASE_SWEEP, +) +from hyperloom.orchestrator.loop.coordinator import Coordinator +from hyperloom.orchestrator.roles import Backend, MockBackend, ScriptedPlan +from hyperloom.orchestrator.state.shared_state import SharedState +from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + +@pytest.fixture +def coord(session_dir) -> Coordinator: + plan = ScriptedPlan( + turns=[], + default_intent=Intent(type=IntentType.SEND_MESSAGE, payload={"topic": "heartbeat", "body_md": "ok"}), + ) + backends: dict[str, Backend] = { + name: MockBackend(plan, name=name) for name in ("orchestration", "critic", "robustness") + } + return Coordinator(session_dir, backends=backends) + + +def _state(rounds, *, cycle: int = 0, tested=None, gaps=None, current_best=None) -> SharedState: + state = SharedState() + state.macro_cycle = cycle + state.specialist_rounds = list(rounds) + state.explore_search = {"tested": dict(tested or {})} + state.gaps = list(gaps or []) + state.current_best = dict(current_best or {}) + return state + + +def _fingerprint(proposal, **base) -> str: + fields = normalize_proposal(proposal) + return effective_fingerprint(fields["extra_args"], fields["extra_envs"], controls=controls_of(fields), **base) + + +def _round(proposals, *, cycle: int = 0, domain: str = "serving_specialist", gap: str = "") -> dict: + return { + "cycle": cycle, + "domain": domain, + "gap_canonical_id": gap, + "proposal_set": list(proposals), + } + + +def test_empty_queue_renders_nothing(): + assert _state([]).to_untested_proposals_summary() == "" + assert _state([_round([{"name": "research-only", "reason": "read it"}])]).to_untested_proposals_summary() == "" + + +def test_only_the_current_cycle_is_rendered(): + state = _state( + [ + _round([{"name": "old", "extra_args": "--old"}], cycle=0), + _round([{"name": "new", "extra_args": "--new"}], cycle=1), + ], + cycle=1, + ) + out = state.to_untested_proposals_summary() + assert "new" in out + assert "old" not in out + + +def test_a_round_missing_its_cycle_field_reads_as_zero(): + entry = _round([{"name": "legacy", "extra_args": "--legacy"}]) + entry.pop("cycle") + assert "legacy" in _state([entry], cycle=0).to_untested_proposals_summary() + assert _state([entry], cycle=1).to_untested_proposals_summary() == "" + + +def test_benched_fingerprints_are_dropped(): + proposal = {"name": "benched", "extra_args": "--a 1"} + tested = {_fingerprint(proposal): {"outcome": "REVERT"}} + assert _state([_round([proposal])], tested=tested).to_untested_proposals_summary() == "" + + +def test_the_tested_lookup_folds_in_the_stack_base_controls(): + """A naive fingerprint misses the ledger whenever the stack removes a flag.""" + proposal = {"name": "with-removal", "extra_args": "--a 1", "remove_args": ["--v"]} + current_best = {"remove_args": ["--b"], "unset_envs": ["BE"]} + effective = _fingerprint(proposal, base_remove_args=["--b"], base_unset_envs=["BE"]) + naive = canonical_fingerprint("--a 1", {}, remove_args=["--v"]) + assert effective != naive + + hidden = _state([_round([proposal])], tested={effective: {}}, current_best=current_best) + assert hidden.to_untested_proposals_summary() == "" + + still_shown = _state([_round([proposal])], tested={naive: {}}, current_best=current_best) + assert "with-removal" in still_shown.to_untested_proposals_summary() + + +def test_duplicate_fingerprints_collapse_across_rounds(): + proposal = {"name": "dup", "extra_args": "--a 1"} + out = _state([_round([proposal]), _round([dict(proposal, name="dup-again")])]).to_untested_proposals_summary() + assert out.count("•") == 1 + + +def test_every_control_field_reaches_the_line(): + proposal = { + "name": "coupled", + "extra_args": "--a 1", + "extra_envs": {"E": "2"}, + "remove_args": ["--drop"], + "unset_envs": ["DROP_ENV"], + "args_mode": "replace", + "atomic": True, + "reason": "splitting it OOMs", + } + line = _state([_round([proposal])]).to_untested_proposals_summary() + assert "ATOMIC" in line + assert "+args=--a 1" in line + assert "+envs=E=2" in line + assert "-args=--drop" in line + assert "-envs=DROP_ENV" in line + assert "mode=replace" in line + assert "why=splitting it OOMs" in line + + +def test_a_removal_only_proposal_does_not_render_as_a_no_op(): + proposal = {"name": "drop-prefix-caching", "remove_args": ["--enable-prefix-caching"]} + line = _state([_round([proposal])]).to_untested_proposals_summary() + assert "-args=--enable-prefix-caching" in line + assert "+args=" not in line + + +def test_ranking_is_gap_severity_then_recency(): + state = _state( + [ + _round([{"name": "low-old", "extra_args": "--1"}], gap="gap.low"), + _round([{"name": "high", "extra_args": "--2"}], gap="gap.high"), + _round([{"name": "low-new", "extra_args": "--3"}], gap="gap.low"), + ], + gaps=[ + {"canonical_id": "gap.high", "severity": "high"}, + {"canonical_id": "gap.low", "severity": "low"}, + ], + ) + names = [line.split()[1] for line in state.to_untested_proposals_summary().splitlines() if line.startswith("•")] + assert names == ["high", "low-new", "low-old"] + + +def test_a_pruned_gap_sorts_last_but_is_not_dropped(): + state = _state( + [ + _round([{"name": "orphan", "extra_args": "--1"}], gap="gap.gone"), + _round([{"name": "known", "extra_args": "--2"}], gap="gap.low"), + ], + gaps=[{"canonical_id": "gap.low", "severity": "low"}], + ) + out = state.to_untested_proposals_summary() + names = [line.split()[1] for line in out.splitlines() if line.startswith("•")] + assert names == ["known", "orphan"] + assert "sev?" in out + + +def test_overflow_is_truncated_and_counted(): + proposals = [{"name": f"v{i}", "extra_args": f"--flag {i}"} for i in range(20)] + out = _state([_round(proposals)]).to_untested_proposals_summary(max_entries=12) + assert out.count("•") == 12 + assert "(+8 more not shown)" in out + + +@pytest.mark.parametrize( + "phase,expected", + [ + (PHASE_PRELUDE, False), + (PHASE_FRAMEWORK_AGENT, False), + (PHASE_EXPLORE, True), + (PHASE_KERNEL_AGENT, False), + (PHASE_SWEEP, False), + (PHASE_CLOSE, False), + ], +) +@pytest.mark.asyncio +async def test_the_block_is_injected_only_in_explore(coord, phase, expected): + coord.shared_state.phase = phase + coord.shared_state.macro_cycle = 0 + coord.shared_state.specialist_rounds = [_round([{"name": "queued", "extra_args": "--a 1"}])] + out = await coord._compose_prompt("orchestration") + assert ("=== Untested proposals (current cycle) ===" in out) is expected + + +@pytest.mark.asyncio +async def test_the_block_survives_a_delta_turn(coord, monkeypatch): + coord.shared_state.phase = PHASE_EXPLORE + coord.shared_state.macro_cycle = 0 + coord.shared_state.specialist_rounds = [_round([{"name": "queued", "extra_args": "--a 1"}])] + seed = await coord._compose_prompt("orchestration") + + monkeypatch.setattr(type(coord.conversation), "_orchestration_conversational", lambda self: True) + coord._orchestration_seeded = True + delta = await coord._compose_prompt("orchestration") + + assert "=== Shared session state ===" in seed + assert "=== Shared session state ===" not in delta + assert "queued" in seed + assert "queued" in delta diff --git a/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py b/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py new file mode 100644 index 000000000..8b01c396e --- /dev/null +++ b/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""One identity for a specialist proposal and the explore variant it becomes. + +The proposal is a mapping keyed ``extra_args``; the variant is a ``GridVariant`` +keyed ``extra_server_args``. Both fingerprint into ``explore_search["tested"]`` +under the variant's own args and envs folded together with the union of its +removal controls and the current stack's, so the key is stack-relative and a +proposal hashed on its own would not match the ledger. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from hyperloom.common.coerce import to_str_list + +from ._canonical_fingerprint import canonical_fingerprint + + +__all__ = [ + "controls_of", + "effective_fingerprint", + "is_executable", + "normalize_proposal", +] + + +def _args_mode_of(value: Any) -> str: + """Coerce an args-mode to ``"replace"`` or ``"append"``.""" + return "replace" if str(value or "").strip().lower() == "replace" else "append" + + +def normalize_proposal(proposal: Mapping[str, Any]) -> dict[str, Any]: + """Project a ``proposal_set`` entry onto the variant field set. + + Args: + proposal: One ``specialist_done.proposal_set`` entry. + + Returns: + ``name`` / ``extra_args`` / ``extra_envs`` / ``remove_args`` / + ``unset_envs`` / ``args_mode`` / ``atomic`` / ``reason``, with the + ``extra_args`` / ``extra_server_args`` alias resolved. + """ + args = str(proposal.get("extra_args") or proposal.get("extra_server_args") or "").strip() + envs = proposal.get("extra_envs") + return { + "name": str(proposal.get("name") or "").strip(), + "extra_args": args, + "extra_envs": {str(k): str(v) for k, v in envs.items()} if isinstance(envs, Mapping) else {}, + "remove_args": to_str_list(proposal.get("remove_args")), + "unset_envs": to_str_list(proposal.get("unset_envs")), + "args_mode": _args_mode_of(proposal.get("args_mode")), + "atomic": bool(proposal.get("atomic")), + "reason": str(proposal.get("reason") or "").strip(), + } + + +def is_executable(fields: Mapping[str, Any]) -> bool: + """Whether a server restart could apply these fields. + + A removal-only entry qualifies; a research-only entry does not. + + Args: + fields: A :func:`normalize_proposal` result. + + Returns: + ``True`` when the entry carries args, envs, or a removal/replacement + control. + """ + return bool( + fields["extra_args"] + or fields["extra_envs"] + or fields["remove_args"] + or fields["unset_envs"] + or fields["args_mode"] == "replace" + ) + + +def controls_of(fields: Mapping[str, Any]) -> dict[str, Any]: + """Return only the non-default removal/replacement controls. + + Args: + fields: A :func:`normalize_proposal` result. + + Returns: + The controls that differ from the default; all-default yields ``{}`` so + a plain variant fingerprints unchanged. + """ + out: dict[str, Any] = {} + if fields["remove_args"]: + out["remove_args"] = list(fields["remove_args"]) + if fields["unset_envs"]: + out["unset_envs"] = list(fields["unset_envs"]) + if fields["args_mode"] == "replace": + out["args_mode"] = "replace" + return out + + +def effective_fingerprint( + extra_args: Any, + extra_envs: Any, + *, + controls: Mapping[str, Any] | None = None, + base_remove_args: Any = None, + base_unset_envs: Any = None, + base_args_mode: Any = None, +) -> str: + """Fingerprint a variant against the stack it will be launched on. + + Removals union base-first with order preserved; a ``replace`` base + args-mode wins over the variant's, since the base is what it launches on. + + Args: + extra_args: The variant's own server-args string. + extra_envs: The variant's own env mapping. + controls: The variant's own non-default controls. + base_remove_args: ``base_remove_args`` from the current stack. + base_unset_envs: ``base_unset_envs`` from the current stack. + base_args_mode: ``base_args_mode`` from the current stack. + + Returns: + The 16-char fingerprint ``explore_search["tested"]`` is keyed on. + """ + identity = dict(controls or {}) + remove_args = list(dict.fromkeys(to_str_list(base_remove_args) + to_str_list(identity.get("remove_args")))) + unset_envs = list(dict.fromkeys(to_str_list(base_unset_envs) + to_str_list(identity.get("unset_envs")))) + if remove_args: + identity["remove_args"] = remove_args + if unset_envs: + identity["unset_envs"] = unset_envs + if _args_mode_of(base_args_mode) == "replace": + identity["args_mode"] = "replace" + return canonical_fingerprint(extra_args, extra_envs, **identity) diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 1f2b08ff7..624aa895b 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -59,7 +59,8 @@ parse_eval_results, ) from . import _framework_switch_manifest as _switch_manifest -from ._canonical_fingerprint import canonical_fingerprint, workload_signature +from ._canonical_fingerprint import workload_signature +from ._proposal_identity import effective_fingerprint from ._grid_runner import ( _MN_BACKENDS_PRIORITY, _MN_PARAMS_PRIORITY, @@ -173,7 +174,7 @@ def _carry_variant_metadata(src: Any, dst: Any) -> Any: def _variant_control_fields(variant: Any) -> dict[str, Any]: - """Return non-default remove/unset/replace controls for ledger rows.""" + """Return non-default remove/unset/replace controls for identity and ledger rows.""" remove_args = to_str_list(getattr(variant, "remove_args", [])) unset_envs = to_str_list(getattr(variant, "unset_envs", [])) args_mode = str(getattr(variant, "args_mode", "append") or "append").strip().lower() @@ -994,23 +995,13 @@ async def __call__(self, ctx) -> dict[str, Any]: unique_in_round: dict[str, GridVariant] = {} skipped_dup: list[dict[str, Any]] = [] for gv in grid: - identity_controls = _variant_control_fields(gv) - identity_remove_args = list( - dict.fromkeys(base_remove_args + to_str_list(identity_controls.get("remove_args"))) - ) - identity_unset_envs = list( - dict.fromkeys(base_unset_envs + to_str_list(identity_controls.get("unset_envs"))) - ) - if identity_remove_args: - identity_controls["remove_args"] = identity_remove_args - if identity_unset_envs: - identity_controls["unset_envs"] = identity_unset_envs - if base_args_mode == "replace": - identity_controls["args_mode"] = "replace" - fp = canonical_fingerprint( + fp = effective_fingerprint( gv.extra_server_args, gv.extra_envs, - **identity_controls, + controls=_variant_control_fields(gv), + base_remove_args=base_remove_args, + base_unset_envs=base_unset_envs, + base_args_mode=base_args_mode, ) gv.canonical_fp = fp # type: ignore[attr-defined] if fp in unique_in_round: diff --git a/src/hyperloom/orchestrator/loop/conversation.py b/src/hyperloom/orchestrator/loop/conversation.py index bce606daf..30e30650e 100644 --- a/src/hyperloom/orchestrator/loop/conversation.py +++ b/src/hyperloom/orchestrator/loop/conversation.py @@ -547,6 +547,12 @@ async def _compose_prompt(self, agent_name: str, *, system_prompt: str | None = denial_summary = self.shared_state.to_policy_denial_summary(top_k=6) if denial_summary: sections.append(denial_summary) + # Outside the SEED gate: a queue seen once is the amnesia it fixes. + if (self.shared_state.phase or "").strip().upper() == _phase_state.PHASE_EXPLORE: + untested_block = self.shared_state.to_untested_proposals_summary() + if untested_block: + sections.append("=== Untested proposals (current cycle) ===") + sections.append(untested_block) # Recipe KB T0 warm-start snapshot + structured gaps[] ledger. if agent_name == "orchestration" and push_full: @@ -1186,8 +1192,12 @@ def _recent_proposed_variants( return out def _research_scout_seed_block(self) -> str: - """Render all persisted research-scout findings for an Orchestration SEED.""" - from ..actions.executors._canonical_fingerprint import canonical_fingerprint + """Render the persisted research-scout findings for an Orchestration SEED. + + The scout's executable proposals are not rendered here: they go through + ``=== Untested proposals (current cycle) ===`` alongside every other + domain's, which also drops the ones already benched. + """ from ..knowledge import research_hints as _research_hints hints = _research_hints.load_hints(self.session_dir) @@ -1205,38 +1215,15 @@ def _research_scout_seed_block(self) -> str: for hint in hints: lines.append(json.dumps(hint, sort_keys=True)) - proposals: list[dict[str, Any]] = [] - proposal_names: set[str] = set() - proposal_fingerprints: set[str] = set() questions: list[str] = [] seen_questions: set[str] = set() for row in rounds: - for proposal in row.get("proposal_set") or []: - if not isinstance(proposal, dict): - continue - name = str(proposal.get("name") or "").strip() - fingerprint = canonical_fingerprint( - str(proposal.get("extra_args") or proposal.get("extra_server_args") or ""), - proposal.get("extra_envs") if isinstance(proposal.get("extra_envs"), dict) else {}, - remove_args=proposal.get("remove_args"), - unset_envs=proposal.get("unset_envs"), - args_mode=str(proposal.get("args_mode") or "append"), - ) - if (name and name in proposal_names) or fingerprint in proposal_fingerprints: - continue - if name: - proposal_names.add(name) - proposal_fingerprints.add(fingerprint) - proposals.append(proposal) for question in row.get("residual_questions") or []: text = str(question).strip() if text and text not in seen_questions: seen_questions.add(text) questions.append(text) - if proposals: - lines.append("Untested executable proposals:") - lines.extend(json.dumps(proposal, sort_keys=True) for proposal in proposals) if questions: lines.append("Residual questions:") lines.extend(f"- {question}" for question in questions) diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 4c7eb3422..004ba1b83 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -1472,6 +1472,7 @@ async def _maybe_materialize_mn_explore( if bool((getattr(task, "params", None) or {}).get("framework_config_generation")): return from ..actions.executors._multi_node_env import is_multi_node + from ..actions.executors._proposal_identity import controls_of, is_executable, normalize_proposal if not is_multi_node() or not proposals: return @@ -1479,37 +1480,17 @@ async def _maybe_materialize_mn_explore( for i, p in enumerate(proposals[: self._MN_AUTO_EXPLORE_GRID_CAP]): if not isinstance(p, dict): continue - args = str(p.get("extra_args") or p.get("extra_server_args") or "").strip() - envs_raw = p.get("extra_envs") - envs = {str(k): str(v) for k, v in envs_raw.items()} if isinstance(envs_raw, dict) else {} - controls: dict[str, Any] = {} - for key in ("remove_args", "unset_envs"): - raw = p.get(key) - if isinstance(raw, str): - vals = [raw.strip()] if raw.strip() else [] - elif isinstance(raw, (list, tuple, set)): - vals = [str(v).strip() for v in raw if str(v).strip()] - else: - vals = [] - if vals: - controls[key] = vals - mode = str(p.get("args_mode") or "append").strip().lower() - if mode == "replace": - controls["args_mode"] = "replace" - # Drop entries with neither a server-arg nor an env override — - # nothing for the restart to apply (e.g. research-only items) - # unless the entry removes inherited args/envs. - if not args and not envs and not controls: + fields = normalize_proposal(p) + if not is_executable(fields): continue - name = str(p.get("name") or "").strip() or (f"{domain or 'specialist'}-{task.task_id[:8]}-{i}") grid.append( { - "name": name, - "extra_args": args, - "extra_envs": envs, - **controls, + "name": fields["name"] or f"{domain or 'specialist'}-{task.task_id[:8]}-{i}", + "extra_args": fields["extra_args"], + "extra_envs": fields["extra_envs"], + **controls_of(fields), "provenance": f"specialist:{domain}" if domain else "specialist", - "note": str(p.get("reason") or "")[:200], + "note": fields["reason"][:200], } ) if not grid: diff --git a/src/hyperloom/orchestrator/state/_shared_state/render.py b/src/hyperloom/orchestrator/state/_shared_state/render.py index 3060bb7f2..7373971df 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/render.py +++ b/src/hyperloom/orchestrator/state/_shared_state/render.py @@ -453,6 +453,112 @@ def to_gaps_summary(self, *, max_entries: int = 10, max_attempts: int = 0) -> st rows.append(f" · (+{len(ordered) - max_entries} older gaps elided; see state.json `gaps[]`)") return "\n".join(rows) + def _untested_proposal_rows(self) -> list[dict[str, Any]]: + """Executable proposals from this cycle that no explore round has benched. + + Returns: + Rows ranked by gap severity then recency, each carrying the + normalized proposal fields plus ``domain`` / ``severity``. + """ + from ...actions.executors._proposal_identity import ( + controls_of, + effective_fingerprint, + is_executable, + normalize_proposal, + ) + from ..shared_state import stack_base_params + + base = stack_base_params(self.current_best) + tested = set((self.explore_search or {}).get("tested") or {}) + severity_of = { + str(g.get("canonical_id") or ""): str(g.get("severity") or "").strip().lower() + for g in (self.gaps or []) + if isinstance(g, dict) + } + rank = {"high": 3, "medium": 2, "low": 1} + + ranked: list[tuple[int, int, dict[str, Any]]] = [] + seen: set[str] = set() + for order, entry in enumerate(self.specialist_rounds or []): + if not isinstance(entry, dict) or int(entry.get("cycle") or 0) != int(self.macro_cycle or 0): + continue + domain = str(entry.get("domain") or "?").removesuffix("_specialist") + severity = severity_of.get(str(entry.get("gap_canonical_id") or ""), "") + for proposal in entry.get("proposal_set") or []: + if not isinstance(proposal, dict): + continue + row = normalize_proposal(proposal) + if not is_executable(row): + continue + fingerprint = effective_fingerprint( + row["extra_args"], + row["extra_envs"], + controls=controls_of(row), + base_remove_args=base.get("base_remove_args"), + base_unset_envs=base.get("base_unset_envs"), + base_args_mode=base.get("base_args_mode"), + ) + if fingerprint in tested or fingerprint in seen: + continue + seen.add(fingerprint) + row["domain"] = domain + row["severity"] = severity + ranked.append((rank.get(severity, 0), order, row)) + ranked.sort(key=lambda r: (-r[0], -r[1])) + return [row for _, _, row in ranked] + + @staticmethod + def _untested_proposal_line(row: dict[str, Any]) -> str: + """Render one queue row, marking each field it carries. + + Args: + row: One row from :meth:`_untested_proposal_rows`. + + Returns: + A single ``•``-prefixed line. + """ + parts = [f"• {row['name'] or '(unnamed)'} [{row['domain']}·{row['severity'] or 'sev?'}]"] + if row["atomic"]: + parts.append("ATOMIC") + if row["extra_args"]: + parts.append(f"+args={row['extra_args']}") + if row["extra_envs"]: + parts.append("+envs=" + ",".join(f"{k}={v}" for k, v in sorted(row["extra_envs"].items()))) + if row["remove_args"]: + parts.append("-args=" + " ".join(row["remove_args"])) + if row["unset_envs"]: + parts.append("-envs=" + ",".join(row["unset_envs"])) + if row["args_mode"] == "replace": + parts.append("mode=replace") + reason = row["reason"].replace("\n", " ").strip()[:80].rstrip() + if reason: + parts.append(f"why={reason}") + return _flatten_for_prompt(" ".join(parts)) + + def to_untested_proposals_summary(self, *, max_entries: int = 12) -> str: + """Render the specialist proposals still waiting for a benchmark slot. + + Args: + max_entries (int): Rows to render before collapsing the rest into a + count. + + Returns: + str: The rendered queue, or ``""`` when nothing is waiting. + """ + rows = self._untested_proposal_rows() + if not rows: + return "" + out = [ + "Executable specialist proposals from this cycle that no explore round has benched.", + "Ranked by gap severity, then most recent. Compose the next `explore` grid from these;", + "dispatch an ATOMIC entry verbatim as one variant — never split or re-derive its flags.", + "", + ] + out.extend(self._untested_proposal_line(row) for row in rows[:max_entries]) + if len(rows) > max_entries: + out.append(f"(+{len(rows) - max_entries} more not shown)") + return "\n".join(out) + def to_proposal_scores_summary(self, *, max_rounds: int = 2) -> str: """Render advisory multi-model proposal scores for Orchestration; no mean/sorting, rater identities anonymized. Empty when no recent round carries scores. From 491855f0df38887147757aff62bcab9ea42c7dc8 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 19 Aug 2026 07:02:02 +0000 Subject: [PATCH 3/9] feat(orchestration): point the grid author at the queue and bound the grid Both decision lists told the author that "the proposal_set drives the next explore grid" when an explore round had just finished. Neither half was true: the proposal_set was never rendered, and finishing a round has nothing to do with what a specialist produced. Name the block that now carries them. Give the grid a size while we are here. It had one until the per-round breadth caps were removed in favour of a resource-derived bound, but explore reserves server_lifecycle and benchmark_lane, never research_lane, so the bound that was supposed to take over does not reach it. What is left is the session deadline skipping whatever the round cannot finish, from the end of the grid rather than the bottom of the ranking. Rounds of twenty variants followed. State the target and the ceiling on both surfaces the author reads, with the serial lane cost that makes them what they are. The removed gate's own hint told the author to defer runners-up to a later round, which only means something now that a later round can still see them. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_grid_size_prompt_unit.py | 33 +++++++++++++++++++ .../orchestrator/prompts/orchestration.md | 17 ++++++++-- .../orchestrator/prompts/prompt_builder.py | 19 +++++++++-- 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py diff --git a/src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py b/src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py new file mode 100644 index 000000000..e00148438 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_grid_size_prompt_unit.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""The grid author's two prompt surfaces agree on where a grid comes from and how big it gets.""" + +from __future__ import annotations + + +def _grid_prompt_surfaces() -> list[str]: + from hyperloom.orchestrator.prompts.prompt_builder import ( + _idea_generation_lines, + _format_grid_injection_hint, + ) + + return [_format_grid_injection_hint("explore") or "", "\n".join(_idea_generation_lines())] + + +def test_both_grid_surfaces_state_the_same_target_and_ceiling(): + for surface in _grid_prompt_surfaces(): + assert "4" in surface + assert "maximum 6" in surface + assert all("Untested proposals (current cycle)" in s for s in _grid_prompt_surfaces()[1:]) + + +def test_the_stale_proposal_set_wording_is_gone(): + from pathlib import Path + + from hyperloom.orchestrator.prompts import prompt_builder + + stale = "proposal_set drives the next" + assert stale not in Path(prompt_builder.__file__).read_text(encoding="utf-8") + md = Path(prompt_builder.__file__).parent / "orchestration.md" + assert "specialist proposal_set" not in md.read_text(encoding="utf-8") diff --git a/src/hyperloom/orchestrator/prompts/orchestration.md b/src/hyperloom/orchestrator/prompts/orchestration.md index 822089099..2d2ba14b2 100644 --- a/src/hyperloom/orchestrator/prompts/orchestration.md +++ b/src/hyperloom/orchestrator/prompts/orchestration.md @@ -185,9 +185,10 @@ phases' goals are omitted because you cannot act on them from here. **Decision priority**: pick the next action by reading facts in this order: (a) current phase + `allowed_actions`, (b) gaps / KB sub-graph / recent -winners / specialist proposal_set, (c) mandatory ordering (baseline first; -`explore` revalidates the stack inline — no separate rebench step), -(d) `phase_budget_remaining_pct` as the urgency signal. +winners / `=== Untested proposals (current cycle) ===`, (c) mandatory +ordering (baseline first; `explore` revalidates the stack inline — no +separate rebench step), (d) `phase_budget_remaining_pct` as the urgency +signal. ### PRELUDE — phase goal @@ -213,6 +214,16 @@ provide KB/PR/source evidence for `explore` grids and may produce patches for `integrate_patch`. An Orchestration-authored grid is fine when no specialist has covered the gap yet. +**Where a grid comes from.** `=== Untested proposals (current cycle) ===` +carries every executable specialist proposal this cycle that no explore round +has benched, ranked by gap severity, with the ones already benched removed. +Draw from it first and copy an entry's fields verbatim — an entry marked +ATOMIC is a coupled set that must go in as one variant, never split or +re-authored. Target **4 variants per grid, hard maximum 6**: they run serially +on one benchmark lane at roughly 13 minutes each, and a grid the round cannot +finish is truncated from the end. Top up from the idea-generation moves only +after the queue holds nothing else worth running. + **GPU specialists** hold the same cards as the serving stack and acquire `gpu_research_lane` (mutually exclusive with benchmark/profile/serving lanes). Use them opportunistically in the idle research window — while diff --git a/src/hyperloom/orchestrator/prompts/prompt_builder.py b/src/hyperloom/orchestrator/prompts/prompt_builder.py index 8ca28a90b..8e7f9cd1c 100644 --- a/src/hyperloom/orchestrator/prompts/prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/prompt_builder.py @@ -465,7 +465,13 @@ def _format_grid_injection_hint(name: str) -> str | None: "Use remove_args/unset_envs to ablate harmful base flags; " "args_mode='replace' to drop inherited server args. " "provenance values: 'llm_direct', 'default_grid', " - "'specialist:' (audit/advisory, not a gate)." + "'specialist:' (audit/advisory, not a gate). " + "SIZE: target 4 variants, hard maximum 6. Variants run serially " + "on a single benchmark lane at ~13min each, so a 4-variant round " + "is about an hour of GPU. Submit a 5th or 6th only when it still " + "beats the median of the four you already have; a grid the round " + "cannot finish is truncated from the end, dropping whatever you " + "ranked last rather than whatever is worth least." ) if name == "sweep": return ( @@ -590,8 +596,11 @@ def _section_decision_framework(*, kernel_enabled: bool, phase: str = "", transp " `last_action_failures` + `explore_search.winners_history`.", " c. **KB sub-graphs + warm-start recipe** when present —", " cross-session priors carry " + "*qualitative* hints (what worked / what failed last time).", - " d. **specialist proposal_set** — when an explore round just", - " finished, the proposal_set drives the next `explore` grid.", + " d. **`=== Untested proposals (current cycle) ===`** — every", + " executable specialist proposal this cycle that no explore", + " round has benched, ranked by gap severity. This is the", + " grid's primary source; an entry marked ATOMIC goes in", + " verbatim as one variant.", " e. **Ordering facts**: baseline runs before anything else", " (invariant). ``analysis.md`` / ``last_profile_trace`` arrive", " automatically from the Coordinator-owned analysis task at", @@ -712,6 +721,10 @@ def _idea_generation_lines() -> list[str]: "`extra_server_args` is framework-neutral (routed to EXTRA_SGLANG_ARGS", "/ EXTRA_VLLM_ARGS / EXTRA_ATOM_ARGS by `--framework`).", "", + "Draw first from `=== Untested proposals (current cycle) ===`; the", + "five moves above are for topping the grid up to its target of 4", + "(hard maximum 6) once the queue is drained of anything worth running.", + "", "An explore round that produces zero new ideas is a bug — heartbeat", "with body_md='idea-pipeline-empty' so Robustness can intervene.", ] From 12b1214b0cd2a1d6cf7d6ece83c8e454a2ecc93e Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 19 Aug 2026 09:31:55 +0000 Subject: [PATCH 4/9] fix(specialist): make the proposal target a bound and give the run a way to stop The ceiling had already become the target once: after it was restated in Section 1, no run exceeded it and a quarter landed exactly on it. A target states the same invitation more directly, so say which direction it binds -- one real proposal is a better round than two padded, and an empty one is better than a single padded entry. Nothing told the specialist when to stop looking. Section 1 spent its budget encouraging depth, Section 2a said turns are not the stop signal, and the only remaining bound was the wall clock, which is how a round with nothing to show still spends its full hour. Name the other stop: rounds that stop yielding findings are done. The cold-start branch fires exactly when there are no priors, which is when having nothing to propose is most likely, and it opened by forbidding an empty set before walking it back a paragraph later. Lead with the rule the two paragraphs agreed on and drop the contradiction. Its fallback bullets live in Section 1 whether or not the dispatch has a domain focus block, so point there rather than at a block a free-form specialist never gets. Also corrects the numbers this commit's predecessor claimed but never applied. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_critic_verdict_map.py | 9 ++- .../prompts/specialist_prompt_builder.py | 58 +++++++++---------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index add175fd6..12cf15538 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -2077,9 +2077,12 @@ def test_specialist_prompt_renders_proposal_target_and_ceiling(): text = _build_specialist_prompt_text() # Section 8 states both numbers; Section 1 repeats them so the target is # visible before the specialist starts working, not only at exit time. - assert "**target 4 entries, hard maximum 6.**" in text - assert "**aim for 4** ranked proposals" in text - assert "**6 is a hard ceiling, never a target**" in text + assert "**2 entries is the norm, 4 the hard cap.**" in text + assert "**2 proposals is the norm, 4 the hard" in text + # Both the padding and the keep-going pressures need a stated counterweight. + assert "``empty=true`` is better than one" in text + assert "stop once" in text and "not the only stop" in text + assert "a coin-flip proposal is worse than none" in text assert "reviews each surviving variant" in text diff --git a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py index cabe978b4..ea19ea56f 100644 --- a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py @@ -1029,9 +1029,10 @@ def _section_identity(inp: SpecialistPromptInputs) -> list[str]: "source roots (Section 7), search any public GitHub repo or NVIDIA PR,", capability_line, "to be thorough. Be creative. Investigate deeply. One-turn shortcuts", - "are discouraged when a real bottleneck is on the table. Quality is", - "scored over quantity: **aim for 4** ranked proposals in your final", - "``proposal_set``. **6 is a hard ceiling, never a target** (Section 8).", + "are discouraged when a real bottleneck is on the table — but stop once", + "rounds stop yielding new findings; the wall clock is not the only stop", + "signal. Quality over quantity: **2 proposals is the norm, 4 the hard", + "cap**. One real beats two padded; ``empty=true`` beats one padded.", "", "Division of labour: the Coordinator owns the serving GPU, runs the E2E", "benchmark, and decides KEEP/REVERT — you do not have to validate final", @@ -1508,27 +1509,22 @@ def _section_kb_subgraph(inp: SpecialistPromptInputs) -> list[str]: "- Warm-start recipe: ``(none)`` (Section 5).", "- Use ``mcp__pr_monitor__*`` tools (Section 6) to query PRs on demand.", "", - "**Directive — DO NOT return an empty proposal_set.** " - + "Treat the *Winning techniques* + *Pitfalls* in your " - + "**domain focus** block (Section 1) as your fallback " - + "prior. Pick the **1–2 most conservative, " - + "well-attested defaults** from those bullets that are " - + "compatible with the hardware (Section 2) and the " - + "gap symptom (Section 3); flag each with " - + "``provenance: domain_focus_default`` in the proposal " - + "and say it is an unvalidated fallback prior in the " - + "proposal's ``reason``. Do NOT add a ``confidence`` " - + "field: self-reported confidence / gain fields are " - + "stripped from your output before review. Use the " - + "``residual_questions`` field to record what RecipeKB, " - + "research, or ``mcp__pr_monitor__*`` query a future round should pursue.", - "", - "If the *Winning techniques* block is generic enough " - + "that no proposal is safer than a coin-flip, you may " - + "still emit ``empty=true`` — but you MUST cite which " - + "bullets you considered and why each was rejected " - + "(in ``summary``). A bare empty exit with no rationale " - + "will be treated as a tool failure by the Coordinator.", + "**Directive — a coin-flip proposal is worse than none.** " + + "Treat the *Winning techniques* + *Pitfalls* bullets in " + + "Section 1 as your fallback prior and take the **1–2 " + + "most conservative, well-attested defaults** that fit " + + "the hardware (Section 2) and the gap symptom " + + "(Section 3); flag each ``provenance: " + + "domain_focus_default`` and call it an unvalidated " + + "fallback in the proposal's ``reason``. If none clears " + + "that bar, emit ``empty=true`` and cite in ``summary`` " + + "which you considered and why each was rejected — a " + + "bare empty exit with no rationale reads as a tool " + + "failure. Do NOT add a ``confidence`` field: " + + "self-reported confidence / gain fields are stripped " + + "before review. Record in ``residual_questions`` what " + + "RecipeKB, research, or ``mcp__pr_monitor__*`` query a " + + "future round should pursue.", ] ) else: @@ -2227,14 +2223,14 @@ def _section_output_protocol(inp: SpecialistPromptInputs) -> list[str]: "coupling across several proposals." ), ( - "- ``proposal_set``: **target 4 entries, hard maximum 6.** You are " - "a curator, not a brainstormer: rank candidates by expected gain x " - "your confidence, drop everything that contradicts ``kb_subgraph`` " - "/ ``pr_evidence`` already in your prompt, and cut at 4. Emit a 5th " - "or 6th ONLY if it still beats the median of the four you already " - "have. Filling the ceiling is a failure, not thoroughness: every " + "- ``proposal_set``: **2 entries is the norm, 4 the hard cap.** You " + "are a curator, not a brainstormer: rank by expected gain x " + "confidence, drop anything contradicting ``kb_subgraph`` / " + "``pr_evidence``, and stop at 2. A 3rd or 4th must beat the median " + "of the first two. Padding is a failure, not thoroughness: each " "weak entry costs a Critic reject and a slot on the serial " - "benchmark queue." + "benchmark queue. One real proposal is a better round than two " + "padded ones, and ``empty=true`` is better than one." ), ( "- The Critic reviews each surviving variant against the KB " From abe1c5b1b2a06bf716d9c4b85cc172b92b0e78cd Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 19 Aug 2026 09:56:13 +0000 Subject: [PATCH 5/9] chore: drop the throwaway pricing self-check from the repo root Committed alongside a one-hunk baseline fix, and the file says what it is in its own first line: a scratch script that prints ok/PREDICTED while sweeping the phase-pricing helpers for disagreements. Nothing imports it, no workflow or config names it, and it is the only module at the root of a src-layout tree. Co-Authored-By: Claude Opus 5 (1M context) --- selfcheck_gates.py | 124 --------------------------------------------- 1 file changed, 124 deletions(-) delete mode 100644 selfcheck_gates.py diff --git a/selfcheck_gates.py b/selfcheck_gates.py deleted file mode 100644 index c5fcce8f9..000000000 --- a/selfcheck_gates.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Throwaway self-review: sweep the real pricing functions for disagreements.""" - -from __future__ import annotations - -from types import SimpleNamespace - -from hyperloom.orchestrator.phases import machine_state as ms - -BOOT, COLD_BENCH, HOT = 350.0, 550.0, 400.0 -COLD_ROUND = BOOT + COLD_BENCH - - -def state(usable, *, phase="PRELUDE", warm=HOT, post_ready=COLD_BENCH, marked=False, double=True): - return SimpleNamespace( - phase=phase, - max_minutes=180, - baseline_tput=1000.0, - baseline_runtime_sec=COLD_ROUND, - baseline_post_ready_runtime_sec=post_ready, - baseline_warm_runtime_sec=warm, - baseline_measure_round_dropped=marked, - baseline_double_run=double, - session_budget_usable_sec=lambda: usable, - ) - - -def gate1_need(s): - """What the pre-ignition gate demands, mirroring _round_affordable.""" - cold = ms.measured_seconds(s, "baseline_runtime_sec") - rnd = ms.baseline_round_cost_sec(s, double_run=bool(s.baseline_double_run)) - if cold is None or rnd is None: - return None - use = (ms.one_more_measurement_sec(s) or cold) if s.phase == "PRELUDE" else 0.0 - return rnd + use - - -def gate2_need(s, *, warmup_sec, warmup_post_ready): - """What the post-warmup gate demands, mirroring _measure_round_affordable.""" - bench = ms.measured_seconds(s, "baseline_warm_runtime_sec") - if bench is None: - bench = warmup_post_ready - if bench is None or warmup_sec is None: - return None - use = (ms.one_more_measurement_sec(s) or warmup_sec) if s.phase == "PRELUDE" else 0.0 - return bench + use - - -def main() -> int: - bad = 0 - - # 1. The band: is there a budget gate 1 admits and gate 2 then certainly refuses? - # Gate 2 is asked after the warmup has spent a cold pass. - band = [] - for usable in range(0, 6001, 10): - s = state(float(usable)) - need1 = gate1_need(s) - admitted = need1 is not None and usable >= need1 - if not admitted: - continue - after = state(float(usable) - COLD_ROUND) - need2 = gate2_need(after, warmup_sec=COLD_ROUND, warmup_post_ready=COLD_BENCH) - if need2 is not None and (usable - COLD_ROUND) < need2: - band.append(usable) - if band: - bad += 1 - print(f"BAND gate 1 admits and gate 2 refuses for usable in {band[0]}..{band[-1]}") - else: - print("ok no budget is admitted before ignition only to be refused after the cold pass") - - # 2. Livelock: with the mark set, does every budget either close or admit a retry? - stuck = [] - for usable in range(0, 8001, 10): - s = state(float(usable), marked=True) - closes = ms.exit_cold_anchor_prelude(s) is not None - need1 = gate1_need(s) - admits = need1 is not None and usable >= need1 - if not closes and not admits: - stuck.append(usable) - if stuck: - bad += 1 - print(f"LIVELOCK neither closes nor admits for usable in {stuck[0]}..{stuck[-1]}") - else: - print("ok a marked session always either closes or may retry") - - # 3. A session with no split measured (multi-node / scriptable shape). - s = state(3000.0, warm=0.0, post_ready=0.0) - need = gate1_need(s) - if need is None: - bad += 1 - print("UNGATED a round with no boot boundary is waved through") - else: - print(f"ok a round with no split is priced at {need:.0f}s (whole cold rounds)") - - # 4. A first baseline must never be judged. - first = SimpleNamespace( - phase="PRELUDE", - max_minutes=180, - baseline_tput=0.0, - baseline_runtime_sec=0.0, - baseline_post_ready_runtime_sec=0.0, - baseline_warm_runtime_sec=0.0, - baseline_measure_round_dropped=False, - baseline_double_run=True, - session_budget_usable_sec=lambda: 60.0, - ) - if gate1_need(first) is not None: - bad += 1 - print("PREDICTED a first baseline was priced from measurements it cannot have") - else: - print("ok a first baseline is not judged") - - # 5. Later phases ask only whether the round fits. - later = state(2000.0, phase="EXPLORE") - if gate1_need(later) != ms.baseline_round_cost_sec(later, double_run=True): - bad += 1 - print("SCOPED a re-baseline outside PRELUDE was charged for a successor") - else: - print("ok a re-baseline outside PRELUDE pays only for itself") - - return bad - - -if __name__ == "__main__": - raise SystemExit(main()) From b0ca347a8a6fc48c0ca51bd70294304af84d668f Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 19 Aug 2026 09:57:32 +0000 Subject: [PATCH 6/9] chore: clear the lint backlog so the gate speaks for this branch Eleven unused imports, a missing trailing newline, and one E402, all present on main and none of them from this branch's changes. A gate that is already red cannot report on anything. Two needed more than the autofix. forge_gemm_tuning imported truthy behind a sys.path insert/pop pair whose only purpose was that import, so the whole block goes rather than leaving the path dance wrapped around nothing; sys and Path keep other callers. build_actions imported re beside the regexes it builds rather than at the top, which is not the cycle-safe late import the per-file E402 list is for, so it moves up instead of joining that list. Co-Authored-By: Claude Opus 5 (1M context) --- docs/conf.py | 2 +- src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py | 6 ------ .../inference_optimizer/tests/test_accuracy_gate_units.py | 1 - .../inference_optimizer/tests/test_coverage_boost2_unit.py | 1 - .../inference_optimizer/tests/test_custom_framework.py | 1 - .../inference_optimizer/tests/test_per_domain_prompts.py | 1 - .../inference_optimizer/tests/test_specialist_lifecycle.py | 1 - src/hyperloom/inference_optimizer/tests/test_ssh_client.py | 1 - src/hyperloom/orchestrator/framework/build_actions.py | 3 +-- src/hyperloom/orchestrator/framework/targeted_build.py | 4 ---- 10 files changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index eb9ac8992..4f8ffe0a0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,4 +85,4 @@ html_title = f"{project} {version_number} documentation" -external_projects_current_project = "Hyperloom" \ No newline at end of file +external_projects_current_project = "Hyperloom" diff --git a/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py b/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py index d0279da02..d8ce9165d 100644 --- a/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py +++ b/src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py @@ -19,12 +19,6 @@ from pathlib import Path from typing import Any -# Sibling import: kernel-agent tools cannot rely on the ``hyperloom`` import root. -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from _io_utils import truthy # noqa: E402 - -sys.path.pop(0) - def _load_input_json(path: str) -> dict[str, Any]: if not path: diff --git a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py index e74a41faa..fe6a9df5a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py @@ -317,7 +317,6 @@ def test_crash_and_success_produce_same_fingerprint(self, tmp_path): def test_workload_change_changes_fingerprint(self, tmp_path): """A change to workload shape (ISL) changes the fingerprint.""" - import yaml as _yaml cfg1 = tmp_path / "c1.yaml" cfg2 = tmp_path / "c2.yaml" diff --git a/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py index 141f3b578..78252a18c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py @@ -6,7 +6,6 @@ from __future__ import annotations import json -from types import SimpleNamespace # --------------------------------------------------------------------------- # diff --git a/src/hyperloom/inference_optimizer/tests/test_custom_framework.py b/src/hyperloom/inference_optimizer/tests/test_custom_framework.py index 30072ea8e..f52f91860 100644 --- a/src/hyperloom/inference_optimizer/tests/test_custom_framework.py +++ b/src/hyperloom/inference_optimizer/tests/test_custom_framework.py @@ -13,7 +13,6 @@ from __future__ import annotations import os -from pathlib import Path import pytest import yaml diff --git a/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py b/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py index 2c8669e72..3d8f7aa97 100644 --- a/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py +++ b/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py @@ -465,7 +465,6 @@ async def test_runner_does_not_log_generic_template_for_any_domain(tmp_path): PolicyDenied, PolicyGate, SPECIALIST_ACTION_NAME, - SPECIALIST_FROM_AGENT_PREFIX, ) from hyperloom.orchestrator.bus.resource_lock import ( KNOWN_LANES, diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py index 3905170ee..1635d7c91 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py @@ -19,7 +19,6 @@ from hyperloom.inference_optimizer.protocol.intent import ( Intent, - IntentType, ) from hyperloom.orchestrator.policy.gate import SPECIALIST_FROM_AGENT_PREFIX diff --git a/src/hyperloom/inference_optimizer/tests/test_ssh_client.py b/src/hyperloom/inference_optimizer/tests/test_ssh_client.py index 41f385ce3..f6be0d756 100644 --- a/src/hyperloom/inference_optimizer/tests/test_ssh_client.py +++ b/src/hyperloom/inference_optimizer/tests/test_ssh_client.py @@ -12,7 +12,6 @@ from __future__ import annotations import base64 -import os import shlex import pytest diff --git a/src/hyperloom/orchestrator/framework/build_actions.py b/src/hyperloom/orchestrator/framework/build_actions.py index 30063a36e..dd2c86d23 100644 --- a/src/hyperloom/orchestrator/framework/build_actions.py +++ b/src/hyperloom/orchestrator/framework/build_actions.py @@ -17,6 +17,7 @@ from __future__ import annotations +import re as _re from dataclasses import dataclass, field from typing import Any, Literal, Mapping @@ -252,8 +253,6 @@ def build_novelty_key( ) -import re as _re - _GITHUB_PR_RE = _re.compile(r"https?://github\.com/([^/]+/[^/]+)/pull/(\d+)", _re.IGNORECASE) _PR_REF_RE = _re.compile(r"^PR:(\d+)$") # An issue is a discussion thread, not a branch: GitHub publishes diff --git a/src/hyperloom/orchestrator/framework/targeted_build.py b/src/hyperloom/orchestrator/framework/targeted_build.py index baa0e7b5b..40528a6ad 100644 --- a/src/hyperloom/orchestrator/framework/targeted_build.py +++ b/src/hyperloom/orchestrator/framework/targeted_build.py @@ -337,9 +337,6 @@ def run_aiter_build( so the coordinator tick loop is never blocked). All subprocess calls go through the injectable ``run`` shim for testability. """ - import json - import shutil - import subprocess as _subprocess import time as _time from .build_utils import ( @@ -1058,7 +1055,6 @@ def _driver_main(argv: list[str] | None = None) -> int: """Driver subprocess entry: load plan.json, call run_aiter_build, write result.json.""" import argparse import json - import sys as _sys parser = argparse.ArgumentParser(description="Off-loop targeted-build driver") parser.add_argument("--attempt-root", required=True, help="Attempt directory") From d100fed8e122cda924e5941ead1f7161034733d0 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 20 Aug 2026 03:43:07 +0000 Subject: [PATCH 7/9] fix(executors): give the queue and the grid parser one args coercion The parser space-joins a list-form extra_args because the LLM sometimes emits the flags as JSON and str(list) yields a repr the server rejects. The queue built its own identity with a bare str(), so the same proposal hashed two ways: it never matched the ledger it was meant to be filtered against, and the line it rendered carried the repr back to whoever copied it into a grid. Move the coercion next to the identity it feeds and let the parser build its variant from the same projection, which also drops the duplicate alias and list handling it kept alongside it. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_proposal_identity_unit.py | 26 +++++++++++++ .../actions/executors/_proposal_identity.py | 21 +++++++++- .../orchestrator/actions/executors/explore.py | 39 ++++--------------- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py b/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py index 179f244cc..840db5f32 100644 --- a/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_proposal_identity_unit.py @@ -120,3 +120,29 @@ def test_controls_of_drops_defaults(): "remove_args": ["--x"], "args_mode": "replace", } + + +@pytest.mark.parametrize( + "value,expected", + [ + (["--max-num-seqs", "128"], "--max-num-seqs 128"), + (("--a", "1"), "--a 1"), + (["--a", "", " ", "1"], "--a 1"), + ("--a 1", "--a 1"), + (None, ""), + ], +) +def test_list_form_args_are_space_joined_not_repr(value, expected): + assert normalize_proposal({"extra_args": value})["extra_args"] == expected + + +def test_the_queue_and_the_grid_parser_agree_on_a_list_form_variant(): + from hyperloom.orchestrator.actions.executors.explore import _grid_variants_from_payload + + payload = {"name": "v", "extra_args": ["--max-num-seqs", "128"], "extra_envs": {"E": "1"}} + parsed = _grid_variants_from_payload([payload])[0] + fields = normalize_proposal(payload) + assert fields["extra_args"] == parsed.extra_server_args + assert effective_fingerprint( + fields["extra_args"], fields["extra_envs"], controls=controls_of(fields) + ) == effective_fingerprint(parsed.extra_server_args, parsed.extra_envs, controls={}) diff --git a/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py b/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py index 8b01c396e..97e27b4fa 100644 --- a/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py +++ b/src/hyperloom/orchestrator/actions/executors/_proposal_identity.py @@ -20,6 +20,7 @@ __all__ = [ + "coerce_args", "controls_of", "effective_fingerprint", "is_executable", @@ -27,6 +28,23 @@ ] +def coerce_args(value: Any) -> str: + """Coerce a payload ``extra_args`` / ``extra_server_args`` value to a shell-arg string. + + The LLM sometimes emits the flags as a JSON list; ``str(list)`` would yield + a Python repr the server rejects, so lists are space-joined into tokens. + + Args: + value: The raw payload value (string, list/tuple, or ``None``). + + Returns: + The coerced shell-arg string. + """ + if isinstance(value, (list, tuple)): + return " ".join(str(v).strip() for v in value if str(v).strip()) + return str(value or "").strip() + + def _args_mode_of(value: Any) -> str: """Coerce an args-mode to ``"replace"`` or ``"append"``.""" return "replace" if str(value or "").strip().lower() == "replace" else "append" @@ -43,11 +61,10 @@ def normalize_proposal(proposal: Mapping[str, Any]) -> dict[str, Any]: ``unset_envs`` / ``args_mode`` / ``atomic`` / ``reason``, with the ``extra_args`` / ``extra_server_args`` alias resolved. """ - args = str(proposal.get("extra_args") or proposal.get("extra_server_args") or "").strip() envs = proposal.get("extra_envs") return { "name": str(proposal.get("name") or "").strip(), - "extra_args": args, + "extra_args": coerce_args(proposal.get("extra_args") or proposal.get("extra_server_args")), "extra_envs": {str(k): str(v) for k, v in envs.items()} if isinstance(envs, Mapping) else {}, "remove_args": to_str_list(proposal.get("remove_args")), "unset_envs": to_str_list(proposal.get("unset_envs")), diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 624aa895b..e58a21237 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -60,7 +60,7 @@ ) from . import _framework_switch_manifest as _switch_manifest from ._canonical_fingerprint import workload_signature -from ._proposal_identity import effective_fingerprint +from ._proposal_identity import effective_fingerprint, normalize_proposal from ._grid_runner import ( _MN_BACKENDS_PRIORITY, _MN_PARAMS_PRIORITY, @@ -124,27 +124,6 @@ def _initial_explore_search_state() -> dict[str, Any]: } -def _coerce_args_str(value: Any) -> str: - """Coerce a payload ``extra_args`` / ``extra_server_args`` value into a - shell-arg string. - - The LLM sometimes emits the server flags as a JSON list instead of a single - string; a naive ``str(list)`` yields the Python repr which the server - rejects. Lists/tuples are space-joined into individual tokens. - - Args: - value: The raw payload value (string, list/tuple, or None). - - Returns: - The coerced shell-arg string ("" when ``value`` is None). - """ - if value is None: - return "" - if isinstance(value, (list, tuple)): - return " ".join(str(v).strip() for v in value if str(v).strip()) - return str(value) - - # Audit/provenance metadata stashed on a GridVariant that must survive being # rebuilt into a derived variant. _CARRIED_VARIANT_ATTRS: tuple[str, ...] = ( @@ -221,17 +200,15 @@ def _grid_variants_from_payload(payload: list[Any]) -> list[GridVariant]: for raw in payload or []: if not isinstance(raw, dict) or not raw.get("name"): continue - args = _coerce_args_str(raw.get("extra_args") or raw.get("extra_server_args") or "").strip() - envs_raw = raw.get("extra_envs") or {} - envs = {str(k): str(v) for k, v in envs_raw.items()} if isinstance(envs_raw, dict) else {} + fields = normalize_proposal(raw) gv = GridVariant( - name=str(raw["name"]), - extra_server_args=args, - extra_envs=envs, + name=fields["name"], + extra_server_args=fields["extra_args"], + extra_envs=fields["extra_envs"], note=str(raw.get("note") or raw.get("provenance") or ""), - remove_args=to_str_list(raw.get("remove_args")), - unset_envs=to_str_list(raw.get("unset_envs")), - args_mode=str(raw.get("args_mode") or "append"), + remove_args=fields["remove_args"], + unset_envs=fields["unset_envs"], + args_mode=fields["args_mode"], ) # Stash extra metadata on the GridVariant so the ledger writer can # pull provenance/evidence. From 93724781571f5f59762233f6c65bd62c15ae95f7 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 20 Aug 2026 03:44:40 +0000 Subject: [PATCH 8/9] fix(state): match the untested queue on proposal content, not the ledger key The ledger key folds in whatever removal controls the stack carried when the round opened, and a KEEP mid-round rewrites those. Every candidate benched before that KEEP then hashes differently from what the queue recomputes off the finished current_best, so the queue offers them again as untried. Around one session in eight ends with a stack carrying such a control. The queue asks a narrower question than the executor's in-round dedup does -- whether these flags have been tried, not whether this variant collides with another under one base -- and the ledger rows already store the variant's own fields. Match on those and the stack drops out of the comparison entirely, along with the base plumbing the renderer needed to reproduce it. Two smaller ones on the same path: a proposal with no name was offered as executable although the grid parser skips unnamed entries, so it now gets the same derived name the multi-node materialiser gives it; and a non-numeric cycle raised out of prompt assembly, which the shared int coercion settles. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_untested_proposal_queue_unit.py | 51 ++++++++++++++----- .../state/_shared_state/render.py | 38 ++++++++------ 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py b/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py index 70dbc2062..50e4bd404 100644 --- a/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_untested_proposal_queue_unit.py @@ -7,7 +7,6 @@ import pytest -from hyperloom.orchestrator.actions.executors._canonical_fingerprint import canonical_fingerprint from hyperloom.orchestrator.actions.executors._proposal_identity import ( controls_of, effective_fingerprint, @@ -49,6 +48,16 @@ def _state(rounds, *, cycle: int = 0, tested=None, gaps=None, current_best=None) return state +def _ledger_row(proposal) -> dict: + """A tested row as the executor writes it: variant-own fields, server-args key.""" + fields = normalize_proposal(proposal) + return { + "extra_server_args": fields["extra_args"], + "extra_envs": fields["extra_envs"], + **controls_of(fields), + } + + def _fingerprint(proposal, **base) -> str: fields = normalize_proposal(proposal) return effective_fingerprint(fields["extra_args"], fields["extra_envs"], controls=controls_of(fields), **base) @@ -88,25 +97,41 @@ def test_a_round_missing_its_cycle_field_reads_as_zero(): assert _state([entry], cycle=1).to_untested_proposals_summary() == "" -def test_benched_fingerprints_are_dropped(): +def test_benched_proposals_are_dropped(): proposal = {"name": "benched", "extra_args": "--a 1"} - tested = {_fingerprint(proposal): {"outcome": "REVERT"}} + tested = {"whatever-key": dict(_ledger_row(proposal), outcome="REVERT")} assert _state([_round([proposal])], tested=tested).to_untested_proposals_summary() == "" -def test_the_tested_lookup_folds_in_the_stack_base_controls(): - """A naive fingerprint misses the ledger whenever the stack removes a flag.""" +def test_a_keep_that_changes_the_stack_does_not_resurrect_benched_proposals(): + """The ledger key is stack-relative; matching on it would miss after a KEEP.""" proposal = {"name": "with-removal", "extra_args": "--a 1", "remove_args": ["--v"]} - current_best = {"remove_args": ["--b"], "unset_envs": ["BE"]} - effective = _fingerprint(proposal, base_remove_args=["--b"], base_unset_envs=["BE"]) - naive = canonical_fingerprint("--a 1", {}, remove_args=["--v"]) - assert effective != naive + round_start = _fingerprint(proposal) + after_keep = _fingerprint(proposal, base_remove_args=["--b"], base_args_mode="replace") + assert round_start != after_keep + + state = _state( + [_round([proposal])], + tested={round_start: dict(_ledger_row(proposal), outcome="REVERT")}, + current_best={"remove_args": ["--b"], "args_mode": "replace"}, + ) + assert state.to_untested_proposals_summary() == "" + + +def test_a_nameless_proposal_gets_a_stable_name_the_grid_parser_accepts(): + from hyperloom.orchestrator.actions.executors.explore import _grid_variants_from_payload + + entry = _round([{"extra_args": "--z"}], domain="comm_specialist") + entry["task_id"] = "deadbeef99" + line = _state([entry]).to_untested_proposals_summary() + assert "comm-deadbeef-0" in line + assert _grid_variants_from_payload([{"name": "comm-deadbeef-0", "extra_args": "--z"}]) - hidden = _state([_round([proposal])], tested={effective: {}}, current_best=current_best) - assert hidden.to_untested_proposals_summary() == "" - still_shown = _state([_round([proposal])], tested={naive: {}}, current_best=current_best) - assert "with-removal" in still_shown.to_untested_proposals_summary() +def test_a_non_numeric_cycle_does_not_break_prompt_assembly(): + entry = _round([{"name": "v", "extra_args": "--a"}]) + entry["cycle"] = "bad" + assert "v" in _state([entry], cycle=0).to_untested_proposals_summary() def test_duplicate_fingerprints_collapse_across_rounds(): diff --git a/src/hyperloom/orchestrator/state/_shared_state/render.py b/src/hyperloom/orchestrator/state/_shared_state/render.py index 7c61c8434..7caac94f3 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/render.py +++ b/src/hyperloom/orchestrator/state/_shared_state/render.py @@ -446,20 +446,33 @@ def to_gaps_summary(self, *, max_entries: int = 10, max_attempts: int = 0) -> st def _untested_proposal_rows(self) -> list[dict[str, Any]]: """Executable proposals from this cycle that no explore round has benched. + Matched on the proposal's own content. The ledger is keyed on the + variant folded together with whatever removal controls the stack + carried at the time, so a KEEP that changes those controls mid-round + would otherwise make everything benched before it look untried. + Returns: Rows ranked by gap severity then recency, each carrying the normalized proposal fields plus ``domain`` / ``severity``. """ + from hyperloom.common.coerce import to_int + from ...actions.executors._proposal_identity import ( controls_of, effective_fingerprint, is_executable, normalize_proposal, ) - from ..shared_state import stack_base_params - base = stack_base_params(self.current_best) - tested = set((self.explore_search or {}).get("tested") or {}) + def content_fingerprint(fields: dict[str, Any]) -> str: + return effective_fingerprint(fields["extra_args"], fields["extra_envs"], controls=controls_of(fields)) + + cycle = to_int(self.macro_cycle, default=0) + benched = { + content_fingerprint(normalize_proposal(row)) + for row in ((self.explore_search or {}).get("tested") or {}).values() + if isinstance(row, dict) + } severity_of = { str(g.get("canonical_id") or ""): str(g.get("severity") or "").strip().lower() for g in (self.gaps or []) @@ -470,27 +483,22 @@ def _untested_proposal_rows(self) -> list[dict[str, Any]]: ranked: list[tuple[int, int, dict[str, Any]]] = [] seen: set[str] = set() for order, entry in enumerate(self.specialist_rounds or []): - if not isinstance(entry, dict) or int(entry.get("cycle") or 0) != int(self.macro_cycle or 0): + if not isinstance(entry, dict) or to_int(entry.get("cycle"), default=0) != cycle: continue domain = str(entry.get("domain") or "?").removesuffix("_specialist") severity = severity_of.get(str(entry.get("gap_canonical_id") or ""), "") - for proposal in entry.get("proposal_set") or []: + task_id = str(entry.get("task_id") or "")[:8] + for index, proposal in enumerate(entry.get("proposal_set") or []): if not isinstance(proposal, dict): continue row = normalize_proposal(proposal) if not is_executable(row): continue - fingerprint = effective_fingerprint( - row["extra_args"], - row["extra_envs"], - controls=controls_of(row), - base_remove_args=base.get("base_remove_args"), - base_unset_envs=base.get("base_unset_envs"), - base_args_mode=base.get("base_args_mode"), - ) - if fingerprint in tested or fingerprint in seen: + fingerprint = content_fingerprint(row) + if fingerprint in benched or fingerprint in seen: continue seen.add(fingerprint) + row["name"] = row["name"] or f"{domain or 'specialist'}-{task_id}-{index}" row["domain"] = domain row["severity"] = severity ranked.append((rank.get(severity, 0), order, row)) @@ -507,7 +515,7 @@ def _untested_proposal_line(row: dict[str, Any]) -> str: Returns: A single ``•``-prefixed line. """ - parts = [f"• {row['name'] or '(unnamed)'} [{row['domain']}·{row['severity'] or 'sev?'}]"] + parts = [f"• {row['name']} [{row['domain']}·{row['severity'] or 'sev?'}]"] if row["atomic"]: parts.append("ATOMIC") if row["extra_args"]: From ba909e0ef3b2ee806a74ee40e4b6c2382b6f9cc4 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 20 Aug 2026 03:45:06 +0000 Subject: [PATCH 9/9] docs(prompts): stop describing the queue and the cold-start exit as they were The two decision lists called the queue "every executable proposal", which reads as exhaustive next to a block that truncates and prints the count of what it withheld. And two comments still describe the cold-start branch as the thing that stops a specialist returning an empty set, which is the rule that branch now states the other way round. Co-Authored-By: Claude Opus 5 (1M context) --- src/hyperloom/orchestrator/prompts/orchestration.md | 4 ++-- src/hyperloom/orchestrator/prompts/prompt_builder.py | 10 +++++----- .../orchestrator/prompts/specialist_prompt_builder.py | 7 +++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/hyperloom/orchestrator/prompts/orchestration.md b/src/hyperloom/orchestrator/prompts/orchestration.md index 2d2ba14b2..eaf34669b 100644 --- a/src/hyperloom/orchestrator/prompts/orchestration.md +++ b/src/hyperloom/orchestrator/prompts/orchestration.md @@ -215,8 +215,8 @@ provide KB/PR/source evidence for `explore` grids and may produce patches for has covered the gap yet. **Where a grid comes from.** `=== Untested proposals (current cycle) ===` -carries every executable specialist proposal this cycle that no explore round -has benched, ranked by gap severity, with the ones already benched removed. +carries the executable specialist proposals this cycle that no explore round +has benched, ranked by gap severity and truncated to a count the block states. Draw from it first and copy an entry's fields verbatim — an entry marked ATOMIC is a coupled set that must go in as one variant, never split or re-authored. Target **4 variants per grid, hard maximum 6**: they run serially diff --git a/src/hyperloom/orchestrator/prompts/prompt_builder.py b/src/hyperloom/orchestrator/prompts/prompt_builder.py index d4b882c84..15b7c21a9 100644 --- a/src/hyperloom/orchestrator/prompts/prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/prompt_builder.py @@ -595,11 +595,11 @@ def _section_decision_framework(*, kernel_enabled: bool, phase: str = "", transp " `last_action_failures` + `explore_search.winners_history`.", " c. **KB sub-graphs + warm-start recipe** when present —", " cross-session priors carry " + "*qualitative* hints (what worked / what failed last time).", - " d. **`=== Untested proposals (current cycle) ===`** — every", - " executable specialist proposal this cycle that no explore", - " round has benched, ranked by gap severity. This is the", - " grid's primary source; an entry marked ATOMIC goes in", - " verbatim as one variant.", + " d. **`=== Untested proposals (current cycle) ===`** — the", + " executable specialist proposals this cycle that no explore", + " round has benched, ranked by gap severity and truncated to", + " a count the block states. This is the grid's primary", + " source; an entry marked ATOMIC goes in verbatim.", " e. **Ordering facts**: baseline runs before anything else", " (invariant). ``analysis.md`` / ``last_profile_trace`` arrive", " automatically from the Coordinator-owned analysis task at", diff --git a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py index ea19ea56f..eef69f7af 100644 --- a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py @@ -1448,9 +1448,8 @@ def _section_gap(inp: SpecialistPromptInputs) -> list[str]: # Section 4 — optional KB context def _is_cold_start(inp: SpecialistPromptInputs) -> bool: - """Return True when every prior KB/PR/research source is empty, so a - cold-start directive is injected instead of letting specialists return - an empty proposal_set. + """Return True when every prior KB/PR/research source is empty, so the + cold-start directive is injected in place of the KB block. Args: inp: The specialist prompt inputs. @@ -1496,7 +1495,7 @@ def _section_kb_subgraph(inp: SpecialistPromptInputs) -> list[str]: ) return rows if cold: - # Cold-start directive: propose domain-focus defaults, not an empty set. + # Cold-start directive: fall back to the Section 1 defaults, or exit empty with a rationale. rows.extend( [ "**COLD-START MODE — no priors available.**",