fix(forge-gemm): MoE runtime key, E2E faults, warm-replay nogit, model path

src/hyperloom/agents/kernel/tests/test_forge_gemm_tuning.py

@@ -36,6 +36,7 @@ def _payload() -> dict:
         "global_timeout": 456,
         "tuner": "fmoe_ck",
         "untuned_csv": "/tmp/in.csv",
+        "moe_untuned_csv": "/tmp/untuned_fmoe_from_runtime.csv",
         "shapes_json": "/tmp/shapes.json",
         "tunableop_input": "/tmp/tunable.txt",
         "kernel_signature_log": "/tmp/server.log",
@@ -57,12 +58,59 @@ def test_build_cmd_maps_all_options():
     assert cmd[cmd.index("--quant-type") + 1] == "auto"
     assert cmd[cmd.index("--mp") + 1] == "8"
     assert cmd[cmd.index("--tuner") + 1] == "fmoe_ck"
+    assert cmd[cmd.index("--untuned-csv") + 1] == "/tmp/in.csv"
+    assert cmd[cmd.index("--kernel-signature-log") + 1] == "/tmp/server.log"
+    assert cmd[cmd.index("--tp") + 1] == "1"
+    assert cmd[cmd.index("--conc") + 1] == "256"
+    assert cmd[cmd.index("--timeout") + 1] == "123"
+    assert cmd[cmd.index("--global-timeout") + 1] == "456"
     assert cmd[cmd.index("--tokens") + 1] == "64,128"
     assert "--skip-gpu-check" in cmd
     assert "--verbose" in cmd
     assert "--thorough" in cmd
 
 
+def test_build_cmd_forwards_the_moe_untuned_csv():
+    """The runtime-derived MoE key reaches forge only through this option.
+
+    The orchestrator derives the CSV from the dispatch tuple in the server log;
+    without the option forge infers the key from the model config instead --
+    the exact failure this lane exists to remove, and one that leaves no trace
+    because the tuning still reports success.
+    """
+    cmd = forge_gemm_tuning._build_cmd(_payload())
+
+    assert cmd[cmd.index("--moe-untuned-csv") + 1] == "/tmp/untuned_fmoe_from_runtime.csv"
+
+
+def test_build_cmd_omits_the_moe_untuned_csv_when_absent():
+    """No runtime key observed: forge must not receive an empty option."""
+    payload = _payload()
+    payload.pop("moe_untuned_csv")
+
+    assert "--moe-untuned-csv" not in forge_gemm_tuning._build_cmd(payload)
+
+
+def test_build_cmd_asserts_every_option_it_can_emit():
+    """Meta-guard: an option added to _build_cmd must be asserted in this file.
+
+    This file is the only guard on the agent-tool argv, and it had drifted to
+    covering 10 of the options it emits -- which is how the MoE CSV option went
+    unasserted while being the whole point of this lane. Comparing the emitted
+    flags against a declared set makes the next omission fail here.
+    """
+    emitted = {tok for tok in forge_gemm_tuning._build_cmd(_payload()) if tok.startswith("--")}
+    declared = {
+        "--model-path", "--framework", "--precision", "--quant-type", "--gpu-type",
+        "--tp", "--conc", "--mp", "--output-dir", "--iters", "--warmup",
+        "--min-improvement-pct", "--timeout", "--global-timeout", "--tuner",
+        "--untuned-csv", "--moe-untuned-csv", "--shapes-json", "--tunableop-input",
+        "--kernel-signature-log", "--gpu-ids", "--skip-gpu-check", "--verbose",
+        "--thorough", "--tokens", "--kb-current-lib",
+    }
+    assert emitted <= declared, f"option(s) not declared here: {sorted(emitted - declared)}"
+
+
 def test_build_cmd_forwards_provenance_but_no_knowledge_base_options(monkeypatch):
     """Tuning has no knowledge base; asking it to consult one aborts the run."""
     payload = _payload()

src/hyperloom/agents/kernel/tools/forge_gemm_tuning.py

@@ -57,6 +57,7 @@ def _build_cmd(args: dict[str, Any]) -> list[str]:
     _add_opt(cmd, args, "global_timeout", "--global-timeout")
     _add_opt(cmd, args, "tuner", "--tuner")
     _add_opt(cmd, args, "untuned_csv", "--untuned-csv")
+    _add_opt(cmd, args, "moe_untuned_csv", "--moe-untuned-csv")
     _add_opt(cmd, args, "shapes_json", "--shapes-json")
     _add_opt(cmd, args, "tunableop_input", "--tunableop-input")
     _add_opt(cmd, args, "kernel_signature_log", "--kernel-signature-log")

src/hyperloom/common/model_paths.py

@@ -19,7 +19,9 @@
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
+from typing import Any
 
 
 def _identity_leaf(seg: str) -> str:
@@ -128,3 +130,60 @@
     if isinstance(hit, str) and Path(hit).is_file():
         return Path(hit).parent
     return None
+
+
+def resolve_serving_model_path(raw: str) -> str:
+    """Resolve a session model identity to a path suitable for launching servers.
+
+    Precedence mirrors ``run_hyperloom.sbatch``: an existing directory wins,
+    then ``HL_MODEL_BASE/<repo-tail>``, then the HuggingFace hub cache via
+    :func:`resolve_local_model_dir`. When nothing resolves, the original
+    string is returned unchanged.
+    """
+    text = str(raw or "").strip()
+    if not text:
+        return ""
+    try:
+        direct = Path(text).expanduser()
+        if direct.is_dir():
+            return str(direct)
+    except OSError:
+        pass
+    base = os.environ.get("HL_MODEL_BASE", "").strip()
+    if base:
+        leaf = text.rstrip("/").split("/")[-1]
+        if leaf:
+            candidate = Path(base) / leaf
+            try:
+                if candidate.is_dir():
+                    return str(candidate)
+            except OSError:
+                pass
+    resolved = resolve_local_model_dir(text)
+    if resolved is not None:
+        return str(resolved)
+    return text
+
+
+def resolve_session_model_path(
+    *,
+    params: dict[str, Any] | None = None,
+    state_model_path: str = "",
+    for_serving: bool = False,
+) -> str:
+    """Unified session model-path precedence for executors and handlers.
+
+    Order: ``params['model_path']`` → ``$MODEL_PATH`` → ``state.model_path``.
+    When ``for_serving`` is true, :func:`resolve_serving_model_path` is applied
+    to the chosen raw value.
+    """
+    raw = (
+        str((params or {}).get("model_path") or "").strip()
+        or os.environ.get("MODEL_PATH", "").strip()
+        or str(state_model_path or "").strip()
+    )
+    if not raw:
+        return ""
+    if for_serving:
+        return resolve_serving_model_path(raw)
+    return raw

src/hyperloom/inference_optimizer/breakdown/collectors/kernels.py

@@ -1326,10 +1326,15 @@ def collect_gemm_tuning(state: dict[str, Any]) -> dict[str, Any]:
         if not isinstance(raw, dict):
             continue
         engine = _resolve_gemm_engine(raw)
+        e2e_gain_pct = _to_float(raw.get("e2e_gain_pct"))
         speedup = _to_float(raw.get("best_speedup"))
         gain_pct: float | None = None
         tuned_tput: float | None = None
-        if speedup is not None:
+        if e2e_gain_pct is not None:
+            gain_pct = e2e_gain_pct
+            if baseline_tput is not None:
+                tuned_tput = baseline_tput * (1.0 + e2e_gain_pct / 100.0)
+        elif speedup is not None:
             gain_pct = (speedup - 1.0) * 100.0
             if baseline_tput is not None:
                 tuned_tput = baseline_tput * speedup
@@ -1360,6 +1365,7 @@ def collect_gemm_tuning(state: dict[str, Any]) -> dict[str, Any]:
             "gpu_type": str(raw.get("gpu_type") or gpu_type),
             "baseline_tput": baseline_tput,
             "best_speedup": speedup,
+            "e2e_gain_pct": e2e_gain_pct,
             "gain_pct": gain_pct,
             "tuned_tput": tuned_tput,
             "tuned_file": tuned_file,

src/hyperloom/inference_optimizer/cli/__init__.py

@@ -1985,7 +1985,9 @@ async def _run_optimize(args: argparse.Namespace) -> int:
             )
             sys.exit(2)
         # Re-export so subprocess executors inject the resolved model into the Magpie YAML, not its hardcoded model.
-        os.environ["MODEL_PATH"] = str(args.model)
+        from hyperloom.common.model_paths import resolve_serving_model_path
+
+        os.environ["MODEL_PATH"] = resolve_serving_model_path(str(args.model)) or str(args.model)
 
         # Quantization prelude (one-shot, before any session/baseline work):
         # if --quantize was passed, quantize the source model now and rewrite