Improve Hyperloom Roofline & Kernel-Opt based on Real Workload HW measurements

src/hyperloom/inference_optimizer/breakdown/collectors/telemetry.py

@@ -233,13 +233,30 @@ def _max(key: str) -> float:
         vals = [v for v in vals if v is not None]
         return round(max(vals), 2) if vals else 0.0
 
+    def _count(key: str) -> int:
+        """Number of samples carrying a numeric value for *key*.
+
+        Args:
+            key (str): Sample field name.
+
+        Returns:
+            int: The count of present values.
+        """
+        return sum(1 for s in samples if _to_float(s.get(key)) is not None)
+
     return {
         "samples": len(samples),
         "avg_power_w": _avg("power_w") or _avg("power"),
         "max_power_w": _max("power_w") or _max("power"),
         "avg_temp_c": _avg("temperature_c") or _avg("temperature"),
         "max_temp_c": _max("temperature_c") or _max("temperature"),
         "avg_clock_mhz": _avg("clock_mhz") or _avg("sclk_mhz"),
+        "max_clock_mhz": _max("clock_mhz") or _max("sclk_mhz"),
+        "avg_mclk_mhz": _avg("mclk_mhz"),
+        # Distinct from ``samples``: only a subset of samples may carry clocks
+        # (older samplers omitted ``--showclocks``), and the effective-frequency
+        # roofline derate keys off this count.
+        "clock_samples": _count("clock_mhz") or _count("sclk_mhz"),
     }
 
 

src/hyperloom/inference_optimizer/breakdown/schema.py

@@ -1110,7 +1110,13 @@ class GpuMonitorAggregate(TypedDict, total=False):
         max_power_w (float): Peak power draw (watts).
         avg_temp_c (float): Average temperature (Celsius).
         max_temp_c (float): Peak temperature (Celsius).
-        avg_clock_mhz (float): Average clock frequency (MHz).
+        avg_clock_mhz (float): Average engine (sclk) frequency (MHz).
+        max_clock_mhz (float): Peak engine (sclk) frequency (MHz).
+        avg_mclk_mhz (float): Average memory (mclk) frequency (MHz). Recorded
+            for provenance; MI300-series parts expose a single mclk DPM state,
+            so this is expected to be constant.
+        clock_samples (int): Samples that carried an engine clock. Lower than
+            ``samples`` when the sampler predates ``--showclocks``.
     """
 
     samples: int
@@ -1119,6 +1125,9 @@ class GpuMonitorAggregate(TypedDict, total=False):
     avg_temp_c: float
     max_temp_c: float
     avg_clock_mhz: float
+    max_clock_mhz: float
+    avg_mclk_mhz: float
+    clock_samples: int
 
 
 class LaneTimelineEntry(TypedDict, total=False):

src/hyperloom/inference_optimizer/multi_node/scripts/launch_infera_node.py

@@ -766,14 +766,18 @@ def _start_gpu_sampler(out_csv: Path, pid_file: Path, interval_s: int) -> None:
     q_csv = shlex.quote(str(out_csv))
     q_rocm = shlex.quote(rocm)
     interval = max(1, int(interval_s))
+    # ``--showclocks`` carries sclk/mclk, which the roofline uses to anchor the
+    # compute ceiling to the clock the workload actually sustained instead of
+    # the vendor boost clock.
+    query = "--showuse --showmemuse --showpower --showtemp --showclocks --csv"
     # header once, then loop: prepend epoch ts to each rocm-smi --csv data row.
     script = (
         "set +e; "
-        f'H="ts,$({q_rocm} --showuse --showmemuse --showpower --showtemp --csv 2>/dev/null | head -1)"; '
+        f'H="ts,$({q_rocm} {query} 2>/dev/null | head -1)"; '
         f'[ -s {q_csv} ] || echo "$H" > {q_csv}; '
         "while true; do "
         "TS=$(date +%s); "
-        f"{q_rocm} --showuse --showmemuse --showpower --showtemp --csv 2>/dev/null "
+        f"{q_rocm} {query} 2>/dev/null "
         f'| tail -n +2 | sed "s/^/$TS,/" >> {q_csv}; '
         f"sleep {interval}; done"
     )

src/hyperloom/inference_optimizer/tests/conftest.py

@@ -32,6 +32,24 @@ def _clear_kernel_request_handler_caches():
     krh._default_kernel_batch_parallel.cache_clear()
 
 
+@pytest.fixture(autouse=True)
+def _disable_gpu_hw_probe(request, monkeypatch):
+    """Keep roofline assertions off real hardware measurements.
+
+    The probe layer feeds measured compute and bandwidth roofs into every
+    ceiling. On a machine that has run a probe, a cached result would silently
+    replace the table values these tests assert against, so results would depend
+    on whether the developer happens to own a GPU. Tests that exercise the probe
+    itself opt out by name.
+    """
+    from hyperloom.orchestrator.kernel import hw_probe
+
+    if request.node.module.__name__.endswith("test_hw_probe"):
+        return
+    monkeypatch.setenv(hw_probe.DISABLE_ENV, "1")
+    hw_probe.clear_caches()
+
+
 def _bootstrap_kernel_agent_env() -> None:
     """Point HYPERLOOM_KERNEL_AGENT_ROOT at the in-repo kernel-agent checkout."""
     if os.environ.get("HYPERLOOM_KERNEL_AGENT_ROOT"):

src/hyperloom/inference_optimizer/tests/test_benchmark_result_branches_unit.py

@@ -338,6 +338,62 @@ def test_row_to_gpu_sample_empty_when_no_numeric():
     assert br._row_to_gpu_sample(["timestamp", "note"], ["1000", "n/a"]) == {}
 
 
+# Verbatim ``rocm-smi --showclocks --csv`` layout from an MI355X node: every
+# clock arrives as a "(1412Mhz)" speed cell paired with a small-integer DPM
+# level cell.
+_MI355X_CLOCK_HEADER = [
+    "ts",
+    "device",
+    "Temperature (Sensor junction) (C)",
+    "Temperature (Sensor memory) (C)",
+    "fclk clock speed:",
+    "fclk clock level:",
+    "mclk clock speed:",
+    "mclk clock level:",
+    "sclk clock speed:",
+    "sclk clock level:",
+    "socclk clock speed:",
+    "socclk clock level:",
+    "Current Socket Graphics Package Power (W)",
+    "GPU use (%)",
+    "GPU Memory Allocated (VRAM%)",
+]
+_MI355X_CLOCK_ROW = [
+    "1700000000",
+    "card0",
+    "42.0",
+    "25.0",
+    "(1250Mhz)",
+    "0",
+    "(2000Mhz)",
+    "0",
+    "(1412Mhz)",
+    "1",
+    "(38Mhz)",
+    "S",
+    "254.0",
+    "0",
+    "92",
+]
+
+
+def test_row_to_gpu_sample_reads_showclocks_speeds_not_dpm_levels():
+    s = br._row_to_gpu_sample(_MI355X_CLOCK_HEADER, _MI355X_CLOCK_ROW)
+    # The neighbouring "level" cell is a small integer that would otherwise be
+    # recorded as a frequency and silently derate the roofline to nothing.
+    assert s["clock_mhz"] == 1412
+    assert s["mclk_mhz"] == 2000
+    assert s["temperature_c"] == 42.0
+    assert s["power_w"] == 254.0
+
+
+def test_row_to_gpu_sample_omits_mclk_when_not_sampled():
+    header = ["timestamp", "Average Socket Power (W)", "sclk clock (MHz)"]
+    s = br._row_to_gpu_sample(header, ["1000", "310.5", "1400"])
+    assert s["clock_mhz"] == 1400
+    assert "mclk_mhz" not in s
+
+
 # ---- _aggregate_gpu_samples_by_role ---------------------------------------
 def test_aggregate_gpu_samples_by_role():
     samples = [