fix: resolve framework version from an isolated framework venv

src/hyperloom/common/provenance.py

@@ -21,9 +21,9 @@
 * **stdlib-only**: any package may import it without an import cycle.
 
 ``session/manifest.py`` and the TraceShapeManifest producer both call
-``build_provenance`` for gfx/EP/graph-mode/server-args, but ``manifest.py``
-still keeps its own ``_detect_stack_fingerprint`` / ``_detect_image`` /
-``_git_revision`` for the fields it writes directly, so those detectors are
+``build_provenance`` for gfx/EP/graph-mode/server-args and the stack
+fingerprint. ``manifest.py`` still keeps its own ``_detect_image`` /
+``_git_revision`` for the fields it writes directly, so those two detectors are
 currently duplicated here.
 """
 
@@ -52,6 +52,16 @@
     "vllm": ("VLLM_VERSION",),
 }
 
+# Where a component lives when it was installed into a venv of its own instead
+# of the interpreter running this code. ``--framework-env isolated`` is the
+# default for vLLM, whose ROCm wheel pins its own torch and so must not share
+# the orchestrator's environment -- which also means ``importlib.metadata``
+# here cannot see it, and the version silently degraded to "unknown" on the
+# default bare-metal vLLM path. Setup records the root it installed into.
+_STACK_VENV_ROOT_ENVS: dict[str, tuple[str, ...]] = {
+    "vllm": ("VLLM_VENV_ROOT",),
+}
+
 #: Runtime-arch overrides only. ``PYTORCH_ROCM_ARCH`` is deliberately absent:
 #: it names the archs a wheel is *compiled* for, not the installed device, and
 #: ``framework/targeted_build.py`` sets it for exactly that purpose.
@@ -184,27 +194,58 @@ def detect_stack_fingerprint(env: Mapping[str, str], *, probe: bool = True) -> d
                     val = v
                     break
         if not val and probe:
-            val = _probe_pkg_version(component)
+            val = _probe_pkg_version(component, _venv_site_packages(env, component))
         out[component] = val or "unknown"
     return out
 
 
-def _probe_pkg_version(component: str) -> str:
+def _venv_site_packages(env: Mapping[str, str], component: str) -> list[str]:
+    """``site-packages`` dirs of the venv a component was installed into.
+
+    Empty when the component has no isolated-venv convention or the run did not
+    use one, which is the shared-environment case the interpreter already
+    covers.
+    """
+    root = _env_first(env, *_STACK_VENV_ROOT_ENVS.get(component, ()))
+    if not root:
+        return []
+    try:
+        return [str(p) for p in sorted(Path(root).glob("lib/python*/site-packages"))]
+    except OSError:
+        return []
+
+
+def _probe_pkg_version(component: str, venv_path: list[str] | None = None) -> str:
     """Best-effort installed-package version for a stack component.
 
     Uses ``importlib.metadata`` (reads the installed distribution's metadata)
     instead of importing the package: ``import vllm``/``import aiter`` are heavy
     (seconds; may touch the GPU/driver or trigger JIT module loads), and this
     runs on the session-manifest build path. Env vars (e.g. ``VLLM_VERSION``,
     ``AITER_COMMIT``) still take priority in ``detect_stack_fingerprint``.
+
+    Falls back to ``venv_path`` when the running interpreter has no such
+    distribution: an isolated framework venv is invisible to this process
+    otherwise, and reporting a framework the run actually served with as
+    "unknown" is worse than the extra directory scan.
     """
     dist = {"sglang": "sglang", "vllm": "vllm", "aiter": "aiter"}.get(component)
     if not dist:
         return ""
     try:
         return (_im.version(dist) or "").strip()
     except Exception:  # noqa: BLE001 — a missing package is normal.
+        pass
+    if not venv_path:
         return ""
+    try:
+        for found in _im.distributions(path=list(venv_path)):
+            name = (found.metadata["Name"] or "").strip().lower().replace("_", "-")
+            if name == dist:
+                return (found.version or "").strip()
+    except Exception:  # noqa: BLE001 — an unreadable venv is not a failure.
+        return ""
+    return ""
 
 
 def detect_code_revision(env: Mapping[str, str], *, probe: bool = True) -> str:

src/hyperloom/inference_optimizer/session/manifest.py

@@ -36,81 +36,6 @@
 SCHEMA_VERSION = 4
 
 
-# Env vars consulted by _detect_stack_fingerprint (operator pins that
-# bypass the import/marker-file auto-detect).
-_STACK_FINGERPRINT_ENVS: dict[str, tuple[str, ...]] = {
-    "rocm": ("ROCM_VERSION", "HIP_VERSION"),
-    "aiter": ("AITER_COMMIT", "AITER_VERSION"),
-    "sglang": ("SGLANG_VERSION", "SGL_VERSION"),
-    "vllm": ("VLLM_VERSION",),
-}
-
-
-def _read_first_line(path: Path) -> str:
-    """Return the first non-empty, stripped line of a file.
-
-    Args:
-        path (Path): File to read.
-
-    Returns:
-        str: First non-blank line stripped of surrounding whitespace, or an
-        empty string when the file is missing, empty, or unreadable.
-    """
-    try:
-        if not path.exists():
-            return ""
-        for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
-            s = line.strip()
-            if s:
-                return s
-    except OSError:
-        return ""
-    return ""
-
-
-def _detect_stack_fingerprint() -> dict[str, str]:
-    """Best-effort ``stack_fingerprint``. Per component, first non-empty wins:
-    env var -> /opt/rocm marker (rocm only) -> package __version__/__commit__.
-    Missing components map to ``"unknown"``.
-
-    Returns:
-        Mapping of component name to detected version/commit (``"unknown"``
-        when not found).
-    """
-    out: dict[str, str] = {}
-    for component, env_vars in _STACK_FINGERPRINT_ENVS.items():
-        val = ""
-        for var in env_vars:
-            candidate = (os.environ.get(var) or "").strip()
-            if candidate:
-                val = candidate
-                break
-        if not val and component == "rocm":
-            for marker in ("/opt/rocm/.info/version", "/opt/rocm/.info/version-utils"):
-                v = _read_first_line(Path(marker))
-                if v:
-                    val = v
-                    break
-        if not val:
-            try:
-                if component == "sglang":
-                    import sglang as _mod  # type: ignore
-
-                    val = str(getattr(_mod, "__version__", "")).strip()
-                elif component == "vllm":
-                    import vllm as _mod  # type: ignore
-
-                    val = str(getattr(_mod, "__version__", "")).strip()
-                elif component == "aiter":
-                    import aiter as _mod  # type: ignore
-
-                    val = str(getattr(_mod, "__commit__", None) or getattr(_mod, "__version__", "")).strip()
-            except Exception:  # noqa: BLE001 — defensive, missing pkg is normal.
-                val = ""
-        out[component] = val or "unknown"
-    return out
-
-
 def _git_revision() -> str:
     """Best-effort source revision of the repo containing this package.
 
@@ -447,8 +372,12 @@ def build_manifest(
         "pid": os.getpid(),
         "host": platform.node() or socket.gethostname() or "",
         "image": _detect_image(),
-        # Snapshotted so resume-after-redeploy can detect drift.
-        "stack_fingerprint": _detect_stack_fingerprint(),
+        # Snapshotted so resume-after-redeploy can detect drift. Taken from the
+        # shared builder because the manifest is what the KB row, the specialist
+        # prompt and resume all read, and each of them drops "unknown": a
+        # detector that only sees this interpreter leaves an isolated framework
+        # venv missing from every one of them.
+        "stack_fingerprint": _prov.get("stack_fingerprint") or {},
         # Locked at session start; resume reads it back so a restart can't
         # change concurrency semantics.
         "research_lane_capacity": int(getattr(args, "research_lane_capacity", 1) or 1) if args is not None else 1,

src/hyperloom/inference_optimizer/tests/test_common_provenance.py

@@ -280,3 +280,49 @@ def _boom(*a, **k):
 
     monkeypatch.setattr(_prov.subprocess, "run", _boom)
     assert _prov.detect_code_revision({"HYPERLOOM_CODE_REVISION": "envrev"}, probe=True) == "envrev"
+
+
+# --- isolated framework venv ------------------------------------------------
+
+
+def _installed(venv_root, name: str, version: str):
+    """A distribution installed under ``venv_root`` and nowhere this process looks."""
+    site = venv_root / "lib" / "python3.12" / "site-packages"
+    info = site / f"{name}-{version}.dist-info"
+    info.mkdir(parents=True)
+    (info / "METADATA").write_text(
+        f"Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n"
+    )
+    return venv_root
+
+
+def test_a_framework_in_its_own_venv_is_still_versioned(tmp_path):
+    """``--framework-env isolated`` is the default for vLLM, whose ROCm wheel
+    pins its own torch. The orchestrator's interpreter cannot see that venv, so
+    without following ``VLLM_VENV_ROOT`` every bare-metal vLLM report recorded
+    the framework it actually served with as "unknown"."""
+    root = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723")
+    fp = _prov.detect_stack_fingerprint({"VLLM_VENV_ROOT": str(root)}, probe=True)
+    assert fp["vllm"] == "0.27.1+rocm723"
+
+
+def test_an_operator_pin_still_wins_over_the_venv(tmp_path):
+    root = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723")
+    fp = _prov.detect_stack_fingerprint(
+        {"VLLM_VENV_ROOT": str(root), "VLLM_VERSION": "0.28.0-rc1"}, probe=True
+    )
+    assert fp["vllm"] == "0.28.0-rc1"
+
+
+def test_a_venv_root_that_is_not_there_is_not_a_failure(tmp_path):
+    fp = _prov.detect_stack_fingerprint(
+        {"VLLM_VENV_ROOT": str(tmp_path / "gone")}, probe=True
+    )
+    assert fp["vllm"] == "unknown"
+
+
+def test_the_venv_is_not_scanned_under_probe_false(tmp_path):
+    """probe=False is the hermetic contract: env only, no filesystem."""
+    root = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723")
+    fp = _prov.detect_stack_fingerprint({"VLLM_VENV_ROOT": str(root)}, probe=False)
+    assert fp["vllm"] == "unknown"

src/hyperloom/inference_optimizer/tests/test_manifest_unit.py

@@ -30,64 +30,6 @@ def test_utc_now_compact_and_session_id():
     assert mf.build_session_id("").startswith("session_")
 
 
-def test_read_first_line(tmp_path):
-    assert mf._read_first_line(tmp_path / "missing.txt") == ""
-    f = tmp_path / "v.txt"
-    f.write_text("\n\n  hello \nworld\n", encoding="utf-8")
-    assert mf._read_first_line(f) == "hello"
-
-
-def test_read_first_line_blank_only(tmp_path):
-    f = tmp_path / "blank.txt"
-    f.write_text("\n\n   \n", encoding="utf-8")
-    assert mf._read_first_line(f) == ""
-
-
-def test_read_first_line_oserror(tmp_path):
-    assert mf._read_first_line(tmp_path) == ""
-
-
-def test_detect_stack_fingerprint_package_imports(monkeypatch):
-    import sys
-
-    for var in (
-        "SGLANG_VERSION",
-        "SGL_VERSION",
-        "VLLM_VERSION",
-        "AITER_COMMIT",
-        "AITER_VERSION",
-        "ROCM_VERSION",
-        "HIP_VERSION",
-    ):
-        monkeypatch.delenv(var, raising=False)
-    monkeypatch.setattr(mf, "_read_first_line", lambda p: "")
-    monkeypatch.setitem(sys.modules, "sglang", SimpleNamespace(__version__="0.5"))
-    monkeypatch.setitem(sys.modules, "vllm", SimpleNamespace(__version__="0.7"))
-    monkeypatch.setitem(sys.modules, "aiter", SimpleNamespace(__commit__="cafe"))
-    out = mf._detect_stack_fingerprint()
-    assert out["sglang"] == "0.5"
-    assert out["vllm"] == "0.7"
-    assert out["aiter"] == "cafe"
-
-
-# ---- stack fingerprint ----------------------------------------------------
-def test_detect_stack_fingerprint_env_and_marker(monkeypatch, tmp_path):
-    monkeypatch.setenv("SGLANG_VERSION", "0.4.1")
-    monkeypatch.setenv("VLLM_VERSION", "0.6.0")
-    monkeypatch.delenv("ROCM_VERSION", raising=False)
-    monkeypatch.delenv("HIP_VERSION", raising=False)
-    monkeypatch.delenv("AITER_COMMIT", raising=False)
-    monkeypatch.delenv("AITER_VERSION", raising=False)
-    marker = tmp_path / "version"
-    marker.write_text("6.2.0\n", encoding="utf-8")
-    monkeypatch.setattr(mf, "_read_first_line", lambda p: "6.2.0" if "version" in str(p) else "")
-    out = mf._detect_stack_fingerprint()
-    assert out["sglang"] == "0.4.1"
-    assert out["vllm"] == "0.6.0"
-    assert out["rocm"] == "6.2.0"
-    assert out["aiter"] == "unknown"
-
-
 # ---- git helpers ----------------------------------------------------------
 def test_git_revision_at_success(monkeypatch):
     monkeypatch.setattr(mf.subprocess, "run", lambda *a, **k: _Proc(0, "abc1234\n"))
@@ -290,7 +232,6 @@ def test_build_manifest_without_args(monkeypatch):
     monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
     monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
     monkeypatch.setattr(mf, "_detect_image", lambda: None)
-    monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {})
     m = mf.build_manifest(Path("/tmp/sd"))
     assert m["schema_version"] == mf.SCHEMA_VERSION
     assert m["framework"] == "sglang"
@@ -301,7 +242,6 @@ def test_build_manifest_with_args(monkeypatch):
     monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
     monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
     monkeypatch.setattr(mf, "_detect_image", lambda: None)
-    monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {})
     for v in ("ISL", "OSL", "CONC", "TP", "MAX_MODEL_LEN"):
         monkeypatch.delenv(v, raising=False)
     args = argparse.Namespace(
@@ -336,7 +276,6 @@ def test_build_manifest_shared_provenance_fields(monkeypatch):
     monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
     monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
     monkeypatch.setattr(mf, "_detect_image", lambda: None)
-    monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {})
     monkeypatch.setattr(mf, "build_provenance", lambda *a, **k: {
         "gfx_arch": "gfx950", "ep": 8, "graph_mode": "graph_capture",
         "server_args": ["--tp", "1"], "server_args_hash": "abc123",
@@ -350,11 +289,32 @@ def test_build_manifest_shared_provenance_fields(monkeypatch):
     assert m["server_args_hash"] == "abc123"
 
 
+def test_manifest_versions_a_framework_installed_in_its_own_venv(monkeypatch, tmp_path):
+    """``--framework-env isolated`` is the default for vLLM, so the framework is
+    installed where the orchestrator's interpreter cannot see it. The manifest is
+    the copy the KB row, the specialist prompt and resume all read, and each of
+    them drops ``unknown`` -- so a fingerprint that degrades here is absent from
+    all three, not just from the run report.
+    """
+    monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
+    monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
+    monkeypatch.setattr(mf, "_detect_image", lambda: None)
+    venv_root = tmp_path / "vllm-venv"
+    info = venv_root / "lib" / "python3.12" / "site-packages" / "vllm-0.27.1+rocm723.dist-info"
+    info.mkdir(parents=True)
+    (info / "METADATA").write_text(
+        "Metadata-Version: 2.4\nName: vllm\nVersion: 0.27.1+rocm723\n", encoding="utf-8"
+    )
+    monkeypatch.delenv("VLLM_VERSION", raising=False)
+    monkeypatch.setenv("VLLM_VENV_ROOT", str(venv_root))
+    m = mf.build_manifest(Path("/tmp/sd"))
+    assert m["stack_fingerprint"]["vllm"] == "0.27.1+rocm723"
+
+
 def test_build_manifest_snapshots_user_data_path_from_env(monkeypatch, tmp_path):
     monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
     monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
     monkeypatch.setattr(mf, "_detect_image", lambda: None)
-    monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {})
     monkeypatch.setenv(mf._paths.ENV_USER_DATA_PATH, str(tmp_path / "ud"))
     m = mf.build_manifest(tmp_path / "ud" / "sess")
     assert m["user_data_path"] == str(tmp_path / "ud")
@@ -364,7 +324,6 @@ def test_build_manifest_user_data_path_falls_back_to_workspace_root(monkeypatch)
     monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
     monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
     monkeypatch.setattr(mf, "_detect_image", lambda: None)
-    monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {})
     monkeypatch.delenv(mf._paths.ENV_USER_DATA_PATH, raising=False)
     m = mf.build_manifest(Path("/tmp/sd"))
     assert m["user_data_path"] == str(mf._paths.workspace_root())
@@ -375,7 +334,6 @@ def test_write_and_load_manifest_roundtrip(monkeypatch, tmp_path):
     monkeypatch.setattr(mf, "_git_revision", lambda: "rev1")
     monkeypatch.setattr(mf, "_build_dependencies", lambda: {})
     monkeypatch.setattr(mf, "_detect_image", lambda: None)
-    monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {})
     written = mf.write_manifest(tmp_path, session_id="sid-x")
     loaded = mf.load_manifest(tmp_path)
     assert loaded["session_id"] == "sid-x"