diff --git a/src/hyperloom/common/provenance.py b/src/hyperloom/common/provenance.py index ad0342383f..1becc90922 100644 --- a/src/hyperloom/common/provenance.py +++ b/src/hyperloom/common/provenance.py @@ -21,9 +21,9 @@ * **stdlib-only**: any package may import it without an import cycle. ``session/manifest.py`` and the TraceShapeManifest producer both call -``build_provenance`` for gfx/EP/graph-mode/server-args, but ``manifest.py`` -still keeps its own ``_detect_stack_fingerprint`` / ``_detect_image`` / -``_git_revision`` for the fields it writes directly, so those detectors are +``build_provenance`` for gfx/EP/graph-mode/server-args and the stack +fingerprint. ``manifest.py`` still keeps its own ``_detect_image`` / +``_git_revision`` for the fields it writes directly, so those two detectors are currently duplicated here. """ @@ -52,6 +52,16 @@ "vllm": ("VLLM_VERSION",), } +# Where a component lives when it was installed into a venv of its own instead +# of the interpreter running this code. ``--framework-env isolated`` is the +# default for vLLM, whose ROCm wheel pins its own torch and so must not share +# the orchestrator's environment -- which also means ``importlib.metadata`` +# here cannot see it, and the version silently degraded to "unknown" on the +# default bare-metal vLLM path. Setup records the root it installed into. +_STACK_VENV_ROOT_ENVS: dict[str, tuple[str, ...]] = { + "vllm": ("VLLM_VENV_ROOT",), +} + #: Runtime-arch overrides only. ``PYTORCH_ROCM_ARCH`` is deliberately absent: #: it names the archs a wheel is *compiled* for, not the installed device, and #: ``framework/targeted_build.py`` sets it for exactly that purpose. @@ -184,12 +194,28 @@ def detect_stack_fingerprint(env: Mapping[str, str], *, probe: bool = True) -> d val = v break if not val and probe: - val = _probe_pkg_version(component) + val = _probe_pkg_version(component, _venv_site_packages(env, component)) out[component] = val or "unknown" return out -def _probe_pkg_version(component: str) -> str: +def _venv_site_packages(env: Mapping[str, str], component: str) -> list[str]: + """``site-packages`` dirs of the venv a component was installed into. + + Empty when the component has no isolated-venv convention or the run did not + use one, which is the shared-environment case the interpreter already + covers. + """ + root = _env_first(env, *_STACK_VENV_ROOT_ENVS.get(component, ())) + if not root: + return [] + try: + return [str(p) for p in sorted(Path(root).glob("lib/python*/site-packages"))] + except OSError: + return [] + + +def _probe_pkg_version(component: str, venv_path: list[str] | None = None) -> str: """Best-effort installed-package version for a stack component. Uses ``importlib.metadata`` (reads the installed distribution's metadata) @@ -197,6 +223,11 @@ def _probe_pkg_version(component: str) -> str: (seconds; may touch the GPU/driver or trigger JIT module loads), and this runs on the session-manifest build path. Env vars (e.g. ``VLLM_VERSION``, ``AITER_COMMIT``) still take priority in ``detect_stack_fingerprint``. + + Falls back to ``venv_path`` when the running interpreter has no such + distribution: an isolated framework venv is invisible to this process + otherwise, and reporting a framework the run actually served with as + "unknown" is worse than the extra directory scan. """ dist = {"sglang": "sglang", "vllm": "vllm", "aiter": "aiter"}.get(component) if not dist: @@ -204,7 +235,17 @@ def _probe_pkg_version(component: str) -> str: try: return (_im.version(dist) or "").strip() except Exception: # noqa: BLE001 — a missing package is normal. + pass + if not venv_path: return "" + try: + for found in _im.distributions(path=list(venv_path)): + name = (found.metadata["Name"] or "").strip().lower().replace("_", "-") + if name == dist: + return (found.version or "").strip() + except Exception: # noqa: BLE001 — an unreadable venv is not a failure. + return "" + return "" def detect_code_revision(env: Mapping[str, str], *, probe: bool = True) -> str: diff --git a/src/hyperloom/inference_optimizer/session/manifest.py b/src/hyperloom/inference_optimizer/session/manifest.py index b72b6e6faa..d3432bff59 100644 --- a/src/hyperloom/inference_optimizer/session/manifest.py +++ b/src/hyperloom/inference_optimizer/session/manifest.py @@ -36,81 +36,6 @@ SCHEMA_VERSION = 4 -# Env vars consulted by _detect_stack_fingerprint (operator pins that -# bypass the import/marker-file auto-detect). -_STACK_FINGERPRINT_ENVS: dict[str, tuple[str, ...]] = { - "rocm": ("ROCM_VERSION", "HIP_VERSION"), - "aiter": ("AITER_COMMIT", "AITER_VERSION"), - "sglang": ("SGLANG_VERSION", "SGL_VERSION"), - "vllm": ("VLLM_VERSION",), -} - - -def _read_first_line(path: Path) -> str: - """Return the first non-empty, stripped line of a file. - - Args: - path (Path): File to read. - - Returns: - str: First non-blank line stripped of surrounding whitespace, or an - empty string when the file is missing, empty, or unreadable. - """ - try: - if not path.exists(): - return "" - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - s = line.strip() - if s: - return s - except OSError: - return "" - return "" - - -def _detect_stack_fingerprint() -> dict[str, str]: - """Best-effort ``stack_fingerprint``. Per component, first non-empty wins: - env var -> /opt/rocm marker (rocm only) -> package __version__/__commit__. - Missing components map to ``"unknown"``. - - Returns: - Mapping of component name to detected version/commit (``"unknown"`` - when not found). - """ - out: dict[str, str] = {} - for component, env_vars in _STACK_FINGERPRINT_ENVS.items(): - val = "" - for var in env_vars: - candidate = (os.environ.get(var) or "").strip() - if candidate: - val = candidate - break - if not val and component == "rocm": - for marker in ("/opt/rocm/.info/version", "/opt/rocm/.info/version-utils"): - v = _read_first_line(Path(marker)) - if v: - val = v - break - if not val: - try: - if component == "sglang": - import sglang as _mod # type: ignore - - val = str(getattr(_mod, "__version__", "")).strip() - elif component == "vllm": - import vllm as _mod # type: ignore - - val = str(getattr(_mod, "__version__", "")).strip() - elif component == "aiter": - import aiter as _mod # type: ignore - - val = str(getattr(_mod, "__commit__", None) or getattr(_mod, "__version__", "")).strip() - except Exception: # noqa: BLE001 — defensive, missing pkg is normal. - val = "" - out[component] = val or "unknown" - return out - - def _git_revision() -> str: """Best-effort source revision of the repo containing this package. @@ -447,8 +372,12 @@ def build_manifest( "pid": os.getpid(), "host": platform.node() or socket.gethostname() or "", "image": _detect_image(), - # Snapshotted so resume-after-redeploy can detect drift. - "stack_fingerprint": _detect_stack_fingerprint(), + # Snapshotted so resume-after-redeploy can detect drift. Taken from the + # shared builder because the manifest is what the KB row, the specialist + # prompt and resume all read, and each of them drops "unknown": a + # detector that only sees this interpreter leaves an isolated framework + # venv missing from every one of them. + "stack_fingerprint": _prov.get("stack_fingerprint") or {}, # Locked at session start; resume reads it back so a restart can't # change concurrency semantics. "research_lane_capacity": int(getattr(args, "research_lane_capacity", 1) or 1) if args is not None else 1, diff --git a/src/hyperloom/inference_optimizer/tests/test_common_provenance.py b/src/hyperloom/inference_optimizer/tests/test_common_provenance.py index 9ab00eb3aa..c415317c12 100644 --- a/src/hyperloom/inference_optimizer/tests/test_common_provenance.py +++ b/src/hyperloom/inference_optimizer/tests/test_common_provenance.py @@ -280,3 +280,49 @@ def _boom(*a, **k): monkeypatch.setattr(_prov.subprocess, "run", _boom) assert _prov.detect_code_revision({"HYPERLOOM_CODE_REVISION": "envrev"}, probe=True) == "envrev" + + +# --- isolated framework venv ------------------------------------------------ + + +def _installed(venv_root, name: str, version: str): + """A distribution installed under ``venv_root`` and nowhere this process looks.""" + site = venv_root / "lib" / "python3.12" / "site-packages" + info = site / f"{name}-{version}.dist-info" + info.mkdir(parents=True) + (info / "METADATA").write_text( + f"Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n" + ) + return venv_root + + +def test_a_framework_in_its_own_venv_is_still_versioned(tmp_path): + """``--framework-env isolated`` is the default for vLLM, whose ROCm wheel + pins its own torch. The orchestrator's interpreter cannot see that venv, so + without following ``VLLM_VENV_ROOT`` every bare-metal vLLM report recorded + the framework it actually served with as "unknown".""" + root = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723") + fp = _prov.detect_stack_fingerprint({"VLLM_VENV_ROOT": str(root)}, probe=True) + assert fp["vllm"] == "0.27.1+rocm723" + + +def test_an_operator_pin_still_wins_over_the_venv(tmp_path): + root = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723") + fp = _prov.detect_stack_fingerprint( + {"VLLM_VENV_ROOT": str(root), "VLLM_VERSION": "0.28.0-rc1"}, probe=True + ) + assert fp["vllm"] == "0.28.0-rc1" + + +def test_a_venv_root_that_is_not_there_is_not_a_failure(tmp_path): + fp = _prov.detect_stack_fingerprint( + {"VLLM_VENV_ROOT": str(tmp_path / "gone")}, probe=True + ) + assert fp["vllm"] == "unknown" + + +def test_the_venv_is_not_scanned_under_probe_false(tmp_path): + """probe=False is the hermetic contract: env only, no filesystem.""" + root = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723") + fp = _prov.detect_stack_fingerprint({"VLLM_VENV_ROOT": str(root)}, probe=False) + assert fp["vllm"] == "unknown" diff --git a/src/hyperloom/inference_optimizer/tests/test_manifest_unit.py b/src/hyperloom/inference_optimizer/tests/test_manifest_unit.py index 861d3d143b..72a84bd71e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_manifest_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_manifest_unit.py @@ -30,64 +30,6 @@ def test_utc_now_compact_and_session_id(): assert mf.build_session_id("").startswith("session_") -def test_read_first_line(tmp_path): - assert mf._read_first_line(tmp_path / "missing.txt") == "" - f = tmp_path / "v.txt" - f.write_text("\n\n hello \nworld\n", encoding="utf-8") - assert mf._read_first_line(f) == "hello" - - -def test_read_first_line_blank_only(tmp_path): - f = tmp_path / "blank.txt" - f.write_text("\n\n \n", encoding="utf-8") - assert mf._read_first_line(f) == "" - - -def test_read_first_line_oserror(tmp_path): - assert mf._read_first_line(tmp_path) == "" - - -def test_detect_stack_fingerprint_package_imports(monkeypatch): - import sys - - for var in ( - "SGLANG_VERSION", - "SGL_VERSION", - "VLLM_VERSION", - "AITER_COMMIT", - "AITER_VERSION", - "ROCM_VERSION", - "HIP_VERSION", - ): - monkeypatch.delenv(var, raising=False) - monkeypatch.setattr(mf, "_read_first_line", lambda p: "") - monkeypatch.setitem(sys.modules, "sglang", SimpleNamespace(__version__="0.5")) - monkeypatch.setitem(sys.modules, "vllm", SimpleNamespace(__version__="0.7")) - monkeypatch.setitem(sys.modules, "aiter", SimpleNamespace(__commit__="cafe")) - out = mf._detect_stack_fingerprint() - assert out["sglang"] == "0.5" - assert out["vllm"] == "0.7" - assert out["aiter"] == "cafe" - - -# ---- stack fingerprint ---------------------------------------------------- -def test_detect_stack_fingerprint_env_and_marker(monkeypatch, tmp_path): - monkeypatch.setenv("SGLANG_VERSION", "0.4.1") - monkeypatch.setenv("VLLM_VERSION", "0.6.0") - monkeypatch.delenv("ROCM_VERSION", raising=False) - monkeypatch.delenv("HIP_VERSION", raising=False) - monkeypatch.delenv("AITER_COMMIT", raising=False) - monkeypatch.delenv("AITER_VERSION", raising=False) - marker = tmp_path / "version" - marker.write_text("6.2.0\n", encoding="utf-8") - monkeypatch.setattr(mf, "_read_first_line", lambda p: "6.2.0" if "version" in str(p) else "") - out = mf._detect_stack_fingerprint() - assert out["sglang"] == "0.4.1" - assert out["vllm"] == "0.6.0" - assert out["rocm"] == "6.2.0" - assert out["aiter"] == "unknown" - - # ---- git helpers ---------------------------------------------------------- def test_git_revision_at_success(monkeypatch): monkeypatch.setattr(mf.subprocess, "run", lambda *a, **k: _Proc(0, "abc1234\n")) @@ -290,7 +232,6 @@ def test_build_manifest_without_args(monkeypatch): monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) monkeypatch.setattr(mf, "_detect_image", lambda: None) - monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {}) m = mf.build_manifest(Path("/tmp/sd")) assert m["schema_version"] == mf.SCHEMA_VERSION assert m["framework"] == "sglang" @@ -301,7 +242,6 @@ def test_build_manifest_with_args(monkeypatch): monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) monkeypatch.setattr(mf, "_detect_image", lambda: None) - monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {}) for v in ("ISL", "OSL", "CONC", "TP", "MAX_MODEL_LEN"): monkeypatch.delenv(v, raising=False) args = argparse.Namespace( @@ -336,7 +276,6 @@ def test_build_manifest_shared_provenance_fields(monkeypatch): monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) monkeypatch.setattr(mf, "_detect_image", lambda: None) - monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {}) monkeypatch.setattr(mf, "build_provenance", lambda *a, **k: { "gfx_arch": "gfx950", "ep": 8, "graph_mode": "graph_capture", "server_args": ["--tp", "1"], "server_args_hash": "abc123", @@ -350,11 +289,32 @@ def test_build_manifest_shared_provenance_fields(monkeypatch): assert m["server_args_hash"] == "abc123" +def test_manifest_versions_a_framework_installed_in_its_own_venv(monkeypatch, tmp_path): + """``--framework-env isolated`` is the default for vLLM, so the framework is + installed where the orchestrator's interpreter cannot see it. The manifest is + the copy the KB row, the specialist prompt and resume all read, and each of + them drops ``unknown`` -- so a fingerprint that degrades here is absent from + all three, not just from the run report. + """ + monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") + monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) + monkeypatch.setattr(mf, "_detect_image", lambda: None) + venv_root = tmp_path / "vllm-venv" + info = venv_root / "lib" / "python3.12" / "site-packages" / "vllm-0.27.1+rocm723.dist-info" + info.mkdir(parents=True) + (info / "METADATA").write_text( + "Metadata-Version: 2.4\nName: vllm\nVersion: 0.27.1+rocm723\n", encoding="utf-8" + ) + monkeypatch.delenv("VLLM_VERSION", raising=False) + monkeypatch.setenv("VLLM_VENV_ROOT", str(venv_root)) + m = mf.build_manifest(Path("/tmp/sd")) + assert m["stack_fingerprint"]["vllm"] == "0.27.1+rocm723" + + def test_build_manifest_snapshots_user_data_path_from_env(monkeypatch, tmp_path): monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) monkeypatch.setattr(mf, "_detect_image", lambda: None) - monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {}) monkeypatch.setenv(mf._paths.ENV_USER_DATA_PATH, str(tmp_path / "ud")) m = mf.build_manifest(tmp_path / "ud" / "sess") assert m["user_data_path"] == str(tmp_path / "ud") @@ -364,7 +324,6 @@ def test_build_manifest_user_data_path_falls_back_to_workspace_root(monkeypatch) monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) monkeypatch.setattr(mf, "_detect_image", lambda: None) - monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {}) monkeypatch.delenv(mf._paths.ENV_USER_DATA_PATH, raising=False) m = mf.build_manifest(Path("/tmp/sd")) assert m["user_data_path"] == str(mf._paths.workspace_root()) @@ -375,7 +334,6 @@ def test_write_and_load_manifest_roundtrip(monkeypatch, tmp_path): monkeypatch.setattr(mf, "_git_revision", lambda: "rev1") monkeypatch.setattr(mf, "_build_dependencies", lambda: {}) monkeypatch.setattr(mf, "_detect_image", lambda: None) - monkeypatch.setattr(mf, "_detect_stack_fingerprint", lambda: {}) written = mf.write_manifest(tmp_path, session_id="sid-x") loaded = mf.load_manifest(tmp_path) assert loaded["session_id"] == "sid-x"