Bind serving receipts to TraceLens image lineage

Magpie/benchmark_config.yaml.example

@@ -20,6 +20,10 @@ benchmark:
 
   # Run mode: docker (default), local, or ray
   run_mode: docker
+
+  # Evidence lane: profiler-free measurement or non-reward diagnostic.
+  # "auto" infers diagnostic when any heavyweight profiler is enabled.
+  run_kind: measurement
 
   # Environment variables (passed to container/process)
   envs:
@@ -36,6 +40,15 @@ benchmark:
       enabled: false
     system_profiler:
       enabled: false
+
+    # Diagnostic-only selected-kernel evidence. Requires torch_profiler=true.
+    targeted_trace:
+      enabled: false
+      backend: torch_profiler
+      run_seed: magpie-targeted-trace
+      sample_rate: 1.0
+      max_records_per_shard: 100000
+      targets: []
 
   # Timeout (seconds)
   timeout_seconds: 3600

Magpie/main.py

@@ -930,8 +930,29 @@ def load_benchmark_config(benchmark_config_path: Path) -> Dict[str, Any]:
     Returns:
         Dictionary with benchmark configuration
     """
-    data = load_yaml(benchmark_config_path)
-    return data.get("benchmark", {})
+    config, _input_sha256 = load_benchmark_config_with_sha256(
+        benchmark_config_path
+    )
+    return config
+
+
+def load_benchmark_config_with_sha256(
+    benchmark_config_path: Path,
+) -> tuple[Dict[str, Any], str]:
+    """Parse and hash the same exact benchmark YAML bytes once."""
+
+    from .modes.benchmark.serving_runtime import sha256_bytes
+
+    if not benchmark_config_path.exists():
+        return {}, ""
+    raw = benchmark_config_path.read_bytes()
+    loaded = yaml.safe_load(raw.decode("utf-8")) or {}
+    if not isinstance(loaded, dict):
+        return {}, sha256_bytes(raw)
+    benchmark = loaded.get("benchmark", {})
+    if not isinstance(benchmark, dict):
+        return {}, sha256_bytes(raw)
+    return benchmark, sha256_bytes(raw)
 
 
 def run_gap_analysis_standalone(args) -> int:
@@ -1032,10 +1053,13 @@ def run_benchmark(args, config: Dict[str, Any]) -> int:
 
     # Build benchmark config
     benchmark_cfg = {}
+    input_config_sha256 = None
 
     if args.benchmark_config:
         # Load from config file
-        benchmark_cfg = load_benchmark_config(args.benchmark_config)
+        benchmark_cfg, input_config_sha256 = load_benchmark_config_with_sha256(
+            args.benchmark_config
+        )
         if not benchmark_cfg:
             logger.error(f"No benchmark config found in {args.benchmark_config}")
             return 1
@@ -1045,6 +1069,7 @@ def run_benchmark(args, config: Dict[str, Any]) -> int:
             "framework": args.framework,
             "model": args.model,
             "precision": args.precision,
+            "run_kind": args.run_kind or "auto",
             "envs": {
                 "TP": args.tp,
                 "CONC": args.concurrency,
@@ -1077,6 +1102,9 @@ def run_benchmark(args, config: Dict[str, Any]) -> int:
     run_mode = getattr(args, "run_mode", None)
     if run_mode:
         benchmark_cfg["run_mode"] = run_mode
+    run_kind = getattr(args, "run_kind", None)
+    if run_kind:
+        benchmark_cfg["run_kind"] = run_kind
 
     # Get benchmark settings from framework config
     bench_settings = config.get("benchmark", {})
@@ -1099,6 +1127,7 @@ def run_benchmark(args, config: Dict[str, Any]) -> int:
         benchmarker = BenchmarkMode(
             config=benchmark_config,
             output_dir=str(args.output_dir),
+            input_config_sha256=input_config_sha256,
         )
         result = benchmarker.run()
 
@@ -1156,6 +1185,10 @@ def create_parser() -> argparse.ArgumentParser:
 
     subparsers = parser.add_subparsers(dest="mode", help="Evaluation mode")
 
+    from .targeted_trace.cli import add_targeted_trace_parser
+
+    add_targeted_trace_parser(subparsers)
+
     # Analyze subcommand
     analyze_parser = subparsers.add_parser(
         "analyze", help="Analyze kernel(s) - requires testcase"
@@ -1268,6 +1301,13 @@ def create_parser() -> argparse.ArgumentParser:
         help="Execution mode: 'docker' (default) runs inside a container; "
              "'local' runs directly on the host (useful inside pods/containers)"
     )
+    benchmark_parser.add_argument(
+        "--run-kind",
+        choices=["measurement", "diagnostic"],
+        default=None,
+        help="Evidence lane: measurement rejects profilers; diagnostic artifacts "
+        "are never reward eligible",
+    )
     benchmark_parser.add_argument(
         "--docker-image", type=str, help="Override Docker image"
     )
@@ -1371,6 +1411,10 @@ def main() -> int:
         return run_compare(args, config)
     elif args.mode == "benchmark":
         return run_benchmark(args, config)
+    elif args.mode == "targeted-trace":
+        from .targeted_trace.cli import run_targeted_trace
+
+        return run_targeted_trace(args)
     else:
         logger.error(f"Unknown mode: {args.mode}")
         return 1

Magpie/modes/benchmark/__init__.py

@@ -15,6 +15,7 @@
 from .config import (
     BenchmarkConfig,
     DEFAULT_SHARED_STORAGE_PATH,
+    LmEvalRuntimeConfig,
     ProfilerConfig,
     TorchProfilerConfig,
     SystemProfilerConfig,
@@ -26,22 +27,23 @@
 from .image_selector import ImageSelector
 from .inferencex import InferenceXManager, ensure_inferencex_available
 from .gap_analysis import GapAnalyzer, GapAnalysisResult
+from ...targeted_trace.config import TargetedTraceConfig
 
 __all__ = [
     "BenchmarkMode",
     "BenchmarkConfig",
     "DEFAULT_SHARED_STORAGE_PATH",
+    "LmEvalRuntimeConfig",
     "BenchmarkResult",
     "ProfilerConfig",
     "TorchProfilerConfig",
     "SystemProfilerConfig",
     "GapAnalysisConfig",
+    "TargetedTraceConfig",
     "GapAnalyzer",
     "GapAnalysisResult",
     "WorkspaceManager",
     "ImageSelector",
     "InferenceXManager",
     "ensure_inferencex_available",
 ]
-
-