diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b2c01fa9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Evaluation-tool images need only their locked Docker assets and the immutable +# eval-tools framework. The repository commonly contains multi-gigabyte task +# workspaces under experiments/, so an allow-list keeps builds fast and prevents +# candidate/run artifacts from entering the Docker build context. +** +!docker/ +!docker/eval-tools/ +!docker/eval-tools/** +!src/ +!src/eval_tools/ +!src/eval_tools/** +**/__pycache__/ +**/*.py[cod] diff --git a/.gitignore b/.gitignore index a743dc66..0e3b8365 100755 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build/ .claude workspace_* /experiments/ +/.eval-tool-artifacts/ # Auto-cloned agent dependencies agents/geak_optimagentv2/GEAK-agent/ @@ -41,6 +42,10 @@ traj.json # Generated held-out test data (methodology is in src/held_out/, data is private) held_out_tests/ +# Repository-level quality_loop audit state and isolated git worktrees +quality_loop_runs/ +.quality_loop_worktrees/ + # Documentation build environment and output .docvenv/ docs/_build/ diff --git a/Makefile b/Makefile index d1b2c9f0..b55db2ea 100755 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ SHELL := /bin/bash -.PHONY: help docker-shell docker-check-agents docker-smoke docker-run docker-parallel-run docker-setup-flydsl docker-setup-geak \ +.PHONY: help docker-shell docker-check-agents docker-smoke docker-run docker-parallel-run docker-quality-loop docker-setup-flydsl docker-setup-geak \ check-docker-runner check-evaluator check-held-out check-visualization \ visualization-build visualization-serve visualization-run \ sync-perf-helpers check-perf-helpers materialize-perf-workspace \ @@ -23,6 +23,7 @@ help: @echo "make docker-smoke - Verify Docker Python, ROCm tools, imports, and GPU access" @echo "make docker-run CONFIG=example_configs/quickstart_claude_mi300.yaml RUN_ARGS=\"--run-suffix test\" - Run an experiment in Docker" @echo "make docker-parallel-run CONFIG=example_configs/benchmark_cursor_mi355x.yaml GPU_IDS=0,1 - Run an experiment across one worker container per GPU" + @echo "make docker-quality-loop QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml - Audit and harden tasks with Codex, then open one draft PR" @echo " Default CONFIG is the MI300/MI300X Claude quickstart" @echo " On other GPUs, pass a matching CONFIG explicitly" @echo " Images: gfx942->mi30x, gfx950->mi35x; override with AKA_DOCKER_IMAGE=..." @@ -45,6 +46,8 @@ help: DOCKER_RUNNER := src/scripts/docker_benchmark.sh CONFIG ?= example_configs/quickstart_claude_mi300.yaml RUN_ARGS ?= +QUALITY_LOOP_CONFIG ?= agents/quality_loop/agent_config.yaml +QUALITY_LOOP_ARGS ?= AGENTS ?= WORKSPACES ?= $(WORKSPACE) TASKS ?= $(TASK) @@ -70,6 +73,9 @@ docker-run: docker-parallel-run: @GPU_IDS="$(GPU_IDS)" $(DOCKER_RUNNER) parallel-run --config_name $(CONFIG) $(RUN_ARGS) +docker-quality-loop: + @$(DOCKER_RUNNER) quality-loop --config $(QUALITY_LOOP_CONFIG) $(QUALITY_LOOP_ARGS) + # Install FlyDSL into the container's persistent pip user-base when the selected # image does not ship it. Needed by all three FlyDSL task types. docker-setup-flydsl: diff --git a/agents/quality_loop/README.md b/agents/quality_loop/README.md new file mode 100644 index 00000000..44ffd3ce --- /dev/null +++ b/agents/quality_loop/README.md @@ -0,0 +1,97 @@ +# quality_loop agent + +`quality_loop` is a repository-level task curator. It audits every selected task, +attempts one repair for blocking validator failures, runs exactly one Codex +optimization iteration, sends the result to an independent Codex reviewer, and +optionally hardens an easy baseline or task cases behind fail-closed correctness +gates. It files idempotent GitHub issues for unrepairable task defects and bundles +all accepted task changes into one draft pull request. + +Unlike normal Arena agents, `quality_loop` is not registered in `AgentType`. +Normal launchers operate once inside one copied task workspace; this workflow owns +the full repository campaign, isolated git worktree, issue deduplication, resume +manifest, and final PR. + +## Hard preflight + +A real run stops before creating a branch or modifying a task unless all of these +pass: + +- `gh auth status -h github.com` +- the authenticated account has repository write permission +- GitHub Issues are enabled +- `git`, `gh`, and `codex` are installed +- Git has a usable author identity for task commits +- the source worktree is clean +- the configured GPU/runtime is available through the Docker runner + +Only the host-side deterministic publisher uses `gh`. The Docker runner performs +GitHub preflight and creates the audit worktree on the host, runs Codex/GPU work +without mounting GitHub credentials, then returns to the host to create issues, +commit accepted task changes, push, and open the draft PR. The main checkout is +read-only inside that container; only this run's artifact and isolated worktree +directories are writable. Codex login state is copied into an ephemeral container +home instead of being writable in place. + +## Run + +Inspect task selection without credentials, GPU work, or mutations: + +```bash +python3 -m agents.quality_loop \ + --config example_configs/quality_loop_mi300.yaml \ + --plan +``` + +Run through the supported Docker environment: + +```bash +make docker-quality-loop QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml +``` + +Use `example_configs/quality_loop_mi355x.yaml` on MI355X (`gfx950`). + +Limit a smoke run to several tasks: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--tasks hip2hip/gpumode/GELU triton2triton/vllm/triton_rms_norm" +``` + +Resume after interruption: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--resume 20260803_120000" +``` + +`--no-publish` still requires the GitHub login/write preflight, but suppresses +issues, push, and PR creation. `--plan` is the only intentionally offline mode. + +## Per-task gates + +1. Run the existing 10-check validator in a fresh workspace. +2. Record WARN findings without repairing them. +3. For FAIL, allow one task-local repair and re-run the full validator in another + fresh workspace. File one fingerprinted issue if it still fails. +4. Measure the baseline, run one Codex optimization candidate, protect the + harness, and use the centralized compile/correctness/performance evaluator. +5. Run an independent read-only Codex review. Deterministic evaluator failures + always override an agent acceptance. +6. Treat the task as easy only when three measurements of the single candidate + have median speedup at least 5x, use consistent benchmark methods and case + counts, and the reviewer accepts logic equivalence. +7. Promote only committed standalone optimization kernels. Translation/generation + tasks report 5x hardening as not applicable. +8. Case changes may touch only test/harness paths and are accepted only when both + the pre-audit kernel and candidate pass the updated cases. +9. Before any host commit, the complete worktree diff must exactly match the + accepted per-task paths recorded in `state.yaml`; unexpected edits abort + publication. + +Run artifacts are written under `quality_loop_runs//`; the isolated audit +branch lives under `.quality_loop_worktrees//`. Both are ignored by Git. +Tasks pinned to another GPU architecture are reported as `platform_deferred`; run +the matching MI300/MI355X campaign to audit those tasks. diff --git a/agents/quality_loop/__init__.py b/agents/quality_loop/__init__.py new file mode 100644 index 00000000..128c95f3 --- /dev/null +++ b/agents/quality_loop/__init__.py @@ -0,0 +1,6 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +"""Repository-level task quality audit and hardening workflow.""" + +from .config import QualityLoopConfig, load_config + +__all__ = ["QualityLoopConfig", "load_config"] diff --git a/agents/quality_loop/__main__.py b/agents/quality_loop/__main__.py new file mode 100644 index 00000000..f25f7f47 --- /dev/null +++ b/agents/quality_loop/__main__.py @@ -0,0 +1,80 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import argparse +import dataclasses +import logging +from pathlib import Path + +import yaml + +from .config import load_config +from .orchestrator import QualityLoop + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = Path(__file__).with_name("agent_config.yaml") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Audit, repair, harden, and publish AgentKernelArena tasks" + ) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument( + "--tasks", + nargs="+", + help="Override task selectors from the config (paths relative to tasks/)", + ) + parser.add_argument( + "--plan", + action="store_true", + help="List runnable/deferred tasks without GitHub, GPU, or agent execution", + ) + parser.add_argument("--resume", metavar="RUN_ID", help="Resume a prior run") + parser.add_argument( + "--no-publish", + action="store_true", + help="Run all hard preflights and audits but do not create issues, push, or open a PR", + ) + parser.add_argument("--defer-github", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--skip-preflight", action="store_true", help=argparse.SUPPRESS) + return parser + + +def configure_logging() -> logging.Logger: + logger = logging.getLogger("quality_loop") + logger.setLevel(logging.INFO) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(handler) + return logger + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + config_path = args.config if args.config.is_absolute() else REPO_ROOT / args.config + config = load_config(config_path) + if args.tasks: + config = dataclasses.replace(config, tasks=tuple(args.tasks)) + if args.no_publish: + config = dataclasses.replace( + config, + github=dataclasses.replace(config.github, publish=False), + ) + logger = configure_logging() + workflow = QualityLoop(REPO_ROOT, config, logger=logger, defer_github=args.defer_github) + if args.plan: + print(yaml.safe_dump(workflow.plan(), sort_keys=False, allow_unicode=True)) + return 0 + report = workflow.run( + resume_run_id=args.resume, + skip_preflight=args.skip_preflight, + ) + logger.info("quality_loop report: %s", report) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/quality_loop/agent_config.yaml b/agents/quality_loop/agent_config.yaml new file mode 100644 index 00000000..6b586a3c --- /dev/null +++ b/agents/quality_loop/agent_config.yaml @@ -0,0 +1,41 @@ +# quality_loop is a repository-level audit workflow. It defaults to auditing all +# tasks that are runnable on the selected target GPU. +tasks: + - all +target_gpu_model: MI300 + +quality_loop: + backend: + name: codex + model: null # null uses the Codex CLI default/config + effort: xhigh + timeout_seconds: 3600 + reviewer: + name: codex # independent, read-only Codex session + model: null + effort: xhigh + timeout_seconds: 1800 + + max_repair_attempts: 1 + optimization_iterations: 1 # enforced; any other value is rejected + easy_speedup_threshold: 5.0 + easy_confirmation_runs: 3 + case_enhancement: true + + # Only optimization tasks with a committed runnable baseline can promote a + # first-iteration 5x candidate into the new task baseline. Translation and + # authoring tasks still receive validation, optimization, review, and case audit. + promotion_task_types: + - hip2hip + - triton2triton + - flydsl2flydsl + + artifact_root: quality_loop_runs + worktree_root: .quality_loop_worktrees + + github: + publish: true + draft_pr: true + branch_prefix: quality-loop + base_branch: null # null resolves the repository default branch + issue_labels: [] # labels must already exist in the repository diff --git a/agents/quality_loop/backend.py b/agents/quality_loop/backend.py new file mode 100644 index 00000000..76d3aee9 --- /dev/null +++ b/agents/quality_loop/backend.py @@ -0,0 +1,107 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import json +import logging +import os +import shutil +import subprocess +from pathlib import Path +from typing import Protocol + +from .config import BackendConfig + + +class AgentBackend(Protocol): + def run(self, prompt: str, workspace: Path, *, role: str) -> str: ... + + +def _format_event(line: str) -> str: + try: + payload = json.loads(line) + except json.JSONDecodeError: + return line + if not isinstance(payload, dict): + return line + if payload.get("type") in {"item.completed", "item.updated"}: + item = payload.get("item") or {} + if isinstance(item, dict) and item.get("type") == "agent_message": + return str(item.get("text") or "") + if payload.get("type") in {"turn.failed", "error"}: + return str(payload.get("error") or payload.get("message") or line) + return line + + +class CodexBackend: + """Role-scoped Codex runner with GitHub credentials removed from children.""" + + def __init__(self, config: BackendConfig, logger: logging.Logger): + self.config = config + self.logger = logger + + def run(self, prompt: str, workspace: Path, *, role: str) -> str: + if not shutil.which("codex"): + raise RuntimeError("codex CLI is required for quality_loop") + workspace = workspace.resolve() + no_gh_dir = workspace / ".quality_loop_no_gh" + no_gh_dir.mkdir(exist_ok=True) + env = os.environ.copy() + for key in ( + "GH_TOKEN", + "GITHUB_TOKEN", + "SSH_AUTH_SOCK", + "GIT_ASKPASS", + "GIT_SSH_COMMAND", + ): + env.pop(key, None) + env["GH_CONFIG_DIR"] = str(no_gh_dir) + env["GIT_CONFIG_GLOBAL"] = os.devnull + env["GIT_CONFIG_NOSYSTEM"] = "1" + + command = [ + "codex", + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "-c", + "features.memories=false", + "--cd", + str(workspace), + ] + if self.config.model: + command.extend(["--model", self.config.model]) + if self.config.effort: + command.extend(["-c", f'model_reasoning_effort="{self.config.effort}"']) + command.append(prompt) + + self.logger.info( + "Starting Codex role=%s model=%s effort=%s workspace=%s", + role, + self.config.model or "", + self.config.effort, + workspace, + ) + try: + result = subprocess.run( + command, + cwd=workspace, + env=env, + capture_output=True, + text=True, + timeout=self.config.timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + f"Codex role {role} timed out after {self.config.timeout_seconds}s" + ) from exc + output = "\n".join( + _format_event(line) for line in result.stdout.splitlines() if line.strip() + ) + if result.returncode != 0: + detail = (result.stderr or output).strip() + raise RuntimeError(f"Codex role {role} failed ({result.returncode}): {detail[-4000:]}") + if result.stderr.strip(): + self.logger.warning("Codex role=%s stderr: %s", role, result.stderr[-1000:]) + return output diff --git a/agents/quality_loop/config.py b/agents/quality_loop/config.py new file mode 100644 index 00000000..ad92e2b3 --- /dev/null +++ b/agents/quality_loop/config.py @@ -0,0 +1,156 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + + +def _runtime_root(value: Any, name: str) -> str: + path = Path(str(value)) + if path.is_absolute() or not path.parts or any(part == ".." for part in path.parts): + raise ValueError(f"{name} must be a repository-relative path without '..'") + return path.as_posix() + + +@dataclass(frozen=True) +class BackendConfig: + name: str = "codex" + model: str | None = None + effort: str = "xhigh" + timeout_seconds: int = 3600 + + @classmethod + def from_dict(cls, raw: dict[str, Any] | None) -> "BackendConfig": + raw = raw or {} + name = str(raw.get("name", "codex")).strip().lower() + if name != "codex": + raise ValueError("quality_loop currently supports only the codex backend") + timeout = int(raw.get("timeout_seconds", 3600)) + if timeout <= 0: + raise ValueError("backend.timeout_seconds must be positive") + model = raw.get("model") + return cls( + name=name, + model=str(model) if model else None, + effort=str(raw.get("effort", "xhigh")), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class GitHubConfig: + publish: bool = True + draft_pr: bool = True + issue_labels: tuple[str, ...] = () + branch_prefix: str = "quality-loop" + base_branch: str | None = None + + @classmethod + def from_dict(cls, raw: dict[str, Any] | None) -> "GitHubConfig": + raw = raw or {} + labels = raw.get("issue_labels", []) + if isinstance(labels, str): + labels = [labels] + prefix = str(raw.get("branch_prefix", "quality-loop")).strip(" /-") + if not prefix: + raise ValueError("github.branch_prefix must not be empty") + base = raw.get("base_branch") + return cls( + publish=bool(raw.get("publish", True)), + draft_pr=bool(raw.get("draft_pr", True)), + issue_labels=tuple(str(label) for label in labels), + branch_prefix=prefix, + base_branch=str(base) if base else None, + ) + + +@dataclass(frozen=True) +class QualityLoopConfig: + tasks: tuple[str, ...] = ("all",) + target_gpu_model: str = "MI300" + backend: BackendConfig = field(default_factory=BackendConfig) + reviewer: BackendConfig = field(default_factory=BackendConfig) + github: GitHubConfig = field(default_factory=GitHubConfig) + max_repair_attempts: int = 1 + optimization_iterations: int = 1 + easy_speedup_threshold: float = 5.0 + easy_confirmation_runs: int = 3 + case_enhancement: bool = True + artifact_root: str = "quality_loop_runs" + worktree_root: str = ".quality_loop_worktrees" + evaluation_tools: dict[str, Any] = field(default_factory=dict) + promotion_task_types: tuple[str, ...] = ( + "hip2hip", + "triton2triton", + "flydsl2flydsl", + ) + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> "QualityLoopConfig": + tasks = raw.get("tasks", ["all"]) + if isinstance(tasks, str): + tasks = [tasks] + if not isinstance(tasks, list) or not tasks: + raise ValueError("tasks must be a non-empty string or list") + target = str(raw.get("target_gpu_model", "")).strip() + if not target: + raise ValueError("target_gpu_model is required") + + audit = raw.get("quality_loop", {}) or {} + if not isinstance(audit, dict): + raise ValueError("quality_loop must be a mapping") + iterations = int(audit.get("optimization_iterations", 1)) + if iterations != 1: + raise ValueError( + "quality_loop enforces exactly one optimization iteration; " + "optimization_iterations must be 1" + ) + repair_attempts = int(audit.get("max_repair_attempts", 1)) + if repair_attempts < 0 or repair_attempts > 1: + raise ValueError("max_repair_attempts must be 0 or 1") + confirmations = int(audit.get("easy_confirmation_runs", 3)) + if confirmations < 1: + raise ValueError("easy_confirmation_runs must be at least 1") + threshold = float(audit.get("easy_speedup_threshold", 5.0)) + if threshold <= 1.0: + raise ValueError("easy_speedup_threshold must be greater than 1.0") + + promotion_types = audit.get( + "promotion_task_types", + ["hip2hip", "triton2triton", "flydsl2flydsl"], + ) + evaluation_tools = raw.get("evaluation_tools") or {} + if not isinstance(evaluation_tools, dict): + raise ValueError("evaluation_tools must be a mapping") + return cls( + tasks=tuple(str(task) for task in tasks), + target_gpu_model=target, + backend=BackendConfig.from_dict(audit.get("backend")), + reviewer=BackendConfig.from_dict(audit.get("reviewer", audit.get("backend"))), + github=GitHubConfig.from_dict(audit.get("github")), + max_repair_attempts=repair_attempts, + optimization_iterations=iterations, + easy_speedup_threshold=threshold, + easy_confirmation_runs=confirmations, + case_enhancement=bool(audit.get("case_enhancement", True)), + artifact_root=_runtime_root( + audit.get("artifact_root", "quality_loop_runs"), + "quality_loop.artifact_root", + ), + worktree_root=_runtime_root( + audit.get("worktree_root", ".quality_loop_worktrees"), + "quality_loop.worktree_root", + ), + evaluation_tools=dict(evaluation_tools), + promotion_task_types=tuple(str(value) for value in promotion_types), + ) + + +def load_config(path: Path) -> QualityLoopConfig: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(raw, dict): + raise ValueError(f"quality_loop config must be a mapping: {path}") + return QualityLoopConfig.from_dict(raw) diff --git a/agents/quality_loop/filesystem.py b/agents/quality_loop/filesystem.py new file mode 100644 index 00000000..1bdcff0f --- /dev/null +++ b/agents/quality_loop/filesystem.py @@ -0,0 +1,144 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import hashlib +import shutil +from dataclasses import dataclass +from pathlib import Path + +from src.perf_helper_materialization import ( + MARK_END, + MARK_STARTS, + ROCMBENCH_HELPER_STUB, + VLLM_HELPER_STUB_BLOCK, + replace_marked_region, +) + + +GENERATED_NAMES = { + "validation_report.yaml", + "task_result.yaml", + "baseline_perf.yaml", + "optimized_perf.yaml", + "quality_loop_review.yaml", + "performance_report.json", + "compile_report.json", +} +GENERATED_DIRS = { + ".git", + ".pytest_cache", + ".quality_loop_no_gh", + ".quality_loop_original_sources", + ".rocprofv3", + "__pycache__", + "build", +} + + +def _digest(path: Path) -> str: + if path.is_symlink(): + return "link:" + str(path.readlink()) + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def snapshot_tree(root: Path) -> dict[str, str]: + result: dict[str, str] = {} + for path in sorted(root.rglob("*")): + rel = path.relative_to(root) + if any(part in GENERATED_DIRS for part in rel.parts): + continue + if path.is_file() or path.is_symlink(): + result[rel.as_posix()] = _digest(path) + return result + + +@dataclass(frozen=True) +class TreeChanges: + added: tuple[str, ...] + modified: tuple[str, ...] + deleted: tuple[str, ...] + + @property + def paths(self) -> tuple[str, ...]: + return self.added + self.modified + self.deleted + + @property + def empty(self) -> bool: + return not self.paths + + +def diff_trees(before: dict[str, str], after: dict[str, str]) -> TreeChanges: + return TreeChanges( + added=tuple(sorted(set(after) - set(before))), + modified=tuple(sorted(k for k in set(before) & set(after) if before[k] != after[k])), + deleted=tuple(sorted(set(before) - set(after))), + ) + + +def is_generated_path(relative: str, *, repo_subdir: str | None = None) -> bool: + path = Path(relative) + if path.name in GENERATED_NAMES: + return True + if any(part in GENERATED_DIRS for part in path.parts): + return True + if repo_subdir and path.parts and path.parts[0] == repo_subdir: + return True + return False + + +def is_case_path(relative: str) -> bool: + path = Path(relative) + parts = set(path.parts[:-1]) + name = path.name + return bool( + parts & {"script", "scripts", "test", "tests"} + or name.startswith("test_") + or name.endswith(("_test.py", "_harness.py")) + ) + + +def apply_changes(source: Path, destination: Path, changes: TreeChanges) -> None: + """Apply an already-validated, root-relative change set.""" + source = source.resolve() + destination = destination.resolve() + for relative in changes.added + changes.modified: + src = (source / relative).resolve() + dst = (destination / relative).resolve() + if not src.is_relative_to(source) or not dst.is_relative_to(destination): + raise ValueError(f"unsafe quality_loop path: {relative}") + if not src.is_file() and not src.is_symlink(): + raise ValueError(f"changed path is not a file: {relative}") + dst.parent.mkdir(parents=True, exist_ok=True) + if src.is_symlink(): + if dst.exists() or dst.is_symlink(): + dst.unlink() + dst.symlink_to(src.readlink()) + else: + shutil.copy2(src, dst) + for relative in changes.deleted: + dst = (destination / relative).resolve() + if not dst.is_relative_to(destination): + raise ValueError(f"unsafe quality_loop deletion: {relative}") + if dst.is_file() or dst.is_symlink(): + dst.unlink() + + +def restore_committed_perf_stubs(task_root: Path) -> None: + """Undo runtime helper materialization before a task diff is committed.""" + for helper in task_root.rglob("performance_utils_pytest.py"): + if helper.is_file(): + helper.write_text(ROCMBENCH_HELPER_STUB, encoding="utf-8") + for runner in task_root.rglob("task_runner.py"): + if not runner.is_file(): + continue + current = runner.read_text(encoding="utf-8") + if not (any(marker in current for marker in MARK_STARTS) or MARK_END in current): + continue + replaced = replace_marked_region(current, VLLM_HELPER_STUB_BLOCK) + if replaced is None: + raise RuntimeError(f"invalid AKA-GENERATED markers after audit: {runner}") + runner.write_text(replaced, encoding="utf-8") diff --git a/agents/quality_loop/github.py b/agents/quality_loop/github.py new file mode 100644 index 00000000..74f17526 --- /dev/null +++ b/agents/quality_loop/github.py @@ -0,0 +1,346 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import json +import logging +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +from .config import GitHubConfig + + +class CommandError(RuntimeError): + pass + + +def run_command( + args: Sequence[str], + *, + cwd: Path, + check: bool = True, + timeout: int = 120, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + list(args), + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + if check and result.returncode != 0: + rendered = " ".join(args) + detail = (result.stderr or result.stdout).strip() + raise CommandError(f"command failed ({result.returncode}): {rendered}\n{detail}") + return result + + +def parse_github_slug(remote: str) -> str: + remote = remote.strip() + patterns = ( + r"^git@github\.com:([^/]+/[^/]+?)(?:\.git)?$", + r"^ssh://git@github\.com/([^/]+/[^/]+?)(?:\.git)?$", + r"^https?://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$", + ) + for pattern in patterns: + match = re.match(pattern, remote) + if match: + return match.group(1) + raise ValueError(f"origin is not a supported GitHub remote: {remote!r}") + + +@dataclass(frozen=True) +class PreflightResult: + repo_slug: str + default_branch: str + base_sha: str + viewer_permission: str + + +class GitHubPublisher: + """The only quality_loop component allowed to use GitHub credentials.""" + + def __init__(self, repo_root: Path, config: GitHubConfig, logger: logging.Logger): + self.repo_root = repo_root.resolve() + self.config = config + self.logger = logger + + def preflight(self) -> PreflightResult: + for command in ("git", "gh", "codex"): + if not shutil.which(command): + raise RuntimeError(f"required command not found: {command}") + + status = run_command(["git", "status", "--porcelain"], cwd=self.repo_root) + if status.stdout.strip(): + raise RuntimeError( + "quality_loop requires a clean source worktree before creating " + "its isolated audit worktree" + ) + run_command(["git", "var", "GIT_AUTHOR_IDENT"], cwd=self.repo_root) + + run_command(["gh", "auth", "status", "-h", "github.com"], cwd=self.repo_root) + remote = run_command( + ["git", "remote", "get-url", "origin"], cwd=self.repo_root + ).stdout.strip() + slug = parse_github_slug(remote) + repo_data = json.loads( + run_command(["gh", "api", f"repos/{slug}"], cwd=self.repo_root).stdout + ) + permissions = repo_data.get("permissions") or {} + permission = str(repo_data.get("viewer_permission") or "").upper() + push_allowed = bool(permissions.get("push")) or permission in { + "WRITE", + "MAINTAIN", + "ADMIN", + } + if not push_allowed: + raise RuntimeError( + f"authenticated GitHub user lacks write permission for {slug} " + f"(viewer_permission={permission or 'unknown'})" + ) + if repo_data.get("has_issues") is False: + raise RuntimeError(f"GitHub issues are disabled for {slug}") + + default_branch = self.config.base_branch or repo_data.get("default_branch") + if not default_branch: + raise RuntimeError(f"could not determine the default branch for {slug}") + run_command(["git", "fetch", "origin", default_branch], cwd=self.repo_root, timeout=600) + base_sha = run_command( + ["git", "rev-parse", f"origin/{default_branch}"], cwd=self.repo_root + ).stdout.strip() + return PreflightResult(slug, str(default_branch), base_sha, permission or "WRITE") + + def create_worktree( + self, + *, + path: Path, + branch: str, + base_branch: str, + ) -> None: + if path.exists(): + raise RuntimeError(f"quality_loop worktree already exists: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + run_command( + [ + "git", + "worktree", + "add", + "-b", + branch, + str(path), + f"origin/{base_branch}", + ], + cwd=self.repo_root, + timeout=600, + ) + + def commit_task(self, worktree: Path, task_id: str) -> str | None: + relative = f"tasks/{task_id}" + run_command(["git", "add", "--", relative], cwd=worktree) + staged = run_command( + ["git", "diff", "--cached", "--quiet", "--", relative], + cwd=worktree, + check=False, + ) + if staged.returncode == 0: + return None + if staged.returncode != 1: + raise CommandError(f"could not inspect staged changes for {task_id}") + run_command( + ["git", "commit", "-m", f"fix(tasks): quality audit {task_id}"], + cwd=worktree, + timeout=600, + ) + return run_command(["git", "rev-parse", "HEAD"], cwd=worktree).stdout.strip() + + def verify_pending_changes( + self, + *, + worktree: Path, + branch: str, + base_sha: str, + expected_paths: set[str], + ) -> None: + top = Path( + run_command(["git", "rev-parse", "--show-toplevel"], cwd=worktree) + .stdout.strip() + ).resolve() + if top != worktree.resolve(): + raise RuntimeError(f"quality_loop worktree identity changed: {top}") + current_branch = run_command( + ["git", "branch", "--show-current"], cwd=worktree + ).stdout.strip() + if current_branch != branch: + raise RuntimeError( + f"quality_loop worktree branch changed: expected {branch}, got {current_branch}" + ) + ancestor = run_command( + ["git", "merge-base", "--is-ancestor", base_sha, "HEAD"], + cwd=worktree, + check=False, + ) + if ancestor.returncode != 0: + raise RuntimeError("quality_loop worktree no longer descends from its recorded base") + + changed: set[str] = set() + for args in ( + ["git", "diff", "--name-only", "HEAD"], + ["git", "diff", "--cached", "--name-only"], + ["git", "ls-files", "--others", "--exclude-standard"], + ): + output = run_command(args, cwd=worktree).stdout + changed.update(line for line in output.splitlines() if line) + if changed != expected_paths: + unexpected = sorted(changed - expected_paths) + missing = sorted(expected_paths - changed) + raise RuntimeError( + "quality_loop worktree diff does not match accepted task changes; " + f"unexpected={unexpected}, missing={missing}" + ) + + @staticmethod + def issue_marker(task_id: str, fingerprint: str) -> str: + return f"" + + def ensure_issue( + self, + *, + repo_slug: str, + task_id: str, + fingerprint: str, + title: str, + body: str, + artifact_dir: Path, + ) -> str: + marker = self.issue_marker(task_id, fingerprint) + existing_raw = run_command( + [ + "gh", + "issue", + "list", + "--repo", + repo_slug, + "--state", + "all", + "--limit", + "1000", + "--json", + "number,body,state,url", + ], + cwd=self.repo_root, + ).stdout + for issue in json.loads(existing_raw or "[]"): + if marker not in str(issue.get("body") or ""): + continue + if str(issue.get("state", "")).upper() == "CLOSED": + run_command( + ["gh", "issue", "reopen", str(issue["number"]), "--repo", repo_slug], + cwd=self.repo_root, + ) + return str(issue.get("url") or "") + + artifact_dir.mkdir(parents=True, exist_ok=True) + body_path = artifact_dir / "issue_body.md" + body_path.write_text(f"{marker}\n\n{body.rstrip()}\n", encoding="utf-8") + args = [ + "gh", + "issue", + "create", + "--repo", + repo_slug, + "--title", + title, + "--body-file", + str(body_path), + ] + for label in self.config.issue_labels: + args.extend(["--label", label]) + return run_command(args, cwd=self.repo_root).stdout.strip() + + def publish_draft_pr( + self, + *, + worktree: Path, + repo_slug: str, + branch: str, + base_branch: str, + title: str, + body: str, + artifact_dir: Path, + ) -> str | None: + ahead = int( + run_command( + ["git", "rev-list", "--count", f"origin/{base_branch}..HEAD"], + cwd=worktree, + ).stdout.strip() + or "0" + ) + if ahead == 0: + self.logger.info("No accepted task changes; skipping empty pull request") + return None + + # Use gh's credential helper explicitly so an SSH origin does not require + # forwarding private SSH keys into the GPU container. + https_remote = f"https://github.com/{repo_slug}.git" + run_command( + [ + "git", + "-c", + "credential.helper=!gh auth git-credential", + "push", + "--set-upstream", + https_remote, + branch, + ], + cwd=worktree, + timeout=1200, + ) + + artifact_dir.mkdir(parents=True, exist_ok=True) + body_path = artifact_dir / "pull_request_body.md" + body_path.write_text(body.rstrip() + "\n", encoding="utf-8") + args = [ + "gh", + "pr", + "create", + "--repo", + repo_slug, + "--head", + branch, + "--base", + base_branch, + "--title", + title, + "--body-file", + str(body_path), + ] + if self.config.draft_pr: + args.append("--draft") + existing = json.loads( + run_command( + [ + "gh", + "pr", + "list", + "--repo", + repo_slug, + "--head", + branch, + "--state", + "all", + "--limit", + "1", + "--json", + "url", + ], + cwd=worktree, + ).stdout + or "[]" + ) + if existing: + return str(existing[0].get("url") or "") + return run_command(args, cwd=worktree, timeout=300).stdout.strip() diff --git a/agents/quality_loop/host.py b/agents/quality_loop/host.py new file mode 100644 index 00000000..c647a977 --- /dev/null +++ b/agents/quality_loop/host.py @@ -0,0 +1,212 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +"""Host-side GitHub preflight and publication for credential isolation.""" +from __future__ import annotations + +import argparse +import dataclasses +import logging +from datetime import datetime, timezone +from pathlib import Path + +from .config import QualityLoopConfig, load_config +from .github import GitHubPublisher +from .orchestrator import QualityLoop, _task_slug +from .state import AuditState, resolve_worktree, stable_fingerprint, validate_run_id + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = Path(__file__).with_name("agent_config.yaml") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="quality_loop host publication boundary") + parser.add_argument("action", choices=("start", "check", "paths", "finalize")) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--tasks", nargs="+") + parser.add_argument("--no-publish", action="store_true") + parser.add_argument("--resume") + parser.add_argument("--run-id") + return parser + + +def _effective_config(args: argparse.Namespace) -> QualityLoopConfig: + path = args.config if args.config.is_absolute() else REPO_ROOT / args.config + config = load_config(path) + if args.tasks: + config = dataclasses.replace(config, tasks=tuple(args.tasks)) + if args.no_publish: + config = dataclasses.replace( + config, + github=dataclasses.replace(config.github, publish=False), + ) + return config + + +def _logger() -> logging.Logger: + logger = logging.getLogger("quality_loop.host") + logger.setLevel(logging.INFO) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(handler) + return logger + + +def _paths(config: QualityLoopConfig, run_id: str) -> tuple[Path, Path, Path]: + artifact = (REPO_ROOT / config.artifact_root / run_id).resolve() + worktree = (REPO_ROOT / config.worktree_root / run_id).resolve() + return artifact, worktree, artifact / "state.yaml" + + +def start(config: QualityLoopConfig, run_id: str, logger: logging.Logger) -> str: + validate_run_id(run_id) + publisher = GitHubPublisher(REPO_ROOT, config.github, logger) + preflight = publisher.preflight() + artifact, worktree, state_path = _paths(config, run_id) + if artifact.exists() or worktree.exists(): + raise RuntimeError(f"quality_loop run already exists: {run_id}") + branch = f"{config.github.branch_prefix}/{run_id}" + publisher.create_worktree( + path=worktree, + branch=branch, + base_branch=preflight.default_branch, + ) + AuditState.create( + state_path, + run_id=run_id, + config_fingerprint=stable_fingerprint(config), + repo_slug=preflight.repo_slug, + base_sha=preflight.base_sha, + base_branch=preflight.default_branch, + branch=branch, + worktree=worktree.relative_to(REPO_ROOT), + ) + return run_id + + +def check(config: QualityLoopConfig, run_id: str, logger: logging.Logger) -> str: + validate_run_id(run_id) + publisher = GitHubPublisher(REPO_ROOT, config.github, logger) + preflight = publisher.preflight() + _, worktree, state_path = _paths(config, run_id) + state = AuditState.load(state_path) + if state.data.get("config_fingerprint") != stable_fingerprint(config): + raise RuntimeError("resume config does not match the original quality_loop run") + if state.data.get("repo_slug") != preflight.repo_slug: + raise RuntimeError("resume repository does not match the original quality_loop run") + if ( + resolve_worktree(REPO_ROOT, str(state.data.get("worktree"))) != worktree + or not worktree.is_dir() + ): + raise RuntimeError(f"resume worktree is missing or changed: {worktree}") + return run_id + + +def finalize(config: QualityLoopConfig, run_id: str, logger: logging.Logger) -> str: + validate_run_id(run_id) + publisher = GitHubPublisher(REPO_ROOT, config.github, logger) + preflight = publisher.preflight() + artifact, worktree, state_path = _paths(config, run_id) + state = AuditState.load(state_path) + if state.data.get("status") != "awaiting_publication": + raise RuntimeError( + f"quality_loop run is not ready for publication: {state.data.get('status')}" + ) + if state.data.get("config_fingerprint") != stable_fingerprint(config): + raise RuntimeError("publication config does not match the original quality_loop run") + if state.data.get("repo_slug") != preflight.repo_slug: + raise RuntimeError("publication repository does not match the original quality_loop run") + if ( + resolve_worktree(REPO_ROOT, str(state.data.get("worktree"))) != worktree + or not worktree.is_dir() + ): + raise RuntimeError(f"publication worktree is missing or changed: {worktree}") + + expected_paths = { + f"tasks/{task_id}/{relative}" + for task_id, record in state.data.get("tasks", {}).items() + if record.get("state") == "completed" and record.get("commit_pending") + for relative in record.get("changes", []) + } + publisher.verify_pending_changes( + worktree=worktree, + branch=str(state.data["branch"]), + base_sha=str(state.data["base_sha"]), + expected_paths=expected_paths, + ) + + for task_id, record in state.data.get("tasks", {}).items(): + if record.get("state") != "completed" or not record.get("commit_pending"): + continue + commit = publisher.commit_task(worktree, task_id) + if not commit: + raise RuntimeError(f"accepted changes disappeared before commit: {task_id}") + record["commit"] = commit + record["commit_pending"] = False + state.save() + + if config.github.publish: + for task_id, record in state.data.get("tasks", {}).items(): + if record.get("state") != "issue_pending": + continue + request = record.get("issue_request") or {} + issue_url = publisher.ensure_issue( + repo_slug=str(state.data["repo_slug"]), + task_id=task_id, + fingerprint=str(request["fingerprint"]), + title=str(request["title"]), + body=str(request["body"]), + artifact_dir=artifact / "tasks" / _task_slug(task_id), + ) + state.transition(task_id, "issue_filed", issue_url=issue_url) + + workflow = QualityLoop(REPO_ROOT, config, logger=logger, publisher=publisher) + workflow.state = state + workflow.artifact_dir = artifact + workflow.worktree = worktree + workflow.preflight = preflight + report_path = workflow._write_report() + pr_url = None + if config.github.publish: + pr_url = publisher.publish_draft_pr( + worktree=worktree, + repo_slug=str(state.data["repo_slug"]), + branch=str(state.data["branch"]), + base_branch=str(state.data["base_branch"]), + title="audit(tasks): quality_loop task quality pass", + body=workflow._pull_request_body(report_path), + artifact_dir=artifact, + ) + state.finish("completed", pull_request_url=pr_url) + return pr_url or "no pull request (no accepted changes or publication disabled)" + + +def main(argv: list[str] | None = None) -> int: + args, unknown = _parser().parse_known_args(argv) + if unknown: + raise ValueError(f"unknown quality_loop host arguments: {unknown}") + config = _effective_config(args) + run_id = args.run_id or args.resume + if args.action == "start": + run_id = run_id or datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + print(start(config, run_id, _logger())) + elif args.action == "check": + if not run_id: + raise ValueError("check requires --resume or --run-id") + print(check(config, run_id, _logger())) + elif args.action == "paths": + if not run_id: + raise ValueError("paths requires --resume or --run-id") + validate_run_id(run_id) + artifact, worktree, _ = _paths(config, run_id) + print(artifact.relative_to(REPO_ROOT)) + print(worktree.relative_to(REPO_ROOT)) + else: + if not run_id: + raise ValueError("finalize requires --run-id") + print(finalize(config, run_id, _logger())) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/quality_loop/orchestrator.py b/agents/quality_loop/orchestrator.py new file mode 100644 index 00000000..74678a61 --- /dev/null +++ b/agents/quality_loop/orchestrator.py @@ -0,0 +1,948 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import logging +import os +import shutil +import statistics +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + +from agents.quality_loop.backend import AgentBackend, CodexBackend +from agents.quality_loop.config import QualityLoopConfig +from agents.quality_loop.filesystem import ( + TreeChanges, + apply_changes, + diff_trees, + is_case_path, + is_generated_path, + restore_committed_perf_stubs, + snapshot_tree, +) +from agents.quality_loop.github import GitHubPublisher, PreflightResult +from agents.quality_loop.prompts import ( + case_enhancement_prompt, + optimizer_prompt, + repair_prompt, + reviewer_prompt, +) +from agents.quality_loop.state import ( + AuditState, + resolve_worktree, + stable_fingerprint, + validate_run_id, +) +from agents.task_validator.validation_prompt import build_validation_prompt +from src.evaluator import ( + evaluate_compilation, + evaluate_correctness, + evaluate_kernel, + measure_baseline, + write_task_result, +) +from src.harness_guard import snapshot_workspace_harness, verify_workspace_harness +from src.perf_helper_materialization import materialize_perf_helpers_in_workspace +from src.preprocessing import _resolve_gfx_arch, setup_workspace +from src.prompt_builder import prompt_builder +from src.testcases import collect_benchmark_methods +from src.eval_tools.config import EvalToolsConfig +from src.eval_tools.contracts import SourceEvidence +from src.eval_tools.evidence import capture_submission_evidence + + +def _task_slug(task_id: str) -> str: + return task_id.replace("/", "__") + + +def _source_paths(config: dict[str, Any]) -> tuple[str, ...]: + raw = config.get("source_file_path", []) + if isinstance(raw, str): + raw = [raw] + return tuple(str(path) for path in raw if str(path).strip()) + + +def _repo_subdir(config: dict[str, Any]) -> str | None: + if config.get("repo_subdir"): + return str(config["repo_subdir"]) + if config.get("image_repo_path"): + return Path(str(config["image_repo_path"])).name + if config.get("repo_url"): + name = str(config["repo_url"]).rstrip("/").rsplit("/", 1)[-1] + return name[:-4] if name.endswith(".git") else name + return None + + +def _filtered_changes( + before: dict[str, str], + after: dict[str, str], + *, + repo_subdir: str | None, +) -> TreeChanges: + changes = diff_trees(before, after) + return TreeChanges( + added=tuple(p for p in changes.added if not is_generated_path(p, repo_subdir=repo_subdir)), + modified=tuple( + p for p in changes.modified if not is_generated_path(p, repo_subdir=repo_subdir) + ), + deleted=tuple( + p for p in changes.deleted if not is_generated_path(p, repo_subdir=repo_subdir) + ), + ) + + +def _validation_warnings(report: dict[str, Any]) -> list[str]: + warnings: list[str] = [] + for name, check in (report.get("checks") or {}).items(): + if isinstance(check, dict) and str(check.get("status", "")).upper() == "WARN": + warnings.append(f"{name}: {check.get('details') or check.get('analysis') or 'warning'}") + return warnings + + +def _review_is_valid(review: Any) -> bool: + if not isinstance(review, dict): + return False + for key in ( + "accepted", + "logic_equivalent", + "evidence_sufficient", + "case_enhancement_needed", + ): + if not isinstance(review.get(key), bool): + return False + return isinstance(review.get("summary"), str) and isinstance( + review.get("case_rationale"), str + ) + + +def difficulty_is_easy( + *, + task_type: str, + speedups: list[float], + result: dict[str, Any], + review: dict[str, Any], + config: QualityLoopConfig, +) -> bool: + """Return true only for a reproducible, review-approved first-iteration 5x gain.""" + return bool( + task_type in config.promotion_task_types + and len(speedups) == config.easy_confirmation_runs + and all(value > 0 for value in speedups) + and statistics.median(speedups) >= config.easy_speedup_threshold + and result.get("pass_compilation") is True + and result.get("pass_correctness") is True + and result.get("pass_tool_gate", True) is True + and result.get("tool_policy_satisfied", True) is True + and result.get("benchmark_method_consistent") is True + and int(result.get("valid_baseline_cases", 0)) > 0 + and result.get("valid_baseline_cases") == result.get("valid_optimized_cases") + and review.get("accepted") is True + and review.get("logic_equivalent") is True + and review.get("evidence_sufficient") is True + ) + + +class QualityLoop: + def __init__( + self, + repo_root: Path, + config: QualityLoopConfig, + *, + logger: logging.Logger, + backend: AgentBackend | None = None, + reviewer_backend: AgentBackend | None = None, + publisher: GitHubPublisher | None = None, + defer_github: bool = False, + ): + self.repo_root = repo_root.resolve() + self.config = config + self.logger = logger + self.backend = backend or CodexBackend(config.backend, logger) + self.reviewer_backend = reviewer_backend or CodexBackend(config.reviewer, logger) + self.publisher = publisher or GitHubPublisher(self.repo_root, config.github, logger) + self.defer_github = defer_github + self.state: AuditState | None = None + self.artifact_dir: Path | None = None + self.worktree: Path | None = None + self.preflight: PreflightResult | None = None + + def discover_tasks(self, root: Path | None = None) -> dict[str, Path]: + tasks_root = (root or self.repo_root) / "tasks" + discovered = { + str(path.parent.relative_to(tasks_root)): path + for path in tasks_root.rglob("config.yaml") + } + if "all" in self.config.tasks: + return dict(sorted(discovered.items())) + selected: dict[str, Path] = {} + missing: list[str] = [] + for selector in self.config.tasks: + matches = { + task_id: path + for task_id, path in discovered.items() + if task_id == selector or task_id.startswith(selector.rstrip("/") + "/") + } + if not matches: + missing.append(selector) + selected.update(matches) + if missing: + raise ValueError(f"task selector(s) matched nothing: {missing}") + return dict(sorted(selected.items())) + + def plan(self) -> dict[str, Any]: + tasks = self.discover_tasks() + gfx_arch = _resolve_gfx_arch(self.config.target_gpu_model) + runnable: list[str] = [] + deferred: list[str] = [] + for task_id, config_path in tasks.items(): + task_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + if self._platform_matches(task_config, gfx_arch): + runnable.append(task_id) + else: + deferred.append(task_id) + return { + "total": len(tasks), + "runnable": runnable, + "platform_deferred": deferred, + "target_gpu_model": self.config.target_gpu_model, + "gfx_arch": gfx_arch, + "backend": self.config.backend.name, + "optimization_iterations": self.config.optimization_iterations, + } + + @staticmethod + def _platform_matches(task_config: dict[str, Any], gfx_arch: str | None) -> bool: + platform = task_config.get("platform_support") + if not isinstance(platform, dict): + return True + if str(platform.get("status", "active")).strip().lower() == "skip": + return False + required = platform.get("required_arch") + return not required or (gfx_arch is not None and str(required).strip() == gfx_arch) + + def run( + self, + *, + resume_run_id: str | None = None, + skip_preflight: bool = False, + ) -> Path: + run_id = resume_run_id or datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + validate_run_id(run_id) + self.artifact_dir = (self.repo_root / self.config.artifact_root / run_id).resolve() + state_path = self.artifact_dir / "state.yaml" + fingerprint = stable_fingerprint(self.config) + + if resume_run_id: + self.state = AuditState.load(state_path) + if self.state.data.get("config_fingerprint") != fingerprint: + raise RuntimeError("resume config does not match the original quality_loop run") + self.worktree = resolve_worktree( + self.repo_root, str(self.state.data["worktree"]) + ) + if not self.worktree.is_dir(): + raise RuntimeError(f"resume worktree is missing: {self.worktree}") + if skip_preflight: + self.preflight = PreflightResult( + repo_slug=str(self.state.data["repo_slug"]), + default_branch=str(self.state.data["base_branch"]), + base_sha=str(self.state.data["base_sha"]), + viewer_permission="WRITE", + ) + else: + self.preflight = self.publisher.preflight() + else: + if skip_preflight: + raise ValueError("skip_preflight is only valid for a host-initialized resume run") + self.preflight = self.publisher.preflight() + branch = f"{self.config.github.branch_prefix}/{run_id}" + self.worktree = (self.repo_root / self.config.worktree_root / run_id).resolve() + self.publisher.create_worktree( + path=self.worktree, + branch=branch, + base_branch=self.preflight.default_branch, + ) + self.state = AuditState.create( + state_path, + run_id=run_id, + config_fingerprint=fingerprint, + repo_slug=self.preflight.repo_slug, + base_sha=self.preflight.base_sha, + base_branch=self.preflight.default_branch, + branch=branch, + worktree=self.worktree.relative_to(self.repo_root), + ) + + assert self.state is not None and self.worktree is not None + tasks = self.discover_tasks(self.worktree) + gfx_arch = _resolve_gfx_arch(self.config.target_gpu_model) + for index, (task_id, config_path) in enumerate(tasks.items(), 1): + if self.state.is_terminal(task_id): + self.logger.info("Resume: skipping terminal task %s", task_id) + continue + task_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + if not self._platform_matches(task_config, gfx_arch): + self.state.transition( + task_id, + "platform_deferred", + reason=f"requires a different platform than {gfx_arch or 'unknown'}", + ) + continue + self.logger.info("quality_loop task %d/%d: %s", index, len(tasks), task_id) + try: + self._process_task(task_id, config_path.parent) + except Exception as exc: + self.logger.exception("quality_loop task failed: %s", task_id) + # Tooling/agent/runtime failures are not evidence that the task is + # defective. Keep them resumable and never file a task issue. + self.state.transition(task_id, "infrastructure_failed", error=str(exc)) + + report_path = self._write_report() + infrastructure_failures = [ + task_id + for task_id, record in self.state.data.get("tasks", {}).items() + if record.get("state") == "infrastructure_failed" + ] + if infrastructure_failures: + self.state.finish("incomplete") + raise RuntimeError( + "quality_loop cannot publish until infrastructure failures are resumed: " + + ", ".join(infrastructure_failures) + ) + if self.defer_github: + self.state.finish("awaiting_publication") + return report_path + pr_url = None + if self.config.github.publish and not self.defer_github: + pr_url = self.publisher.publish_draft_pr( + worktree=self.worktree, + repo_slug=str(self.state.data["repo_slug"]), + branch=str(self.state.data["branch"]), + base_branch=str(self.state.data["base_branch"]), + title="audit(tasks): quality_loop task quality pass", + body=self._pull_request_body(report_path), + artifact_dir=self.artifact_dir, + ) + self.state.finish("completed", pull_request_url=pr_url) + return report_path + + def _process_task(self, task_id: str, canonical_task: Path) -> None: + assert self.state is not None + assert self.artifact_dir is not None + assert self.worktree is not None + task_artifacts = self.artifact_dir / "tasks" / _task_slug(task_id) + original_task = task_artifacts / "original_task" + candidate_task = task_artifacts / "candidate_task" + self._reset_path(task_artifacts) + task_artifacts.mkdir(parents=True) + shutil.copytree(canonical_task, original_task) + shutil.copytree(canonical_task, candidate_task) + original_tree = snapshot_tree(original_task) + original_validation_status = "FAIL" + + self.state.transition(task_id, "validating") + validation_workspace, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_initial" + ) + original_validation_status = str(validation.get("overall_status", "FAIL")).upper() + warnings = _validation_warnings(validation) + + if original_validation_status == "FAIL": + if self.config.max_repair_attempts == 0: + self._handle_unrepairable(task_id, validation, task_artifacts) + return + self.state.transition(task_id, "repairing", warnings=warnings) + task_config = self._load_task_config(candidate_task) + before = snapshot_tree(validation_workspace) + self.backend.run( + repair_prompt(validation, task_id), validation_workspace, role="repair" + ) + after = snapshot_tree(validation_workspace) + changes = _filtered_changes( + before, after, repo_subdir=_repo_subdir(task_config) + ) + if changes.empty: + self._handle_unrepairable(task_id, validation, task_artifacts) + return + apply_changes(validation_workspace, candidate_task, changes) + restore_committed_perf_stubs(candidate_task) + _, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_repaired" + ) + warnings = _validation_warnings(validation) + if str(validation.get("overall_status", "FAIL")).upper() == "FAIL": + self._handle_unrepairable(task_id, validation, task_artifacts) + return + + self.state.transition(task_id, "optimizing", warnings=warnings) + optimization_workspace, baseline_cases, result = self._optimize_once( + task_id, candidate_task, task_artifacts / "optimization" + ) + review = self._review(task_id, optimization_workspace, result) + speedups = [float(result.get("speedup_ratio") or 0.0)] + if speedups[0] >= self.config.easy_speedup_threshold and review.get("accepted"): + for _ in range(1, self.config.easy_confirmation_runs): + eval_result = evaluate_kernel( + optimization_workspace, + self._load_task_config(candidate_task), + baseline_cases, + self.logger, + ) + baseline_methods = set(collect_benchmark_methods(baseline_cases)) + optimized_methods = set(eval_result.get("optimized_benchmark_methods") or []) + repeated_valid = bool( + eval_result.get("pass_compilation") + and eval_result.get("pass_correctness") + and baseline_methods + and baseline_methods == optimized_methods + and int(eval_result.get("valid_baseline_cases", 0)) > 0 + and eval_result.get("valid_baseline_cases") + == eval_result.get("valid_optimized_cases") + ) + speedups.append( + float(eval_result.get("average_speedup") or 0.0) + if repeated_valid + else 0.0 + ) + + hardened = False + task_config = self._load_task_config(candidate_task) + if difficulty_is_easy( + task_type=str(task_config.get("task_type", "")), + speedups=speedups, + result=result, + review=review, + config=self.config, + ): + hardened = self._promote_baseline( + task_id, + original_task, + candidate_task, + optimization_workspace, + task_artifacts, + ) + + cases_enhanced = False + if ( + self.config.case_enhancement + and review.get("accepted") is True + and review.get("case_enhancement_needed") is True + and original_validation_status in {"PASS", "WARN"} + ): + cases_enhanced = self._enhance_cases( + task_id, + original_task, + candidate_task, + str(review.get("case_rationale", "")), + task_artifacts, + ) + + restore_committed_perf_stubs(candidate_task) + candidate_tree = snapshot_tree(candidate_task) + final_changes = _filtered_changes( + original_tree, + candidate_tree, + repo_subdir=_repo_subdir(task_config), + ) + commit = None + commit_pending = False + if not final_changes.empty: + apply_changes(candidate_task, canonical_task, final_changes) + restore_committed_perf_stubs(canonical_task) + if self.defer_github: + # A linked worktree's .git file contains a host-absolute path, + # which is intentionally unavailable inside the GPU container. + # The credential-bearing host finalizer verifies and commits it. + commit_pending = True + else: + commit = self.publisher.commit_task(self.worktree, task_id) + self.state.transition( + task_id, + "completed", + warnings=warnings, + changes=list(final_changes.paths), + commit=commit, + commit_pending=commit_pending, + speedups=speedups, + reviewer=review, + baseline_hardened=hardened, + cases_enhanced=cases_enhanced, + ) + + def _validate( + self, task_id: str, task_dir: Path, stage_dir: Path + ) -> tuple[Path, dict[str, Any]]: + workspace = self._make_workspace(task_id, task_dir, stage_dir) + prompt = build_validation_prompt( + str(task_dir / "config.yaml"), + str(workspace), + self._eval_config(), + ) + self.backend.run(prompt, workspace, role="validator") + report_path = workspace / "validation_report.yaml" + if not report_path.exists(): + report = { + "task_name": task_id, + "overall_status": "FAIL", + "checks": {}, + "summary": "validator backend did not produce validation_report.yaml", + } + report_path.write_text(yaml.safe_dump(report), encoding="utf-8") + return workspace, report + report = yaml.safe_load(report_path.read_text(encoding="utf-8")) or {} + if not isinstance(report, dict) or str(report.get("overall_status", "")).upper() not in { + "PASS", + "WARN", + "FAIL", + }: + raise RuntimeError(f"invalid validator report for {task_id}: {report_path}") + return workspace, report + + def _optimize_once( + self, task_id: str, task_dir: Path, stage_dir: Path + ) -> tuple[Path, list[Any], dict[str, Any]]: + workspace = self._make_workspace(task_id, task_dir, stage_dir) + task_config = self._load_task_config(task_dir) + eval_tools_config = EvalToolsConfig.from_mapping(self._eval_config()) + submission_evidence = None + if eval_tools_config.enabled: + submission_evidence = capture_submission_evidence( + workspace, + task_config, + stage_dir / "submission_evidence", + ) + original_sources = stage_dir / "original_sources" + original_sources.mkdir() + source_manifest: dict[str, str] = {} + for relative in _source_paths(task_config): + source = (workspace / relative).resolve() + if not source.is_relative_to(workspace.resolve()) or not source.is_file(): + source_manifest[relative] = "missing" + continue + destination = original_sources / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + source_manifest[relative] = "copied" + (original_sources / "manifest.yaml").write_text( + yaml.safe_dump(source_manifest, sort_keys=True), encoding="utf-8" + ) + original_source_tree = snapshot_tree(original_sources) + task_type = str(task_config.get("task_type", "")) + if task_type == "torch2hip": + baseline_cases = measure_baseline(workspace, task_config, self.logger) + else: + compiled, error = evaluate_compilation(workspace, task_config, self.logger) + if not compiled: + raise RuntimeError(f"baseline compilation failed after validation: {error}") + baseline_cases = measure_baseline(workspace, task_config, self.logger) + + harness = snapshot_workspace_harness(workspace) + base_prompt = prompt_builder( + str(task_dir / "config.yaml"), + str(workspace), + self._eval_config(), + self.logger, + ) + self.backend.run( + optimizer_prompt(base_prompt, task_id), workspace, role="optimizer" + ) + verify_workspace_harness(harness) + if snapshot_tree(original_sources) != original_source_tree: + raise RuntimeError("optimizer modified the protected original-source snapshot") + materialize_perf_helpers_in_workspace(workspace, logger=self.logger) + tool_manager = None + tool_source_evidence = None + if eval_tools_config.enabled: + assert submission_evidence is not None + submission_evidence.verify() + tool_source_evidence = SourceEvidence( + original_root=str(submission_evidence.files_dir), + original_fingerprint=submission_evidence.fingerprint, + candidate_fingerprint=submission_evidence.candidate_fingerprint(), + metadata={ + "manifest": str(submission_evidence.storage_dir / "manifest.json"), + "quality_loop_task": task_id, + }, + ) + from src.eval_tools.factory import ( + create_default_manager, + task_artifact_root, + ) + + tool_manager = create_default_manager() + tool_report_root = task_artifact_root(workspace) + else: + tool_report_root = None + evaluation = evaluate_kernel( + workspace, + task_config, + baseline_cases, + self.logger, + tool_manager=tool_manager, + eval_tools_config=eval_tools_config, + tool_source_evidence=tool_source_evidence, + tool_artifact_root=tool_report_root, + gpu_arch=_resolve_gfx_arch(self.config.target_gpu_model), + ) + write_task_result( + workspace, + evaluation, + baseline_cases, + task_id, + "quality_loop/codex", + self.logger, + create_plots=False, + ) + shutil.copytree( + original_sources, + workspace / ".quality_loop_original_sources", + ) + return workspace, baseline_cases, yaml.safe_load( + (workspace / "task_result.yaml").read_text(encoding="utf-8") + ) + + def _review( + self, task_id: str, workspace: Path, result: dict[str, Any] + ) -> dict[str, Any]: + output_name = "quality_loop_review.yaml" + before = snapshot_tree(workspace) + evidence_names = ( + "task_result.yaml", + "baseline_perf.yaml", + "optimized_perf.yaml", + ) + evidence_before = { + name: (workspace / name).read_bytes() + for name in evidence_names + if (workspace / name).is_file() + } + original_sources = workspace / ".quality_loop_original_sources" + original_before = snapshot_tree(original_sources) + self.reviewer_backend.run( + reviewer_prompt(task_id, workspace / "task_result.yaml", output_name), + workspace, + role="reviewer", + ) + after = snapshot_tree(workspace) + evidence_after = { + name: (workspace / name).read_bytes() + for name in evidence_names + if (workspace / name).is_file() + } + if ( + evidence_after != evidence_before + or snapshot_tree(original_sources) != original_before + ): + raise RuntimeError("reviewer modified protected evaluation evidence") + changes = diff_trees(before, after) + unexpected = [path for path in changes.paths if path != output_name] + if unexpected: + raise RuntimeError(f"reviewer modified non-review files: {unexpected}") + review_path = workspace / output_name + review = ( + yaml.safe_load(review_path.read_text(encoding="utf-8")) + if review_path.exists() + else None + ) + if not _review_is_valid(review): + raise RuntimeError(f"reviewer returned an invalid decision for {task_id}") + if not ( + result.get("pass_compilation") + and result.get("pass_correctness") + and result.get("benchmark_method_consistent") + ): + review["accepted"] = False + review["evidence_sufficient"] = False + review["summary"] = ( + "Deterministic evaluator gate rejected the candidate. " + + review["summary"] + ) + return review + + def _promote_baseline( + self, + task_id: str, + original_task: Path, + candidate_task: Path, + optimized_workspace: Path, + task_artifacts: Path, + ) -> bool: + task_config = self._load_task_config(candidate_task) + sources = _source_paths(task_config) + if not sources or any(not (candidate_task / path).is_file() for path in sources): + self.logger.warning("Task %s has no promotable committed source baseline", task_id) + return False + if any(not (optimized_workspace / path).is_file() for path in sources): + self.logger.warning("Task %s optimizer omitted a declared source file", task_id) + return False + source_backup = task_artifacts / "baseline_before_promotion" + self._reset_path(source_backup) + source_backup.mkdir(parents=True) + for relative in sources: + backup_file = source_backup / relative + backup_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(candidate_task / relative, backup_file) + for relative in sources: + source = optimized_workspace / relative + if not source.is_file(): + return False + destination = candidate_task / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + restore_committed_perf_stubs(candidate_task) + + _, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_hardened" + ) + if str(validation.get("overall_status", "FAIL")).upper() == "FAIL": + # Restore the previously accepted source files; an unverified faster + # candidate must never become the task baseline. + for relative in sources: + shutil.copy2(source_backup / relative, candidate_task / relative) + return False + accepted = self._dual_correctness_gate( + task_id, original_task, candidate_task, task_artifacts / "hardening_gate" + ) + if not accepted: + for relative in sources: + shutil.copy2(source_backup / relative, candidate_task / relative) + return accepted + + def _enhance_cases( + self, + task_id: str, + original_task: Path, + candidate_task: Path, + rationale: str, + task_artifacts: Path, + ) -> bool: + case_workspace = self._make_workspace( + task_id, candidate_task, task_artifacts / "case_candidate" + ) + task_config = self._load_task_config(candidate_task) + before = snapshot_tree(case_workspace) + self.backend.run( + case_enhancement_prompt(task_id, rationale), + case_workspace, + role="case_enhancer", + ) + after = snapshot_tree(case_workspace) + changes = _filtered_changes( + before, after, repo_subdir=_repo_subdir(task_config) + ) + if changes.empty: + return False + if any( + not is_case_path(path) or Path(path).name == "performance_utils_pytest.py" + for path in changes.paths + ): + self.logger.warning("Rejecting non-case changes from case enhancer: %s", changes.paths) + return False + + backup = task_artifacts / "candidate_before_cases" + shutil.copytree(candidate_task, backup) + apply_changes(case_workspace, candidate_task, changes) + restore_committed_perf_stubs(candidate_task) + if not self._dual_correctness_gate( + task_id, original_task, candidate_task, task_artifacts / "case_gate" + ): + self._replace_directory(candidate_task, backup) + return False + _, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_cases" + ) + if str(validation.get("overall_status", "FAIL")).upper() == "FAIL": + self._replace_directory(candidate_task, backup) + return False + return True + + def _dual_correctness_gate( + self, task_id: str, original_task: Path, candidate_task: Path, stage_dir: Path + ) -> bool: + task_config = self._load_task_config(candidate_task) + sources = _source_paths(task_config) + if not sources or any(not (original_task / path).is_file() for path in sources): + self.logger.warning( + "Cannot prove new cases/baseline against a committed original kernel for %s", + task_id, + ) + return False + original_with_cases = stage_dir / "original_with_cases" + self._reset_path(stage_dir) + shutil.copytree(candidate_task, original_with_cases) + for relative in sources: + destination = original_with_cases / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(original_task / relative, destination) + + for label, task_dir in ( + ("original", original_with_cases), + ("candidate", candidate_task), + ): + workspace = self._make_workspace(task_id, task_dir, stage_dir / label) + config = self._load_task_config(task_dir) + compiled, _ = evaluate_compilation(workspace, config, self.logger) + correct, _ = evaluate_correctness(workspace, config, self.logger) + if not compiled or not correct: + self.logger.warning( + "Dual correctness gate rejected %s (%s): compile=%s correctness=%s", + task_id, + label, + compiled, + correct, + ) + return False + return True + + def _handle_unrepairable( + self, task_id: str, report: dict[str, Any], task_artifacts: Path + ) -> None: + assert self.state is not None + fingerprint = stable_fingerprint( + { + "task": task_id, + "base_sha": self.state.data["base_sha"], + "checks": report.get("checks"), + } + ) + body = ( + f"`quality_loop` could not repair task `{task_id}`.\n\n" + f"Base commit: `{self.state.data['base_sha']}`\n\n" + "Validator report:\n\n```yaml\n" + + yaml.safe_dump(report, sort_keys=False) + + "```" + ) + issue_url = None + if self.defer_github and self.config.github.publish: + state = "issue_pending" + elif self.config.github.publish: + issue_url = self.publisher.ensure_issue( + repo_slug=str(self.state.data["repo_slug"]), + task_id=task_id, + fingerprint=fingerprint, + title=f"[quality_loop] Invalid task: {task_id}", + body=body, + artifact_dir=task_artifacts, + ) + state = "issue_filed" + else: + state = "reported_failure" + self.state.transition( + task_id, + state, + issue_url=issue_url, + validation_report=report, + issue_request={ + "task_id": task_id, + "fingerprint": fingerprint, + "title": f"[quality_loop] Invalid task: {task_id}", + "body": body, + } + if state == "issue_pending" + else None, + ) + + def _make_workspace(self, task_id: str, task_dir: Path, stage_dir: Path) -> Path: + self._reset_path(stage_dir) + stage_dir.mkdir(parents=True, exist_ok=True) + return setup_workspace( + str(task_dir / "config.yaml"), + stage_dir, + "qualityloop", + self.logger, + task_name=task_id, + ) + + def _eval_config(self) -> dict[str, Any]: + result = { + "target_gpu_model": self.config.target_gpu_model, + "agent": { + "template": "codex", + "python_path": os.environ.get("AGENT_KERNEL_ARENA_PYTHON"), + "compile_timeout": 600, + "correctness_timeout": 600, + "performance_timeout": 600, + "max_iterations": 1, + }, + } + if self.config.evaluation_tools: + result["evaluation_tools"] = dict(self.config.evaluation_tools) + return result + + @staticmethod + def _load_task_config(task_dir: Path) -> dict[str, Any]: + value = yaml.safe_load((task_dir / "config.yaml").read_text(encoding="utf-8")) or {} + if not isinstance(value, dict): + raise ValueError(f"invalid task config: {task_dir / 'config.yaml'}") + return value + + @staticmethod + def _reset_path(path: Path) -> None: + if not path.exists() and not path.is_symlink(): + return + if path.is_symlink() or path.is_file(): + path.unlink() + else: + shutil.rmtree(path) + + @classmethod + def _replace_directory(cls, destination: Path, source: Path) -> None: + cls._reset_path(destination) + shutil.copytree(source, destination) + + def _write_report(self) -> Path: + assert self.state is not None and self.artifact_dir is not None + records = self.state.data.get("tasks", {}) + counts: dict[str, int] = {} + warning_count = 0 + for record in records.values(): + status = str(record.get("state", "unknown")) + counts[status] = counts.get(status, 0) + 1 + warning_count += len(record.get("warnings") or []) + report = { + "run_id": self.state.data["run_id"], + "repo": self.state.data["repo_slug"], + "base_sha": self.state.data["base_sha"], + "target_gpu_model": self.config.target_gpu_model, + "backend": self.config.backend.name, + "optimization_iterations": 1, + "counts": counts, + "warning_count": warning_count, + "tasks": records, + } + path = self.artifact_dir / "audit_report.yaml" + path.write_text( + yaml.safe_dump(report, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + return path + + def _pull_request_body(self, report_path: Path) -> str: + assert self.state is not None + records = self.state.data.get("tasks", {}) + completed = [task for task, value in records.items() if value.get("state") == "completed"] + changed = [task for task in completed if records[task].get("changes")] + issues = [value.get("issue_url") for value in records.values() if value.get("issue_url")] + warnings = sum(len(value.get("warnings") or []) for value in records.values()) + return f"""## Summary + +- Audited tasks: {len(records)} +- Accepted task changes: {len(changed)} +- Validator warnings recorded: {warnings} +- Unrepairable task issues: {len(issues)} +- Optimizer: Codex, exactly one iteration per task +- Easy-task threshold: reproducible {self.config.easy_speedup_threshold:.1f}x + +## Changed tasks + +{chr(10).join(f'- `{task}`' for task in changed) or '- None'} + +## Issues + +{chr(10).join(f'- {url}' for url in issues) or '- None'} + +The full machine-readable report is stored in the local run artifact +`{report_path}`. Every promoted baseline and case change passed the fail-closed +dual correctness gate against the pre-audit kernel. +""" diff --git a/agents/quality_loop/prompts.py b/agents/quality_loop/prompts.py new file mode 100644 index 00000000..2145d2f2 --- /dev/null +++ b/agents/quality_loop/prompts.py @@ -0,0 +1,82 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +from pathlib import Path + +import yaml + + +def repair_prompt(report: dict, task_id: str) -> str: + return f"""# quality_loop validation repair + +Task: `{task_id}` + +The task validator found blocking failures. Fix only the FAIL/TIMEOUT findings in +this task workspace. Do not spend time fixing WARN-only findings. Preserve the +task's intended computation and public contract. Run the relevant compile and +correctness commands after editing. Do not use GitHub, git, network services, or +edit anything outside this workspace. + +Validation report: +```yaml +{yaml.safe_dump(report, sort_keys=False)} +``` +""" + + +def optimizer_prompt(base_prompt: str, task_id: str) -> str: + return base_prompt.rstrip() + f""" + +## quality_loop single-iteration boundary + +This is the one and only optimization iteration for `{task_id}`. Produce at most +one candidate implementation. Complete all analysis, implementation, compile, +correctness, and performance checks needed for that candidate in this single +iteration. You may edit only declared kernel/source files. Do not edit config, +tests, scripts, performance helpers, or any other harness file. Do not use git or +GitHub. Preserve the exact computation, outputs, dtypes, shapes, aliasing, and +side effects of the original task. +""" + + +def reviewer_prompt(task_id: str, result_file: Path, output_name: str) -> str: + return f"""# quality_loop independent evaluation review + +Review task `{task_id}` independently. You are a read-only evaluator, not the +optimizer. Compare the candidate's declared source paths with their pre-optimizer +copies under `.quality_loop_original_sources/`. Inspect the config, test harness, +and centralized evaluator evidence in `{result_file.name}`. Decide whether the +candidate preserves the task's computation and whether the evidence is strong +enough to accept it. Also decide whether task cases have material coverage gaps. + +Do not edit any existing file. Write exactly one new YAML file `{output_name}`: + +```yaml +accepted: true # boolean +logic_equivalent: true # boolean +evidence_sufficient: true # boolean +case_enhancement_needed: false # boolean +case_rationale: "..." +summary: "..." +``` + +Fail closed: set accepted false when behavior is ambiguous, evidence is missing, +the harness changed, performance methods differ, valid case counts shrink, or +the candidate depends on untested assumptions. Do not use git, GitHub, or network +services. +""" + + +def case_enhancement_prompt(task_id: str, rationale: str) -> str: + return f"""# quality_loop task-case hardening + +Task: `{task_id}` +Reviewer rationale: {rationale} + +Strengthen correctness coverage only where the rationale identifies a real gap. +Add a small, targeted set of valid boundary/shape/dtype cases. Do not modify the +kernel/source implementation, computation contract, tolerances merely to accept +wrong answers, benchmark timing helpers, or performance methodology. Every new +case must be valid for and pass the original pre-audit kernel. Run the appropriate +correctness command. Do not use git, GitHub, or network services. +""" diff --git a/agents/quality_loop/state.py b/agents/quality_loop/state.py new file mode 100644 index 00000000..371bdb62 --- /dev/null +++ b/agents/quality_loop/state.py @@ -0,0 +1,134 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + + +TERMINAL_TASK_STATES = { + "completed", + "issue_filed", + "issue_pending", + "reported_failure", + "platform_deferred", +} + +RUN_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + + +def validate_run_id(run_id: str) -> str: + if not RUN_ID_PATTERN.fullmatch(run_id) or run_id in {".", ".."}: + raise ValueError( + "quality_loop run ID must contain only letters, digits, '.', '_', or '-'" + ) + return run_id + + +def resolve_worktree(repo_root: Path, stored_path: str | Path) -> Path: + path = Path(stored_path) + return path.resolve() if path.is_absolute() else (repo_root / path).resolve() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def stable_fingerprint(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, default=str).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +class AuditState: + """Crash-safe YAML manifest used for resume and publication summaries.""" + + def __init__(self, path: Path, data: dict[str, Any]): + self.path = path + self.data = data + + @classmethod + def create( + cls, + path: Path, + *, + run_id: str, + config_fingerprint: str, + repo_slug: str, + base_sha: str, + base_branch: str, + branch: str, + worktree: Path, + ) -> "AuditState": + validate_run_id(run_id) + state = cls( + path, + { + "schema_version": 1, + "run_id": run_id, + "created_at": utc_now(), + "updated_at": utc_now(), + "config_fingerprint": config_fingerprint, + "repo_slug": repo_slug, + "base_sha": base_sha, + "base_branch": base_branch, + "branch": branch, + "worktree": str(worktree), + "status": "running", + "tasks": {}, + "pull_request_url": None, + }, + ) + state.save() + return state + + @classmethod + def load(cls, path: Path) -> "AuditState": + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(raw, dict) or raw.get("schema_version") != 1: + raise ValueError(f"unsupported or corrupt quality_loop state: {path}") + return cls(path, raw) + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.data["updated_at"] = utc_now() + tmp = self.path.with_name(f".{self.path.name}.{os.getpid()}.tmp") + tmp.write_text( + yaml.safe_dump(self.data, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + os.replace(tmp, self.path) + + def task(self, task_id: str) -> dict[str, Any]: + tasks = self.data.setdefault("tasks", {}) + return tasks.setdefault( + task_id, + { + "state": "pending", + "events": [], + "warnings": [], + "changes": [], + "issue_url": None, + }, + ) + + def transition(self, task_id: str, state: str, **fields: Any) -> None: + record = self.task(task_id) + record["state"] = state + record.update(fields) + record.setdefault("events", []).append({"at": utc_now(), "state": state}) + self.save() + + def is_terminal(self, task_id: str) -> bool: + return self.task(task_id).get("state") in TERMINAL_TASK_STATES + + def finish(self, status: str, *, pull_request_url: str | None = None) -> None: + self.data["status"] = status + if pull_request_url: + self.data["pull_request_url"] = pull_request_url + self.save() diff --git a/docker/eval-tools/gpu-asan/Dockerfile b/docker/eval-tools/gpu-asan/Dockerfile new file mode 100644 index 00000000..a3c2bf17 --- /dev/null +++ b/docker/eval-tools/gpu-asan/Dockerfile @@ -0,0 +1,30 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena ROCm GPU ASan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.version="rocm-7.2.0-asan" + +COPY docker/eval-tools/gpu-asan/packages.sha256 /tmp/packages.sha256 + +RUN mkdir -p /tmp/gpu-asan-debs \ + && cd /tmp/gpu-asan-debs \ + && apt-get update \ + && apt-get download \ + rocm-core-asan=7.2.0.70200-43~22.04 \ + comgr-asan=3.0.0.70200-43~22.04 \ + hsa-rocr-asan=1.18.0.70200-43~22.04 \ + hip-runtime-amd-asan=7.2.26015.70200-43~22.04 \ + && sha256sum --check /tmp/packages.sha256 \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ./*.deb \ + && rm -rf /tmp/gpu-asan-debs /tmp/packages.sha256 /var/lib/apt/lists/* + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + AKA_GPU_ASAN_RUNTIME_DIR=/opt/rocm-7.2.0/lib/asan \ + AKA_GPU_ASAN_HIP_RUNTIME=/opt/rocm-7.2.0/lib/asan/libamdhip64.so \ + AKA_GPU_ASAN_HOST_PRELOAD=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux/libclang_rt.asan-x86_64.so \ + AKA_GPU_ASAN_HOST_LIB_DIR=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux \ + AKA_GPU_ASAN_NORMAL_ROCM_LIB_DIR=/opt/rocm-7.2.0/lib diff --git a/docker/eval-tools/gpu-asan/packages.sha256 b/docker/eval-tools/gpu-asan/packages.sha256 new file mode 100644 index 00000000..6d8078c4 --- /dev/null +++ b/docker/eval-tools/gpu-asan/packages.sha256 @@ -0,0 +1,4 @@ +3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83 rocm-core-asan_7.2.0.70200-43~22.04_amd64.deb +31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179 comgr-asan_3.0.0.70200-43~22.04_amd64.deb +c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc hsa-rocr-asan_1.18.0.70200-43~22.04_amd64.deb +fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300 hip-runtime-amd-asan_7.2.26015.70200-43~22.04_amd64.deb diff --git a/docker/eval-tools/hip-fpsan/Dockerfile b/docker/eval-tools/hip-fpsan/Dockerfile new file mode 100644 index 00000000..d7b4a4e9 --- /dev/null +++ b/docker/eval-tools/hip-fpsan/Dockerfile @@ -0,0 +1,31 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +ARG HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e + +LABEL org.opencontainers.image.title="AgentKernelArena HIP-FpSan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="0ac9be8a1539a473ba21dfa686564c3be33c890e" + +COPY src/eval_tools/probes/hip_fpsan_probe.hip /tmp/hip_fpsan_probe.hip + +RUN git init /opt/hip-fpsan \ + && git -C /opt/hip-fpsan remote add origin https://github.com/ROCm/hip-fpsan.git \ + && git -C /opt/hip-fpsan fetch --depth 1 origin ${HIP_FPSAN_COMMIT} \ + && git -C /opt/hip-fpsan checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/hip-fpsan rev-parse HEAD)" = "${HIP_FPSAN_COMMIT}" \ + && mkdir -p /opt/eval-tools/probes \ + && /opt/rocm/bin/hipcc -O2 --offload-arch=gfx950 \ + -I/opt/hip-fpsan/include \ + /tmp/hip_fpsan_probe.hip \ + -o /opt/eval-tools/probes/hip_fpsan_probe \ + && rm -f /tmp/hip_fpsan_probe.hip + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + HIP_FPSAN_ROOT=/opt/hip-fpsan \ + AKA_HIP_FPSAN_INCLUDE_DIR=/opt/hip-fpsan/include \ + AKA_HIP_FPSAN_PROBE=/opt/eval-tools/probes/hip_fpsan_probe \ + AKA_HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e diff --git a/docker/eval-tools/hip-fpsan/sources.lock.yaml b/docker/eval-tools/hip-fpsan/sources.lock.yaml new file mode 100644 index 00000000..87f39fe3 --- /dev/null +++ b/docker/eval-tools/hip-fpsan/sources.lock.yaml @@ -0,0 +1 @@ +hip_fpsan: 0ac9be8a1539a473ba21dfa686564c3be33c890e diff --git a/docker/eval-tools/images.lock.yaml b/docker/eval-tools/images.lock.yaml new file mode 100644 index 00000000..90a243ab --- /dev/null +++ b/docker/eval-tools/images.lock.yaml @@ -0,0 +1,74 @@ +schema_version: 1 + +# The scoring runtime remains unchanged. Every tool image is an immutable child +# of this verified MI355X/gfx950 image. +base: + gfx950: + reference: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705 + digest: sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 + rocm: 7.2.0 + +tools: + triton_fpsan: + triton: + version: 3.7.0+amd.rocm7.2.0.gitd0d77a509 + sha256: 3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687 + triton_kernels: + version: 1.0.0+amd.rocm7.2.0.gitd0d77a509 + sha256: df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a + + gpu_asan: + packages: + rocm-core-asan: + version: 7.2.0.70200-43~22.04 + sha256: 3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83 + comgr-asan: + version: 3.0.0.70200-43~22.04 + sha256: 31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179 + hsa-rocr-asan: + version: 1.18.0.70200-43~22.04 + sha256: c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc + hip-runtime-amd-asan: + version: 7.2.26015.70200-43~22.04 + sha256: fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300 + + rocjitsu: + repository: https://github.com/ROCm/rocm-systems.git + commit: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140 + gcc: 13.4.0-6ubuntu1~22~ppa2 + googletest: + repository: https://github.com/google/googletest.git + commit: b514bdc898e2951020cbdca1304b75f5950d1f59 + flatbuffers: + repository: https://github.com/google/flatbuffers.git + commit: 595bf0007ab1929570c7671f091313c8fc20644e + + rocjitsu_waitcheck: + repository: https://github.com/ROCm/rocm-systems.git + commit: ed35c0b54547c98bab359c8732529d9f5e8fd1ae + target: gfx950 + interface: waitcheck-c-api-v1 + gcc: 13.4.0-6ubuntu1~22~ppa2 + zstd: + version: 1.5.7 + sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + + rocjitsu_consan: + repository: https://github.com/ROCm/rocm-systems.git + commit: ed35c0b54547c98bab359c8732529d9f5e8fd1ae + target: gfx950 + mode: record-replay + policy: strict + gcc: 13.4.0-6ubuntu1~22~ppa2 + zstd: + version: 1.5.7 + sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + + hip_fpsan: + repository: https://github.com/ROCm/hip-fpsan.git + commit: 0ac9be8a1539a473ba21dfa686564c3be33c890e + image_probe: src/eval_tools/probes/hip_fpsan_probe.hip + +verification: + gfx950: verified + gfx942: unverified diff --git a/docker/eval-tools/rocjitsu-sanitizers/Dockerfile b/docker/eval-tools/rocjitsu-sanitizers/Dockerfile new file mode 100644 index 00000000..9178b765 --- /dev/null +++ b/docker/eval-tools/rocjitsu-sanitizers/Dockerfile @@ -0,0 +1,117 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} AS builder + +ARG ROCJITSU_SANITIZERS_COMMIT=ed35c0b54547c98bab359c8732529d9f5e8fd1ae +ARG GOOGLETEST_COMMIT=b514bdc898e2951020cbdca1304b75f5950d1f59 +ARG FLATBUFFERS_COMMIT=595bf0007ab1929570c7671f091313c8fc20644e +ARG ZSTD_VERSION=1.5.7 +ARG ZSTD_SHA256=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git software-properties-common zlib1g-dev \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gcc-13=13.4.0-6ubuntu1~22~ppa2 \ + g++-13=13.4.0-6ubuntu1~22~ppa2 \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN git init /src/rocm-systems \ + && git -C /src/rocm-systems remote add origin https://github.com/ROCm/rocm-systems.git \ + && git -C /src/rocm-systems fetch --depth 1 origin ${ROCJITSU_SANITIZERS_COMMIT} \ + && git -C /src/rocm-systems checkout --detach FETCH_HEAD \ + && test "$(git -C /src/rocm-systems rev-parse HEAD)" = "${ROCJITSU_SANITIZERS_COMMIT}" \ + && git init /src/googletest \ + && git -C /src/googletest remote add origin https://github.com/google/googletest.git \ + && git -C /src/googletest fetch --depth 1 origin ${GOOGLETEST_COMMIT} \ + && git -C /src/googletest checkout --detach FETCH_HEAD \ + && git init /src/flatbuffers \ + && git -C /src/flatbuffers remote add origin https://github.com/google/flatbuffers.git \ + && git -C /src/flatbuffers fetch --depth 1 origin ${FLATBUFFERS_COMMIT} \ + && git -C /src/flatbuffers checkout --detach FETCH_HEAD + +RUN curl -sSfL -o /tmp/zstd.tar.gz \ + "https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz" \ + && printf '%s %s\n' "${ZSTD_SHA256}" /tmp/zstd.tar.gz | sha256sum --check --quiet \ + && mkdir -p /src/zstd \ + && tar -xzf /tmp/zstd.tar.gz -C /src/zstd --strip-components=1 \ + && cmake -S /src/zstd/build/cmake -B /build/zstd -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/zstd \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DZSTD_BUILD_SHARED=OFF \ + -DZSTD_BUILD_STATIC=ON \ + -DZSTD_BUILD_PROGRAMS=OFF \ + -DZSTD_BUILD_TESTS=OFF \ + && cmake --build /build/zstd --parallel \ + && cmake --install /build/zstd \ + && rm -f /tmp/zstd.tar.gz + +RUN cmake -S /src/rocm-systems/emulation/rocjitsu -B /build/rocjitsu -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-13 \ + -DCMAKE_CXX_COMPILER=g++-13 \ + -DCMAKE_INSTALL_PREFIX=/opt/rocjitsu \ + -Dzstd_DIR=/opt/zstd/lib/cmake/zstd \ + -DFETCHCONTENT_SOURCE_DIR_GOOGLETEST=/src/googletest \ + -DFETCHCONTENT_SOURCE_DIR_FLATBUFFERS=/src/flatbuffers \ + -DBUILD_TESTING=OFF \ + && cmake --build /build/rocjitsu --parallel \ + --target rj_waitcheck rocjitsu_waitcheck rocjitsu_dbi_hooks \ + && mkdir -p /opt/rocjitsu/include /opt/rocjitsu/lib \ + && cp -a /src/rocm-systems/emulation/rocjitsu/lib/rocjitsu/include/rocjitsu \ + /opt/rocjitsu/include/ \ + && cp -a /build/rocjitsu/librocjitsu_waitcheck.so* /opt/rocjitsu/lib/ \ + && install -D -m 0755 \ + /build/rocjitsu/lib/rocjitsu/src/rocjitsu/hooks/librocjitsu_dbi_hooks.so \ + /opt/rocjitsu/lib/librocjitsu_dbi_hooks.so \ + && install -D -m 0755 /build/rocjitsu/tools/rj_waitcheck /opt/rocjitsu/bin/rj_waitcheck \ + && mkdir -p /opt/rocjitsu/runtime \ + && cp -a /usr/lib/x86_64-linux-gnu/libstdc++.so.6* /opt/rocjitsu/runtime/ \ + && cp -a /lib/x86_64-linux-gnu/libgcc_s.so.1 /opt/rocjitsu/runtime/ + +COPY docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp /src/waitcheck-capi.cpp +RUN g++-13 -std=c++17 -O2 -Wall -Wextra -Werror \ + -I/opt/rocjitsu/include /src/waitcheck-capi.cpp \ + -L/opt/rocjitsu/lib -Wl,-rpath,/opt/rocjitsu/lib \ + -lrocjitsu_waitcheck -o /opt/rocjitsu/bin/aka-waitcheck-capi \ + && /opt/rocjitsu/bin/aka-waitcheck-capi --help >/dev/null + +FROM ${BASE_IMAGE} AS runtime-common + +LABEL org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="ed35c0b54547c98bab359c8732529d9f5e8fd1ae" + +COPY --from=builder /opt/rocjitsu /opt/rocjitsu +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + LD_LIBRARY_PATH=/opt/rocjitsu/lib:/opt/rocjitsu/runtime \ + PATH=/opt/rocjitsu/bin:${PATH} \ + AKA_ROCJITSU_SANITIZERS_COMMIT=ed35c0b54547c98bab359c8732529d9f5e8fd1ae + +FROM runtime-common AS rocjitsu-waitcheck-runtime + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu Waitcheck runtime" + +ENV AKA_WAITCHECK_BINARY=/opt/rocjitsu/bin/rj_waitcheck \ + AKA_WAITCHECK_CAPI_WRAPPER=/opt/rocjitsu/bin/aka-waitcheck-capi + +RUN /opt/rocjitsu/bin/rj_waitcheck --help >/dev/null \ + && /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/waitcheck_entrypoint.py \ + --help >/dev/null + +FROM runtime-common AS rocjitsu-consan-runtime + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu ConSan runtime" + +ENV AKA_CONSAN_HOOK=/opt/rocjitsu/lib/librocjitsu_dbi_hooks.so + +RUN test -f /opt/rocjitsu/lib/librocjitsu_dbi_hooks.so \ + && /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/consan_entrypoint.py \ + --help >/dev/null diff --git a/docker/eval-tools/rocjitsu-sanitizers/sources.lock.yaml b/docker/eval-tools/rocjitsu-sanitizers/sources.lock.yaml new file mode 100644 index 00000000..700509c6 --- /dev/null +++ b/docker/eval-tools/rocjitsu-sanitizers/sources.lock.yaml @@ -0,0 +1,7 @@ +rocm_systems: ed35c0b54547c98bab359c8732529d9f5e8fd1ae +googletest: b514bdc898e2951020cbdca1304b75f5950d1f59 +flatbuffers: 595bf0007ab1929570c7671f091313c8fc20644e +gcc_13: 13.4.0-6ubuntu1~22~ppa2 +zstd: + version: 1.5.7 + sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 diff --git a/docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp b/docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp new file mode 100644 index 00000000..9e86b060 --- /dev/null +++ b/docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp @@ -0,0 +1,213 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Diagnostic { + std::string code; + std::string kernel_name; + uint64_t kernel_entry = 0; + int counter = 0; + int access = 0; + int register_class = 0; + unsigned register_index = 0; + unsigned register_width = 0; + std::string section_name; + uint64_t section_offset = 0; + uint64_t file_offset = 0; + std::string instruction; + uint64_t producer_section_offset = 0; + uint64_t producer_file_offset = 0; + std::string producer_instruction; + uint32_t required_count = 0; + std::string message; +}; + +std::string copy_string(const char *value) { return value == nullptr ? "" : value; } + +void write_json_string(std::ostream &output, std::string_view value) { + static constexpr char hex[] = "0123456789abcdef"; + output << '"'; + for (const unsigned char character : value) { + switch (character) { + case '"': output << "\\\""; break; + case '\\': output << "\\\\"; break; + case '\b': output << "\\b"; break; + case '\f': output << "\\f"; break; + case '\n': output << "\\n"; break; + case '\r': output << "\\r"; break; + case '\t': output << "\\t"; break; + default: + if (character < 0x20) { + output << "\\u00" << hex[character >> 4] << hex[character & 0xf]; + } else { + output << character; + } + } + } + output << '"'; +} + +void diagnostic_callback(const rj_waitcheck_diagnostic_t *input, void *opaque) { + if (input == nullptr || opaque == nullptr) + return; + auto &diagnostics = *static_cast *>(opaque); + Diagnostic value; + value.code = copy_string(rj_waitcheck_diagnostic_code_name(input->code)); + value.kernel_name = copy_string(input->kernel_name); + value.kernel_entry = input->kernel_entry_offset; + value.counter = static_cast(input->counter); + value.access = static_cast(input->access); + value.register_class = static_cast(input->reg.register_class); + value.register_index = input->reg.index; + value.register_width = input->reg.width; + value.section_name = copy_string(input->section_name); + value.section_offset = input->section_offset; + value.file_offset = input->file_offset; + value.instruction = copy_string(input->instruction); + value.producer_section_offset = input->producer_section_offset; + value.producer_file_offset = input->producer_file_offset; + value.producer_instruction = copy_string(input->producer_instruction); + value.required_count = input->required_count; + value.message = copy_string(input->message); + diagnostics.push_back(std::move(value)); +} + +void error_callback(const char *message, void *opaque) { + if (opaque != nullptr) + *static_cast(opaque) = copy_string(message); +} + +bool parse_uint64(std::string_view text, uint64_t &value) { + try { + size_t consumed = 0; + value = std::stoull(std::string(text), &consumed, 0); + return consumed == text.size(); + } catch (...) { + return false; + } +} + +} // namespace + +int main(int argc, char **argv) { + std::string path; + uint64_t kernel_entry = 0; + bool has_entry = false; + for (int index = 1; index < argc; ++index) { + const std::string_view argument(argv[index]); + if (argument == "--code-object" && index + 1 < argc) { + path = argv[++index]; + } else if (argument == "--kernel-entry" && index + 1 < argc) { + has_entry = parse_uint64(argv[++index], kernel_entry); + } else if (argument == "--help") { + std::cout << "usage: aka-waitcheck-capi --code-object PATH --kernel-entry OFFSET\n"; + return 0; + } else { + std::cerr << "invalid argument: " << argument << "\n"; + return 2; + } + } + if (path.empty() || !has_entry) { + std::cerr << "code object and kernel entry are required\n"; + return 2; + } + + std::ifstream input(path, std::ios::binary); + if (!input) { + std::cerr << "could not open code object\n"; + return 2; + } + const std::vector bytes((std::istreambuf_iterator(input)), {}); + if (bytes.empty()) { + std::cerr << "code object is empty\n"; + return 2; + } + + std::vector diagnostics; + std::string analysis_error; + rj_waitcheck_options_t options{}; + rj_waitcheck_result_t result{}; + if (rj_waitcheck_options_init(&options, sizeof(options)) != ROCJITSU_STATUS_SUCCESS || + rj_waitcheck_result_init(&result, sizeof(result)) != ROCJITSU_STATUS_SUCCESS) { + std::cerr << "could not initialize waitcheck ABI structures\n"; + return 2; + } + options.max_diagnostics = 0; + options.stop_after_first_diagnostic = 0; + options.diagnostic_callback = diagnostic_callback; + options.error_callback = error_callback; + // Both callbacks need distinct storage, so switch user_data immediately + // around the API call through a small aggregate. + struct CallbackState { + std::vector *diagnostics; + std::string *error; + } state{&diagnostics, &analysis_error}; + options.user_data = &state; + options.diagnostic_callback = [](const rj_waitcheck_diagnostic_t *value, void *opaque) { + auto *callback_state = static_cast(opaque); + diagnostic_callback(value, callback_state->diagnostics); + }; + options.error_callback = [](const char *value, void *opaque) { + auto *callback_state = static_cast(opaque); + error_callback(value, callback_state->error); + }; + + const rj_status_t status = rj_waitcheck_analyze_kernel( + bytes.data(), bytes.size(), kernel_entry, &options, &result); + std::cout << "AKA_WAITCHECK_CAPI_RESULT {\"schema_version\":1,\"api_status\":" + << static_cast(status) << ",\"analysis_complete\":" + << (status == ROCJITSU_STATUS_SUCCESS ? "true" : "false") << ",\"target\":"; + write_json_string(std::cout, rj_waitcheck_target_name(result.target)); + std::cout << ",\"instructions_analyzed\":" << result.instructions_analyzed + << ",\"memory_events_tracked\":" << result.memory_events_tracked + << ",\"kernels_discovered\":" << result.kernels_discovered + << ",\"kernels_analyzed\":" << result.kernels_analyzed + << ",\"diagnostics_observed\":" << result.diagnostics_observed + << ",\"diagnostics_reported\":" << result.diagnostics_reported + << ",\"passed\":" << (result.passed ? "true" : "false") + << ",\"diagnostics_truncated\":" + << (result.diagnostics_truncated ? "true" : "false") + << ",\"stopped_early\":" << (result.stopped_early ? "true" : "false") + << ",\"analysis_error\":"; + write_json_string(std::cout, analysis_error); + std::cout << ",\"diagnostics\":["; + for (size_t index = 0; index < diagnostics.size(); ++index) { + if (index != 0) + std::cout << ','; + const Diagnostic &value = diagnostics[index]; + std::cout << "{\"code\":"; + write_json_string(std::cout, value.code); + std::cout << ",\"kernel_name\":"; + write_json_string(std::cout, value.kernel_name); + std::cout << ",\"kernel_entry\":" << value.kernel_entry + << ",\"counter\":" << value.counter << ",\"access\":" << value.access + << ",\"register_class\":" << value.register_class + << ",\"register_index\":" << value.register_index + << ",\"register_width\":" << value.register_width + << ",\"section_name\":"; + write_json_string(std::cout, value.section_name); + std::cout << ",\"section_offset\":" << value.section_offset + << ",\"file_offset\":" << value.file_offset << ",\"instruction\":"; + write_json_string(std::cout, value.instruction); + std::cout << ",\"producer_section_offset\":" << value.producer_section_offset + << ",\"producer_file_offset\":" << value.producer_file_offset + << ",\"producer_instruction\":"; + write_json_string(std::cout, value.producer_instruction); + std::cout << ",\"required_count\":" << value.required_count << ",\"message\":"; + write_json_string(std::cout, value.message); + std::cout << '}'; + } + std::cout << "]}\n"; + return status == ROCJITSU_STATUS_SUCCESS ? 0 : 2; +} diff --git a/docker/eval-tools/rocjitsu/Dockerfile b/docker/eval-tools/rocjitsu/Dockerfile new file mode 100644 index 00000000..7a0c2c0a --- /dev/null +++ b/docker/eval-tools/rocjitsu/Dockerfile @@ -0,0 +1,70 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} AS builder + +ARG ROCJITSU_COMMIT=0bf561a0d8a4a6b88954f2c46bd3a50871cda140 +ARG GOOGLETEST_COMMIT=b514bdc898e2951020cbdca1304b75f5950d1f59 +ARG FLATBUFFERS_COMMIT=595bf0007ab1929570c7671f091313c8fc20644e + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates git software-properties-common \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gcc-13=13.4.0-6ubuntu1~22~ppa2 \ + g++-13=13.4.0-6ubuntu1~22~ppa2 \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN git init /src/rocm-systems \ + && git -C /src/rocm-systems remote add origin https://github.com/ROCm/rocm-systems.git \ + && git -C /src/rocm-systems fetch --depth 1 origin ${ROCJITSU_COMMIT} \ + && git -C /src/rocm-systems checkout --detach FETCH_HEAD \ + && test "$(git -C /src/rocm-systems rev-parse HEAD)" = "${ROCJITSU_COMMIT}" \ + && git init /src/googletest \ + && git -C /src/googletest remote add origin https://github.com/google/googletest.git \ + && git -C /src/googletest fetch --depth 1 origin ${GOOGLETEST_COMMIT} \ + && git -C /src/googletest checkout --detach FETCH_HEAD \ + && git init /src/flatbuffers \ + && git -C /src/flatbuffers remote add origin https://github.com/google/flatbuffers.git \ + && git -C /src/flatbuffers fetch --depth 1 origin ${FLATBUFFERS_COMMIT} \ + && git -C /src/flatbuffers checkout --detach FETCH_HEAD + +RUN cmake -S /src/rocm-systems/emulation/rocjitsu -B /build/rocjitsu -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-13 \ + -DCMAKE_CXX_COMPILER=g++-13 \ + -DCMAKE_INSTALL_PREFIX=/opt/rocjitsu \ + -DFETCHCONTENT_SOURCE_DIR_GOOGLETEST=/src/googletest \ + -DFETCHCONTENT_SOURCE_DIR_FLATBUFFERS=/src/flatbuffers \ + -DCMAKE_HIP_ARCHITECTURES=gfx950 \ + -DBUILD_TESTING=ON \ + -DRJ_INSTALL_TESTS=OFF \ + && cmake --build /build/rocjitsu --parallel \ + && cmake --install /build/rocjitsu \ + && mkdir -p /opt/rocjitsu/runtime \ + && cp -a /usr/lib/x86_64-linux-gnu/libstdc++.so.6* /opt/rocjitsu/runtime/ \ + && cp -a /lib/x86_64-linux-gnu/libgcc_s.so.1 /opt/rocjitsu/runtime/ + +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="0bf561a0d8a4a6b88954f2c46bd3a50871cda140" + +COPY --from=builder /opt/rocjitsu /opt/rocjitsu +COPY docker/eval-tools/rocjitsu/rocjitsu-wrapper /usr/local/bin/rocjitsu +RUN chmod 0755 /usr/local/bin/rocjitsu \ + && /usr/local/bin/rocjitsu --help >/dev/null + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools +RUN /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/rocjitsu_replay_entrypoint.py \ + --help >/dev/null + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + PATH=/opt/rocjitsu/bin:${PATH} \ + AKA_ROCJITSU_BINARY=/usr/local/bin/rocjitsu \ + AKA_ROCJITSU_CONFIG=/opt/rocjitsu/share/rocjitsu/configs/gfx950_cdna4.json \ + AKA_ROCJITSU_COMMIT=0bf561a0d8a4a6b88954f2c46bd3a50871cda140 diff --git a/docker/eval-tools/rocjitsu/rocjitsu-wrapper b/docker/eval-tools/rocjitsu/rocjitsu-wrapper new file mode 100755 index 00000000..d094b3a8 --- /dev/null +++ b/docker/eval-tools/rocjitsu/rocjitsu-wrapper @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +export LD_LIBRARY_PATH="/opt/rocjitsu/runtime:/opt/rocjitsu/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +# rocJITsu's current gfx950 runtime exposes one simulated agent numbered zero. +# A physical host GPU mask (for example ROCR_VISIBLE_DEVICES=4) would otherwise +# filter that simulated agent out of the launched HIP process. +export ROCR_VISIBLE_DEVICES=0 +export HIP_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=0 +export GPU_DEVICE_ORDINAL=0 +exec /opt/rocjitsu/bin/rocjitsu "$@" diff --git a/docker/eval-tools/rocjitsu/sources.lock.yaml b/docker/eval-tools/rocjitsu/sources.lock.yaml new file mode 100644 index 00000000..2975cc96 --- /dev/null +++ b/docker/eval-tools/rocjitsu/sources.lock.yaml @@ -0,0 +1,4 @@ +rocm_systems: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140 +googletest: b514bdc898e2951020cbdca1304b75f5950d1f59 +flatbuffers: 595bf0007ab1929570c7671f091313c8fc20644e +gcc_13: 13.4.0-6ubuntu1~22~ppa2 diff --git a/docker/eval-tools/triton-fpsan/Dockerfile b/docker/eval-tools/triton-fpsan/Dockerfile new file mode 100644 index 00000000..fa7819cc --- /dev/null +++ b/docker/eval-tools/triton-fpsan/Dockerfile @@ -0,0 +1,21 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena Triton FpSan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.version="triton-3.7.0+amd.rocm7.2.0.gitd0d77a509" + +COPY docker/eval-tools/triton-fpsan/requirements.lock /tmp/eval-tool-requirements.lock + +RUN python -m pip uninstall -y \ + triton pytorch-triton pytorch-triton-rocm triton-rocm amd-triton triton-kernels \ + && python -m pip install --no-cache-dir --no-deps --require-hashes \ + -r /tmp/eval-tool-requirements.lock \ + && python -c "import importlib.metadata as m, triton, triton_kernels; assert m.version('triton') == '3.7.0+amd.rocm7.2.0.gitd0d77a509'; assert m.version('triton-kernels') == '1.0.0+amd.rocm7.2.0.gitd0d77a509'" \ + && rm -f /tmp/eval-tool-requirements.lock + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + AKA_TRITON_FPSAN_VERSION=3.7.0+amd.rocm7.2.0.gitd0d77a509 diff --git a/docker/eval-tools/triton-fpsan/requirements.lock b/docker/eval-tools/triton-fpsan/requirements.lock new file mode 100644 index 00000000..d8fc5460 --- /dev/null +++ b/docker/eval-tools/triton-fpsan/requirements.lock @@ -0,0 +1,4 @@ +triton @ https://pypi.amd.com/triton/release_/rocm-7.2.0/packages/triton/triton-3.7.0+amd.rocm7.2.0.gitd0d77a509-cp310-cp310-linux_x86_64.whl \ + --hash=sha256:3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687 +triton-kernels @ https://pypi.amd.com/triton/release_/rocm-7.2.0/packages/triton-kernels/triton_kernels-1.0.0+amd.rocm7.2.0.gitd0d77a509-py3-none-any.whl \ + --hash=sha256:df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a diff --git a/docs/README.md b/docs/README.md index 17bd0d0b..d9c7abc5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,10 +30,12 @@ python -m sphinx -T -b html docs docs/_build/html | `reference/api-reference.md` | Configuration and API reference | Run configuration schema, task `config.yaml` schema, CLI flags, scoring, and the agent registry. | | `reference/benchmark-methodology.md` | Reference | Timing methodology, performance-helper materialization, and speedup interpretation. | | `how-to/run-evaluation.md` | How-to | Choose or create a run configuration, run an experiment through Docker, resume runs, and read results. | +| `how-to/use-evaluation-tools.md` | How-to | Configure isolated sanitizer/analysis sidecars, interpret capability and findings, and understand the strict language/GPU support matrix. | | `how-to/parallel-run.md` | How-to | Run one isolated Docker worker per GPU, use the shared `.parallel/` task queue, resume parallel runs, and parallelize `task_validator`. | | `how-to/agents.md` | How-to | Supported agents, model providers, and A/B testing. | | `how-to/add-task.md` | How-to | Task directory layout, `config.yaml` fields, and task types. | | `how-to/task-validator.md` | How-to | The task_validator agent and its 10 checks. | +| `how-to/quality-loop.md` | How-to | Repository-wide validation, one-pass optimization, task hardening, issue filing, and draft PR publication. | | `how-to/held-out-evaluation.md` | How-to | Generate private shapes and evaluate completed runs for generalization. | | `how-to/visualization.md` | How-to | Build dashboard data and serve the comparison dashboard. | | `examples/examples.md` | Examples | Step-by-step walkthroughs with expected output. | diff --git a/docs/how-to/agents.md b/docs/how-to/agents.md index 26f1d677..039f3384 100644 --- a/docs/how-to/agents.md +++ b/docs/how-to/agents.md @@ -40,6 +40,10 @@ The Cursor, Claude Code, and Codex integrations reuse their host CLI login state. Specialized integrations have additional setup and configuration under their respective `agents//` directories. +`quality_loop` is intentionally not an `agent.template`. It is a repository-level +task maintenance workflow that invokes Codex roles across many tasks, files issues, +and publishes one draft PR. See [Audit and harden tasks](quality-loop.md). + ## Models, providers, and agent settings AgentKernelArena has no shared model/provider field in the run configuration. diff --git a/docs/how-to/quality-loop.md b/docs/how-to/quality-loop.md new file mode 100644 index 00000000..352fe972 --- /dev/null +++ b/docs/how-to/quality-loop.md @@ -0,0 +1,107 @@ +--- +myst: + html_meta: + "description": "Audit, repair, harden, and publish AgentKernelArena tasks with the Codex-based quality_loop workflow." + "keywords": "AgentKernelArena, quality_loop, task audit, Codex, GitHub issues, pull request, GPU kernel" +--- + +# Audit and harden tasks with quality_loop + +`quality_loop` is a repository-level workflow for maintaining the task corpus. +It differs from a normal `agent.template`: normal agents optimize one copied task, +while `quality_loop` owns a complete multi-task campaign and a single Git branch. + +For every selected, platform-compatible task it: + +1. Runs the existing task validator in a fresh workspace. +2. Records WARN results without repairing them. +3. Attempts one repair for FAIL results, revalidates from a fresh copy, and files + a fingerprinted GitHub issue if the task remains invalid. +4. Runs exactly one Codex optimization iteration and the centralized evaluator. +5. Starts a separate, read-only Codex session to review correctness evidence and + case coverage. +6. Promotes a first-iteration candidate only when three measurements have median + speedup at least 5x and all correctness/method/case-count gates pass. +7. Adds targeted cases only when both the original kernel and candidate pass them. +8. Commits accepted task changes to one isolated branch and creates one draft PR. + +## Prerequisites + +Install and authenticate Codex and GitHub CLI on the host. The GitHub identity +must have write permission to this repository: + +```bash +codex --version +gh auth status -h github.com +gh api repos/AMD-AGI/AgentKernelArena --jq '.permissions.push' +``` + +The Docker launcher performs GitHub preflight and creates the audit worktree on +the host. It mounts Codex state, but never mounts GitHub credentials into the GPU +container. The main checkout is mounted read-only, while only the current run's +artifact and isolated worktree directories are writable. After the task campaign +exits, a host-side deterministic publisher verifies the recorded diff, commits +accepted task changes, creates issues, pushes the branch, and opens the draft PR. + +## Inspect a campaign + +Planning is offline and does not create a branch or require GPU access: + +```bash +python3 -m agents.quality_loop \ + --config example_configs/quality_loop_mi300.yaml \ + --plan +``` + +The output lists runnable and platform-deferred tasks. A task with +`platform_support.required_arch` is run only on the matching architecture. + +## Run and resume + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml +``` + +Select `example_configs/quality_loop_mi355x.yaml` on an MI355X host. + +For a bounded smoke campaign: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--tasks hip2hip/gpumode/GELU triton2triton/vllm/triton_rms_norm" +``` + +Resume with the run ID printed in `quality_loop_runs/`: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--resume " +``` + +The crash-safe `state.yaml` skips terminal tasks. `audit_report.yaml` records every +warning, failure, issue URL, speedup confirmation, accepted file change, and commit. + +## Safety boundaries + +- GitHub authentication and write permission are hard preflights. Failure happens + before branch creation or task mutation. +- GitHub credentials never enter the agent container, and Codex state is copied + into an ephemeral writable home. +- The host refuses to commit or publish when the worktree contains a path that is + not in the accepted per-task change manifest. +- Optimizers cannot edit task harness files; the existing harness digest guard is + checked before evaluation. +- Reviewer output is schema checked, and modifications beyond its one YAML result + file invalidate the review. +- External repository/image worktrees and generated benchmark helpers are never + copied into a task commit. +- Translation/authoring tasks do not promote a generated 5x solution as their new + baseline because that would change the task category. +- If equivalence cannot be established against a committed original kernel, the + baseline or case change is rejected. + +See `agents/quality_loop/README.md` and +`agents/quality_loop/agent_config.yaml` for the complete configuration contract. diff --git a/docs/how-to/run-evaluation.md b/docs/how-to/run-evaluation.md index 4f2b0d6a..9538e4db 100644 --- a/docs/how-to/run-evaluation.md +++ b/docs/how-to/run-evaluation.md @@ -15,13 +15,14 @@ resume, and inspect a run. ## Choose or create a run configuration A run configuration selects the agent, tasks, and target GPU. The repository -ships three examples: +ships quickstarts and specialized examples, including: | Configuration | Purpose | | --- | --- | | `example_configs/quickstart_claude_mi300.yaml` | One Claude Code GELU task on MI300/MI300X (`gfx942`). | | `example_configs/quickstart_claude_mi355x.yaml` | One Claude Code GELU task on MI355X (`gfx950`). | | `example_configs/benchmark_cursor_mi355x.yaml` | Curated 60-task Cursor Agent benchmark on MI355X; use only after installing and authenticating Cursor Agent. | +| `example_configs/evaluation_tools_advisory_mi355x.yaml` | Default-disabled template for experimental `gfx950` evaluation-tool sidecars. It still requires task-specific adapters before opt-in. | For a first run, select the quickstart that matches the physical GPU: @@ -78,6 +79,11 @@ select tasks at any level of granularity. See [Configuration and API reference](../reference/api-reference.md) for the full set of run-configuration fields. +Optional sanitizer and analysis sidecars run after ordinary correctness and +before performance. They are disabled by default, verified only on `gfx950`, +and require language/task-specific adapters. See [Check kernels with evaluation +tools](use-evaluation-tools.md) before enabling them. + ## Start a run ```bash diff --git a/docs/how-to/use-evaluation-tools.md b/docs/how-to/use-evaluation-tools.md new file mode 100644 index 00000000..dd7356f3 --- /dev/null +++ b/docs/how-to/use-evaluation-tools.md @@ -0,0 +1,1007 @@ +--- +myst: + html_meta: + "description": "Run Triton FpSan, ROCm GPU AddressSanitizer, rocJITsu Race Detector, Waitcheck, ConSan, and HIP-FpSan as isolated AgentKernelArena evaluation tools." + "keywords": "AgentKernelArena, sanitizer, Triton FpSan, GPU ASan, rocJITsu, Waitcheck, ConSan, HIP-FpSan, ROCm, gfx950" +--- + +# Check kernels with evaluation tools + +AgentKernelArena can run optional kernel-analysis tools after ordinary +compilation and correctness checks and before performance measurement. The +initial tool set is: + +- Triton FpSan for reference-versus-candidate floating-point semantic + comparison. +- ROCm GPU AddressSanitizer (GPU ASan) for invalid device-memory accesses. +- rocJITsu for simulated race detection. +- rocJITsu Waitcheck for static missing or too-weak wait detection on one exact + final code object and kernel entry. +- rocJITsu ConSan for strict dynamic record/replay concurrency checking of one + exact code object launched by a focused native harness. +- HIP-FpSan for explicitly ported HIP/C++ floating-point comparisons. + +This feature is experimental and opt-in. Capability and evidence checks fail +closed; whether an incomplete result blocks performance is controlled by the +`advisory` or `required` policy. It is not a general sanitizer suite: +sanitizer suite: every result is qualified by the kernel language, generated +artifact, adapter, tool image, GPU architecture, and evidence that the intended +kernel was actually instrumented or dispatched. + +> **Current validation boundary:** sidecar build locks, integrated startup +> controls, and end-to-end fixtures exist only for MI355X (`gfx950`). All six +> startup controls passed in the current hardware qualification. Candidate +> readiness still depends on language, artifact, adapter, and attestation. +> `gfx942` is unverified, and the Docker runner currently rejects +> evaluation-tool sidecars on that architecture. Do not interpret normal +> MI300/MI325 task support as sanitizer support. + +## Keep the scoring runtime unchanged + +Evaluation tools do not get installed into the agent or scoring container. +Each enabled tool runs in a separate run- or worker-scoped sidecar, reused +across that worker's tasks, and communicates with the evaluator over a Unix +socket: + +```mermaid +flowchart LR + A["Pinned scoring image
agent + correctness + scoring"] -->|"health and invocation RPC"| B["One tool sidecar"] + B -->|"bounded logs and structured result"| A + C["Repository tree"] -->|"/input read only"| B + D["Per-worker artifact namespace
.eval-tool-artifacts/label"] <-->|"/artifacts read/write"| B + E["Per-tool socket directory"] -->|"one writable UDS path"| B +``` + +The runner creates a narrow repository-root +`.eval-tool-artifacts/` namespace. It mounts only that directory +as writable `/artifacts` in each sidecar and explicitly bind-mounts the same host +directory read/write at `/workspace/.eval-tool-artifacts/` in the +scoring container. The explicit submount keeps reports writable when the broad +repository mount is read-only, as it is in the quality loop, and avoids asking +Docker to create bind sources on a root-squashed NFS home. Task workspaces and +captured source evidence are not reachable through a writable sidecar alias. +In ordinary and parallel runs, the runner also overlays the top-level +`.eval-tool-artifacts` namespace read-only before overlaying only the current +worker child read/write. This prevents the broad writable repository mount from +becoming a second path to sibling workers' reports or bind sources. +All sidecars for that worker still share this artifact namespace. Each sidecar +receives only its own nested writable socket directory, while the scoring +container receives the socket parent read-only. These changes reduce accidental +cross-tool mutation, but do not create an agent/evaluator trust boundary; the +remaining security consequences are described later. + +All six tool images bake the worker, trusted replay helper, and synthetic +probes into the read-only image at `/opt/aka-eval-tools`. Worker startup verifies +that it imported this image-owned tree rather than the repository mounted at +`/input`. This protects the sidecar control-plane code from a task changing the +checkout used by a running worker; candidate-specific commands and inputs remain +separate, explicitly mounted data. + +The verified `gfx950` scoring image remains: + +```text +lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705 +lmsysorg/sglang-rocm@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +``` + +When any evaluation tool is enabled, the runner resolves Docker's immutable +local image ID for both the selected scoring-image reference and the +content-addressed manifest reference above. Startup fails unless the two IDs are +identical, then launches the scoring container by that verified ID rather than +the mutable tag. A different tag that is an exact alias of the same image is +accepted; an upgraded, rebuilt, or retagged scoring image is rejected even if it +looks compatible. The selected reference and verified image ID are recorded under +`plan.source_evidence.metadata.scoring_runtime`, serialized with the report, +and covered by the plan fingerprint. + +The current design deliberately does **not** upgrade that image, FlyDSL 0.2.2, +or AITER `0.1.17.dev110+g9127c94a1`. The Triton FpSan sidecar replaces Triton +only inside its own container; the other tool dependencies are likewise local +to their sidecars. A sidecar is not a replacement scoring image and must not be +used to establish a new performance baseline. + +FlyDSL does not promise that every generated artifact or task remains compatible +across releases. If the scoring image, FlyDSL, AITER, PyTorch, ROCm, or Triton is +upgraded later, treat that as a benchmark-runtime migration: rerun compilation, +correctness, held-out, sanitizer positive/negative fixtures, and performance +baselines rather than assuming backward compatibility. + +The pinned sidecar dependencies are recorded in +`docker/eval-tools/images.lock.yaml`: + +| Sidecar | Isolated dependency change | +| --- | --- | +| `triton_fpsan` | AMD Triton `3.7.0+amd.rocm7.2.0.gitd0d77a509` and matching `triton-kernels` wheels. | +| `gpu_asan` | ROCm 7.2 ASan runtime packages, including `hip-runtime-amd-asan`. | +| `rocjitsu` | rocJITsu from pinned `rocm-systems` commit `0bf561a0...`, built with GCC 13 for `gfx950`. | +| `rocjitsu_waitcheck` | rocJITsu Waitcheck C API and CLI from pinned `rocm-systems` commit `ed35c0b...`; zstd source is separately checksum-locked. | +| `rocjitsu_consan` | rocJITsu ConSan HSA hook from the same pinned `ed35c0b...` source, forced to strict record/replay mode. | +| `hip_fpsan` | HIP-FpSan headers/source from pinned commit `0ac9be8a...`. | + +## Understand support levels + +“The engine can execute a code object” and “the evaluator can safely wrap a +Python submission” are different claims. The report therefore records four +capability dimensions: + +| Dimension | Question | +| --- | --- | +| `engine` | Can this analysis engine reason about the language or generated ISA? | +| `adapter` | Is there a task-specific build, comparison, or replay path that identifies the exact candidate? | +| `runtime` | Is the isolated sidecar reachable, architecture-matched, and carrying the required assets? | +| `effective` | Is the tool ready after combining the other three dimensions? | + +Possible states are `ready`, `adapter_required`, `unsupported`, +`not_applicable`, and `unavailable_runtime`. A non-ready tool is not executed. +Unsupported tooling is also not converted into an ordinary kernel correctness +failure. + +### Use evaluator plugins, not agent skills, for enforcement + +A sanitizer belongs in the evaluator control plane. An agent skill, prompt, or +tool-use recipe may help an optimizer author an adapter or interpret a report, +but it runs on the untrusted optimization side and cannot prove that a required +check happened. It must never be the source of a scoring gate. + +The extension boundary deliberately separates four responsibilities: + +| Layer | Responsibility | +| --- | --- | +| Core manager | Resolve task profiles, combine capability states, enforce policy, fingerprint plans, and serialize one stable report schema. It contains no sanitizer-specific parsing. | +| Tool plugin | Implement deterministic `assess`, `build_invocation`, and `parse` behavior for one tool. It converts raw output into independent execution/finding states. | +| Sidecar runtime provider | Own the pinned image, dependencies, health evidence, runtime-internal paths, architecture guard, and synthetic startup control. | +| Task adapter | Compile or replay the exact candidate and emit candidate-specific build/dispatch attestation. It is selected by language/artifact kind, not merely by tool name. | + +This makes a future analyzer additive: add its typed plugin/parser, isolated +image and lock, worker health/startup control, runner allowlist, task adapters, +support-matrix entry, and positive/negative tests without changing ordinary +correctness or performance code. The current registry is an explicit built-in +allowlist; arbitrary third-party plugin discovery is not implemented. That is +intentional while report provenance and the evaluator/agent isolation boundary +remain experimental. + +### Strict support matrix + +The following table describes the current end-to-end evaluator, not just a +successful standalone experiment. “Ready with adapter” means the task must +provide the dedicated argv/harness and required attestation described later; +it does not mean the ordinary correctness command is automatically reused. + +| Kernel path | Triton FpSan | GPU ASan | rocJITsu | HIP-FpSan | +| --- | --- | --- | --- | --- | +| Editable Triton Python/JIT | Ready with comparison adapter and instrumentation attestation | Ready with dedicated command, fresh JIT cache, XNACK, and build attestation | Trusted `triton_aot` capsule replay is implemented on `gfx950`; whole-Python JIT remains unsupported, and capsule capture/binding to the correctness run is not automatic, so use it only as advisory evidence | Not applicable | +| HIP source controlled by the task | Not applicable | Ready only after recompiling the candidate with `-fsanitize=address -shared-libsan --offload-arch=gfx950:xnack+`, then attesting that artifact | Ready with a dedicated native launcher | Source port and comparison adapter required; both reference and candidate paths must explicitly use `fpsan::Value` | +| FlyDSL 0.2.2 Python/JIT | Unsupported; FlyDSL does not use the Triton FpSan pipeline | Unsupported; the current ROCDL pipeline does not insert AMD GPU ASan instrumentation | Trusted `flydsl_aot` capsule replay is implemented on `gfx950` and detects the seeded LDS race; automatic capsule capture/binding to the correctness run is not ready, so use it only as advisory evidence | Not applicable | +| Editable Triton source inside AITER | Engine may be eligible for the explicitly selected source only, with a dedicated comparison adapter; this does not sanitize AITER library kernels | Unsupported by the current default AITER runtime path | Unsupported by the current Python/AITER runtime | Not applicable | +| AITER or another precompiled HSACO/library kernel | Cannot retrofit instrumentation | Unsupported unless the exact kernel source is rebuilt and attested; preloading the runtime is insufficient | Unsupported by the current evaluator runtime | Cannot retrofit value semantics | +| rocBLAS or RCCL internal kernel | Do not enable; library internals are outside the selected submission | The stock library is not instrumented and is not covered | Not a supported general library-runtime path | Do not enable | + +Waitcheck and ConSan use narrower, explicit final-code-object adapters: + +| Kernel path | rocJITsu Waitcheck | rocJITsu ConSan | +| --- | --- | --- | +| Standalone final `gfx950` HSACO from HIP, Triton, or FlyDSL | Advisory-ready only when `code_object`, `expected_kernel`, and exact `kernel_entry` identify one descriptor in an unbundled final ELF. It is static and receives no GPU device. | Advisory-ready only with `code_object`, a focused native `command` that names and loads that file, and an independent `oracle_command`. Strict record/replay must report the same FNV-1a identity and complete coverage. | +| Whole Python/JIT process | Not automatically captured or bound to the correctness dispatch. Extract and declare the exact final code object first. | Unsupported by this first integration; incidental framework and library code objects make exact candidate attribution ambiguous. | +| AITER, rocBLAS, RCCL, or another broad library runtime | A selected extracted kernel may be inspected statically, but that does not cover the surrounding runtime or prove it was dispatched. | Unsupported; the current adapter deliberately rejects broad library runtimes. | + +Both tools are currently qualified only for `policy: advisory`. The generic +manager can enforce `required` when explicitly configured, but doing so for +these tools is unsupported until the selected task adapter and candidate +provenance have been independently promoted. The evaluator does not +automatically discover optimized kernels or reuse the ordinary correctness +command for either tool. + +This matrix describes engine and adapter support once the corresponding runtime +is qualified. The current `gfx950` startup qualification is stricter: + +| Tool runtime | Current startup qualification | +| --- | --- | +| Triton FpSan | Passing on hardware; eligible task paths can proceed to candidate attestation. | +| HIP-FpSan | Passing on hardware; explicitly ported task paths can proceed to candidate attestation. | +| GPU ASan | Passing on hardware for both HIP and Triton safe/OOB lanes; an applicable candidate still needs its own instrumentation/build attestation. | +| rocJITsu | Passing on hardware with barrier-safe and deliberately racy LDS fixtures; an applicable candidate still needs a native HIP launcher or validated AOT replay capsule. | +| rocJITsu Waitcheck | Passing on hardware: a correct `s_waitcnt lgkmcnt(0)` fixture is clean and a missing-wait fixture produces one exact hazard. Candidate use still requires exact SHA-256, kernel name, and entry attestation. | +| rocJITsu ConSan | Passing on hardware in strict record/replay: a single-wave LDS fixture is clean and a two-wave conflicting fixture produces complete FNV-attributed diagnostics. Candidate use still requires an exact code object, focused loader, and separate oracle. | + +Additional boundaries: + +- Static Triton HSACO fixtures, including dynamic matmul, buffer-async matmul, + and flash-attention fixtures, have executed under the `gfx950` rocJITsu + engine. That evidence does not make `rocjitsu -- python task.py` supported. +- A FlyDSL HSACO with a deliberately missing LDS barrier produced a rocJITsu + race report. Directly wrapping the FlyDSL Python process failed before a + usable dispatch. The implemented adapter therefore extracts that boundary + into a strict, single-dispatch replay capsule rather than wrapping Python. +- For Triton/FlyDSL, the rocJITsu plugin rejects arbitrary launchers, validates + the language-specific capsule and all manifest hashes, requires `gfx950`, and + invokes an image-owned helper that generates and compiles a native launcher. + The parser requires the expected kernel dispatch, capsule/code-object digest + attestation, and a successful replay/golden-output marker. The missing piece + is automatic evaluator-owned capsule capture and proof that the capsule came + from the exact correctness run; a task-supplied capsule is still weak + candidate provenance. +- A deliberately out-of-bounds, **uninstrumented** HIP HSACO exited normally in + a complete GPU ASan runtime. This is why a runtime preload alone is never + accepted as GPU ASan coverage. +- ConSan embeds Waitcheck as a preflight. The ConSan parser records that preflight + as metadata but does not duplicate its diagnostics as ConSan race findings; + enable `rocjitsu_waitcheck` separately when a standalone wait-hazard result is + required. +- `gfx942` has not completed the same image, positive-control, adapter, and + end-to-end validation. Its status is unverified, not unsupported by theory. + +## Build and check the sidecars + +Building requires Docker and network access to the pinned package and source +locations. Runtime sidecars themselves start with networking disabled. + +Build all six local `gfx950` images from the repository root: + +```bash +src/scripts/docker_benchmark.sh build-eval-tool-images +``` + +The default local tags are: + +```text +agent-kernel-arena/eval-tool-triton-fpsan:gfx950 +agent-kernel-arena/eval-tool-gpu-asan:gfx950 +agent-kernel-arena/eval-tool-rocjitsu:gfx950 +agent-kernel-arena/eval-tool-rocjitsu-waitcheck:gfx950 +agent-kernel-arena/eval-tool-rocjitsu-consan:gfx950 +agent-kernel-arena/eval-tool-hip-fpsan:gfx950 +``` + +Check that the workers start and report their pinned assets: + +```bash +src/scripts/docker_benchmark.sh eval-tools-smoke +``` + +To check a subset: + +```bash +AKA_EVAL_TOOLS=gpu_asan,rocjitsu \ + src/scripts/docker_benchmark.sh eval-tools-smoke +``` + +`AKA_EVAL_TOOLS` is a host-side subset override. When it is set for either a +smoke test or a normal run, the runner starts exactly that normalized set and +publishes it through the internal `AKA_EVAL_TOOLS_SELECTED` variable; the +scoring process then plans the same set instead of the YAML `enabled` value. +This prevents a sidecar/plan mismatch. Leave the override unset when YAML should +remain authoritative, and record any override as part of the run invocation. + +Worker startup runs a tool-specific synthetic positive control before the Unix +socket appears. Health output includes its verdict, commands, durations, +bounded log paths/excerpts, and the immutable Docker image ID reported by the +worker: + +| Tool | Startup positive control | +| --- | --- | +| `triton_fpsan` | Compile instrumented reference/candidate kernels and require a known numerical mismatch to produce different digests plus FpSan compiler metadata. | +| `gpu_asan` | Compile and run safe/OOB HIP fixtures and safe/OOB Triton fixtures; the task profile selects the relevant lane. | +| `rocjitsu` | Require a barrier-protected fixture to remain clean and a deliberately racy LDS fixture to report a race. | +| `rocjitsu_waitcheck` | Compile unbundled `gfx950` code objects and run the production entrypoint, inventory, C API, and parser on the correct-wait and missing-wait fixtures; retain a direct CLI hazard check as an independent engine control. | +| `rocjitsu_consan` | Compile raw safe/racy HSACOs plus an image-owned module launcher, run the production entrypoint with separate instrumented/oracle argv, require the oracle environment to be scrubbed, and make the production parser return clean/finding with exact FNV attribution. | +| `hip_fpsan` | Require explicitly ported equivalent expressions to match and a known-wrong expression to produce a different digest. | + +`eval-tools-smoke` prints this evidence and exits nonzero if a worker reports +`degraded`, including a failed nested positive control. Inspect and retain the +JSON summaries before promotion. A normal evaluation with +`positive_control: required` repeats the fail-closed check during the typed +runtime probe. + +As of the current `gfx950` qualification run, all six integrated startup +controls pass on hardware. This qualifies the installed tool runtimes only. It +does not promote a candidate path without the language-specific adapter and +attestation in the strict support matrix. + +The same final image set also passed evaluator-manager-to-sidecar candidate +fixtures on the physical MI355X host: + +| Tool and language | Safe fixture | Seeded bug fixture | +| --- | --- | --- | +| Triton FpSan, Triton | `clean` | Numerical mismatch `found` | +| GPU ASan, HIP | `clean` | Out-of-bounds access `found` | +| GPU ASan, Triton | `clean` | Out-of-bounds access `found` | +| HIP-FpSan, explicitly ported HIP | `clean` | Wrong expression `found` | +| rocJITsu, Triton AOT replay | `clean` | Not exercised in this candidate pair | +| rocJITsu, FlyDSL AOT replay | Not exercised in this candidate pair | Missing-barrier LDS race `found` | + +These are controlled synthetic fixtures that validate the current adapters, +transport, parsing, and attestation paths. They do not qualify any bundled +production task, broaden the strict support matrix, or close the AOT +correctness-dispatch provenance gap. + +## Recommended rollout plan + +Promote one language/tool path at a time. Do not make “all sanitizers enabled” a +global milestone. + +| Phase | Work | Exit criterion | +| --- | --- | --- | +| 0. Freeze baselines | Keep the pinned scoring image, FlyDSL 0.2.2, and AITER version unchanged; build each tool from its lock into a sidecar. | Existing compilation, correctness, held-out, and performance baselines remain unchanged with tools disabled. Sidecar image IDs and the verified scoring-image ID/reference are captured in plans. | +| 1. Qualify installations | Run automatic safe/known-bug startup controls on `gfx950`; repeat the now-passing six-tool qualification on clean hosts. | Both positive and negative lanes pass repeatedly. `eval-tools-smoke` evidence is archived and independently reviewed. | +| 2. Build trusted pilot adapters | Start with one editable Triton task for Triton FpSan, one Triton and one HIP task for GPU ASan, one native HIP task for rocJITsu, one final-HSACO task for Waitcheck, one focused native loader for ConSan, and one explicitly ported HIP-FpSan task. Put harnesses under protected `scripts/` paths and declare all inputs. | Each pilot distinguishes a safe fixture from a seeded bug, identifies the selected candidate, and produces bounded structured artifacts. No precompiled AITER/library kernel is claimed as broadly covered. | +| 3. Finish AOT capture and binding | The trusted `triton_aot`/`flydsl_aot` replay path now validates one-dispatch capsules and generates the launcher. Add evaluator-owned extraction immediately after correctness and bind the capsule to that exact candidate/case. | Safe and racy fixtures pass end to end, malformed capsules fail closed, and a task cannot substitute a different valid capsule for the correctness dispatch. | +| 4. Harden provenance and phase isolation | The runner now uses per-tool writable socket directories, a read-only socket parent in scoring, a narrow per-worker artifact mount, fresh per-invocation artifact directories, a complete serialized plan, and capsule digests in the fingerprint. Next run tools only after the agent exits, freeze the candidate, use evaluator-only/authenticated RPC and evaluator-owned artifacts, strengthen artifact/dispatch binding, and wire resume to plan freshness. | An adversarial task cannot call a worker, overwrite evidence, reach another task's artifacts, spoof a clean result, or reuse a stale report. This phase is required before sanitizer output becomes a reward signal. | +| 5. Advisory campaign | Run qualified paths with `policy: advisory` across representative and private held-out shapes; measure overhead, timeouts, log volume, flakes, false positives, and GPU recovery behavior. | Each task/tool pair has reviewed coverage cases, stable resource limits, and an explicit owner/runbook. Incomplete results remain visible and never score as clean. | +| 6. Narrow required gates | Change only individually qualified task/tool pairs to `required`; leave unsupported and not-yet-qualified paths advisory or disabled. | Required gates block seeded findings and infrastructure failures without changing ordinary correctness semantics or the scoring performance baseline. | +| 7. Add `gfx942` separately | Build architecture-specific images/configs and rerun every startup, adapter, security, and workload fixture on MI300X/MI325X. | Only mark `gfx942` supported after independent qualification; do not infer it from `gfx950`. | + +Phase 0, the six integrated startup controls, complete plan serialization, the +narrower runner mounts, and the trusted AOT replay core exist. No bundled task +has completed production qualification through phases 2–6. Automatic capsule +capture, exact candidate/dispatch provenance, top-level resume freshness, and +the same-phase agent/evaluator boundary remain blockers. + +## Start from the disabled example + +Copy `example_configs/evaluation_tools_advisory_mi355x.yaml`. It is a normal +MI355X run configuration but has `evaluation_tools.enabled: false`, so copying +and running it does not build or start any sidecar when the host +`AKA_EVAL_TOOLS` override is unset: + +```bash +cp example_configs/evaluation_tools_advisory_mi355x.yaml my_sanitized_run.yaml +``` + +After building the images and adding task-specific adapters, opt in to only the +tools that can actually inspect the selected task: + +```yaml +evaluation_tools: + enabled: + - gpu_asan + - rocjitsu + policy: advisory + positive_control: required + timeout_s: 600 + tools: + gpu_asan: + options: {} + rocjitsu: + options: {} +``` + +Do not enable all six tools merely because all six images exist. On a +heterogeneous task set, irrelevant tools become `not_applicable`, unsupported +paths remain visible as unsupported, and missing adapters remain +`adapter_required`. + +Timeouts are exact YAML/JSON integers from 1 through 3600 seconds; booleans, +floats, numeric strings, and larger values are rejected. A task-level timeout +must also be no larger than the run-level timeout. Explicit +`evaluation_tools` sections and per-tool entries reject unknown fields rather +than silently weakening the requested policy. Supplying both `runtime_ref` and +its `image_digest` alias is rejected when the two assertions differ. + +### Select and attest each tool image separately + +The host runner selects the Docker image using the default local tag or an +environment override. For example: + +```bash +export AKA_EVAL_TOOL_IMAGE_GPU_ASAN='registry.example/eval-tool-gpu-asan@sha256:' +make docker-run CONFIG=my_sanitized_run.yaml RUN_ARGS='--run-suffix asan_advisory' +``` + +The override names are `AKA_EVAL_TOOL_IMAGE_TRITON_FPSAN`, +`AKA_EVAL_TOOL_IMAGE_GPU_ASAN`, `AKA_EVAL_TOOL_IMAGE_ROCJITSU`, +`AKA_EVAL_TOOL_IMAGE_ROCJITSU_WAITCHECK`, +`AKA_EVAL_TOOL_IMAGE_ROCJITSU_CONSAN`, and +`AKA_EVAL_TOOL_IMAGE_HIP_FPSAN`. + +After selecting a tool-image reference, the runner resolves its local immutable +image ID with `docker image inspect`, launches the sidecar by that bare +`sha256:...` ID, and injects the same ID as runtime identity evidence. Runtime +health reports that ID, and the typed probe fails with +`RUNTIME_REF_MISMATCH` if the planned and observed values differ. The resolved +ID belongs in the plan fingerprint. Scoring-image verification likewise launches +the scoring container by its verified local ID. A registry reference such as +`name@sha256:...` is not string-equal to Docker's local image ID and must not be +used as a manual substitute. + +YAML `runtime_ref`/`image_digest` fields are identity assertions, not image +selectors. Omit them when using automatic host injection. If supplied manually, +use the exact bare ID returned by: + +```bash +docker image inspect --format '{{.Id}}' +``` + +## Add a task adapter + +Run-level configuration chooses tools, policy, optional image-identity +assertions, and maximum timeout. A task can only add options for an +already-enabled tool and lower its timeout. It cannot enable a tool, change the +image or top-level policy, or raise the run-level timeout. +Reserved framework options are rejected at both run and task level. They are +`positive_control_required`; GPU ASan's `asan_runtime_dir`, +`hip_asan_runtime`, `host_asan_preload`, `host_asan_lib_dir`, and +`normal_rocm_lib_dir`; rocJITsu's `rocjitsu_binary` and `config_path`; and +Waitcheck's `waitcheck_binary` and `waitcheck_capi_wrapper`; ConSan's +`consan_hook`; and HIP-FpSan's `include_dir` and `public_header`. The +host/runtime probe is the only authority for those values. + +Commands must be argv lists, never shell strings. A dedicated tool command is +required because reusing `correctness_command` could instrument the reference, +load a precompiled library kernel, or sanitize the wrong candidate. + +For example, a HIP task can declare the shape of its adapters as follows: + +```yaml +evaluation_profile: + language: hip + artifact_kind: source_aot + framework: standalone + instrumentation_control: recompile + source_available: true + submission_paths: + - optimized_kernel.hip + - scripts/eval_tools/run_gpu_asan.py + - scripts/eval_tools/rocjitsu_launcher + +evaluation_tools: + tools: + gpu_asan: + timeout_s: 300 + options: + command: [python3, scripts/eval_tools/run_gpu_asan.py] + rocjitsu: + options: + launcher: [scripts/eval_tools/rocjitsu_launcher] + expected_kernel: my_kernel +``` + +These are adapter contracts, not automatically generated files. The task +wrapper must build and launch the optimized candidate, exercise representative +inputs, and emit the required evidence. Sidecar health is the only authority +for container-internal ASan libraries and preload, the rocJITsu binary and +architecture config, the Waitcheck CLI/C API wrapper, the ConSan HSA hook, and +the HIP-FpSan include directory. The runtime probe attests and injects those +values into the plugin context; neither run nor task configuration may supply +or override them. + +Common built-in option keys are: + +| Tool | Required adapter options | Additional evidence/options | +| --- | --- | --- | +| `triton_fpsan` | `comparison_command` or `command` | `attestation_path`; command must emit one `AKA_FPSAN_RESULT` JSON line | +| `gpu_asan` | `command` | Candidate `attestation_path`; a HIP command must use the required compile flags. Runtime/preload/library paths come from health. | +| `rocjitsu` | HIP: `launcher` or `command`. Triton/FlyDSL: `capsule` plus an exact profile adapter of `triton_aot` or `flydsl_aot`; user launchers are forbidden on these AOT paths. | HIP may set `expected_kernel` and `race_report`. AOT capsule path must stay below the task workspace and target `gfx950`; the executable/config and trusted replay helper come from the sidecar image. | +| `rocjitsu_waitcheck` | `code_object`, `expected_kernel`, and integer `kernel_entry` | `code_object` must be an unbundled final AMDGPU ELF below the workspace. The image-owned inventory helper must attest exactly the requested `gfx950` descriptor before the C API runs. | +| `rocjitsu_consan` | `code_object`, `command`, and `oracle_command` | `command` must be a focused native argv that explicitly names and loads `code_object`; `oracle_command` runs separately without the hook. Exact SHA-256 and FNV-1a64 identities are required. | +| `hip_fpsan` | `comparison_command` or `command`, plus `evaluation_profile.fpsan_ported: true` | Candidate `attestation_path`; both paths must be instrumented. The include directory comes from health. | + +The plugin validates these adapter options directly, so both tools may inspect +the same declared candidate in one advisory plan. For example: + +```yaml +evaluation_profile: + language: hip + artifact_kind: hsaco_precompiled + framework: standalone + +evaluation_tools: + tools: + rocjitsu_waitcheck: + options: + code_object: build/optimized.hsaco + expected_kernel: optimized_kernel + kernel_entry: 0 + rocjitsu_consan: + options: + code_object: build/optimized.hsaco + command: [scripts/load_hsaco, build/optimized.hsaco] + oracle_command: [scripts/load_hsaco, build/optimized.hsaco, --check] +``` + +With ROCm 7.2, `hipcc --genco` produces a clang bundle by default; use +`--no-gpu-bundle-output` or explicitly extract the final device ELF before +supplying `code_object`. + +`attestation_path` is resolved below the fresh artifact directory for the +current tool invocation and the same resolved path is used for the injected +environment variable, metadata, and parser. Relative paths are recommended; +an absolute path is accepted only when it resolves below that invocation +directory. Native HIP rocJITsu applies the same containment rule to +`race_report` and requires the report filename to remain `race.log`. AOT replay +does not accept a task-configured race-report path. + +For repository or image-kernel tasks, declare every candidate file whose change +must invalidate evidence with `evaluation_profile.submission_paths`. Paths must +be workspace-relative and cannot contain `..`. If this field is absent, capture +falls back to `source_file_path` and `target_file_path`; silently hashing an +entire multi-gigabyte repository is intentionally avoided. + +Put evaluator-owned adapter code under a harness-protected path such as +`scripts/`, not an arbitrary agent-editable `eval_tools/` directory. Also list +adapter scripts, HSACO files, and input blobs in `submission_paths` when their +contents must affect the general candidate fingerprint. A configured replay +capsule or Waitcheck/ConSan code object receives additional handling: +immediately before plan construction the manager records its SHA-256 and size under +`source_evidence.metadata.option_artifacts`, so that digest is covered by the +plan fingerprint. The validated capsule manifest contains and verifies the +HSACO and blob digests. This binds the plan to the supplied capsule bytes; it +does not prove who captured the capsule or that it came from the same dispatch +as ordinary correctness. + +## Require evidence before accepting “clean” + +Process success is not proof that an analysis ran. The built-in parsers require +tool-specific attestation: + +- GPU ASan requires `build_attestation.json` for the declared artifact. HIP + attestation must include all sanitizer/XNACK flags and `HSA_XNACK=1`; Triton + attestation must include `TRITON_ENABLE_ASAN=1` and `HSA_XNACK=1`. +- Triton FpSan and HIP-FpSan currently require one build attestation whose + evidence contains the self-declared `reference_instrumented` and + `candidate_instrumented` booleans, plus an `AKA_FPSAN_RESULT` payload with the + two digests. Exactly one result marker is required; zero or multiple markers, + a timeout, or any nonzero/unknown process exit is a tool error. They do not + validate two independently attested artifacts. +- Native HIP rocJITsu requires a canonical simulator dispatch in the + evaluator-owned report sink, optionally matched to `expected_kernel`. + Task-launcher stdout/stderr cannot attest a clean dispatch, even if it prints + the complete rocJITsu prefix. The task launcher still inherits the report + configuration, so this lane remains advisory rather than trusted reward + evidence. +- Triton/FlyDSL rocJITsu uses the image-owned replay helper instead of a task + launcher. It revalidates the capsule and manifest, generates the native + launcher, and requires an exact capsule/code-object attestation, expected + kernel dispatch, and `AKA_REPLAY_RESULT pass`. Missing or changed evidence is + inconclusive. This is stronger replay integrity, but the task-supplied capsule + is not yet automatically tied to the correctness run. +- rocJITsu Waitcheck re-hashes the selected final code object, inventories its + descriptors with the image-owned CLI, matches the exact kernel name and entry, + and obtains structured diagnostics through the stable C API. Missing, + duplicate, truncated, or incomplete evidence is inconclusive. +- rocJITsu ConSan re-hashes the code object, requires the configured launcher to + name it, matches the hook's FNV-1a64 identity, enforces strict record/replay + completeness and accounting, and runs a separate correctness oracle without + the hook. Embedded Waitcheck text is preflight metadata, not a duplicate + ConSan finding. +- Build attestation records the compiler, compiler version, and target + architecture. The current validator directly checks tool identity, + `instrumented: true`, required build flags/environment, artifact existence, + and artifact SHA-256. Required flags are matched as complete argv tokens + (including supported split-value forms), not as substrings of unrelated + arguments. The host/runtime `gfx950` guards provide the current + architecture boundary; stricter compiler/version/target comparisons remain + future attestation hardening. + +These checks establish integrity evidence, not complete candidate provenance. +Build-attestation validation does not compare every compiler/version/target +field with an expected build or cryptographically bind the declared artifact to +the actual dispatch. It does require the artifact path to be relative to, and +contained below, the directory holding the attestation. A user-controlled build +JSON and native HIP wrapper output can therefore still satisfy checks without +proving which candidate ran. AOT replay binds its generated launcher to a +validated capsule and digest, but the capsule can still have been supplied for +a different candidate/case. Current `required` policy is suitable for trusted +integration diagnostics, not an adversarial reward boundary. + +The default build-attestation location is the fresh invocation directory below +the external per-task/tool artifact root described later. A wrapper executing +in a sidecar must write it through the writable `/artifacts` mount; the +repository/workspace input mount is read-only. GPU ASan, Triton FpSan, and +HIP-FpSan invocations inject `AKA_BUILD_ATTESTATION_PATH`, and the runtime client +translates that output path into the sidecar namespace. The adapter must place +the built artifact beside or below that JSON file and write `artifact_path` as a +relative path such as `build/candidate.hsaco`. The scoring-side parser resolves +the same relative path below its corresponding artifact directory, rejects +absolute/escaping paths, and checks the declared SHA-256. Do not embed either +the sidecar `/artifacts/...` prefix or a scoring-container absolute path. + +### Positive controls and candidate attestation are both required + +`positive_control: required` is the default. Each sidecar runs its synthetic +known-bug control once at worker startup and writes an audit summary and bounded +logs. Health returns that evidence. For every applicable task, the typed runtime +probe selects the relevant control (HIP or Triton for GPU ASan) and returns +`unavailable_runtime/POSITIVE_CONTROL_FAILED` unless it passed. The requirement +and host-resolved tool runtime identities are covered by the plan inputs. The +separately verified scoring-image ID/reference are recorded in plan source +evidence as well. +Control evidence is retained in runtime capability/reporting, but its startup +artifact content is not itself part of the plan fingerprint. + +This proves that the isolated installation detected its synthetic bug; it does +not prove that the optimized candidate was instrumented or replayed. Candidate +build/dispatch attestation remains a separate requirement. A minimally +meaningful clean diagnostic result needs both: + +1. a passing startup positive control for the applicable language lane; and +2. a task result with the declared candidate's build or dispatch attestation. + +Setting `positive_control: optional`, `disabled`, or `false` records +`positive_control_required: false`. The startup probe still runs and remains +visible in health evidence, but a failure no longer blocks runtime capability. +Use that only for tool bring-up, not trusted benchmark results. + +## Choose the policy + +The policy governs whether performance measurement may proceed. It does not +rewrite `pass_correctness`. + +| Policy | Finding, tool error, inconclusive result, missing adapter, unsupported applicable path, or missing runtime | Performance | +| --- | --- | --- | +| `advisory` | Recorded; `policy_satisfied: false` | Continues (`allowed: true`) | +| `required` | Recorded with a reason | Skipped (`allowed: false`) | + +An explicitly `not_applicable` tool is ignored by the gate. Under `required`, +every other selected tool must have effective capability `ready`, execution +`completed`, and finding status `clean`. Start new integrations in `advisory` +mode. Move to `required` only after task adapters, positive controls, and +architecture-specific fixtures are independently reviewed. + +## Read execution and findings separately + +`execution` answers whether the tool invocation completed. `finding` answers +what the parser concluded about the selected kernel: + +| `execution` | Meaning | +| --- | --- | +| `not_run` | No invocation was made. | +| `completed` | The parser obtained a complete tool outcome; a detected bug can still have this state. | +| `tool_error` | The runtime, wrapper, or parser failed without a valid finding. | +| `timeout` | The isolated process group exceeded its timeout. | + +| `finding` | Meaning | +| --- | --- | +| `not_evaluated` | No supported evaluation ran. | +| `clean` | The intended instrumented/simulated kernel ran and no finding was observed. | +| `found` | One or more structured issues were reported. | +| `inconclusive` | Absence of a finding cannot be trusted, often because attestation is missing. | + +A sanitizer can deliberately terminate with a nonzero process status while +still producing a valid finding. Conversely, an uninstrumented out-of-bounds +kernel can return zero and must remain inconclusive. Never use return code alone +as the sanitizer result. + +The report keeps ordinary scoring fields and adds: + +```yaml +pass_tool_gate: true +tool_policy_satisfied: false +tool_evaluation: + schema_version: 1 + plan_fingerprint: "..." + plan: + schema_version: 1 + policy: advisory + profile: {} + tools: + - tool: gpu_asan + runtime_ref: "sha256:..." + plugin_version: "1" + timeout_s: 600 + options: {positive_control_required: true} + fingerprint: "..." + source_evidence: + metadata: + scoring_runtime: + image_id: "sha256:..." + reference: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705" + policy: advisory + overall_status: incomplete + resolved_task_profile: {} + source_evidence: {} # legacy mirror of plan.source_evidence (abbreviated) + decision: + allowed: true + policy_satisfied: false + reasons: ["gpu_asan:capability:adapter_required"] + tools: + gpu_asan: + capability: + engine: {state: ready} + adapter: {state: adapter_required} + runtime: {state: ready} + effective: {state: adapter_required} + result: null +``` + +Large stdout/stderr streams are not embedded in `task_result.yaml`; they remain +bounded artifacts. The report serializes the complete immutable `plan`, +including each tool's runtime reference, plugin version, timeout, options, +profile, source evidence, and fingerprint. This makes a result reconstructable +for audit without relying on the digest alone. + +## Artifact and evidence layout + +For a normal run, the relevant files are: + +```text +/ +├── .eval-tool-artifacts/ +│ └── / # dedicated sidecar/scoring RW submount +│ ├── _eval_tool_runtime/ +│ │ └── //positive-control/ +│ │ ├── summary.json # startup verdict and step metadata +│ │ └── .{stdout,stderr}.log +│ └── -/ +│ ├── gpu_asan/// +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── build_attestation.json # when the adapter supplies it +│ │ └── triton-gpu-asan-cache/ # Triton only +│ ├── triton_fpsan/// +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── build_attestation.json +│ │ └── triton-fpsan-cache/ +│ ├── rocjitsu/// +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── rocjitsu-report/race.log +│ │ └── rocjitsu-replay/ # generated AOT launcher, when used +│ └── hip_fpsan/// +│ ├── stdout.log +│ ├── stderr.log +│ └── build_attestation.json +└── experiments/workspace__/run_/ + ├── .eval-tool-evidence/ + │ └── / + │ ├── manifest.json + │ └── files/ # captured original declared files + └── / + └── task_result.yaml # nested tool_evaluation summary +``` + +Each tool attempt gets a newly created invocation directory, so a repeated plan +cannot consume an old attestation or race report. Each stdout and stderr file is +limited to 64 MiB by default and records truncation metadata. Tool processes run +in a new process group. The Linux worker also acts as a child subreaper and +tracks the invocation's complete descendant tree through `/proc`, including a +child that starts a new session or process group. Cleanup sends termination and +then kill signals; observing descendants after the command exits makes the +execution fail rather than clean. If survivors cannot be removed, the worker +exits so its container is not reused. Per-sidecar scratch and cache directories +outside `.eval-tool-artifacts` are deleted when the sidecar stops. The report +records absolute scoring-side paths below +`/workspace/.eval-tool-artifacts/`. The runner explicitly mounts +that path read/write even when `/workspace` itself is read-only. The per-worker +namespace prevents a writable sidecar alias to task workspaces but is not a +secret or an adversarial integrity boundary. +Raw parser excerpts are omitted from normal JSON/YAML summaries and remain in +the bounded log artifacts. If either stream is truncated, a result that would +otherwise be clean is changed to `inconclusive`; an already observed finding is +preserved. + +## Resume and plan fingerprints + +Every tool report has a SHA-256 `plan_fingerprint` covering: + +- normalized tool configuration, policy, timeouts, options, and configured + runtime references; +- the selected scoring-image reference and its immutable local image ID, after + the runner resolves the pinned SGLang manifest reference to its local image ID + and verifies that the two local IDs match; +- resolved task profile and explicit overrides; +- enabled plugin versions; +- captured-original and optimized-candidate fingerprints for declared paths, + including both the lexical submission path and its current resolved symlink + target; +- the SHA-256 and size of a configured replay-capsule JSON, whose validated + manifest in turn binds the referenced HSACO and blobs. + +This prevents a report from being considered current after a material input +change **when the caller checks it**. The reporting API exposes an exact +fingerprint check for this purpose. + +Most coverage is declaration-based. A pathname stored in ordinary tool options +contributes only its string; adapter, HSACO, and input contents must be declared +in submission evidence or represented by an explicit digest. Replay `capsule` +is the exception: the manager captures its content digest during plan +construction. The current `has_current_plan` helper checks only the fingerprint +and presence of a tools mapping; it does not prove that every tool completed or +that its result is clean. The serialized `plan` makes the inputs auditable but +does not by itself change resume scheduling. + +The top-level `--resume-run` and `--resume-latest` paths currently skip a task +when its `task_result.yaml` already exists; they do not yet rebuild the tool plan +and call that fingerprint check. Until this is wired into run scheduling: + +- use a new `--run-suffix` after changing tool configuration, plugin code, + sidecar image, adapter, source declaration, or positive-control policy; +- do not change `AKA_EVAL_TOOL_IMAGE_*` while resuming a run; +- do not change the selected scoring-image reference while resuming, even to + another tag for the same image ID, because the reference is plan evidence; +- do not assume a changed `runtime_ref` causes a completed task to rerun; +- if a tool-only rerun is required, archive the old report and start a fresh run + rather than silently combining evidence from two plans. + +## AOT replay capsules for Triton and FlyDSL + +rocJITsu can execute supported generated `gfx950` HSACO, but it cannot safely +wrap the current Triton/FlyDSL Python JIT process. The implemented adapter +boundary is a versioned replay capsule containing the exact HSACO and SHA-256, +kernel symbol, launch geometry, declared lowered ABI, allocation snapshots, +pointer relocations, scratch requirements, target architecture, producer +versions, and case identity. Configure the matching adapter and a +workspace-contained capsule: + +```yaml +evaluation_profile: + language: triton # use flydsl for a FlyDSL artifact + artifact_kind: python_jit + adapter: triton_aot # or flydsl_aot + +evaluation_tools: + tools: + rocjitsu: + options: + capsule: eval_capsule/capsule.json +``` + +The plugin validates the capsule in the scoring process, rejects `launcher` or +`command` on AOT paths, and hashes the capsule JSON into the plan. The +image-owned helper rechecks its digest and manifest inside the sidecar, verifies +the adapter identity and `gfx950` target, generates a native HIP launcher, +compiles it with the sidecar toolchain, and reconstructs exactly one dispatch +inside rocJITsu. Post-execution parsing revalidates the capsule and requires the +expected dispatch, exact capsule/code-object marker, and replay success marker. + +The current validator fails closed for at least: + +- more than one kernel dispatch; +- a capsule with no golden expected-output blob; +- non-integral, boolean, negative, or greater-than-`uint32` launch dimensions; +- launch dimensions, block size, or dynamic shared memory that exceed the + selected device's runtime limits; +- opaque/tensor descriptors; +- empty, misordered, or unsupported ABI arguments and unknown implicit refs; +- invalid relocations and out-of-bounds allocation views; +- missing input blobs, mismatched hashes, or an architecture mismatch. + +Do not reduce a Python cache entry to just “HSACO + kernel name.” Without ABI, +launch, allocation, and framework-version evidence, a clean rocJITsu result may +belong to a different execution than the scored task. The replay adapter itself +is implemented and its safe/racy Triton and FlyDSL capsules have run end to end. +What is not implemented is automatic, evaluator-owned extraction from the +ordinary correctness dispatch and a trusted binding between that capture and +the scored candidate/case. Until that provenance exists, treat a clean AOT +result as advisory diagnostics rather than a production-qualified reward gate. + +## Resource, security, and held-out risks + +### Resource controls + +- The worker is sequential: one sidecar executes one GPU command at a time. + Parallel runs create a separate sidecar set per worker/GPU. +- Do not schedule another benchmark or sanitizer on the same physical GPU. + GPU ASan changes allocation behavior and rocJITsu can be much slower than + native execution; neither runtime is a performance measurement environment. +- Begin with one representative case and a bounded timeout. Expand coverage + only after measuring simulator time, HBM/host memory, and artifact growth. +- A timeout or truncated log is not clean evidence. + +### Isolation is a boundary, not proof of safe untrusted execution + +Runtime sidecars use no network, a read-only root filesystem, dropped Linux +capabilities, `no-new-privileges`, the host UID/GID, a read-only repository +mount, image-owned framework/probe code, and separate writable scratch/artifact +mounts. The scoring container gets the per-worker socket parent read-only, not +Docker access or tool-image credentials. Each sidecar gets only its own nested +writable socket directory, so it cannot replace or connect through a sibling's +mounted UDS path. RPC paths are relative and containment-checked, and normal +commands are argv lists. + +Artifact isolation is narrower but not yet per-task/tool. Every sidecar for one +worker gets the same writable repository-root +`.eval-tool-artifacts/` namespace. It no longer receives a +writable alias of the whole `experiments` tree, task workspaces, or +`.eval-tool-evidence`, but candidate code in one sidecar can still modify another +task/tool report in that worker's diagnostic namespace. The runner explicitly +mounts this namespace writable in the scoring container even when the broad +repository mount is read-only. + +However, a GPU kernel can still hang or reset the device, exploit a driver bug, +consume host/GPU memory, or affect another process sharing the GPU. Run +untrusted submissions on dedicated workers with external time, memory, process, +and machine-recovery controls. Do not mount secrets into a tool sidecar. An +explicit `bash -lc` command should be allowed only in a reviewed, trusted +adapter. + +There is also no evaluator-versus-agent trust boundary inside the current +scoring container. Sidecars start before the agent, and their per-tool Unix +sockets remain mounted for the whole container run without client +authentication. An agent process can therefore observe or call the worker RPC +even though it cannot replace the socket path. The external tool-artifact path +is also writable during optimization, so an adversarial submission could +pre-create evidence unless the adapter independently verifies its provenance. +Filesystem placement and a SHA-256 inside an agent-writable JSON file are +integrity checks, not signatures. + +The pre-agent `.eval-tool-evidence` copy is checked against an in-memory +manifest during an uninterrupted task, which detects ordinary mutation. It is +still stored on the current writable run mount and is not signed. After a crash, +an interrupted-run resume loads that on-disk manifest as its starting point, so +it is not a durable trust anchor against an adversarial process that can rewrite +both the manifest and copied files. + +For adversarial or reward-bearing evaluation, use a separate evaluation phase: +stop the agent container, snapshot the candidate, start fresh sidecars with +evaluator-only sockets, mount the candidate read-only, give each task/tool only +its own evaluator-owned artifact directory, authenticate the RPC client, and +generate attestations and AOT capsules there. Per-tool socket directories and a +narrow per-worker artifact mount are already implemented, but the current +single-container runner does not automate this phase boundary, so its tool +results should be treated as diagnostic evidence rather than tamper-resistant +rewards. + +### Held-out integrity + +Sanitizers exercise only the cases launched by their adapter. A clean visible +case is not proof of safety for all shapes, strides, dtypes, schedules, or race +interleavings, and sanitizer runs do not automatically invoke the separate +held-out evaluator. + +Task-local adapter code, commands, capsules, seeds, input blobs, expected +outputs, and positive-control structure may be visible to the optimizing agent. +The sidecar input mount currently contains the repository root, and the socket +is present during agent execution; merely moving a hidden file to another path +inside that mount does not make it private. +For a trusted held-out campaign: + +1. Keep private cases outside the agent-editable workspace and configuration. +2. Generate replay capsules only after the candidate is frozen. +3. Mount private inputs read-only only for the evaluation phase. +4. Keep private seeds and golden outputs out of committed task files and public + `.eval-tool-artifacts` reports. +5. Run ordinary held-out correctness and sanitizer coverage as separate signals; + neither substitutes for the other. + +## Known limitations checklist + +Before treating evaluation tools as a required scoring gate, account for all of +the following: + +- Only `gfx950` has pinned sidecar locks, passing integrated startup controls, + and end-to-end fixtures; `gfx942` is unverified and rejected by the host + runner. +- Every useful task still needs a reviewed adapter command. Tool installation + alone usually produces `adapter_required`. +- All six startup positive controls pass on the current `gfx950` host. This + qualifies tool installation, not candidate coverage. +- Runtime-internal asset paths are injected from verified sidecar health and + cannot be supplied by task configuration. +- Build-attestation artifact paths must be relative to the attestation file; + absolute paths and paths escaping that per-tool artifact directory are + rejected. +- Build attestations and native-HIP rocJITsu dispatch text are weak, + self-reported integrity evidence. Triton/FlyDSL replay validates and attests + the capsule more strongly, but automatic trusted capture still does not bind + it to the ordinary correctness dispatch. +- Sidecars have isolated writable socket directories, a read-only top-level + artifact namespace in scoring, and a narrow writable mount for only the + current worker. Sockets and that worker's agent-writable report paths remain + visible in the same optimizing/scoring container phase, and its tool sidecars + still share the worker artifact namespace. +- Tool startup resolves both the selected scoring-image reference and the + pinned SGLang content-addressed manifest reference to local image IDs, then + rejects the selected image unless those IDs match. The verified ID and + selected reference are recorded in plan evidence; using an alias does not + authorize a different image build. +- YAML `runtime_ref` does not select the image; it is an assertion compared with + the host-injected, worker-reported local image ID. +- Top-level resume does not yet enforce `plan_fingerprint` freshness. +- Fingerprints cover declared file content plus configured capsule and + Waitcheck/ConSan code-object digests; other option-referenced files need + submission evidence or explicit digests. + The report serializes the complete tool plan. +- Triton/FlyDSL AOT replay is implemented for validated, single-dispatch + `gfx950` capsules and forbids arbitrary launchers. Automatic evaluator-owned + capsule extraction and correctness-run provenance are not implemented. +- FlyDSL GPU ASan and Triton FpSan instrumentation are unavailable. +- Precompiled AITER, rocBLAS, RCCL, and other library kernels are not covered + unless the exact source is rebuilt through a supported and attested path. +- “Clean” means no finding in the executed, attested cases; it is not a proof of + memory safety, race freedom, numerical equivalence for all inputs, or + generalization. + +Use `advisory` while any applicable item above remains unresolved. See the +[configuration and API reference](../reference/api-reference.md#evaluation-tools) +for the canonical field schema. diff --git a/docs/index.rst b/docs/index.rst index d3086ef2..7731218f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ repository. .. grid-item-card:: How to * :doc:`Run an experiment ` + * :doc:`Check kernels with evaluation tools ` * :doc:`Run tasks in parallel across multiple GPUs ` * :doc:`Configure agents and models ` * :doc:`Add a task ` diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index ccc1e668..96633c66 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -50,6 +50,79 @@ log_directory: logs workspace_directory_prefix: workspace ``` +## Evaluation tools + +`evaluation_tools` configures optional, isolated kernel-analysis sidecars. The +section is disabled when it is absent, `null`, or `false`. A configured mapping +with no enabled tools is also disabled unless the host runner supplies +`AKA_EVAL_TOOLS`; for a mapping, that host subset replaces its `enabled` value. +The built-in IDs are `triton_fpsan`, `gpu_asan`, `rocjitsu`, +`rocjitsu_waitcheck`, `rocjitsu_consan`, and `hip_fpsan`. + +Sidecar build locks, integrated positive controls, and end-to-end fixtures +currently exist only for `gfx950`; all six startup controls pass in the current +MI355X qualification. Each applicable candidate still needs a task-specific +adapter and attestation, and enabling an image alone does not imply that a +kernel was analyzed. See [Check kernels with evaluation +tools](../how-to/use-evaluation-tools.md) for the support matrix and operational +requirements. + +When tools are enabled, the selected scoring image must resolve to the same +immutable local Docker image ID as the pinned +`lmsysorg/sglang-rocm@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78` +manifest. The runner rejects a different build, launches by the verified ID, +and records both the selected reference and verified ID in plan source evidence. + +Worker reports live at repository-root +`.eval-tool-artifacts/`. The runner mounts that specific host +directory read/write into both sidecars and the scoring container; the latter +submount remains writable when the quality-loop repository root is read-only. + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `evaluation_tools.enabled` | boolean, string, or list of strings | empty | Tool IDs to plan. `true` expands to all six built-ins; `false` disables the feature. A single string is accepted. Hyphens are normalized to underscores. When the host sets `AKA_EVAL_TOOLS`, its normalized subset is authoritative for both started sidecars and the in-container plan. | +| `evaluation_tools.policy` | `advisory` or `required` | `advisory` | `advisory` always permits performance but records an unsatisfied policy. `required` permits performance only when every applicable selected tool is ready, completes, and reports `clean`. | +| `evaluation_tools.positive_control` | `required`, `optional`, `disabled`, or boolean | `required` | Requires the applicable synthetic known-bug startup control to pass before runtime capability is ready. `optional`, `disabled`, and `false` normalize to not required; the worker still runs and reports its control. | +| `evaluation_tools.timeout_s` | exact integer from 1 through 3600 | `3600` | Default maximum execution time for each selected tool. Booleans, floats, and numeric strings are rejected. | +| `evaluation_tools.runtime_profile` | string or `null` | `null` | Fallback runtime-identity assertion for plans whose tool has no `runtime_ref`. It does not select an image and must exactly match worker health when set. | +| `evaluation_tools.tools` | mapping | `{}` | Per-tool configuration keyed by normalized tool ID. Entries do not enable tools. | +| `evaluation_tools.tools..runtime_ref` | string or `null` | automatic host image ID | Exact bare local Docker image ID (`sha256:...`) asserted by the plan and compared with worker health. The `image_digest` key is an alias. This field does not select an image; omit it when using automatic host injection. | +| `evaluation_tools.tools..timeout_s` | exact integer from 1 through 3600 | top-level timeout | Per-tool timeout; it cannot exceed the top-level timeout. | +| `evaluation_tools.tools..options` | mapping | `{}` | Adapter options, including argv lists and candidate-evidence paths. Reserved framework keys are rejected at run and task level: `positive_control_required`; GPU ASan runtime/preload/library keys; rocJITsu binary/config keys; Waitcheck CLI/C API keys; ConSan hook keys; and HIP-FpSan include/header keys. | + +The exact reserved option keys are `positive_control_required` for every tool; +`asan_runtime_dir`, `hip_asan_runtime`, `host_asan_preload`, +`host_asan_lib_dir`, and `normal_rocm_lib_dir` for `gpu_asan`; +`rocjitsu_binary` and `config_path` for `rocjitsu`; `waitcheck_binary` and +`waitcheck_capi_wrapper` for `rocjitsu_waitcheck`; `consan_hook` for +`rocjitsu_consan`; and `include_dir` and `public_header` for `hip_fpsan`. They +are selected and attested by worker health, not YAML. + +An explicit `evaluation_tools` mapping and each per-tool configuration reject +unknown fields. Unknown enabled tool IDs are also rejected. If both +`runtime_ref` and `image_digest` are supplied, they must be identical. + +Example run-level section: + +```yaml +evaluation_tools: + enabled: + - gpu_asan + policy: advisory + positive_control: required + timeout_s: 600 + tools: + gpu_asan: + options: {} +``` + +The image is selected separately with a host override. The runner resolves and +attests its immutable local image ID automatically: + +```bash +export AKA_EVAL_TOOL_IMAGE_GPU_ASAN='registry.example/eval-tool-gpu-asan@sha256:' +``` + ## Command-line flags The in-container `main.py` entrypoint accepts these flags: @@ -157,6 +230,73 @@ For repository-level tasks (`task_type: repository`): See [Add a task](../how-to/add-task.md) for layout and authoring rules. +### Evaluation profile + +The evaluator infers a profile from `task_type`, `repository_language`, source +suffixes, and repository paths. Add `evaluation_profile` only when that +inference is insufficient. Recognized profile overrides, including +`submission_paths`, are recorded in +`resolved_task_profile.explicit_overrides`. Unknown profile fields are currently +ignored, so use only the documented keys. + +| Field | Type | Description | +| --- | --- | --- | +| `evaluation_profile.language` | string | Canonical values are `triton`, `hip`, `flydsl`, and `unknown`. | +| `evaluation_profile.artifact_kind` | string | `source_aot`, `python_jit`, `hsaco_precompiled`, or `unknown`. | +| `evaluation_profile.framework` | string | Framework identity such as `standalone`, `aiter`, `rocblas`, or `rccl`. | +| `evaluation_profile.instrumentation_control` | string | `compiler_controlled`, `recompile`, `none`, or `unknown`. This describes whether the selected candidate can be rebuilt/instrumented. | +| `evaluation_profile.adapter` | string or `null` | Explicit adapter identity, for example `triton_aot`, `flydsl_aot`, or `hip_fpsan_manual`. It is a claim that must still be supported by adapter options/evidence. | +| `evaluation_profile.source_available` | boolean | Whether source for the selected candidate is available to the evaluator. | +| `evaluation_profile.submission_paths` | string or list of strings | Workspace-relative candidate files captured before agent edits and fingerprinted after optimization. Required when repository/image tasks change files beyond the normal source fields. Absolute paths and `..` are rejected. | +| `evaluation_profile.fpsan_ported` | boolean | Explicit evidence that the HIP reference and candidate were manually ported to HIP-FpSan value semantics. | +| `evaluation_profile.rebuilt_from_source` | boolean | Explicit evidence used when a framework/library path is rebuilt from controlled source. It does not replace artifact attestation. | + +### Task-level tool adapters + +A task can add adapter options only for tools enabled by the run. The only +allowed task-level structure is: + +```yaml +evaluation_tools: + tools: + gpu_asan: + timeout_s: 300 + options: + command: [python3, scripts/eval_tools/run_gpu_asan.py] +``` + +`timeout_s` must be between 1 and the run-level value. Task configuration cannot +enable another tool, change the top-level `policy` or `positive_control`, select +another runtime image, increase a timeout, or set any reserved framework option +listed above. Other options are merged over the run-level options. + +Commands are argv lists, not shell strings. The built-in adapter keys are: + +| Tool | Adapter keys | +| --- | --- | +| `triton_fpsan` | `comparison_command` or `command`; optional invocation-artifact-contained `attestation_path`. | +| `gpu_asan` | `command`; optional invocation-artifact-contained candidate `attestation_path`. ASan runtime/preload/library paths come only from verified sidecar health. | +| `rocjitsu` | HIP uses `launcher` or `command`, with optional `expected_kernel` and invocation-artifact-contained `race_report` whose filename must be `race.log`. Triton/FlyDSL requires `capsule` and the exact `triton_aot`/`flydsl_aot` profile adapter; arbitrary launchers and task-configured race-report paths are rejected. The capsule must be workspace-contained, single-dispatch, contain a golden expected output, be manifest-valid, and target `gfx950`. Binary/config and the trusted replay helper come only from the image/health. Automatic trusted capsule capture from correctness is not implemented. | +| `rocjitsu_waitcheck` | `code_object`, `expected_kernel`, and non-negative integer `kernel_entry`. The workspace-contained unbundled final ELF is SHA-256-bound to the plan; an image-owned inventory must match its exact `gfx950` descriptor before structured C API analysis. | +| `rocjitsu_consan` | `code_object`, focused native `command`, and independent `oracle_command`. The instrumented command must explicitly name and load the code object. Strict record/replay, SHA-256/FNV identity, complete accounting, and an oracle pass are required. Broad AITER/rocBLAS/RCCL runtimes are rejected. | +| `hip_fpsan` | `comparison_command` or `command`; optional invocation-artifact-contained candidate `attestation_path`; requires `evaluation_profile.fpsan_ported: true`. The include path comes only from verified sidecar health. | + +Sidecar health attests and injects runtime-internal assets. Candidate/task +configuration cannot override or supply ASan preload/library paths, the +rocJITsu binary/config path, the Waitcheck CLI/C API wrapper, the ConSan hook, +or the HIP-FpSan include path. + +Build-attestation JSON must store `artifact_path` relative to the directory +containing that JSON. The artifact must be beside or below the attestation; +absolute paths, `..`, and symlink resolutions that escape the directory are +rejected. The parser resolves the relative path in the scoring namespace and +verifies the declared SHA-256. + +Each execution uses a fresh +`//` artifact directory. Custom +attestation and race-report paths are resolved against that directory; an +absolute path is accepted only when its resolution remains inside it. + ### Platform support `platform_support.status: skip` excludes a task unconditionally. An active task @@ -175,6 +315,9 @@ Each task produces a `task_result.yaml` in its workspace: | `compilation_error_message` | Error text if compilation failed, else `null` | | `pass_correctness` | Whether correctness passed | | `correctness_error_message` | Error text if correctness failed, else `null` | +| `pass_tool_gate` | Whether the selected evaluation-tool policy allows performance to proceed. Defaults to `true` when tools are disabled. This is independent of `pass_correctness`. | +| `tool_policy_satisfied` | Whether every applicable selected tool was ready and completed with a `clean` finding status. Under `advisory`, this can be `false` while `pass_tool_gate` remains `true`. | +| `tool_evaluation` | Versioned complete plan, plan fingerprint, profile, capability, execution, finding, evidence, and decision data. Omitted when no tool is enabled. | | `base_execution_time` | Baseline runtime in ms | | `best_optimized_execution_time` | Best optimized runtime in ms | | `speedup_ratio` | Speedup over baseline | @@ -187,6 +330,37 @@ Each task produces a `task_result.yaml` in its workspace: | `optimization_summary` | Framework-generated note identifying the optimizing agent and centralized evaluator | | `score` | Computed score (see below) | +`tool_evaluation` uses this high-level shape: + +| Field | Description | +| --- | --- | +| `schema_version` | Evaluation-tool result schema version. | +| `plan_fingerprint` | SHA-256 over normalized configuration, resolved task profile, plugin versions, captured original/candidate evidence for declared paths, verified scoring-image reference/ID, and the content digest of a configured replay capsule. | +| `plan` | Complete immutable plan: schema, policy, profile, ordered tool records (runtime reference, plugin version, timeout, and options), fingerprint, and source evidence. | +| `policy` | `advisory` or `required`. | +| `overall_status` | `clean`, `finding`, `incomplete`, or `not_applicable`. | +| `resolved_task_profile` | Inferred profile plus auditable explicit overrides. | +| `source_evidence` | Captured-original and candidate fingerprints plus manifest metadata, including `metadata.scoring_runtime.image_id` and `.reference` when tools run. | +| `decision` | `allowed`, `policy_satisfied`, and machine-readable reason strings. | +| `tools..capability` | Separate `engine`, `adapter`, `runtime`, and resolved `effective` checks. | +| `tools..result.execution` | `not_run`, `completed`, `tool_error`, or `timeout`. | +| `tools..result.finding` | `not_evaluated`, `clean`, `found`, or `inconclusive`. | +| `tools..result.findings` | Structured finding records. | +| `tools..result.artifacts` | Paths to retained reports/attestations. Raw stdout/stderr is omitted from the YAML summary by default. | + +Execution status and finding status are deliberately independent. A sanitizer +can terminate the candidate while producing a valid finding, and a process can +exit zero without proving that an instrumented kernel ran. + +The complete `plan.tools` records make the selected runtime, plugin version, +timeout, and options reconstructable from the report. Ordinary option path +strings still do not hash referenced adapter, HSACO, or input contents; include +those files in `evaluation_profile.submission_paths` or add explicit digests. A +configured rocJITsu `capsule` is handled specially: its JSON SHA-256/size are +added to source evidence and the fingerprint, while capsule validation verifies +the manifest's HSACO and blob hashes. This does not prove the capsule came from +the ordinary correctness run. + ## Scoring The score is the sum of three components: diff --git a/docs/reference/compatibility-matrix.md b/docs/reference/compatibility-matrix.md index c5225bcd..4a7295f7 100644 --- a/docs/reference/compatibility-matrix.md +++ b/docs/reference/compatibility-matrix.md @@ -37,6 +37,29 @@ The following software versions are required or verified. | AITER | `0.1.17.dev110+g9127c94a1` in the verified `gfx950` image | Required by AITER-backed task oracles and kernels. | | FlyDSL | `0.2.2` in the verified `gfx950` image (or `make docker-setup-flydsl` when absent) | Required for `flydsl2flydsl`, `torch2flydsl`, and `triton2flydsl` tasks. | +## Evaluation-tool sidecars + +Optional Triton FpSan, GPU ASan, rocJITsu Race Detector, rocJITsu Waitcheck, +rocJITsu ConSan, and HIP-FpSan dependencies are kept out of the scoring image +and installed in one isolated sidecar image per tool. The scoring image, +FlyDSL, and AITER versions in the preceding table remain unchanged. + +| GPU architecture | Sidecar status | Notes | +| --- | --- | --- | +| `gfx950` (MI355X) | Runtime-qualified, candidate-dependent | Pinned image/build locks and all six integrated startup controls pass on the current hardware. End-to-end readiness still depends on language, artifact, adapter, and candidate attestation. Waitcheck and ConSan are qualified only for explicitly configured advisory pilots. Trusted single-dispatch Triton/FlyDSL rocJITsu capsule replay is implemented, but automatic evaluator-owned capsule capture and binding to the correctness run remain advisory-only gaps. | +| `gfx942` (MI300X/MI325X) | Unverified | No equivalent image/adapter/positive-control qualification has completed; the host runner currently rejects evaluation-tool sidecars. | + +The runtime base digest and per-tool package/source locks are recorded in +`docker/eval-tools/images.lock.yaml`. See [Check kernels with evaluation +tools](../how-to/use-evaluation-tools.md#strict-support-matrix) for the strict +Triton, HIP, FlyDSL, AITER, rocBLAS, and RCCL matrix. Normal task compatibility +does not imply sanitizer coverage. Tool startup resolves both the selected +scoring-image reference and the pinned `gfx950` SGLang content-addressed +manifest reference to immutable local image IDs and requires those local IDs to +match. Aliases of that exact image are allowed, but rebuilt, upgraded, or +retagged images are rejected. The scoring container is launched by the verified +image ID. + ## Agents The following templates are selectable in the current `AgentType` registry. See diff --git a/docs/reference/release-notes.md b/docs/reference/release-notes.md index 6d229c38..59de682f 100644 --- a/docs/reference/release-notes.md +++ b/docs/reference/release-notes.md @@ -21,6 +21,11 @@ execution, and RL-ready GPU kernel evaluation. - Added first-class A/B experimentation workflows with labeled baseline and treatment runs. - Exposed compilation, correctness, latency, speedup, and score fields as structured signals for external agent-RL systems. +- Added opt-in, per-tool sidecar plumbing and typed reports for Triton FpSan, + ROCm GPU ASan, rocJITsu Race Detector, rocJITsu Waitcheck, rocJITsu ConSan, + and HIP-FpSan. The initial sidecar locks are verified only for `gfx950`. + Workers automatically execute synthetic startup controls, while useful + candidate runs still require task-specific adapters and attestations. - Added run comparison through `src/tools/compare_runs.py` and the standalone visualization dashboard. - Added held-out evaluation for testing kernel generalization on unseen shapes. - Centralized compilation, correctness, performance measurement, result generation, and scoring outside agent-editable code. @@ -115,6 +120,53 @@ The task validator now includes Codex backend support, repository-task validatio - `cuda2hip` is recognized by the prompt system, but no bundled cuda2hip task suite is currently included. - Local vLLM provider configuration remains specific to the selected agent integration. - GPU task execution requires compatible physical AMD hardware and ROCm driver access. +- Evaluation-tool sidecars are experimental and `gfx950`-only. Startup controls + prove a tool installation can detect its synthetic bug, not that a candidate + was instrumented. No bundled task currently supplies a production-qualified + adapter/attestation. All six integrated startup controls pass on the current + MI355X qualification host. Synthetic manager-to-sidecar candidate pairs also + distinguished clean from seeded-bug Triton FpSan, HIP/Triton GPU ASan, and + HIP-FpSan runs; trusted AOT replay produced a clean Triton result and found the + seeded FlyDSL LDS race. Waitcheck distinguished a correct wait from a missing + `lgkmcnt(0)`, and strict ConSan record/replay distinguished clean single-wave + LDS accesses from seeded cross-wave conflicts. These fixtures do not qualify + a bundled task. Keep the policy advisory until each selected candidate path + is independently qualified. +- Evaluation-tool parsing and execution now fail closed on ambiguous FpSan + markers, process failure, non-finite protocol numbers, stale artifact reuse, + unsupported GPU-library kernels, and replay capsules without a golden output. + Each attempt receives a fresh artifact directory. The worker contains the + complete descendant process tree, including session-detached children, and + treats required cleanup as a failed execution. Replay launch geometry is + validated as exact `uint32` input and checked against runtime device limits. +- Native HIP rocJITsu no longer accepts task-launcher stdout/stderr as clean + dispatch attestation; only canonical records in the evaluator-owned report + sink are eligible. Waitcheck and ConSan startup controls now execute their + production entrypoints and parsers. The Waitcheck control covers inventory + and the C API, while ConSan uses exact raw HSACO identities, separate + instrumented/oracle argv, and an oracle that rejects leaked hook state. +- The current runner exposes evaluation-tool sockets and agent-writable report + paths during the same container run. Per-tool writable socket directories and + a read-only top-level artifact namespace plus one writable + `.eval-tool-artifacts/