feat(agents): add task quality loop auditor

.dockerignore

@@ -0,0 +1,13 @@
+# Evaluation-tool images need only their locked Docker assets and the immutable
+# eval-tools framework.  The repository commonly contains multi-gigabyte task
+# workspaces under experiments/, so an allow-list keeps builds fast and prevents
+# candidate/run artifacts from entering the Docker build context.
+**
+!docker/
+!docker/eval-tools/
+!docker/eval-tools/**
+!src/
+!src/eval_tools/
+!src/eval_tools/**
+**/__pycache__/
+**/*.py[cod]

.gitignore

@@ -7,6 +7,7 @@ build/
 .claude
 workspace_*
 /experiments/
+/.eval-tool-artifacts/
 
 # Auto-cloned agent dependencies
 agents/geak_optimagentv2/GEAK-agent/
@@ -41,6 +42,10 @@ traj.json
 # Generated held-out test data (methodology is in src/held_out/, data is private)
 held_out_tests/
 
+# Repository-level quality_loop audit state and isolated git worktrees
+quality_loop_runs/
+.quality_loop_worktrees/
+
 # Documentation build environment and output
 .docvenv/
 docs/_build/

Makefile

@@ -6,7 +6,7 @@
 
 SHELL := /bin/bash
 
-.PHONY: help docker-shell docker-check-agents docker-smoke docker-run docker-parallel-run docker-setup-flydsl docker-setup-geak \
+.PHONY: help docker-shell docker-check-agents docker-smoke docker-run docker-parallel-run docker-quality-loop docker-setup-flydsl docker-setup-geak \
         check-docker-runner check-evaluator check-held-out check-visualization \
         visualization-build visualization-serve visualization-run \
         sync-perf-helpers check-perf-helpers materialize-perf-workspace \
@@ -23,6 +23,7 @@ help:
 	@echo "make docker-smoke        - Verify Docker Python, ROCm tools, imports, and GPU access"
 	@echo "make docker-run CONFIG=example_configs/quickstart_claude_mi300.yaml RUN_ARGS=\"--run-suffix test\" - Run an experiment in Docker"
 	@echo "make docker-parallel-run CONFIG=example_configs/benchmark_cursor_mi355x.yaml GPU_IDS=0,1 - Run an experiment across one worker container per GPU"
+	@echo "make docker-quality-loop QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml - Audit and harden tasks with Codex, then open one draft PR"
 	@echo "                         Default CONFIG is the MI300/MI300X Claude quickstart"
 	@echo "                         On other GPUs, pass a matching CONFIG explicitly"
 	@echo "                         Images: gfx942->mi30x, gfx950->mi35x; override with AKA_DOCKER_IMAGE=..."
@@ -45,6 +46,8 @@ help:
 DOCKER_RUNNER := src/scripts/docker_benchmark.sh
 CONFIG ?= example_configs/quickstart_claude_mi300.yaml
 RUN_ARGS ?=
+QUALITY_LOOP_CONFIG ?= agents/quality_loop/agent_config.yaml
+QUALITY_LOOP_ARGS ?=
 AGENTS ?=
 WORKSPACES ?= $(WORKSPACE)
 TASKS ?= $(TASK)
@@ -70,6 +73,9 @@ docker-run:
 docker-parallel-run:
 	@GPU_IDS="$(GPU_IDS)" $(DOCKER_RUNNER) parallel-run --config_name $(CONFIG) $(RUN_ARGS)
 
+docker-quality-loop:
+	@$(DOCKER_RUNNER) quality-loop --config $(QUALITY_LOOP_CONFIG) $(QUALITY_LOOP_ARGS)
+
 # Install FlyDSL into the container's persistent pip user-base when the selected
 # image does not ship it. Needed by all three FlyDSL task types.
 docker-setup-flydsl:

agents/quality_loop/README.md

@@ -0,0 +1,97 @@
+# quality_loop agent
+
+`quality_loop` is a repository-level task curator. It audits every selected task,
+attempts one repair for blocking validator failures, runs exactly one Codex
+optimization iteration, sends the result to an independent Codex reviewer, and
+optionally hardens an easy baseline or task cases behind fail-closed correctness
+gates. It files idempotent GitHub issues for unrepairable task defects and bundles
+all accepted task changes into one draft pull request.
+
+Unlike normal Arena agents, `quality_loop` is not registered in `AgentType`.
+Normal launchers operate once inside one copied task workspace; this workflow owns
+the full repository campaign, isolated git worktree, issue deduplication, resume
+manifest, and final PR.
+
+## Hard preflight
+
+A real run stops before creating a branch or modifying a task unless all of these
+pass:
+
+- `gh auth status -h github.com`
+- the authenticated account has repository write permission
+- GitHub Issues are enabled
+- `git`, `gh`, and `codex` are installed
+- Git has a usable author identity for task commits
+- the source worktree is clean
+- the configured GPU/runtime is available through the Docker runner
+
+Only the host-side deterministic publisher uses `gh`. The Docker runner performs
+GitHub preflight and creates the audit worktree on the host, runs Codex/GPU work
+without mounting GitHub credentials, then returns to the host to create issues,
+commit accepted task changes, push, and open the draft PR. The main checkout is
+read-only inside that container; only this run's artifact and isolated worktree
+directories are writable. Codex login state is copied into an ephemeral container
+home instead of being writable in place.
+
+## Run
+
+Inspect task selection without credentials, GPU work, or mutations:
+
+```bash
+python3 -m agents.quality_loop \
+  --config example_configs/quality_loop_mi300.yaml \
+  --plan
+```
+
+Run through the supported Docker environment:
+
+```bash
+make docker-quality-loop QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml
+```
+
+Use `example_configs/quality_loop_mi355x.yaml` on MI355X (`gfx950`).
+
+Limit a smoke run to several tasks:
+
+```bash
+make docker-quality-loop \
+  QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \
+  QUALITY_LOOP_ARGS="--tasks hip2hip/gpumode/GELU triton2triton/vllm/triton_rms_norm"
+```
+
+Resume after interruption:
+
+```bash
+make docker-quality-loop \
+  QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \
+  QUALITY_LOOP_ARGS="--resume 20260803_120000"
+```
+
+`--no-publish` still requires the GitHub login/write preflight, but suppresses
+issues, push, and PR creation. `--plan` is the only intentionally offline mode.
+
+## Per-task gates
+
+1. Run the existing 10-check validator in a fresh workspace.
+2. Record WARN findings without repairing them.
+3. For FAIL, allow one task-local repair and re-run the full validator in another
+   fresh workspace. File one fingerprinted issue if it still fails.
+4. Measure the baseline, run one Codex optimization candidate, protect the
+   harness, and use the centralized compile/correctness/performance evaluator.
+5. Run an independent read-only Codex review. Deterministic evaluator failures
+   always override an agent acceptance.
+6. Treat the task as easy only when three measurements of the single candidate
+   have median speedup at least 5x, use consistent benchmark methods and case
+   counts, and the reviewer accepts logic equivalence.
+7. Promote only committed standalone optimization kernels. Translation/generation
+   tasks report 5x hardening as not applicable.
+8. Case changes may touch only test/harness paths and are accepted only when both
+   the pre-audit kernel and candidate pass the updated cases.
+9. Before any host commit, the complete worktree diff must exactly match the
+   accepted per-task paths recorded in `state.yaml`; unexpected edits abort
+   publication.
+
+Run artifacts are written under `quality_loop_runs/<run-id>/`; the isolated audit
+branch lives under `.quality_loop_worktrees/<run-id>/`. Both are ignored by Git.
+Tasks pinned to another GPU architecture are reported as `platform_deferred`; run
+the matching MI300/MI355X campaign to audit those tasks.

agents/quality_loop/__init__.py

@@ -0,0 +1,6 @@
+# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved.
+"""Repository-level task quality audit and hardening workflow."""
+
+from .config import QualityLoopConfig, load_config
+
+__all__ = ["QualityLoopConfig", "load_config"]

agents/quality_loop/__main__.py

@@ -0,0 +1,80 @@
+# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved.
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import logging
+from pathlib import Path
+
+import yaml
+
+from .config import load_config
+from .orchestrator import QualityLoop
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_CONFIG = Path(__file__).with_name("agent_config.yaml")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Audit, repair, harden, and publish AgentKernelArena tasks"
+    )
+    parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
+    parser.add_argument(
+        "--tasks",
+        nargs="+",
+        help="Override task selectors from the config (paths relative to tasks/)",
+    )
+    parser.add_argument(
+        "--plan",
+        action="store_true",
+        help="List runnable/deferred tasks without GitHub, GPU, or agent execution",
+    )
+    parser.add_argument("--resume", metavar="RUN_ID", help="Resume a prior run")
+    parser.add_argument(
+        "--no-publish",
+        action="store_true",
+        help="Run all hard preflights and audits but do not create issues, push, or open a PR",
+    )
+    parser.add_argument("--defer-github", action="store_true", help=argparse.SUPPRESS)
+    parser.add_argument("--skip-preflight", action="store_true", help=argparse.SUPPRESS)
+    return parser
+
+
+def configure_logging() -> logging.Logger:
+    logger = logging.getLogger("quality_loop")
+    logger.setLevel(logging.INFO)
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
+        logger.addHandler(handler)
+    return logger
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    config_path = args.config if args.config.is_absolute() else REPO_ROOT / args.config
+    config = load_config(config_path)
+    if args.tasks:
+        config = dataclasses.replace(config, tasks=tuple(args.tasks))
+    if args.no_publish:
+        config = dataclasses.replace(
+            config,
+            github=dataclasses.replace(config.github, publish=False),
+        )
+    logger = configure_logging()
+    workflow = QualityLoop(REPO_ROOT, config, logger=logger, defer_github=args.defer_github)
+    if args.plan:
+        print(yaml.safe_dump(workflow.plan(), sort_keys=False, allow_unicode=True))
+        return 0
+    report = workflow.run(
+        resume_run_id=args.resume,
+        skip_preflight=args.skip_preflight,
+    )
+    logger.info("quality_loop report: %s", report)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agents/quality_loop/agent_config.yaml

@@ -0,0 +1,41 @@
+# quality_loop is a repository-level audit workflow. It defaults to auditing all
+# tasks that are runnable on the selected target GPU.
+tasks:
+  - all
+target_gpu_model: MI300
+
+quality_loop:
+  backend:
+    name: codex
+    model: null                 # null uses the Codex CLI default/config
+    effort: xhigh
+    timeout_seconds: 3600
+  reviewer:
+    name: codex                 # independent, read-only Codex session
+    model: null
+    effort: xhigh
+    timeout_seconds: 1800
+
+  max_repair_attempts: 1
+  optimization_iterations: 1  # enforced; any other value is rejected
+  easy_speedup_threshold: 5.0
+  easy_confirmation_runs: 3
+  case_enhancement: true
+
+  # Only optimization tasks with a committed runnable baseline can promote a
+  # first-iteration 5x candidate into the new task baseline. Translation and
+  # authoring tasks still receive validation, optimization, review, and case audit.
+  promotion_task_types:
+    - hip2hip
+    - triton2triton
+    - flydsl2flydsl
+
+  artifact_root: quality_loop_runs
+  worktree_root: .quality_loop_worktrees
+
+  github:
+    publish: true
+    draft_pr: true
+    branch_prefix: quality-loop
+    base_branch: null           # null resolves the repository default branch
+    issue_labels: []            # labels must already exist in the repository