diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bffb0d3..73d8dfc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,3 +68,23 @@ jobs: ${{ matrix.project }}/pytest-report.xml ${{ matrix.project }}/coverage.xml if-no-files-found: ignore + + # Multi-harness skill rendering: golden tests (claude-code identity) and + # adapter drift tripwires. Stdlib-only — no install needed. + renderer: + name: skill-renderer (adapters) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + - name: Render all adapters (must succeed cleanly) + run: python3 scripts/render_skills.py --adapter all + + - name: Run renderer tests + run: python3 -m unittest discover -s tests -v diff --git a/.gitignore b/.gitignore index db91526..65491b4 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,7 @@ work/ # Stray dir from a mis-quoted URL (e.g. a test writing "https://..." as a path) https:/ + +# Rendered per-harness skill bundles (generated by scripts/render_skills.py; +# CI renders fresh - commit adapter sources, not dist output). +dist/ diff --git a/README.md b/README.md index 215785e..08843ad 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,24 @@ REM .\uninstall.cmd > [!NOTE] > `install.sh`/`install.cmd` copy files directly (rather than symlinking) because symlinks can break `find`/`glob` functionality inside subagents. Re-run the install script after pulling updates to refresh your local environment. +### Other agent harnesses (experimental) + +The scanner skill can also be rendered for other agent CLIs — Hermes, GitHub +Copilot CLI, and Codex — with the Claude Code path unchanged: + +```bash +./install.sh --target hermes # ~/.hermes/skills/vulnhunt +./install.sh --target copilot # ~/.copilot/skills/vulnhunt +./install.sh --target codex # ~/.codex/skills/vulnhunt +``` + +See [docs/ENGINES.md](docs/ENGINES.md) for headless usage, engine selection +in the runtime agent and benchmark harness, per-harness status, and +[docs/ADAPTER_GUIDE.md](docs/ADAPTER_GUIDE.md) to add another harness. +Non-Claude adapters are experimental until benchmarked against the +ground-truth corpus — VulnHunter's gates are calibrated for Opus-class +reasoning models. + --- ## Usage Guide diff --git a/adapters/claude-code/adapter.json b/adapters/claude-code/adapter.json new file mode 100644 index 0000000..256add0 --- /dev/null +++ b/adapters/claude-code/adapter.json @@ -0,0 +1,11 @@ +{ + "name": "claude-code", + "description": "Reference adapter: identity transform of the repo-root skill sources. Installs to ~/.claude/skills via install.sh (default target). Byte-compatibility with the sources is enforced by tests/test_render_skills.py.", + "skills": ["vulnhunt", "vulnhunt-fix-verify", "vulnhunter-fix"], + "install": { + "target_dir": "~/.claude/skills", + "script": "./install.sh # or: ./install.sh --target claude-code", + "headless": "claude -p '/vulnhunt ' --output-format stream-json --verbose --allowedTools 'Read Write Edit Bash Agent' --permission-mode acceptEdits --add-dir " + }, + "transforms": [] +} diff --git a/adapters/codex/README.md b/adapters/codex/README.md new file mode 100644 index 0000000..194555e --- /dev/null +++ b/adapters/codex/README.md @@ -0,0 +1,52 @@ +# Codex CLI adapter + +Runs the VulnHunter scanner skill under the OpenAI Codex CLI. + +Codex supports the SKILL.md convention under `~/.codex/skills/` and a +headless `codex exec` mode, so the skill installs like the other adapters. +One structural difference drives most of this adapter's rewrites: **Codex +has no subagent-dispatch tool**, so the orchestrator instructions are +rewritten from "dispatch N parallel agents" to "execute the phases +yourself, sequentially, one class-group pass at a time" — the +class-partitioned phase files (`phase2_class_{inj,nav,log}.md`) make that +decomposition natural. + +- Phase loading points at `~/.codex/skills/vulnhunt/phases/` +- Subagent dispatch → sequential self-execution (Phase 2 fan-out becomes + sequential class-group passes; minimum pass count preserved) +- ORCHESTRATOR role → sequential executor with file-based context hygiene +- `/model opus` gating → one-line calibration notice, then proceed on the + selected model (no blocking, no model enforcement) +- `/cost` reporting → one-line progress reports +- Overlay maps Grep/Glob/Read/Bash → `rg`/`find`/file reads/sandboxed shell + +## Install + +```bash +./install.sh --target codex # renders dist/codex + copies to ~/.codex/skills/vulnhunt +``` + +## Headless use + +```bash +cd +codex exec -C . -s workspace-write -m \ + "Read ~/.codex/skills/vulnhunt/SKILL.md and execute the /vulnhunt workflow on this repository" +``` + +Sandbox modes: `-s workspace-write` (default recommendation — the skill +must write its results dir), `-s read-only` (analysis without artifacts), +`-s danger-full-access` (only for exploit-test runs on disposable clones). + +## Interactive use + +In a Codex session inside the target repo: mention the vulnhunt skill / +ask to "run the vulnhunt security audit on this repository". + +## Status + +Experimental — skill layout and `codex exec` flags verified against +codex-cli 0.147.0, but full-scan parity vs Claude Code has not been +benchmarked. Sequential Phase 2 changes the cost/latency profile (no +parallel fan-out) and may stress context discipline on large repos; +benchmark before production use. diff --git a/adapters/codex/adapter.json b/adapters/codex/adapter.json new file mode 100644 index 0000000..bcd783c --- /dev/null +++ b/adapters/codex/adapter.json @@ -0,0 +1,102 @@ +{ + "name": "codex", + "description": "OpenAI Codex CLI. Skills live in ~/.codex/skills (SKILL.md convention); headless via 'codex exec'. No subagent tool: phases execute sequentially in one session.", + "skills": [ + "vulnhunt" + ], + "install": { + "target_dir": "~/.codex/skills", + "script": "./install.sh --target codex", + "headless": "cd && codex exec -C . -s workspace-write -m \"Read ~/.codex/skills/vulnhunt/SKILL.md and execute the /vulnhunt workflow on this repository\"", + "notes": "Use -s workspace-write (results-dir writes) or -s read-only for pure analysis without artifacts; exploit-test runs may need the wider sandbox." + }, + "transforms": [ + { + "type": "prepend", + "files": "vulnhunt/SKILL.md", + "text_file": "overlay.md" + }, + { + "type": "prepend", + "files": "vulnhunt/phases/*.md", + "text_file": "overlay.md" + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "If it is NOT Opus 4.7 or higher,\n**STOP immediately** and tell the user (do not run any tools, resolve the target,\nor offer the mode menu yet):\n\n> ⚠️ VulnHunter is optimized for Claude Opus 4.7/4.8 and may be unreliable on other\n> models. Please switch with the `/model opus` command, then re-run `/vulnhunt`.\n\nWait for the user. Only proceed past this step once they are on Opus, or if they\nexplicitly reply that they want to continue on the current model anyway.", + "replace": "If it is not an Opus-class reasoning model, print this one-line notice, then\nCONTINUE immediately on the currently selected model (do not stop, do not ask\nthe user to switch, do not run any tools for this check):\n\n> ⚠️ Calibration note: VulnHunter's gates are tuned on Claude Opus-class\n> models. Detection quality on the selected model may differ — weigh results\n> accordingly.\n\nProceed directly to binding the scan metadata.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "(using Grep, Glob, and Read)", + "replace": "(using content search, filename search, and file reads)", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**Grep**, **Glob**, and **Read**", + "replace": "content search (`rg`/`grep`), filename search (`find`, shell globs), and file reads", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "- **Agent (Explore)** — for broader codebase exploration when simple searches aren't enough.", + "replace": "- **Shell search loops** — iterate rg/find pipelines for broader codebase exploration when simple searches aren't enough.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**After each phase completes, run `/cost` and report the result to the user.**", + "replace": "**After each phase completes, report one line of progress to the user.**", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Launch a `general-purpose` subagent:", + "replace": "Execute this phase yourself, following the quoted block exactly (no subagent tool available):", + "count": 4 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Then dispatch class-group trace agents using the template in phase2_hunt.md.\n **Minimum agent count = (3 × partition_count) + 1 sink-driven.**", + "replace": "Then execute the class-group trace passes sequentially (one class group at a\n time) using the template in phase2_hunt.md.\n **Minimum trace passes = (3 × partition_count) + 1 sink-driven.**", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Phase files are in `${CLAUDE_SKILL_DIR}/phases/`. Use this as `PHASES_DIR`.", + "replace": "Phase files are in `~/.codex/skills/vulnhunt/phases/`. Use this as `PHASES_DIR`.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**Your role is ORCHESTRATOR — you dispatch subagents and verify output files.\nYou do NOT perform analysis yourself. Keep your context lean.**", + "replace": "**Your role is to execute each phase in sequence and verify output files.\nCodex has no subagent tool — you perform the analysis yourself, one phase (and\none class-group pass) at a time. Keep your context lean: record results to\nfiles between passes instead of carrying them in context.**", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "- If a subagent fails, re-launch it — do NOT diagnose the failure yourself", + "replace": "- If a phase fails, re-run it — do NOT diagnose the failure yourself", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Verify subagent completion by checking output files exist (Glob)", + "replace": "Verify each phase completed by checking output files exist (filename search)", + "count": 1 + } + ] +} diff --git a/adapters/codex/overlay.md b/adapters/codex/overlay.md new file mode 100644 index 0000000..da5e941 --- /dev/null +++ b/adapters/codex/overlay.md @@ -0,0 +1,15 @@ +> **Harness adaptation note (Codex CLI).** This skill was authored for Claude +> Code; the methodology is unchanged. Two structural differences: +> +> 1. **You have no subagent-dispatch tool.** Where the skill says to launch a +> subagent, execute that phase **yourself**, following the quoted block / +> phase file exactly. For Phase 2, run the class-group trace passes +> **sequentially** — one class group at a time — rather than dispatching +> parallel agents; each pass still reads only its own class reference and +> partition data. Keep your own context disciplined: after each pass, +> record results to the results dir and do not carry candidate details +> forward beyond what the phase files require. +> 2. **Tool vocabulary**: "Grep" → content search via shell (`rg` / `grep`); +> "Glob" → filename search (`find`, `rg --files`, shell globs); "Read" → +> file reads (`cat`/`sed -n` or your file-reading tool); "Bash" → the +> sandboxed shell. Phase and skill files live under `~/.codex/skills/vulnhunt/`. diff --git a/adapters/copilot/README.md b/adapters/copilot/README.md new file mode 100644 index 0000000..600e249 --- /dev/null +++ b/adapters/copilot/README.md @@ -0,0 +1,52 @@ +# GitHub Copilot CLI adapter + +Runs the VulnHunter scanner skill under GitHub Copilot CLI. + +Copilot CLI has no global skills directory like Claude Code or Hermes; +customization is driven by custom instruction files and session-attached +directories. This adapter therefore renders a self-contained skill bundle +(`SKILL.md` + `phases/`) whose harness-specific mechanics are rewritten: + +- Phase loading points at the bundle's `phases/` subdir (attach the bundle + directory to the session with `/add-dir`) +- Agent-tool subagent dispatch → parallel subagents (Copilot fleet/delegate) +- `/model opus` gating → one-line calibration notice, then proceed on the + selected model (no blocking, no model enforcement) +- `/cost` reporting → one-line progress reports +- A terminology overlay maps Grep/Glob/Read/Bash vocabulary to Copilot's + search/shell tools + +## Install + +```bash +./install.sh --target copilot # renders dist/copilot + copies to ~/.copilot/skills/vulnhunt +``` + +## Interactive use (Copilot CLI) + +```bash +cd +copilot +# in session: +/add-dir ~/.copilot/skills/vulnhunt +# then: +Read the SKILL.md in the added vulnhunt directory and execute the /vulnhunt +workflow on this repository (read-only mode). +``` + +## Permission preset (shell + writes for results dirs) + +```bash +copilot --allow-tool 'write' --allow-tool 'shell(mkdir:*)' \ + --allow-tool 'shell(git:*)' +``` + +Tighten further for read-only scans: only `write` (for the results directory) +plus the package manager you expect (`shell(npm install:*)`, etc.). + +## Status + +**Experimental.** The Copilot CLI binary was not available when this adapter +was authored, so the non-interactive invocation (headless `-p`-style prompt +flag) is unverified — run `copilot -h` / `copilot help permissions` on your +install and adjust. Full-scan parity vs Claude Code has not been benchmarked. diff --git a/adapters/copilot/adapter.json b/adapters/copilot/adapter.json new file mode 100644 index 0000000..71d2fdb --- /dev/null +++ b/adapters/copilot/adapter.json @@ -0,0 +1,81 @@ +{ + "name": "copilot", + "description": "GitHub Copilot CLI. No global skills directory: the bundle installs to ~/.copilot/skills and is attached to a session via /add-dir. Experimental - flag surface not yet verified against the copilot binary.", + "skills": [ + "vulnhunt" + ], + "install": { + "target_dir": "~/.copilot/skills", + "script": "./install.sh --target copilot", + "headless": "copilot --allow-tool 'write' --allow-tool 'shell(mkdir:*)' -p 'Read ~/.copilot/skills/vulnhunt/SKILL.md and execute the /vulnhunt workflow on this repository (read-only mode)' # verify flags with `copilot -h` on your install", + "notes": "Experimental: Copilot CLI was not available for local verification. Confirm the non-interactive prompt flag and permission flag names (`copilot help permissions`) before scripting." + }, + "transforms": [ + { + "type": "prepend", + "files": "vulnhunt/SKILL.md", + "text_file": "overlay.md" + }, + { + "type": "prepend", + "files": "vulnhunt/phases/*.md", + "text_file": "overlay.md" + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "If it is NOT Opus 4.7 or higher,\n**STOP immediately** and tell the user (do not run any tools, resolve the target,\nor offer the mode menu yet):\n\n> ⚠️ VulnHunter is optimized for Claude Opus 4.7/4.8 and may be unreliable on other\n> models. Please switch with the `/model opus` command, then re-run `/vulnhunt`.\n\nWait for the user. Only proceed past this step once they are on Opus, or if they\nexplicitly reply that they want to continue on the current model anyway.", + "replace": "If it is not an Opus-class reasoning model, print this one-line notice, then\nCONTINUE immediately on the currently selected model (do not stop, do not ask\nthe user to switch, do not run any tools for this check):\n\n> ⚠️ Calibration note: VulnHunter's gates are tuned on Claude Opus-class\n> models. Detection quality on the selected model may differ — weigh results\n> accordingly.\n\nProceed directly to binding the scan metadata.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "(using Grep, Glob, and Read)", + "replace": "(using code search, file search, and file reads)", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**Grep**, **Glob**, and **Read**", + "replace": "code search, file search, and file reads", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "- **Agent (Explore)** — for broader codebase exploration when simple searches aren't enough.", + "replace": "- **Parallel subagents** — dispatch a subagent for broader codebase exploration when simple searches aren't enough.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**After each phase completes, run `/cost` and report the result to the user.**", + "replace": "**After each phase completes, report one line of progress to the user.**", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Launch a `general-purpose` subagent:", + "replace": "Dispatch a parallel subagent (use the quoted block as its prompt):", + "count": 4 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Phase files are in `${CLAUDE_SKILL_DIR}/phases/`. Use this as `PHASES_DIR`.", + "replace": "Phase files are in the `phases/` subdirectory of the vulnhunt skill directory (the added directory containing this SKILL.md). Use that as `PHASES_DIR`.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Verify subagent completion by checking output files exist (Glob)", + "replace": "Verify subagent completion by checking output files exist (file search)", + "count": 1 + } + ] +} diff --git a/adapters/copilot/overlay.md b/adapters/copilot/overlay.md new file mode 100644 index 0000000..7096b1a --- /dev/null +++ b/adapters/copilot/overlay.md @@ -0,0 +1,9 @@ +> **Harness adaptation note (GitHub Copilot CLI).** This skill was authored +> for Claude Code; the methodology is unchanged. Tool vocabulary mapping: +> "Grep" → code/content search (built-in search or `rg` via the shell tool); +> "Glob" → file/pattern search; "Read" → file read; "Bash" → the shell tool; +> "the Agent tool" / "launch a subagent" → your parallel-subagent capability +> (fleet/delegate agents) — dispatch one per quoted block and wait for all +> results before proceeding. This skill directory (containing `SKILL.md` and +> `phases/`) must be added to the session (`/add-dir`) so the phase files are +> readable. diff --git a/adapters/hermes/README.md b/adapters/hermes/README.md new file mode 100644 index 0000000..1e76825 --- /dev/null +++ b/adapters/hermes/README.md @@ -0,0 +1,68 @@ +# Hermes adapter + +Runs the VulnHunter scanner skill under the [Hermes Agent](https://github.com/weav/hermes-agent) +harness. Hermes natively understands Claude-Code-style `SKILL.md` skills, so +this adapter keeps the methodology byte-for-byte and rewrites only the +harness-specific surface: + +- `${CLAUDE_SKILL_DIR}` → `${HERMES_SKILL_DIR}` +- Agent-tool `general-purpose` subagent dispatch → `delegate_task` (Hermes' + subagent tool; children get their own context and terminal session) +- `/cost` reporting → one-line progress reports +- `/model opus` gating → one-line calibration notice, then proceed on the + selected model (no blocking, no model enforcement) +- Frontmatter gains `metadata.hermes.requires_toolsets: [file, terminal, delegation]` +- A short terminology overlay is prepended to `SKILL.md` and every phase file + (Grep/Glob/Read/Bash → search_files / read_file / terminal) + +## Install + +```bash +./install.sh --target hermes # renders dist/hermes + copies to ~/.hermes/skills/vulnhunt +``` + +Verify: `hermes skills list | grep vulnhunt`. + +## Interactive use + +In a Hermes session inside the repo you want scanned: `/vulnhunt .` + +## Headless use + +```bash +cd +hermes chat -Q -s vulnhunt -t file,terminal,delegation \ + --provider anthropic -m claude-opus-4-8 \ + -q '/vulnhunt . --no-read-only' # or omit --no-read-only for read-only mode +``` + +`-Q` is the programmatic contract: stdout carries the final message, +stderr carries `session_id: `, exit code 0/1. Scan success should be +judged by the VulnHunter results contract (`_VULNHUNT_RESULTS_*` +directory + `scan_manifest.json`), not by stdout text. + +## Model policy + +No model enforcement. Run whatever model you select (`--provider`/`-m`, or +your Hermes config default) — the skill prints a one-line calibration note +and proceeds. Hermes' multi-provider routing makes it easy to benchmark the +same skill across models with the harness's ground-truth corpus. + +## Unattended / CI runs + +- Approvals: scans issue shell commands (`mkdir`, package-manager installs) + that Hermes' `DANGEROUS_PATTERNS` may flag. For unattended runs add the + specific commands to `command_allowlist` in `~/.hermes/config.yaml`, or set + `approvals.mode: smart`. Avoid blanket `--yolo` against untrusted code. +- Untrusted targets: run with a `docker` terminal backend + (`terminal.backend: docker`) so dependency installs and exploit tests are + containerized. +- Web lookups: phases reference CVE/vendor-advisory checks; enable the `web` + toolset for best coverage (`-t file,terminal,delegation,web`). + +## Status + +Experimental — smoke-tested for skill discovery, headless preload +(`hermes chat -Q -s vulnhunt`) and `${HERMES_SKILL_DIR}` resolution. +Full-scan parity vs Claude Code has not yet been benchmarked (use +`harness/` benchmark mode with ground truth to compare engines). diff --git a/adapters/hermes/adapter.json b/adapters/hermes/adapter.json new file mode 100644 index 0000000..bfba757 --- /dev/null +++ b/adapters/hermes/adapter.json @@ -0,0 +1,86 @@ +{ + "name": "hermes", + "description": "Hermes Agent (weav). SKILL.md-compatible skills in ~/.hermes/skills; subagents via delegate_task; headless via 'hermes chat -q -Q -s vulnhunt'.", + "skills": [ + "vulnhunt" + ], + "install": { + "target_dir": "~/.hermes/skills", + "script": "./install.sh --target hermes", + "headless": "cd && hermes chat -q -Q -s vulnhunt -t file,terminal,delegation --provider

-m '/vulnhunt .'", + "notes": "Unattended runs: configure approvals (command_allowlist) or approvals.mode smart; scans of untrusted code should use a docker terminal backend." + }, + "transforms": [ + { + "type": "frontmatter_append", + "files": "vulnhunt/SKILL.md", + "text": "metadata:\n hermes:\n requires_toolsets:\n - file\n - terminal\n - delegation" + }, + { + "type": "prepend", + "files": "vulnhunt/SKILL.md", + "text_file": "overlay.md" + }, + { + "type": "prepend", + "files": "vulnhunt/phases/*.md", + "text_file": "overlay.md" + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "If it is NOT Opus 4.7 or higher,\n**STOP immediately** and tell the user (do not run any tools, resolve the target,\nor offer the mode menu yet):\n\n> ⚠️ VulnHunter is optimized for Claude Opus 4.7/4.8 and may be unreliable on other\n> models. Please switch with the `/model opus` command, then re-run `/vulnhunt`.\n\nWait for the user. Only proceed past this step once they are on Opus, or if they\nexplicitly reply that they want to continue on the current model anyway.", + "replace": "If it is not an Opus-class reasoning model, print this one-line notice, then\nCONTINUE immediately on the currently selected model (do not stop, do not ask\nthe user to switch, do not run any tools for this check):\n\n> ⚠️ Calibration note: VulnHunter's gates are tuned on Claude Opus-class\n> models. Detection quality on the selected model may differ — weigh results\n> accordingly.\n\nProceed directly to binding the scan metadata.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "(using Grep, Glob, and Read)", + "replace": "(using content search, filename search, and file reads)", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**Grep**, **Glob**, and **Read**", + "replace": "content search, filename search, and file reads", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "- **Agent (Explore)** — for broader codebase exploration when simple searches aren't enough.", + "replace": "- **delegate_task** — dispatch a subagent for broader codebase exploration when simple searches aren't enough.", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "**After each phase completes, run `/cost` and report the result to the user.**", + "replace": "**After each phase completes, report one line of progress to the user.**", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Launch a `general-purpose` subagent:", + "replace": "Dispatch a subagent via `delegate_task` (use the quoted block as the goal):", + "count": 4 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "${CLAUDE_SKILL_DIR}", + "replace": "${HERMES_SKILL_DIR}", + "count": 1 + }, + { + "type": "substitute", + "files": "vulnhunt/SKILL.md", + "find": "Verify subagent completion by checking output files exist (Glob)", + "replace": "Verify subagent completion by checking output files exist (filename search)", + "count": 1 + } + ] +} diff --git a/adapters/hermes/overlay.md b/adapters/hermes/overlay.md new file mode 100644 index 0000000..1e14a08 --- /dev/null +++ b/adapters/hermes/overlay.md @@ -0,0 +1,20 @@ +> **Harness adaptation note (Hermes).** This skill was authored for Claude +> Code; the methodology is unchanged. Tool vocabulary mapping: "Grep" → +> `search_files` (content search) or `rg` via the terminal tool; "Glob" → +> filename/pattern search; "Read" → `read_file`; "Bash" → the `terminal` +> tool; "the Agent tool" / "launch a subagent" → `delegate_task` (children +> run with their own context and terminal session). Where a step says to +> launch a `general-purpose` subagent, dispatch via `delegate_task` with the +> quoted block as the goal. "Return message" limits ("under 20 words") apply +> to the delegate's final message. Web lookups (CVE/vendor advisories), where +> a phase calls for them, use the web toolset if enabled. +> +> **Delegation protocol (important).** Hermes runs every top-level +> `delegate_task` in the background: the call returns immediately with +> status "dispatched", and the child's result is delivered back to you only +> while your turn is still alive. After dispatching any subagent, do NOT +> conclude, error out, or produce a final answer while children are running. +> Keep the turn alive by periodically calling `delegate_task` with +> `action: "list"` until every child shows completed (its result then +> arrives as a follow-up message), and only then verify that phase's output +> files exist and continue the workflow. diff --git a/docs/ADAPTER_GUIDE.md b/docs/ADAPTER_GUIDE.md new file mode 100644 index 0000000..8f8ded8 --- /dev/null +++ b/docs/ADAPTER_GUIDE.md @@ -0,0 +1,83 @@ +# Adapter guide: adding a new agent harness + +Adapters live in `adapters//` and turn the repo-root skill sources +into a harness-specific bundle. You need three artifacts: + +## 1. `adapter.json` + +```json +{ + "name": "yourharness", + "description": "One line: what the harness is and how it runs the skill.", + "skills": ["vulnhunt"], + "install": { + "target_dir": "~/.yourharness/skills", + "script": "./install.sh --target yourharness", + "headless": "", + "notes": "gotchas" + }, + "transforms": [ ... ] +} +``` + +Transform types (applied in order; see `scripts/render_skills.py`): + +| type | effect | +|---|---| +| `substitute` | literal find/replace on files matching `files` glob. Fails the render when `find` is absent unless `optional: true`. **`count: N` is required on every non-optional substitute** (enforced by `tests/test_render_skills.py`): it pins the expected occurrence total so a source edit that duplicates or half-rewords a phrase trips the drift test instead of silently rewriting the wrong number of sites. | +| `prepend` | insert text (or `text_file`) after the YAML frontmatter | +| `frontmatter_append` | insert lines before the closing `---` | +| `override` | replace/create a file wholesale from the adapter dir | +| `drop` | delete files from the rendered bundle | + +## 2. What you must neutralize + +Audit the rendered `SKILL.md` + phases for these Claude-isms (the existing +adapters are the reference — crib from them): + +- `${CLAUDE_SKILL_DIR}` → your harness's skill-dir token or absolute path +- "Launch a `general-purpose` subagent:" ×4 → your subagent mechanism **or** + sequential self-execution (see `adapters/codex`) +- Phase-2 fan-out wording — keep the minimum pass count meaningful +- `/cost` → a progress line; `/model opus` gating → a calibration notice + that proceeds on the selected model (no model enforcement) +- Opus gating sentence → your model-selection guidance +- Tool vocabulary (Grep/Glob/Read/Bash) → a terminology overlay prepended to + `SKILL.md` and every `phases/*.md` is usually enough +- The ORCHESTRATOR role paragraph — only correct it if your harness cannot + dispatch subagents + +## 3. Wire-up checklist + +- [ ] `install.sh` — add a `case` branch: render + set `SKILLS_PARENT` +- [ ] Renderer tests — extend `tests/test_render_skills.py` with a render + test asserting your key substitutions and overlay presence +- [ ] `vulnhunter-agent/agent/engines/` — add an engine module if the + harness has a CLI; register it in `engines/__init__.py` + (`ENGINE_NAMES` + `get_engine`). If it shells out to a CLI, subclass + `SubprocessEngine` (`engines/_subprocess.py`) and implement only the + hooks (`_binary_name`, `_install_target`, `_binary_hint`, + `_skill_paths`, `_build_command`, `_build_kickoff`) — the shared base + supplies pre-staging, audit, timeout, and the contents-based success + contract. Add the engine to the parametrized contract tests. +- [ ] `harness/local_harness/config.py` — add an `ENGINES` entry +- [ ] `harness/local_harness/scan.py` / `benchmark/judge.py` — add argv + builders + tests +- [ ] `docs/ENGINES.md` + `docs/engine-matrix.md` — capability row and + status (experimental until benchmarked) +- [ ] `adapters//README.md` — install, headless command, permission + preset, model policy, status + +## Invariants (do not break) + +1. `dist/claude-code` stays byte-identical to the repo-root skill sources + (enforced by tests) — the Claude path must not regress. +2. Success is judged by the results contract — a `*_VULNHUNT_RESULTS_*` + dir that actually contains the skill's `README.md` report — never by + engine stdout or by the pre-created directory merely existing. +3. The kickoff prompt always carries the "Pre-resolved scan metadata" + block (results dir, branch label, repo URL, model tag, shell policy). +4. Read-only runs must not enable arbitrary code execution; exploit-test + execution is opt-in only (`--enable-bash` parity). +5. Declare the adapter's status honestly: **experimental** until the + ground-truth benchmark has been run on it. diff --git a/docs/ENGINES.md b/docs/ENGINES.md new file mode 100644 index 0000000..8bac2a0 --- /dev/null +++ b/docs/ENGINES.md @@ -0,0 +1,85 @@ +# Running VulnHunter on multiple agent harnesses + +VulnHunter's methodology is no longer Claude Code-only. The same scanner +skill can be rendered for — and driven by — several agent harnesses +("engines"), with the Claude Code path unchanged as the reference. + +| Engine | Skill install | Headless invocation | Status | +|---|---|---|---| +| claude-code | `./install.sh` → `~/.claude/skills` | `claude -p '/vulnhunt …' --output-format stream-json` | reference (unchanged) | +| hermes | `./install.sh --target hermes` → `~/.hermes/skills` | `hermes chat -Q -s vulnhunt -t file,terminal,delegation -q '/vulnhunt …'` | experimental (skill verified end-to-end; full-scan parity unbenchmarked) | +| copilot | `./install.sh --target copilot` → `~/.copilot/skills` | `copilot -p "Read ~/.copilot/skills/vulnhunt/SKILL.md and execute the /vulnhunt workflow …"` | experimental (CLI flag surface unverified) | +| codex | `./install.sh --target codex` → `~/.codex/skills` | `codex exec -C -s workspace-write -m "Read ~/.codex/skills/vulnhunt/SKILL.md …"` | experimental (flags verified on codex-cli 0.147.0) | + +## How it fits together + +Three layers, each independently extensible: + +1. **Skill rendering** — the repo-root skill directories are the single + source of truth. `adapters//adapter.json` declares declarative + transforms (string substitutions, terminology overlays, frontmatter + additions); `python3 scripts/render_skills.py` renders + `dist///`. The `claude-code` adapter is the identity + transform, and `tests/test_render_skills.py` enforces byte-identity, so + a source edit is a deliberate prompt change every adapter inherits. + Missing find-strings fail the render loudly — upstream prompt edits that + break an adapter surface immediately in tests. + +2. **Headless runtime engines** (`vulnhunter-agent/agent/engines/`) — a + `ScanEngine` protocol (`get_engine(config)`) with `claude-code` (the + existing SDK path), `hermes`, `copilot`, and `codex` subprocess + implementations. All engines share the results-directory success + contract — a `*_VULNHUNT_RESULTS_*` directory containing the scan's + `README.md` report (the subprocess engines pre-create the directory, so + its mere existence is not success; a run that writes no report is a + failure). Downstream publish / issues / audit / verify stages are + engine-agnostic and consume the same `*_VULNHUNT_RESULTS_*` + + `scan_manifest.json` layout. Select with + `[scan] engine = "…"` in the agent TOML; tune with + `engine_command`, `engine_provider`, `engine_timeout_seconds`, + `engine_extra_args`. + +3. **Benchmark harness engines** (`harness/`) — batch scans and the LLM + judge are engine-parametrized (`VULNHUNT_HARNESS_ENGINE`, + `VULNHUNT_JUDGE_ENGINE`). Judging with a *different* engine than the + scanner (e.g. scan with hermes, judge with claude-code) avoids + self-preference bias. Run the ground-truth corpus per engine to compare + detection rates before trusting a new engine. + +## Fan-out semantics per engine + +The skill's Phase 2 parallelism maps differently per harness: + +- **claude-code**: native Agent-tool subagents. +- **hermes**: native `delegate_task` subagents (own context + terminal). +- **copilot**: fleet/delegate subagents. +- **codex**: no subagent tool — the adapter rewrites Phase 2 into + **sequential class-group passes** (the phase files are already + class-partitioned: inj/nav/log). Same minimum pass count, no parallelism; + higher latency, different context profile. + +## Model policy + +**No model enforcement anywhere.** Whichever model you select (per-harness +flag, Hermes provider routing, agent TOML) runs the scan — Opus-class is not +treated as the only capable cyber-vulnerability model. The non-Claude +adapters print a one-line calibration notice ("gates tuned on Claude +Opus-class; quality on the selected model may differ") and proceed; engine +runs never gate on model family. The claude-code render keeps the upstream +interactive STOP gate byte-identically, since that adapter is the identity +transform. Use the benchmark harness (`VULNHUNT_HARNESS_ENGINE` / +`VULNHUNT_HARNESS_JUDGE_ENGINE`) to measure how a given model actually +performs on the ground-truth corpus rather than assuming. + +## Safety notes + +- Unattended runs: prefer per-engine permission presets (hermes + `command_allowlist` / docker terminal backend; codex `-s workspace-write` + sandbox; copilot `--allow-tool` patterns) over blanket bypass flags. +- Scanning untrusted code: use a containerized/sandboxed execution backend + before enabling exploit-test execution. +- The methodology stays analysis-first: PoCs demonstrate reachability, not + weaponized exploitation. + +See `docs/engine-matrix.md` for the verified per-harness capability +matrix, and `docs/ADAPTER_GUIDE.md` to add a new harness. diff --git a/docs/engine-matrix.md b/docs/engine-matrix.md new file mode 100644 index 0000000..6dc2184 --- /dev/null +++ b/docs/engine-matrix.md @@ -0,0 +1,82 @@ +# Engine / Harness Matrix + +VulnHunter's methodology (skills) and headless runtime are being made +harness-agnostic. This matrix records the verified capability surface of each +supported agent harness so adapter authors know what they are targeting. + +Status legend: ✅ verified locally, 📖 documented (not verified locally), +❌ absent, ⬜ planned/unknown (verify before relying on it). + +| Capability | Claude Code (reference) | Hermes | GitHub Copilot CLI | Codex CLI | +|---|---|---|---|---| +| Skill / prompt file format | `SKILL.md` (frontmatter: name, description) in `~/.claude/skills/` | `SKILL.md` (same frontmatter + `metadata.hermes.*`, `required_environment_variables`) in `~/.hermes/skills/`; natively imports Claude skills via `hermes import-agent claude-code` 📖 | Custom instructions (`.github/copilot-instructions.md`, user/path-level files) 📖 | `SKILL.md` convention in `~/.codex/skills/` ✅ (dir exists on codex-cli 0.147.0; no CLI subcommand) | +| Skill-dir template token | `${CLAUDE_SKILL_DIR}` | `${HERMES_SKILL_DIR}` 📖 | n/a (instructions are repo-relative) | n/a | +| Slash-command invocation | `/vulnhunt` | `/` (every installed skill) 📖 | prompt files via `/` menu ⬜ | `/` in TUI 📖 | +| Headless one-shot | `claude -p --output-format stream-json` | `hermes chat -q "" -Q` (stdout = final response, stderr = `session_id: `, exit 0/1) ✅ | non-interactive prompt flag ⬜ (confirm `copilot -h` once installed) | `codex exec [-C

] [-s read-only\|workspace-write\|danger-full-access] [-m ] [--json]` ✅ | +| Preload skill headlessly | `--add-dir` skills dir + slash command in prompt | `-s/--skills ` ✅ | n/a — rely on instruction files | AGENTS.md discovered from cwd 📖 | +| Parallel subagents | `Agent` tool (general-purpose subagents) | `delegate_task` tool (`delegation` toolset; batch `tasks[]`, `output_schema`) 📖 | `/fleet` parallel subagents, `/delegate` cloud agent 📖 | none — use process-level fan-out 📖 | +| File search tools | `Grep` / `Glob` / `Read` | `search_files` / `read_file` (file toolset) 📖 | built-in search + shell 📖 | shell (`rg`, `find`) 📖 | +| Shell execution | `Bash` tool | `terminal` toolset (local/docker/ssh/modal/…) 📖 | shell with approval patterns (`shell(git:*)`) 📖 | sandboxed shell (`--sandbox workspace-write` etc.) 📖 | +| Permission model | `--permission-mode` (`acceptEdits`, …), `--allowedTools` | approval modes `manual|smart|off`, `--yolo`, `command_allowlist`, `DANGEROUS_PATTERNS` 📖 | `--allow-tool` / `--deny-tool` patterns, per-session approvals 📖 | `--sandbox read-only|workspace-write|danger-full-access` 📖 | +| Extra working dirs | `--add-dir` (repeatable) | session cwd; worktrees via `-w` ⬜ | `/add-dir` 📖 | `--add-dir`-equivalent via cwd ⬜ | +| Model selection | `--model claude-opus-4-…` | `-m --provider

`; providers: anthropic, openai-codex, copilot (GITHUB_TOKEN), gemini, openrouter, ollama/vllm (custom), … 📖 | `/model` (Auto, Claude Opus/Sonnet 4.5, GPT-5.2 Codex, org models) 📖 | `-m`, config model/providers 📖 | +| Auth | `ANTHROPIC_API_KEY`, Bedrock OAuth/SigV4 | `~/.hermes/.env` per-provider keys 📖 | GitHub auth (`gh` / device flow) 📖 | `OPENAI_API_KEY` / ChatGPT auth 📖 | +| Programmatic result contract | results dir + `scan_manifest.json` (ours, harness-neutral) | same contract — judge by artifact presence, not stdout ✅ | same | same | + +## Subprocess-engine limitations (hermes / copilot / codex) + +The three subprocess engines (`agent/engines/`, sharing `SubprocessEngine`) +differ from the Claude Agent SDK path in a few operator-visible ways: + +- **No cost/token totals.** The SDK path streams per-message usage into a + `SessionTotals`; the subprocess engines accept the `totals_out` argument + for protocol compatibility but cannot fill it (the CLIs don't expose + structured per-turn usage over the headless contract). Cost/token totals + therefore report **zero** for non-Claude runs. Judge the scan by the + results contract, not by reported cost. +- **Success is contents-based, not exit-code-based.** The engine pre-creates + the `*_VULNHUNT_RESULTS_*` directory, so its existence proves nothing. A + run is a success only when that dir contains the skill's `README.md` + report (`runner._results_dir_is_complete`); a crashed / OOM-killed / + non-zero-exiting engine that wrote nothing is reported as a **failure**, + never as a clean "found nothing". +- **`engine_timeout_seconds` semantics.** A positive value caps one scan; a + value **≤ 0 means "no timeout"** (wait indefinitely), not "time out + instantly". Set a large positive number for a long-but-bounded run. +- **Timeout kill reaches the direct child only.** On timeout the engine + `kill()`s the CLI process it spawned; any subagent/worker processes that + CLI itself launched (hermes `delegate_task`, codex sequential passes) are + not directly reaped and rely on the child's own shutdown. + +## VulnHunter-side coupling inventory (audit date: 2026-08-16) + +- `vulnhunt/SKILL.md`: `${CLAUDE_SKILL_DIR}`, `Grep`/`Glob` tool vocabulary, + Agent-tool subagent dispatch (Phase 2 fan-out), `/cost`, `/model opus` + gating, Opus 4.7+ requirement. +- `vulnhunt/phases/*.md`: `Grep`/`Glob`/`Read` used as capability verbs + (portable wording), `phase4_report.md` embeds a `claude-opus-…` example + model string and `opus46` dir-tag example (illustrative only). +- `vulnhunter-fix/SKILL.md`: Opus gate table, `/model claude-opus-…` + instruction, "Claude Code's Bash tool / sandbox" phrasing. +- `vulnhunt-fix-verify/SKILL.md`: declares tool inventory + ("Read, Write, Edit, Glob, Grep, and Agent"), `${CLAUDE_SKILL_DIR}`, + Agent-tool in-flight semantics. +- `harness/local_harness/scan.py`: shells out to `claude` with + `--allowedTools "Read Write Edit Bash Agent"`, `--permission-mode + acceptEdits`, `--add-dir` (skills + phases dirs). +- `harness/local_harness/benchmark/judge.py`: `claude -p` with + `--system-prompt` and `--output-format text`. +- `vulnhunter-agent/`: `claude_agent_sdk` session construction + (`runner.py`), provider chokepoint `_llm.py:_send_prompt`, + Anthropic/Bedrock auth in `build_settings.py`, + `_MODEL_FAMILIES = ("opus","sonnet","haiku","gpt","o3","o1")` in + `runner.py` (already multi-family). + +## Local environment (this machine) + +- Hermes Agent v0.20.1 — `/home/mark/.local/bin/hermes`; skills at + `~/.hermes/skills`; source with docs at `/home/mark/hermes-agent`. +- codex-cli 0.147.0 — `/home/mark/.local/bin/codex`; `~/.codex` configured. +- `claude`, `copilot` — not installed (rendering/tests do not require them; + smoke tests for those harnesses are documented commands to run where the + CLIs exist). diff --git a/harness/local_harness/benchmark/judge.py b/harness/local_harness/benchmark/judge.py index 6d8285d..5f36459 100644 --- a/harness/local_harness/benchmark/judge.py +++ b/harness/local_harness/benchmark/judge.py @@ -5,6 +5,7 @@ import subprocess import time +from local_harness import config from local_harness.config import ( JUDGE_MAX_RETRIES, JUDGE_RETRY_BACKOFF_MULTIPLIER, @@ -53,13 +54,48 @@ def read_results_report(results_dir): return f.read() -def judge_findings_batch(results_report, findings, model=None): +def build_judge_command(prompt, system_prompt, model, engine=None): + """Build the judge subprocess argv for one engine. + + claude-code replicates the historical argv exactly. Engines without a + --system-prompt flag carry the system prompt inside the prompt text. + """ + engine = engine or config.JUDGE_ENGINE + if engine == "claude-code": + return [ + "claude", "-p", prompt, + "--output-format", "text", + "--model", model, + "--system-prompt", system_prompt, + ] + if engine == "hermes": + return [ + "hermes", "chat", "-Q", + "-m", model, + "-q", f"{system_prompt}\n\n---\n\n{prompt}", + ] + if engine == "copilot": + return ["copilot", "-p", f"{system_prompt}\n\n---\n\n{prompt}"] + if engine == "codex": + # Judge may run outside a git repo — skip the repo check. + return [ + "codex", "exec", + "--skip-git-repo-check", + "-m", model, + f"{system_prompt}\n\n---\n\n{prompt}", + ] + raise ValueError(f"unknown engine {engine!r}") + + +def judge_findings_batch(results_report, findings, model=None, engine=None): """Judge multiple benchmark findings against one scan result in a single LLM call. Args: results_report: Content of the scanner's README.md findings: List of benchmark finding dicts (finding_id, type, description) model: Model to use (defaults to config.MODEL) + engine: Judge engine (defaults to config.JUDGE_ENGINE — may differ + from the scan engine to avoid self-preference bias) Returns: List of judgment dicts, one per finding. """ @@ -90,10 +126,7 @@ def judge_findings_batch(results_report, findings, model=None): for attempt in range(JUDGE_MAX_RETRIES + 1): try: result = subprocess.run( - ["claude", "-p", prompt, - "--output-format", "text", - "--model", model, - "--system-prompt", JUDGE_SYSTEM_PROMPT], + build_judge_command(prompt, JUDGE_SYSTEM_PROMPT, model, engine=engine), capture_output=True, text=True, timeout=JUDGE_TIMEOUT, ) except subprocess.TimeoutExpired: diff --git a/harness/local_harness/config.py b/harness/local_harness/config.py index 125e4ba..d575f76 100644 --- a/harness/local_harness/config.py +++ b/harness/local_harness/config.py @@ -34,6 +34,64 @@ SKILLS_DIR = os.path.expanduser("~/.claude/skills/vulnhunt") PHASES_DIR = os.path.join(SKILLS_DIR, "phases") +# --- Scan/judge engine selection ----------------------------------------- +# Which agent harness drives scans (and, separately, the judge). The skills +# must be installed for that harness (./install.sh --target from the +# repo root). Override via VULNHUNT_HARNESS_ENGINE / VULNHUNT_HARNESS_JUDGE_ENGINE. +# Judging with a DIFFERENT engine than the scanner avoids self-preference +# bias — e.g. scan with hermes, judge with claude-code. +ENGINE = os.environ.get("VULNHUNT_HARNESS_ENGINE", "claude-code") +JUDGE_ENGINE = os.environ.get("VULNHUNT_HARNESS_JUDGE_ENGINE", ENGINE) + +ENGINES = { + "claude-code": { + "binary": "claude", + "skills_dir": "~/.claude/skills/vulnhunt", + }, + "hermes": { + "binary": "hermes", + "skills_dir": "~/.hermes/skills/vulnhunt", + }, + "copilot": { + "binary": "copilot", + "skills_dir": "~/.copilot/skills/vulnhunt", + }, + "codex": { + "binary": "codex", + "skills_dir": "~/.codex/skills/vulnhunt", + }, +} + + +def validate_engine(name): + """Raise a clear ValueError if ``name`` isn't a known engine. + + Deliberately NOT called at import time: a typo'd VULNHUNT_HARNESS_ENGINE + should fail at the point of use (``build_scan_command`` / + ``build_judge_command`` / ``scan_folder``) with a "you asked for engine + X" message, not break every ``import local_harness.config`` — including + ``--help`` paths and unrelated tooling — before anything runs. + """ + if name not in ENGINES: + raise ValueError( + f"unknown harness engine {name!r} (supported: {', '.join(sorted(ENGINES))})" + ) + return name + + +def skills_dir_for(engine): + """Expanded skills directory for ``engine`` (validates first).""" + validate_engine(engine) + return os.path.expanduser(ENGINES[engine]["skills_dir"]) + + +# Engine-aware skills location (same value as SKILLS_DIR for claude-code). +# Computed with a safe fallback so a typo'd env var doesn't KeyError at +# import — validation is deferred to the build/scan call sites above. +ENGINE_SKILLS_DIR = os.path.expanduser( + ENGINES.get(ENGINE, ENGINES["claude-code"])["skills_dir"] +) + # --- Batch scanning (ad-hoc URL list) --- BATCH_CLONE_BASE_DIR = os.path.join(REPO_ROOT, "repos_being_scanned") BATCH_REPO_LIST_FILE = os.path.join(HARNESS_DIR, "batch", "REPO_LIST.txt") diff --git a/harness/local_harness/scan.py b/harness/local_harness/scan.py index 5a677d5..8596a71 100644 --- a/harness/local_harness/scan.py +++ b/harness/local_harness/scan.py @@ -10,6 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime +from . import config from .config import ( MAX_SCAN_WORKERS, MODEL, @@ -174,19 +175,74 @@ def extract_cost_from_log(log_file_path): return {} -def scan_folder(folder_path, log_file=None, readonly=False): +def build_scan_command(folder_path, prompt, engine=None, readonly=False): + """Build the scan subprocess argv for one engine. + + claude-code replicates the historical argv exactly (tests snapshot it). + Other engines drive the same installed skill through their headless + contract; success is still judged by the *_VULNHUNT_RESULTS_* dir. + """ + engine = engine or config.ENGINE + if engine == "claude-code": + return [ + "claude", "-p", prompt, + "--output-format", "stream-json", + "--verbose", + "--allowedTools", "Read", "Write", "Edit", "Bash", "Agent", + "--permission-mode", "acceptEdits", + "--model", MODEL, + "--add-dir", folder_path, + "--add-dir", os.path.dirname(folder_path), + "--add-dir", SKILLS_DIR, + "--add-dir", PHASES_DIR, + ] + if engine == "hermes": + # terminal (shell) only when the run isn't read-only, mirroring the + # Bash policy of the claude-code path. + toolsets = "file,delegation" if readonly else "file,terminal,delegation" + return [ + "hermes", "chat", "-Q", + "-s", "vulnhunt", + "-t", toolsets, + "-m", MODEL, + "-q", prompt, + ] + if engine == "copilot": + # Experimental: verify the flag surface with `copilot -h` / + # `copilot help permissions` before benchmarking. + return ["copilot", "-p", prompt] + if engine == "codex": + # workspace-write even for read-only scans: the skill must write its + # results dir; read-only is enforced by the prompt, mirroring the + # claude-code acceptEdits policy. Flags verified on codex-cli 0.147.0. + return [ + "codex", "exec", + "-C", folder_path, + "-s", "workspace-write", + "-m", MODEL, + prompt, + ] + raise ValueError(f"unknown engine {engine!r}") + + +def scan_folder(folder_path, log_file=None, readonly=False, engine=None): """Run vulnhunt on one folder, stream events to a log file. Returns a ScanResult (folder_path, label, returncode, event_count, elapsed, results_dir, cost_data). """ + engine = engine or config.ENGINE label = os.path.basename(folder_path) if log_file is None: log_file = os.path.join(folder_path, "benchmark_scan.log") - if not os.path.isdir(SKILLS_DIR): - print(f" [{ts()}] [{label}] Error: Skill not installed. Run install.sh first.") + # Resolve the skills dir from the *effective* engine (not the import-time + # default) so an engine= override or a mutated env is honored. + skills_dir = config.skills_dir_for(engine) if engine != "claude-code" else SKILLS_DIR + if not os.path.isdir(skills_dir): + print(f" [{ts()}] [{label}] Error: Skill not installed for engine " + f"'{engine}'. Run install.sh --target {engine} first.") return ScanResult(folder_path, label, 1, 0, 0, None, {}) prompt = ( @@ -204,16 +260,7 @@ def scan_folder(folder_path, log_file=None, readonly=False): start = time.time() proc = subprocess.Popen( - ["claude", "-p", prompt, - "--output-format", "stream-json", - "--verbose", - "--allowedTools", "Read", "Write", "Edit", "Bash", "Agent", - "--permission-mode", "acceptEdits", - "--model", MODEL, - "--add-dir", folder_path, - "--add-dir", os.path.dirname(folder_path), - "--add-dir", SKILLS_DIR, - "--add-dir", PHASES_DIR], + build_scan_command(folder_path, prompt, engine=engine, readonly=readonly), stdout=subprocess.PIPE, # Merge stderr into stdout (which we drain below) rather than piping it # to its own buffer no one reads — an unread stderr pipe deadlocks the @@ -280,7 +327,7 @@ def _kill_on_timeout(): return ScanResult(folder_path, label, proc.returncode, event_count, elapsed, results_dir, cost_data) -def scan_folder_with_retry(folder_path, log_filename=None, readonly=False): +def scan_folder_with_retry(folder_path, log_filename=None, readonly=False, engine=None): """Wrap scan_folder with retry on 429 rate limit failures. Returns: Same ScanResult as scan_folder, with elapsed summed across attempts. @@ -301,7 +348,7 @@ def scan_folder_with_retry(folder_path, log_filename=None, readonly=False): if removed: print(f" [{ts()}] [{label}] Cleaned {len(removed)} partial result(s)", flush=True) - result = scan_folder(folder_path, log_file=log_file, readonly=readonly) + result = scan_folder(folder_path, log_file=log_file, readonly=readonly, engine=engine) total_elapsed += result.elapsed actual_log = os.path.join(folder_path, log_filename or "benchmark_scan.log") @@ -317,7 +364,7 @@ def scan_folder_with_retry(folder_path, log_filename=None, readonly=False): return result._replace(elapsed=total_elapsed) -def scan_targets(targets, max_workers=None, status_interval=300, log_filename=None, readonly=False): +def scan_targets(targets, max_workers=None, status_interval=300, log_filename=None, readonly=False, engine=None): """Scan a list of benchmark targets in parallel with 429 retry. targets: list of dicts with at least 'clone_dir' and 'key' fields. @@ -329,7 +376,7 @@ def scan_targets(targets, max_workers=None, status_interval=300, log_filename=No max_workers = MAX_SCAN_WORKERS print(f"\n[{ts()}] Starting scans for {len(targets)} targets " - f"(max {max_workers} parallel)", flush=True) + f"(engine {engine or config.ENGINE}, max {max_workers} parallel)", flush=True) results = [] completed_keys = set() @@ -338,7 +385,7 @@ def scan_targets(targets, max_workers=None, status_interval=300, log_filename=No with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_target = { - executor.submit(scan_folder_with_retry, t["clone_dir"], log_filename=log_filename, readonly=readonly): t + executor.submit(scan_folder_with_retry, t["clone_dir"], log_filename=log_filename, readonly=readonly, engine=engine): t for t in targets } for future in as_completed(future_to_target): diff --git a/harness/tests/test_engines.py b/harness/tests/test_engines.py new file mode 100644 index 0000000..e7fce88 --- /dev/null +++ b/harness/tests/test_engines.py @@ -0,0 +1,138 @@ +"""Tests for multi-engine scan/judge command construction.""" + +import pytest + +import local_harness.config as config +import local_harness.scan as scan +from local_harness.benchmark import judge + + +class TestScanCommandBuilders: + def test_claude_code_argv_matches_historical_shape(self): + argv = scan.build_scan_command( + "/repos/app", "/vulnhunt /repos/app", engine="claude-code" + ) + assert argv[0] == "claude" + assert argv[1] == "-p" + assert argv[argv.index("--output-format") + 1] == "stream-json" + assert argv[argv.index("--permission-mode") + 1] == "acceptEdits" + assert argv[argv.index("--model") + 1] == config.MODEL + assert "--add-dir" in argv + # tool allow-list stays positional after --allowedTools + i = argv.index("--allowedTools") + assert argv[i + 1 : i + 6] == ["Read", "Write", "Edit", "Bash", "Agent"] + + def test_hermes_argv_read_only(self): + argv = scan.build_scan_command( + "/repos/app", "PROMPT", engine="hermes", readonly=True + ) + assert argv[0] == "hermes" + assert argv[1:5] == ["chat", "-Q", "-s", "vulnhunt"] + assert argv[argv.index("-t") + 1] == "file,delegation" + assert argv[argv.index("-m") + 1] == config.MODEL + assert argv[-2] == "-q" and argv[-1] == "PROMPT" + + def test_hermes_argv_bash_includes_terminal(self): + argv = scan.build_scan_command( + "/repos/app", "PROMPT", engine="hermes", readonly=False + ) + assert argv[argv.index("-t") + 1] == "file,terminal,delegation" + + def test_copilot_argv(self): + argv = scan.build_scan_command("/repos/app", "PROMPT", engine="copilot") + assert argv == ["copilot", "-p", "PROMPT"] + + def test_codex_argv(self): + argv = scan.build_scan_command("/repos/app", "PROMPT", engine="codex") + assert argv[0] == "codex" + assert argv[1] == "exec" + assert argv[argv.index("-C") + 1] == "/repos/app" + assert argv[argv.index("-s") + 1] == "workspace-write" + assert argv[argv.index("-m") + 1] == config.MODEL + assert argv[-1] == "PROMPT" + + def test_unknown_engine_raises(self): + with pytest.raises(ValueError, match="unknown engine"): + scan.build_scan_command("/repos/app", "P", engine="skynet") + + +class TestJudgeCommandBuilders: + def test_claude_code_judge_argv_matches_historical_shape(self): + argv = judge.build_judge_command("PROMPT", "SYS", "claude-opus-4-8", engine="claude-code") + assert argv == [ + "claude", "-p", "PROMPT", + "--output-format", "text", + "--model", "claude-opus-4-8", + "--system-prompt", "SYS", + ] + + def test_hermes_judge_embeds_system_prompt(self): + argv = judge.build_judge_command("PROMPT", "SYS", "m", engine="hermes") + assert argv[0] == "hermes" + assert argv[argv.index("-m") + 1] == "m" + combined = argv[argv.index("-q") + 1] + assert "SYS" in combined and "PROMPT" in combined + + def test_copilot_judge_embeds_system_prompt(self): + argv = judge.build_judge_command("PROMPT", "SYS", "m", engine="copilot") + assert argv == ["copilot", "-p", "SYS\n\n---\n\nPROMPT"] + + def test_codex_judge_skips_git_check(self): + argv = judge.build_judge_command("PROMPT", "SYS", "m", engine="codex") + assert argv[0] == "codex" + assert "--skip-git-repo-check" in argv + assert argv[argv.index("-m") + 1] == "m" + combined = argv[-1] + assert "SYS" in combined and "PROMPT" in combined + + def test_unknown_judge_engine_raises(self): + with pytest.raises(ValueError, match="unknown engine"): + judge.build_judge_command("P", "S", "m", engine="nope") + + +class TestEngineConfig: + def test_default_engine_is_claude_code(self, monkeypatch): + monkeypatch.delenv("VULNHUNT_HARNESS_ENGINE", raising=False) + monkeypatch.delenv("VULNHUNT_HARNESS_JUDGE_ENGINE", raising=False) + import importlib + cfg = importlib.reload(config) + assert cfg.ENGINE == "claude-code" + assert cfg.JUDGE_ENGINE == "claude-code" + assert cfg.ENGINE_SKILLS_DIR == cfg.SKILLS_DIR + importlib.reload(config) # restore for other tests + + def test_engine_env_selection(self, monkeypatch): + monkeypatch.setenv("VULNHUNT_HARNESS_ENGINE", "hermes") + monkeypatch.delenv("VULNHUNT_HARNESS_JUDGE_ENGINE", raising=False) + import importlib + cfg = importlib.reload(config) + assert cfg.ENGINE == "hermes" + assert cfg.JUDGE_ENGINE == "hermes" # follows scan engine by default + assert cfg.ENGINE_SKILLS_DIR.endswith(".hermes/skills/vulnhunt") + importlib.reload(config) + + def test_unknown_engine_env_does_not_break_import(self, monkeypatch): + """A typo'd env var must NOT raise at import time. + + Validation is deferred to the point of use so merely importing the + config (as --help paths and unrelated tooling do) stays safe. No + reload sequencing is needed to clean up global state. + """ + monkeypatch.setenv("VULNHUNT_HARNESS_ENGINE", "bad-engine") + import importlib + cfg = importlib.reload(config) # must not raise + assert cfg.ENGINE == "bad-engine" + # It fails clearly at the point of use instead: + with pytest.raises(ValueError, match="unknown harness engine"): + cfg.validate_engine(cfg.ENGINE) + monkeypatch.delenv("VULNHUNT_HARNESS_ENGINE") + importlib.reload(config) + + def test_validate_engine_accepts_known(self): + for name in ("claude-code", "hermes", "copilot", "codex"): + assert config.validate_engine(name) == name + + def test_skills_dir_for_validates(self): + assert config.skills_dir_for("hermes").endswith(".hermes/skills/vulnhunt") + with pytest.raises(ValueError, match="unknown harness engine"): + config.skills_dir_for("nope") diff --git a/harness/tests/test_scan.py b/harness/tests/test_scan.py index 6fffe57..4df97f5 100644 --- a/harness/tests/test_scan.py +++ b/harness/tests/test_scan.py @@ -252,6 +252,52 @@ def cancel(self): assert "read-only scan" not in captured["prompt"] +def test_scan_folder_reads_engine_default_at_call_time(monkeypatch, tmp_path): + """The env-var → scan_folder() default engine path (real users hit this). + + scan.py reads ``config.ENGINE`` at call time (not a value bound at + import), so a reloaded/overridden config default is honored without + passing engine= explicitly. Regression for the ``from .config import + ENGINE`` value-binding gap. + """ + import importlib + + from local_harness import config + + folder = tmp_path / "repo" + folder.mkdir() + hermes_skills = tmp_path / "hermes_skills" + hermes_skills.mkdir() + + # Select hermes purely via the env var + config reload — no engine= arg. + monkeypatch.setenv("VULNHUNT_HARNESS_ENGINE", "hermes") + importlib.reload(config) + monkeypatch.setattr(scan.config, "ENGINE", "hermes") + monkeypatch.setattr(scan.config, "skills_dir_for", lambda e: str(hermes_skills)) + + captured = {} + + def fake_popen(cmd, *a, **k): + captured["cmd"] = cmd + return _FakePopen([json.dumps({"type": "result"}) + "\n"], returncode=0) + + monkeypatch.setattr(scan.subprocess, "Popen", fake_popen) + + class _NoTimer: + def __init__(self, *a, **k): + pass + def start(self): + pass + def cancel(self): + pass + monkeypatch.setattr(scan.threading, "Timer", _NoTimer) + + scan.scan_folder(str(folder)) # no engine= — must default to hermes + assert captured["cmd"][0] == "hermes" + monkeypatch.delenv("VULNHUNT_HARNESS_ENGINE") + importlib.reload(config) + + def test_scan_folder_timeout(monkeypatch, tmp_path): folder = tmp_path / "repo" folder.mkdir() @@ -319,7 +365,7 @@ def cancel(self): def test_scan_folder_with_retry_success_first_try(monkeypatch, tmp_path): folder = str(tmp_path / "repo") monkeypatch.setattr(scan, "scan_folder", - lambda fp, log_file=None, readonly=False: scan.ScanResult(fp, "repo", 0, 3, 1.0, "rd", {"total_cost_usd": 1})) + lambda fp, log_file=None, readonly=False, engine=None: scan.ScanResult(fp, "repo", 0, 3, 1.0, "rd", {"total_cost_usd": 1})) monkeypatch.setattr(scan, "is_rate_limit_failure", lambda p: False) result = scan.scan_folder_with_retry(folder) assert result.returncode == 0 @@ -329,7 +375,7 @@ def test_scan_folder_with_retry_429_then_success(monkeypatch, tmp_path): folder = str(tmp_path / "repo") calls = {"n": 0} - def fake_scan(fp, log_file=None, readonly=False): + def fake_scan(fp, log_file=None, readonly=False, engine=None): calls["n"] += 1 if calls["n"] == 1: return scan.ScanResult(fp, "repo", 1, 0, 0.5, None, {}) @@ -348,7 +394,7 @@ def fake_scan(fp, log_file=None, readonly=False): def test_scan_folder_with_retry_429_exhausted(monkeypatch, tmp_path): folder = str(tmp_path / "repo") monkeypatch.setattr(scan, "scan_folder", - lambda fp, log_file=None, readonly=False: scan.ScanResult(fp, "repo", 1, 0, 0.5, None, {})) + lambda fp, log_file=None, readonly=False, engine=None: scan.ScanResult(fp, "repo", 1, 0, 0.5, None, {})) monkeypatch.setattr(scan, "is_rate_limit_failure", lambda p: True) monkeypatch.setattr(scan, "clean_prior_results", lambda *a, **k: ["r"]) monkeypatch.setattr(scan.time, "sleep", lambda s: None) diff --git a/install.sh b/install.sh index dc9d133..b9fd9b9 100755 --- a/install.sh +++ b/install.sh @@ -9,7 +9,46 @@ if [ -z "${HOME:-}" ]; then fi SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -SKILLS_PARENT="$HOME/.claude/skills" + +# Target harness. claude-code installs the repo-root skill sources verbatim +# (identity); other targets are rendered by scripts/render_skills.py from +# adapters//adapter.json into dist// first. +TARGET="claude-code" +while [ $# -gt 0 ]; do + case "$1" in + --target) TARGET="${2:-}"; shift 2 ;; + --target=*) TARGET="${1#*=}"; shift ;; + -h|--help) + echo "usage: ./install.sh [--target claude-code|hermes|copilot|codex]" + exit 0 ;; + *) + echo "error: unknown argument: $1" >&2 + exit 1 ;; + esac +done + +case "$TARGET" in + claude-code) + SKILLS_PARENT="$HOME/.claude/skills" ;; + hermes) + SKILLS_PARENT="$HOME/.hermes/skills" + echo "Rendering hermes skill bundle (dist/hermes)..." + python3 "$SCRIPT_DIR/scripts/render_skills.py" --adapter hermes \ + || { echo "error: render_skills.py failed" >&2; exit 1; } ;; + copilot) + SKILLS_PARENT="$HOME/.copilot/skills" + echo "Rendering copilot skill bundle (dist/copilot)..." + python3 "$SCRIPT_DIR/scripts/render_skills.py" --adapter copilot \ + || { echo "error: render_skills.py failed" >&2; exit 1; } ;; + codex) + SKILLS_PARENT="$HOME/.codex/skills" + echo "Rendering codex skill bundle (dist/codex)..." + python3 "$SCRIPT_DIR/scripts/render_skills.py" --adapter codex \ + || { echo "error: render_skills.py failed" >&2; exit 1; } ;; + *) + echo "error: unknown target '$TARGET' (supported: claude-code, hermes, copilot, codex)" >&2 + exit 1 ;; +esac # vulnhunter-fix runtime deps. The skill's scripts/_skill_bootstrap.py expects # a bundled venv at /.venv containing these; without it preflight's @@ -76,12 +115,28 @@ import jsonschema, graphify # noqa: F401 } # Skills shipped from this repo. Format: :. -# Order matters only for output readability — both are independent. -SKILLS=( - "vulnhunt:$SCRIPT_DIR/vulnhunt" - "vulnhunt-fix-verify:$SCRIPT_DIR/vulnhunt-fix-verify" - "vulnhunter-fix:$SCRIPT_DIR/vulnhunter-fix" -) +# claude-code installs the repo-root sources (identity with dist/claude-code, +# enforced by tests/test_render_skills.py); rendered targets install from +# dist// and pick up every skill the adapter declares. +if [ "$TARGET" = "claude-code" ]; then + SKILLS=( + "vulnhunt:$SCRIPT_DIR/vulnhunt" + "vulnhunt-fix-verify:$SCRIPT_DIR/vulnhunt-fix-verify" + "vulnhunter-fix:$SCRIPT_DIR/vulnhunter-fix" + ) +else + DIST_DIR="$SCRIPT_DIR/dist/$TARGET" + SKILLS=() + for d in "$DIST_DIR"/*/; do + [ -f "$d/SKILL.md" ] || continue + name="$(basename "$d")" + SKILLS+=("$name:$d") + done + if [ ${#SKILLS[@]} -eq 0 ]; then + echo "error: no skills found under $DIST_DIR" >&2 + exit 1 + fi +fi # Create the parent skills directory if missing. if [ ! -d "$SKILLS_PARENT" ]; then @@ -138,7 +193,8 @@ done echo "" if [ "$installed_any" -eq 1 ]; then - echo "To update after pulling changes: re-run ./install.sh" + echo "Installed for target: $TARGET" + echo "To update after pulling changes: re-run ./install.sh --target $TARGET" echo "To uninstall: $SCRIPT_DIR/uninstall.sh" else echo "No skills were installed." diff --git a/scripts/render_skills.py b/scripts/render_skills.py new file mode 100644 index 0000000..9e70242 --- /dev/null +++ b/scripts/render_skills.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""Render the harness-neutral skill sources into per-harness skill bundles. + +The skill directories at the repo root (``vulnhunt``, ``vulnhunter-fix``, +``vulnhunt-fix-verify``) are the canonical methodology text and remain the +Claude Code layout. Each adapter lives in ``adapters//adapter.json`` +and declares declarative transforms; this script applies them and writes +``dist///``. + +The ``claude-code`` adapter is the identity transform: its output must stay +byte-identical to the sources (enforced by tests/test_render_skills.py). +Any non-identity change to a source file is therefore a deliberate prompt +change that all adapters inherit. + +Transform types (applied in declared order): + substitute Literal find/replace. Fails loudly when ``find`` is + absent (unless ``optional``), so upstream edits that + break an adapter are caught at render time. + prepend Insert text after the YAML frontmatter (or at the very + top with ``after_frontmatter: false``). + frontmatter_append Insert lines just before the closing ``---``. + override Replace (or create) a file wholesale from the adapter + directory — used when a harness needs a file format the + core does not carry (e.g. an AGENTS.md). + drop Delete files from the rendered bundle. + +File patterns are matched against the path relative to dist// and +support ``**`` (any depth), ``*`` (within one segment) and ``?``. + +Usage: + python3 scripts/render_skills.py [--adapter NAME|all] [--out DIR] + python3 scripts/render_skills.py --adapter NAME --check --baseline DIR + +``--check`` compares a fresh render against ``--baseline`` (a previously +rendered tree). ``dist/`` is gitignored and rendered fresh in CI, so +``--check`` has no committed tree to diff against and requires an explicit +baseline; CI verifies byte-identity through ``tests/test_render_skills.py``. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import re +import shutil +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +ALL_SKILLS = ("vulnhunt", "vulnhunt-fix-verify", "vulnhunter-fix") +COPY_IGNORE = shutil.ignore_patterns(".installed-from", ".venv", "__pycache__", "*.pyc") + + +def is_ignored(path: Path) -> bool: + """True when ``path`` matches a COPY_IGNORE pattern. + + ``shutil.ignore_patterns`` returns a callable expecting + ``(dir, names_iterable)`` and returning the set of ignored names — it + must be given an iterable, never a bare str (which it would iterate + character-by-character, matching nothing). Centralizing the call here + keeps the correct shape in one place for both the renderer and tests. + """ + return path.name in COPY_IGNORE(str(path.parent), [path.name]) + + +class TransformError(Exception): + """A declared transform could not be applied to the current sources.""" + + +def _glob_to_regex(pattern: str) -> re.Pattern: + parts = pattern.split("/") + out = [] + for i, part in enumerate(parts): + if part == "**": + out.append("(?:[^/]+/)*" if i < len(parts) - 1 else ".*") + else: + out.append(fnmatch.translate(part).replace(r"\Z", "")) + # fnmatch.translate anchors with (?s:...)\Z; rebuild without it. + return re.compile("(?s:" + "/".join(out) + r")\Z") + + +@dataclass +class RenderedFile: + path: str # relative to dist// + text: str | None = None # None → binary, kept as raw bytes + bytes: bytes = field(default=b"", repr=False) + + +def load_adapter(adapter_dir: Path) -> dict: + manifest = adapter_dir / "adapter.json" + if not manifest.is_file(): + raise TransformError(f"no adapter.json in {adapter_dir}") + with manifest.open(encoding="utf-8") as fh: + cfg = json.load(fh) + for key in ("name", "skills", "transforms"): + if key not in cfg: + raise TransformError(f"{manifest}: missing required key '{key}'") + if cfg["name"] != adapter_dir.name: + raise TransformError( + f"{manifest}: adapter name {cfg['name']!r} does not match directory {adapter_dir.name!r}" + ) + unknown = [s for s in cfg["skills"] if s not in ALL_SKILLS] + if unknown: + raise TransformError(f"{manifest}: unknown skills {unknown}") + return cfg + + +def _resolve_text(spec: dict, adapter_dir: Path) -> str: + if "text" in spec: + return spec["text"] + if "text_file" in spec: + return (adapter_dir / spec["text_file"]).read_text(encoding="utf-8") + raise TransformError(f"transform needs 'text' or 'text_file': {spec}") + + +def _iter_targets(spec: dict, files: dict[str, RenderedFile]) -> list[str]: + pattern = spec.get("files") + if not pattern: + raise TransformError(f"transform missing 'files': {spec}") + rx = _glob_to_regex(pattern) + if any(c in pattern for c in "*?"): + matches = sorted(p for p in files if rx.match(p)) + else: + matches = [pattern] if pattern in files else [] + if not matches: + raise TransformError(f"no rendered files match pattern {pattern!r}") + return matches + + +def _frontmatter_split(text: str) -> tuple[str, str] | None: + """Split '---\n\n---\n' into (frontmatter block incl. both + fences, remainder). Returns None when the file has no frontmatter.""" + if not text.startswith("---\n"): + return None + end = text.find("\n---\n", 3) + if end == -1: + return None + return text[: end + 5], text[end + 5 :] + + +def _apply_substitute(spec: dict, files: dict[str, RenderedFile], adapter_dir: Path, stats: dict): + find, replace = spec.get("find"), spec.get("replace", "") + if not find: + raise TransformError(f"substitute missing 'find': {spec}") + expected = spec.get("count") + optional = spec.get("optional", False) + hits = 0 + for rel in _iter_targets(spec, files): + f = files[rel] + if f.text is None: + continue + n = f.text.count(find) + if n == 0: + continue + hits += n + f.text = f.text.replace(find, replace) + if hits == 0 and not optional: + raise TransformError(f"substitute 'find' not present in any target: {find!r}") + if expected is not None and hits != expected: + raise TransformError( + f"substitute expected {expected} occurrence(s), found {hits}: {find!r}" + ) + stats["substituted"] += hits + + +def _apply_prepend(spec: dict, files: dict[str, RenderedFile], adapter_dir: Path, stats: dict): + text = _resolve_text(spec, adapter_dir).rstrip("\n") + "\n\n" + after_fm = spec.get("after_frontmatter", True) + for rel in _iter_targets(spec, files): + f = files[rel] + if f.text is None: + continue + if after_fm: + fm = _frontmatter_split(f.text) + if fm: + f.text = fm[0] + text + fm[1].lstrip("\n") + stats["prepended"] += 1 + continue + f.text = text + f.text + stats["prepended"] += 1 + + +def _apply_frontmatter_append(spec: dict, files: dict[str, RenderedFile], adapter_dir: Path, stats: dict): + text = _resolve_text(spec, adapter_dir).rstrip("\n") + for rel in _iter_targets(spec, files): + f = files[rel] + if f.text is None: + continue + fm = _frontmatter_split(f.text) + if not fm: + raise TransformError(f"frontmatter_append: {rel} has no frontmatter") + block, rest = fm + if not block.endswith("---\n"): + raise TransformError(f"frontmatter_append: malformed closing fence in {rel}") + f.text = block[:-4] + text + "\n---\n" + rest + stats["frontmatter"] += 1 + + +def _apply_override(spec: dict, files: dict[str, RenderedFile], adapter_dir: Path, stats: dict): + src = adapter_dir / spec["from"] + if not src.is_file(): + raise TransformError(f"override source missing: {src}") + rel = spec["path"] + try: + text = src.read_text(encoding="utf-8") + except UnicodeDecodeError: + files[rel] = RenderedFile(path=rel, text=None, bytes=src.read_bytes()) + else: + files[rel] = RenderedFile(path=rel, text=text) + stats["overridden"] += 1 + + +def _apply_drop(spec: dict, files: dict[str, RenderedFile], stats: dict): + pattern = spec.get("files") + if not pattern: + raise TransformError(f"drop missing 'files': {spec}") + rx = _glob_to_regex(pattern) + doomed = [p for p in files if rx.match(p)] + if not doomed: + raise TransformError(f"drop matched no rendered files: {pattern!r}") + for p in doomed: + del files[p] + stats["dropped"] += len(doomed) + + +_APPLICATORS = { + "substitute": _apply_substitute, + "prepend": _apply_prepend, + "frontmatter_append": _apply_frontmatter_append, + "override": _apply_override, + "drop": _apply_drop, +} + + +def render_adapter(adapter_dir: Path, repo_root: Path, out_root: Path) -> dict: + cfg = load_adapter(adapter_dir) + files: dict[str, RenderedFile] = {} + for skill in cfg["skills"]: + src = repo_root / skill + if not (src / "SKILL.md").is_file(): + raise TransformError(f"skill source missing SKILL.md: {src}") + for p in sorted(src.rglob("*")): + if not p.is_file() or is_ignored(p): + continue + rel = f"{skill}/{p.relative_to(src).as_posix()}" + try: + files[rel] = RenderedFile(path=rel, text=p.read_text(encoding="utf-8")) + except UnicodeDecodeError: + files[rel] = RenderedFile(path=rel, text=None, bytes=p.read_bytes()) + + stats = {"substituted": 0, "prepended": 0, "frontmatter": 0, "overridden": 0, "dropped": 0} + for i, spec in enumerate(cfg["transforms"]): + kind = spec.get("type") + applicator = _APPLICATORS.get(kind) + if applicator is None: + raise TransformError(f"transform[{i}]: unknown type {kind!r}") + applicator(spec, files, adapter_dir, stats) + + out_dir = out_root / cfg["name"] + if out_dir.exists(): + shutil.rmtree(out_dir) + for rel in sorted(files): + dest = out_dir / rel + dest.parent.mkdir(parents=True, exist_ok=True) + f = files[rel] + if f.text is None: + dest.write_bytes(f.bytes) + else: + dest.write_text(f.text, encoding="utf-8") + return {"adapter": cfg["name"], "skills": cfg["skills"], "files": len(files), **stats} + + +def check_render(adapter_dir: Path, repo_root: Path, baseline_root: Path) -> bool: + """Render into a temp dir and compare with a supplied baseline. + + ``baseline_root`` holds a previously rendered ``/`` tree (an + installed bundle, a pinned reference render, etc.). ``dist/`` is + gitignored and rendered fresh in CI, so there is no committed tree to + diff against — the caller must point ``--baseline`` at whatever + reference it wants to verify (the same fresh render is emitted with + ``render_skills.py --out

`` first). + """ + cfg = load_adapter(adapter_dir) + with tempfile.TemporaryDirectory() as tmp: + tmp_out = Path(tmp) + render_adapter(adapter_dir, repo_root, tmp_out) + baseline = baseline_root / cfg["name"] + fresh = tmp_out / cfg["name"] + if not baseline.is_dir(): + print( + f"{baseline}: baseline not found — render one first " + f"(render_skills.py --adapter {cfg['name']} --out {baseline_root})", + file=sys.stderr, + ) + return False + baseline_files = { + p.relative_to(baseline).as_posix() + for p in baseline.rglob("*") + if p.is_file() + } + fresh_files = {p.relative_to(fresh).as_posix() for p in fresh.rglob("*") if p.is_file()} + if baseline_files != fresh_files: + print( + f"{baseline}: file sets differ " + f"(missing={sorted(baseline_files - fresh_files)} " + f"extra={sorted(fresh_files - baseline_files)})", + file=sys.stderr, + ) + return False + for rel in sorted(baseline_files): + if (baseline / rel).read_bytes() != (fresh / rel).read_bytes(): + print(f"{baseline}/{rel}: content differs from fresh render", file=sys.stderr) + return False + return True + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--adapter", default="all", help="adapter name or 'all' (default)") + ap.add_argument("--source", default=str(REPO_ROOT), help="repo root holding the skill sources") + ap.add_argument("--out", default=None, help="output root (default /dist)") + ap.add_argument( + "--check", + action="store_true", + help="verify a fresh render matches --baseline; no files written", + ) + ap.add_argument( + "--baseline", + default=None, + help="baseline render root to compare against with --check " + "(holds / subtrees; e.g. a prior --out dir or installed bundle)", + ) + args = ap.parse_args(argv) + + repo_root = Path(args.source).resolve() + out_root = Path(args.out).resolve() if args.out else repo_root / "dist" + adapters_root = repo_root / "adapters" + if not adapters_root.is_dir(): + print(f"error: {adapters_root} not found", file=sys.stderr) + return 2 + if args.check and not args.baseline: + print( + "error: --check requires --baseline DIR (dist/ is gitignored and " + "rendered fresh, so there is no committed tree to diff against)", + file=sys.stderr, + ) + return 2 + baseline_root = Path(args.baseline).resolve() if args.baseline else None + + names = ( + sorted(p.name for p in adapters_root.iterdir() if (p / "adapter.json").is_file()) + if args.adapter == "all" + else [args.adapter] + ) + for name in names: + adapter_dir = adapters_root / name + if not adapter_dir.is_dir(): + print(f"error: unknown adapter {name!r}", file=sys.stderr) + return 2 + if args.check: + assert baseline_root is not None # guaranteed by the check above + ok = check_render(adapter_dir, repo_root, baseline_root) + print(f"{name}: {'OK' if ok else 'STALE'}") + if not ok: + return 1 + else: + stats = render_adapter(adapter_dir, repo_root, out_root) + print( + f"rendered dist/{name}: {stats['files']} files " + f"(subs={stats['substituted']} prepends={stats['prepended']} " + f"fm={stats['frontmatter']} overrides={stats['overridden']} drops={stats['dropped']})" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_render_skills.py b/tests/test_render_skills.py new file mode 100644 index 0000000..d2abb6c --- /dev/null +++ b/tests/test_render_skills.py @@ -0,0 +1,286 @@ +"""Tests for scripts/render_skills.py. + +The critical invariant: the claude-code adapter renders byte-identical to the +repo-root skill sources, so any source edit is a deliberate prompt change that +every adapter inherits. Other adapters are checked for their declared marker +substitutions so upstream edits that break an adapter fail here, not in a scan. +""" + +from __future__ import annotations + +import filecmp +import json +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import render_skills # noqa: E402 + +ADAPTERS = REPO_ROOT / "adapters" + + +def render_to_tmp(adapter: str) -> Path: + tmp = Path(tempfile.mkdtemp(prefix=f"vulnhunt-dist-{adapter}-")) + render_skills.render_adapter(ADAPTERS / adapter, REPO_ROOT, tmp) + return tmp / adapter + + +class TestClaudeCodeIdentity(unittest.TestCase): + """dist/claude-code must equal the repo-root skill sources byte-for-byte.""" + + def setUp(self): + self.dist = render_to_tmp("claude-code") + + def _source_tree(self, skill: str) -> Path: + return REPO_ROOT / skill + + def test_all_declared_skills_rendered(self): + for skill in ("vulnhunt", "vulnhunt-fix-verify", "vulnhunter-fix"): + self.assertTrue((self.dist / skill / "SKILL.md").is_file(), skill) + + def test_byte_identical_to_sources(self): + for skill in ("vulnhunt", "vulnhunt-fix-verify", "vulnhunter-fix"): + src = self._source_tree(skill) + dst = self.dist / skill + src_files = sorted( + p.relative_to(src).as_posix() + for p in src.rglob("*") + if p.is_file() and not render_skills.is_ignored(p) + ) + dst_files = sorted(p.relative_to(dst).as_posix() for p in dst.rglob("*") if p.is_file()) + self.assertEqual(src_files, dst_files, f"{skill}: file lists differ") + for rel in src_files: + self.assertTrue( + filecmp.cmp(src / rel, dst / rel, shallow=False), + f"{skill}/{rel}: differs from source", + ) + + +class TestDropValidation(unittest.TestCase): + """`drop` must fail loudly like the other applicators, not silently no-op. + + Matches the fail-loud invariant the renderer relies on: a missing + ``files`` key or a pattern that matches nothing is a broken transform, + not a clean render. + """ + + def _files(self): + return { + "vulnhunt/SKILL.md": render_skills.RenderedFile(path="vulnhunt/SKILL.md", text="x"), + "vulnhunt/phases/p1.md": render_skills.RenderedFile(path="vulnhunt/phases/p1.md", text="y"), + } + + def test_drop_missing_files_key_raises(self): + with self.assertRaises(render_skills.TransformError): + render_skills._apply_drop({"type": "drop"}, self._files(), {"dropped": 0}) + + def test_drop_matching_nothing_raises(self): + with self.assertRaises(render_skills.TransformError): + render_skills._apply_drop( + {"type": "drop", "files": "vulnhunt/nope-*.md"}, self._files(), {"dropped": 0} + ) + + def test_drop_removes_matches(self): + files = self._files() + stats = {"dropped": 0} + render_skills._apply_drop({"type": "drop", "files": "vulnhunt/phases/*.md"}, files, stats) + self.assertNotIn("vulnhunt/phases/p1.md", files) + self.assertIn("vulnhunt/SKILL.md", files) + self.assertEqual(stats["dropped"], 1) + + +class TestCopyIgnore(unittest.TestCase): + """COPY_IGNORE must actually exclude build detritus from rendered bundles. + + Regression for the ``COPY_IGNORE(p, p.name)`` arg-shape bug: passing a + bare str made ``fnmatch.filter`` iterate its characters, so the ignore + never fired and ``__pycache__``/``*.pyc``/``.venv`` leaked into dist/. + """ + + def test_is_ignored_matches_patterns(self): + self.assertTrue(render_skills.is_ignored(Path("/x/__pycache__"))) + self.assertTrue(render_skills.is_ignored(Path("/x/junk.pyc"))) + self.assertTrue(render_skills.is_ignored(Path("/x/.venv"))) + self.assertTrue(render_skills.is_ignored(Path("/x/.installed-from"))) + self.assertFalse(render_skills.is_ignored(Path("/x/SKILL.md"))) + self.assertFalse(render_skills.is_ignored(Path("/x/phase1.md"))) + + def test_pyc_detritus_excluded_from_render(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + # Minimal skill source + planted build detritus. + skill_src = root / "vulnhunt" + (skill_src / "phases").mkdir(parents=True) + (skill_src / "SKILL.md").write_text("# vulnhunt\n") + (skill_src / "phases" / "phase1.md").write_text("phase\n") + junk_dir = skill_src / "__pycache__" + junk_dir.mkdir() + (junk_dir / "junk.pyc").write_text("bytecode") + # A one-skill adapter manifest (identity transform). + adapter_dir = root / "adapters" / "probe" + adapter_dir.mkdir(parents=True) + (adapter_dir / "adapter.json").write_text( + json.dumps({"name": "probe", "skills": ["vulnhunt"], "transforms": []}) + ) + out = root / "dist" + render_skills.render_adapter(adapter_dir, root, out) + rendered = out / "probe" + leaked = [p for p in rendered.rglob("*") if p.suffix == ".pyc" or p.name == "__pycache__"] + self.assertEqual(leaked, [], f"detritus leaked into render: {leaked}") + self.assertTrue((rendered / "vulnhunt" / "SKILL.md").is_file()) + + +class TestAdapterManifests(unittest.TestCase): + def test_manifests_well_formed(self): + for adapter_dir in sorted(ADAPTERS.iterdir()): + if not (adapter_dir / "adapter.json").is_file(): + continue + cfg = render_skills.load_adapter(adapter_dir) + self.assertEqual(cfg["name"], adapter_dir.name) + for skill in cfg["skills"]: + self.assertTrue((REPO_ROOT / skill / "SKILL.md").is_file()) + + def test_substitute_find_strings_exist_in_sources(self): + """Every non-optional 'find' must be present in the current sources + AND declare a matching ``count``. + + This is the drift tripwire: when someone edits a prompt line that an + adapter rewrites, this fails with the exact missing string. Requiring + ``count`` (and asserting the exact occurrence total) makes the + tripwire actually trip on a *partial* rewrite too — a source edit + that duplicates or half-rewords a phrase changes the occurrence + count, so a bare "present at least once" check would pass while the + transform silently rewrites the wrong number of sites. + """ + for adapter_dir in sorted(ADAPTERS.iterdir()): + manifest = adapter_dir / "adapter.json" + if not manifest.is_file(): + continue + cfg = json.loads(manifest.read_text(encoding="utf-8")) + for skill in cfg["skills"]: + for i, spec in enumerate(cfg["transforms"]): + if spec.get("type") != "substitute" or spec.get("optional"): + continue + self.assertIn( + "count", + spec, + f"{adapter_dir.name} transform[{i}]: non-optional substitute " + f"must declare 'count' (drift tripwire): {spec['find']!r}", + ) + pattern = spec["files"] + rx = render_skills._glob_to_regex(pattern) + targets = [ + f"{skill}/{p.relative_to(REPO_ROOT / skill).as_posix()}" + for p in sorted((REPO_ROOT / skill).rglob("*")) + if p.is_file() + and not render_skills.is_ignored(p) + and rx.match(f"{skill}/{p.relative_to(REPO_ROOT / skill).as_posix()}") + ] + self.assertTrue(targets, f"{adapter_dir.name} transform[{i}]: no files match {pattern}") + found = sum( + (REPO_ROOT / rel).read_text(encoding="utf-8").count(spec["find"]) + for rel in targets + ) + self.assertGreater( + found, + 0, + f"{adapter_dir.name} transform[{i}]: find-string no longer in sources: {spec['find']!r}", + ) + + def test_every_nonoptional_substitute_declares_count(self): + """Independent, adapter-wide guard: no non-optional substitute may + omit ``count``. Kept separate from the source-presence check so a new + adapter that forgets a count fails even if its find-string exists.""" + for adapter_dir in sorted(ADAPTERS.iterdir()): + manifest = adapter_dir / "adapter.json" + if not manifest.is_file(): + continue + cfg = json.loads(manifest.read_text(encoding="utf-8")) + for i, spec in enumerate(cfg["transforms"]): + if spec.get("type") != "substitute" or spec.get("optional"): + continue + self.assertIn( + "count", + spec, + f"{adapter_dir.name} transform[{i}] is missing 'count'", + ) + + +class TestHermesRender(unittest.TestCase): + """The hermes adapter must neutralize every Claude Code-ism in vulnhunt.""" + + @classmethod + def setUpClass(cls): + cls.dist = render_to_tmp("hermes") + + def test_skill_present_with_phases(self): + skill = self.dist / "vulnhunt" + self.assertTrue((skill / "SKILL.md").is_file()) + self.assertTrue((skill / "phases" / "phase1_recon.md").is_file()) + + def test_no_claude_skill_dir_token(self): + for md in (self.dist / "vulnhunt").rglob("*.md"): + self.assertNotIn( + "${CLAUDE_SKILL_DIR}", md.read_text(encoding="utf-8"), f"leftover token in {md.name}" + ) + + def test_hermes_token_and_frontmatter(self): + text = (self.dist / "vulnhunt" / "SKILL.md").read_text(encoding="utf-8") + self.assertIn("${HERMES_SKILL_DIR}", text) + self.assertIn("requires_toolsets", text) + self.assertIn("delegation", text) + # frontmatter stays first in the file + self.assertTrue(text.startswith("---\n")) + + def test_claude_commands_neutralized(self): + text = (self.dist / "vulnhunt" / "SKILL.md").read_text(encoding="utf-8") + self.assertNotIn("/cost", text) + self.assertNotIn("/model opus", text) + self.assertNotIn("Launch a `general-purpose` subagent:", text) + self.assertIn("delegate_task", text) + + def test_terminology_overlay_prepended(self): + for name in ("SKILL.md", "phases/phase1_recon.md", "phases/phase2_hunt.md"): + text = (self.dist / "vulnhunt" / name).read_text(encoding="utf-8") + self.assertIn("Harness adaptation note", text, f"overlay missing from {name}") + + def test_delegation_protocol_in_overlay(self): + text = (self.dist / "vulnhunt" / "SKILL.md").read_text(encoding="utf-8") + self.assertIn('action: "list"', text) + self.assertIn("dispatched", text) + self.assertIn("conclude, error out, or produce a final answer", text) + + +class TestModelGateSoftened(unittest.TestCase): + """Non-Claude adapters must not block on model choice: Step 0 becomes a + calibration notice that proceeds on the selected model. (The claude-code + render keeps upstream's interactive STOP gate, byte-identical.)""" + + def test_gate_is_notice_not_stop(self): + for adapter in ("hermes", "codex", "copilot"): + dist = render_to_tmp(adapter) + text = (dist / "vulnhunt" / "SKILL.md").read_text(encoding="utf-8") + self.assertNotIn( + "**STOP immediately**", text, f"{adapter}: Step 0 still blocks" + ) + self.assertNotIn("/model opus", text, f"{adapter}: claude command leaked") + self.assertIn( + "CONTINUE immediately on the currently selected model", + text, + f"{adapter}: proceed-on-selected-model wording missing", + ) + self.assertIn("Calibration note", text, f"{adapter}: notice missing") + + def test_claude_code_keeps_upstream_gate(self): + dist = render_to_tmp("claude-code") + text = (dist / "vulnhunt" / "SKILL.md").read_text(encoding="utf-8") + self.assertIn("**STOP immediately**", text) + self.assertIn("/model opus", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/vulnhunter-agent/agent/__main__.py b/vulnhunter-agent/agent/__main__.py index 1b80387..449d1aa 100644 --- a/vulnhunter-agent/agent/__main__.py +++ b/vulnhunter-agent/agent/__main__.py @@ -72,6 +72,7 @@ run_vulnhunt, set_verbosity, ) +from .engines import ScanSpec, get_engine from ._stream_events import SessionTotals from .issues import PostSummary from .issues_extract import Finding @@ -907,16 +908,37 @@ def _persist_manifest(final_exit_code: int) -> None: github_token=get_github_token("scan", config), github_host=config.github.host, ) - results_dir = await run_vulnhunt( - clone_dir, - config, - model_override=args.model, - scan_id=args.scan_id, - read_only=effective_read_only, - enable_bash=args.enable_bash, - audit_writer=audit_writer, - totals_out=scan_totals, - ) + # Engine routing: the default claude-code path calls the + # module-level run_vulnhunt directly so existing monkeypatch + # seams (tests patch agent.__main__.run_vulnhunt) keep working + # and behavior stays byte-identical. Other engines go through + # the ScanEngine protocol (agent/engines/). + engine = get_engine(config) + if engine.name == "claude-code": + results_dir = await run_vulnhunt( + clone_dir, + config, + model_override=args.model, + scan_id=args.scan_id, + read_only=effective_read_only, + enable_bash=args.enable_bash, + audit_writer=audit_writer, + totals_out=scan_totals, + ) + else: + spec = ScanSpec( + clone_dir=clone_dir, + config=config, + model=args.model or config.anthropic.model, + scan_id=args.scan_id, + read_only=effective_read_only, + enable_bash=args.enable_bash, + ) + results_dir = await engine.run_scan( + spec, + audit_writer=audit_writer, + totals_out=scan_totals, + ) print() print(f"Clone: {clone_dir}") if results_dir is None: diff --git a/vulnhunter-agent/agent/config.py b/vulnhunter-agent/agent/config.py index 49206b3..b22bcb7 100644 --- a/vulnhunter-agent/agent/config.py +++ b/vulnhunter-agent/agent/config.py @@ -22,11 +22,14 @@ from __future__ import annotations import os +import shlex import tomllib -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any +from agent.engines import ENGINE_NAMES + @dataclass(frozen=True) class AnthropicConfig: @@ -117,6 +120,23 @@ class ScanConfig: # the bundled CLI bypasses any inherited HTTP proxy for these hosts/CIDRs. # Default covers loopback + private ranges; add your own internal zones. no_proxy: str = "localhost,127.0.0.1,169.254.169.254,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" + # Which agent harness drives the scan. "claude-code" is the existing + # Claude Agent SDK path (unchanged); "hermes"/"copilot" shell out to the + # respective CLI with the same results-directory contract. See + # agent/engines/. + engine: str = "claude-code" + # Override for the engine binary path (default: $PATH lookup). + engine_command: str = "" + # Hermes model-routing provider (hermes engine only; empty = hermes + # config default), e.g. "anthropic", "openai-codex", "openrouter". + engine_provider: str = "" + # Hard cap for one engine-driven scan, seconds (CLI engines only). + engine_timeout_seconds: int = 21_600 + # Extra CLI flags appended to the engine invocation (e.g. Copilot + # ``--allow-tool`` patterns). Give a native TOML list — + # ``["--allow-tool", "shell(ls,cat)"]`` — so args with commas, spaces, + # or parens survive intact; a bare string is shlex-split as a fallback. + engine_extra_args: list[str] = field(default_factory=list) @dataclass(frozen=True) @@ -409,6 +429,28 @@ def _resolve( return value +def _parse_engine_extra_args(raw: Any) -> list[str]: + """Normalize ``[scan] engine_extra_args`` to a clean argv list. + + A native TOML list (``["--allow-tool", "shell(ls,cat)"]``) is the + canonical form and is passed through verbatim — this preserves args + that contain commas, spaces, or parentheses, exactly the Copilot + ``--allow-tool shell(ls,cat)`` case a naive comma split would shred. + + A bare string (e.g. from an env override, which can't express a list) + is tokenized with ``shlex.split`` so quoting works the way a shell + user expects: ``"--allow-tool 'shell(ls,cat)'"`` → two args. + """ + if raw is None: + return [] + if isinstance(raw, (list, tuple)): + return [str(a) for a in raw if str(a).strip()] + text = str(raw).strip() + if not text: + return [] + return [a for a in shlex.split(text) if a.strip()] + + def load_config(path: str | os.PathLike[str] | None = None) -> AgentConfig: """Load and validate the agent's configuration. @@ -604,7 +646,29 @@ def load_config(path: str | os.PathLike[str] | None = None) -> AgentConfig: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", ) ), + engine=str( + _resolve(scan_raw, "scan", "engine", default="claude-code") + ), + engine_command=str( + _resolve(scan_raw, "scan", "engine_command", default="") + ), + engine_provider=str( + _resolve(scan_raw, "scan", "engine_provider", default="") + ), + engine_timeout_seconds=int( + _resolve( + scan_raw, "scan", "engine_timeout_seconds", kind=int, default=21_600 + ) + ), + engine_extra_args=_parse_engine_extra_args( + _resolve(scan_raw, "scan", "engine_extra_args", default=[]) + ), ) + if scan.engine not in ENGINE_NAMES: + raise ValueError( + f"[scan] engine must be one of {', '.join(ENGINE_NAMES)}; " + f"got {scan.engine!r}" + ) github = GitHubConfig( host=str(_resolve(github_raw, "github", "host", default="github.com")), diff --git a/vulnhunter-agent/agent/engines/__init__.py b/vulnhunter-agent/agent/engines/__init__.py new file mode 100644 index 0000000..d1987bb --- /dev/null +++ b/vulnhunter-agent/agent/engines/__init__.py @@ -0,0 +1,122 @@ +"""Pluggable scan engines for the vulnhunter-agent runtime. + +An *engine* is the agent harness that actually drives the /vulnhunt skill +against a clone: the Claude Agent SDK (reference), the Hermes CLI, the +GitHub Copilot CLI, ... All engines share one contract: + + - success is judged by the VulnHunter results contract — a + ``*_VULNHUNT_RESULTS_*`` directory that actually contains the skill's + ``README.md`` report — never by stdout text, which differs per + harness, and never by the directory's mere existence (the subprocess + engines pre-create it, so an engine that crashes before writing must + still be judged a failure); + - the kickoff prompt carries the same "Pre-resolved scan metadata" + block the skill's Mandatory First Actions expect (results dir, branch + label, repo URL, model tag, shell availability), so the skill runs + identically regardless of engine; + - downstream stages (manifest, publish, issues, audit, verify) are + engine-agnostic and consume only the results contract. + +Select via ``[scan] engine = "claude-code" | "hermes" | "copilot" | +"codex"`` in the agent TOML (default ``claude-code`` — the existing SDK +path, unchanged). + +The three subprocess engines (hermes/copilot/codex) share a single +``SubprocessEngine`` base (``agent/engines/_subprocess.py``); each concrete +engine only declares its binary name, skill path, argv, and kickoff prompt. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from agent._stream_events import SessionTotals + from agent.audit import AuditWriter + from agent.config import AgentConfig + +ENGINE_NAMES = ("claude-code", "hermes", "copilot", "codex") + + +class EngineError(RuntimeError): + """Engine-level failure (binary missing, timeout, non-zero/empty result). + + Lives here — next to ``ScanSpec`` / ``ScanEngine`` — rather than in any + one engine module, so importing a single engine doesn't drag in an + unrelated one purely for the exception type. + """ + + +@dataclass(frozen=True) +class ScanSpec: + """Everything an engine needs to run one scan.""" + + clone_dir: Path + config: "AgentConfig" + model: str + scan_id: str = "" + read_only: bool = True + enable_bash: bool = False + backoffs: tuple[float, ...] = () + + +@runtime_checkable +class ScanEngine(Protocol): + """The engine contract implemented by every harness adapter.""" + + name: str + + async def run_scan( + self, + spec: ScanSpec, + *, + audit_writer: "AuditWriter | None" = None, + totals_out: "SessionTotals | None" = None, + ) -> Path | None: + """Run /vulnhunt against ``spec.clone_dir``; return the results dir. + + Raises on pre-flight failures (missing skill, prior results, + engine binary absent, engine timeout) and on a run that finishes + without a complete results directory. A None return means the + engine finished cleanly but produced no results directory. + """ + ... # pragma: no cover + + +def get_engine(config: "AgentConfig") -> ScanEngine: + """Instantiate the engine selected by ``[scan] engine``.""" + # Local imports keep module import cheap and avoid cycles: the engine + # modules import runner helpers, runner imports config, config imports + # this package — so the concrete engines must not load at package import. + name = config.scan.engine + if name == "claude-code": + from agent.engines.claude_code import ClaudeCodeEngine + + return ClaudeCodeEngine() + if name == "hermes": + from agent.engines.hermes import HermesEngine + + return HermesEngine() + if name == "copilot": + from agent.engines.copilot import CopilotCliEngine + + return CopilotCliEngine() + if name == "codex": + from agent.engines.codex import CodexEngine + + return CodexEngine() + raise ValueError( + f"unknown scan engine {name!r} (supported: {', '.join(ENGINE_NAMES)}); " + "set [scan] engine in the agent TOML" + ) + + +__all__ = [ + "ENGINE_NAMES", + "EngineError", + "ScanEngine", + "ScanSpec", + "get_engine", +] diff --git a/vulnhunter-agent/agent/engines/_subprocess.py b/vulnhunter-agent/agent/engines/_subprocess.py new file mode 100644 index 0000000..a5ff4b6 --- /dev/null +++ b/vulnhunter-agent/agent/engines/_subprocess.py @@ -0,0 +1,203 @@ +"""Shared base for the subprocess-driven scan engines (hermes/copilot/codex). + +Each of those engines drives /vulnhunt by shelling out to an agent CLI and +judging success by the VulnHunter results contract. The orchestration is +identical across all three — skill check, binary check, prior-results +guard, results-dir staging, git context, audit start, prompt/command +build, subprocess launch with timeout, and the completion contract — so it +lives here once. A concrete engine supplies only what actually differs: + + - ``name`` class attribute, the ``[scan] engine`` value; + - ``_binary_name`` PATH lookup name (``"hermes"`` etc.); + - ``_skill_paths()`` candidate SKILL.md locations to verify install; + - ``_install_target`` the ``install.sh --target X`` name for errors; + - ``_binary_hint`` engine-specific "binary not found" remedy text; + - ``_build_command()`` the argv for the subprocess; + - ``_build_kickoff()`` the kickoff prompt. + +Success contract (the load-bearing invariant): the engine pre-creates the +results directory, so ``_find_results_dir`` will always *find* it. Success +is therefore judged by ``_results_dir_is_complete`` — the dir must hold the +skill's ``README.md`` report — never by the directory merely existing. A +crashed / OOM-killed / non-zero-exiting engine leaves an empty shell behind +and is correctly reported as a failure, not a clean "found nothing". +""" + +from __future__ import annotations + +import asyncio +import logging +import shutil +import time +from pathlib import Path +from typing import TYPE_CHECKING + +from agent import audit as _audit +from agent import runner as _runner +from agent.engines import EngineError, ScanSpec + +if TYPE_CHECKING: + from agent._stream_events import SessionTotals + from agent.audit import AuditWriter + +logger = logging.getLogger(__name__) + + +class SubprocessEngine: + """Template base for CLI-driven engines. Subclasses fill the hooks.""" + + name: str = "" + _binary_name: str = "" + _install_target: str = "" + _binary_hint: str = "" + + # --- hooks a subclass must / may override ----------------------------- + + def _skill_paths(self) -> tuple[Path, ...]: + """SKILL.md locations to accept as "installed" (first hit wins).""" + raise NotImplementedError # pragma: no cover + + def _build_command(self, spec: ScanSpec, binary: str, prompt: str) -> list[str]: + """Full argv for the subprocess (binary already resolved).""" + raise NotImplementedError # pragma: no cover + + def _build_kickoff( + self, spec: ScanSpec, *, results_dir: Path, git_ctx: dict[str, str] + ) -> str: + """The kickoff prompt handed to the engine.""" + raise NotImplementedError # pragma: no cover + + def _log_launch(self, cmd: list[str]) -> None: + """Emit a launch log line. Overridable (e.g. experimental warnings).""" + logger.info("%s engine: %s … ", self.name, cmd[0]) + + # --- shared machinery ------------------------------------------------- + + def _resolve_binary(self, spec: ScanSpec) -> str: + configured = spec.config.scan.engine_command + binary = configured or shutil.which(self._binary_name) + if not binary: + raise EngineError( + f"{self._binary_name} binary not found on PATH — {self._binary_hint} " + "or set [scan] engine_command in the agent TOML" + ) + return binary + + def _check_skill_installed(self) -> None: + paths = self._skill_paths() + if not any(p.is_file() for p in paths): + shown = paths[0] + raise EngineError( + f"vulnhunt skill not found at {shown}. " + f"Run ./install.sh --target {self._install_target} from the " + "vulnhunter repo first." + ) + + async def run_scan( + self, + spec: ScanSpec, + *, + audit_writer: "AuditWriter | None" = None, + totals_out: "SessionTotals | None" = None, # noqa: ARG002 (SDK-only) + ) -> Path | None: + self._check_skill_installed() + binary = self._resolve_binary(spec) # fail fast before any pre-staging + clone_dir = spec.clone_dir + model = spec.model + + # Same pre-staging contract as the SDK path: compute + create the + # results dir, refuse to shadow prior results, resolve git context, + # and hand the skill every value it must not recompute. + _runner._check_no_prior_results(clone_dir) + results_dir = _runner._compute_results_dir(clone_dir, model) + results_dir.mkdir(exist_ok=False) + git_ctx = _runner._git_context(clone_dir) + repo_slug = _runner._repo_slug_from_url(git_ctx["repo_url"], clone_dir.name) + report_id = _audit.report_id_from(results_dir) + wall_start = time.time() + + if audit_writer is not None: + audit_writer.emit_audit( + _audit.build_scan_started( + app_id=spec.config.audit.app_id, + actor=spec.config.audit.actor, + repo_slug=repo_slug, + report_id=report_id, + model_version=model, + target_sha=git_ctx["head_sha"], + ) + ) + + prompt = self._build_kickoff(spec, results_dir=results_dir, git_ctx=git_ctx) + cmd = self._build_command(spec, binary, prompt) + self._log_launch(cmd) + + # engine_timeout_seconds <= 0 means "no timeout" (wait indefinitely), + # not "time out instantly". asyncio.wait_for(timeout=None) waits + # forever; a positive value caps the run. + raw_timeout = spec.config.scan.engine_timeout_seconds + timeout = raw_timeout if raw_timeout and raw_timeout > 0 else None + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=str(clone_dir), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except TimeoutError: + proc.kill() + await proc.wait() + raise EngineError( + f"{self.name} exceeded engine_timeout_seconds={raw_timeout}" + ) from None + + out_text = stdout.decode("utf-8", errors="replace").strip() + err_text = stderr.decode("utf-8", errors="replace").strip() + logger.info("%s engine exit=%s final=%r", self.name, proc.returncode, out_text[:200]) + if err_text: + logger.debug("%s engine stderr: %s", self.name, err_text[-2000:]) + + found = _runner._find_results_dir(clone_dir) + # Contents-based success: the results dir the engine pre-created + # always *exists*, so judge on whether it actually holds a report. + # An empty results dir (crash / OOM / early exit) is a failure even + # when the process returned 0, and any non-zero exit is a failure. + complete = _runner._results_dir_is_complete(found) + error: Exception | None = None + if proc.returncode != 0 or not complete: + tail = (out_text + "\n" + err_text)[-800:] + if not complete: + reason = ( + "no results directory produced" + if found is None + else f"results directory {found.name} has no README.md report" + ) + error = EngineError( + f"{self.name} exited {proc.returncode} but {reason} — " + f"the scan did not complete. Output tail:\n{tail}" + ) + else: + error = EngineError( + f"{self.name} exited {proc.returncode} with a results dir. " + f"Output tail:\n{tail}" + ) + + # Only a complete results dir is reported to the audit trail and + # returned as the scan's output; a failure records no results dir. + reported = found if complete else None + _runner._emit_scan_completed_safely( + audit_writer, + config=spec.config, + repo_slug=repo_slug, + report_id=report_id, + model=model, + target_sha=git_ctx["head_sha"], + results_dir=reported, + session_result=None, + error=error, + wall_start=wall_start, + ) + if error is not None: + raise error + return reported diff --git a/vulnhunter-agent/agent/engines/claude_code.py b/vulnhunter-agent/agent/engines/claude_code.py new file mode 100644 index 0000000..c60b78b --- /dev/null +++ b/vulnhunter-agent/agent/engines/claude_code.py @@ -0,0 +1,45 @@ +"""Reference engine: the existing Claude Agent SDK path, unchanged. + +``run_vulnhunt`` in ``agent/runner.py`` remains the single implementation; +this class only adapts it to the ScanEngine protocol so the CLI can treat +every engine uniformly. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from agent import runner as _runner +from agent.engines import ScanSpec + +if TYPE_CHECKING: + from agent._stream_events import SessionTotals + from agent.audit import AuditWriter + +logger = logging.getLogger(__name__) + + +class ClaudeCodeEngine: + name = "claude-code" + + async def run_scan( + self, + spec: ScanSpec, + *, + audit_writer: "AuditWriter | None" = None, + totals_out: "SessionTotals | None" = None, + ) -> Path | None: + backoffs = spec.backoffs or _runner._SCAN_RETRY_BACKOFFS + return await _runner.run_vulnhunt( + spec.clone_dir, + spec.config, + model_override=spec.model, + scan_id=spec.scan_id, + read_only=spec.read_only, + enable_bash=spec.enable_bash, + backoffs=backoffs, + audit_writer=audit_writer, + totals_out=totals_out, + ) diff --git a/vulnhunter-agent/agent/engines/codex.py b/vulnhunter-agent/agent/engines/codex.py new file mode 100644 index 0000000..959e016 --- /dev/null +++ b/vulnhunter-agent/agent/engines/codex.py @@ -0,0 +1,78 @@ +"""OpenAI Codex CLI engine. + +Drives /vulnhunt through a headless ``codex exec`` run against the skill +bundle installed at ``~/.codex/skills/vulnhunt`` +(``./install.sh --target codex``). Codex has no slash-command registry or +subagent tool, so the kickoff prompt points the agent at the bundle's +SKILL.md explicitly and the codex-rendered skill itself carries the +sequential-execution instructions. + +Sandbox: always ``workspace-write`` — even read-only scans must write the +results directory; the read-only contract is enforced by the prompt +(exploit tests written, not run), mirroring the Claude path's acceptEdits +policy. Flags verified against codex-cli 0.147.0. +""" + +from __future__ import annotations + +from pathlib import Path + +from agent import runner as _runner +from agent.engines import ScanSpec +from agent.engines._subprocess import SubprocessEngine, logger + +_CODEX_SKILL = Path.home() / ".codex" / "skills" / "vulnhunt" / "SKILL.md" + + +class CodexEngine(SubprocessEngine): + name = "codex" + _binary_name = "codex" + _install_target = "codex" + _binary_hint = "install the Codex CLI" + + def _skill_paths(self) -> tuple[Path, ...]: + return (_CODEX_SKILL,) + + def _build_command(self, spec: ScanSpec, binary: str, prompt: str) -> list[str]: + cmd = [ + binary, + "exec", + "-C", + str(spec.clone_dir), + "-s", + "workspace-write", + ] + if spec.model: + cmd += ["-m", spec.model] + cmd += list(spec.config.scan.engine_extra_args) + cmd += [prompt] + return cmd + + def _build_kickoff( + self, spec: ScanSpec, *, results_dir: Path, git_ctx: dict[str, str] + ) -> str: + shell_line = ( + "The sandboxed shell is AVAILABLE for exploit-test execution " + "(--enable-bash was passed)." + if spec.enable_bash + else "This is a read-only scan: use shell searches and file reads for " + "analysis; write exploit tests but do NOT run them." + ) + tag = _runner._model_tag(spec.model) + return ( + f"{_runner._VULNHUNT_PROMPT_PREAMBLE}\n\n" + f"Read {_CODEX_SKILL} and execute the /vulnhunt workflow it defines " + f"on {spec.clone_dir}. Follow the SKILL.md and its phase files " + f"exactly (phase files are under ~/.codex/skills/vulnhunt/phases/).\n\n" + f"Use the model tag `{tag}` for this scan. Name the results " + f"directory and any other artifacts with that exact tag.\n\n" + "Pre-resolved scan metadata (use these literal values — do NOT " + "run shell commands to recompute them):\n" + f"- VULNHUNT_DIR: {results_dir}\n" + f"- VULNHUNT_BRANCH: {git_ctx['branch_label']}\n" + f"- Repository URL: {git_ctx['repo_url']}\n" + f"- {shell_line}" + ) + + def _log_launch(self, cmd: list[str]) -> None: + logger.info("codex engine: %s … exec ", cmd[0]) diff --git a/vulnhunter-agent/agent/engines/copilot.py b/vulnhunter-agent/agent/engines/copilot.py new file mode 100644 index 0000000..3f131cd --- /dev/null +++ b/vulnhunter-agent/agent/engines/copilot.py @@ -0,0 +1,75 @@ +"""GitHub Copilot CLI engine (EXPERIMENTAL). + +Drives /vulnhunt through a non-interactive ``copilot`` run against the +skill bundle installed at ``~/.copilot/skills/vulnhunt`` +(``./install.sh --target copilot``). Copilot has no skill/slash-command +registry, so the kickoff prompt points the agent at the bundle's +SKILL.md explicitly. + +Status: the Copilot CLI's headless flag surface (non-interactive prompt +flag, permission flags) could not be verified against a live binary when +this engine was authored. ``_build_command`` uses ``-p`` for the prompt +plus any flags configured via ``[scan] engine_extra_args`` (e.g. +``--allow-tool`` patterns). Verify with ``copilot -h`` / +``copilot help permissions`` on your install before relying on it. +""" + +from __future__ import annotations + +from pathlib import Path + +from agent import runner as _runner +from agent.engines import ScanSpec +from agent.engines._subprocess import SubprocessEngine, logger + +_COPILOT_SKILL = Path.home() / ".copilot" / "skills" / "vulnhunt" / "SKILL.md" + + +class CopilotCliEngine(SubprocessEngine): + name = "copilot" + _binary_name = "copilot" + _install_target = "copilot" + _binary_hint = "install GitHub Copilot CLI" + + def _skill_paths(self) -> tuple[Path, ...]: + return (_COPILOT_SKILL,) + + def _build_command(self, spec: ScanSpec, binary: str, prompt: str) -> list[str]: + cmd = [binary, "-p", prompt] + cmd += list(spec.config.scan.engine_extra_args) + return cmd + + def _build_kickoff( + self, spec: ScanSpec, *, results_dir: Path, git_ctx: dict[str, str] + ) -> str: + shell_line = ( + "The shell tool is AVAILABLE for exploit-test execution " + "(--enable-bash was passed)." + if spec.enable_bash + else "The shell tool is NOT available for this read-only scan — use " + "your file search/read/write tools only; write exploit tests but do " + "not run them." + ) + tag = _runner._model_tag(spec.model) + return ( + f"{_runner._VULNHUNT_PROMPT_PREAMBLE}\n\n" + f"Read {_COPILOT_SKILL} and execute the /vulnhunt workflow it defines " + f"on {spec.clone_dir}. Follow the SKILL.md and its phase files " + f"exactly; add the directory containing the SKILL.md to your session " + f"so the phases/ files are readable.\n\n" + f"Use the model tag `{tag}` for this scan. Name the results " + f"directory and any other artifacts with that exact tag.\n\n" + "Pre-resolved scan metadata (use these literal values — do NOT " + "run shell commands to recompute them):\n" + f"- VULNHUNT_DIR: {results_dir}\n" + f"- VULNHUNT_BRANCH: {git_ctx['branch_label']}\n" + f"- Repository URL: {git_ctx['repo_url']}\n" + f"- {shell_line}" + ) + + def _log_launch(self, cmd: list[str]) -> None: + logger.warning( + "copilot engine is EXPERIMENTAL — verify its flag surface " + "(`copilot -h`) before production use" + ) + logger.info("copilot engine: %s … -p ", cmd[0]) diff --git a/vulnhunter-agent/agent/engines/hermes.py b/vulnhunter-agent/agent/engines/hermes.py new file mode 100644 index 0000000..8edf0f4 --- /dev/null +++ b/vulnhunter-agent/agent/engines/hermes.py @@ -0,0 +1,104 @@ +"""Hermes CLI engine: drive /vulnhunt via a headless ``hermes chat`` run. + +The vulnhunt skill must be installed for Hermes first +(``./install.sh --target hermes`` → ``~/.hermes/skills/vulnhunt``). +This engine pre-stages the same metadata the Claude SDK path provides +(results dir, branch label, repo URL, model tag, shell availability), +launches one headless session, and judges success by the results +directory's contents — Hermes' ``-Q`` contract (stdout = final message, +stderr = session id) is used only for logging/diagnostics. + +Subagent fan-out happens inside Hermes (``delegate_task``), so no +process-level fan-out is needed here; the kickoff teaches the async- +delegation wait protocol instead. +""" + +from __future__ import annotations + +from pathlib import Path + +from agent import runner as _runner +from agent.engines import EngineError, ScanSpec +from agent.engines._subprocess import SubprocessEngine, logger + +# Re-exported for backwards compatibility: callers and tests historically +# imported EngineError from this module. Its home is now engines/__init__. +__all__ = ["EngineError", "HermesEngine"] + +_HERMES_SKILL = Path.home() / ".hermes" / "skills" / "vulnhunt" / "SKILL.md" +# A tuple so additional fallback locations can be added without touching the +# base-class lookup (any-of semantics); tests monkeypatch this to a temp path. +_HERMES_SKILL_CANDIDATES = (_HERMES_SKILL,) + +# Toolsets mirroring the Claude path's tool policy: no terminal for +# read-only scans (the engine pre-creates the results dir); terminal is +# added only with --enable-bash, exactly like ``Bash`` on the SDK path. +_TOOLSETS_READ_ONLY = "file,delegation" +_TOOLSETS_BASH = "file,terminal,delegation" + +# Vocabulary the hermes skill's adaptation overlay maps to Hermes tools +# (search_files / read_file / write_file / patch). Rendered into the +# "Bash is NOT available — use X only" line. +_EFFECTIVE_TOOLS = ["Read", "Write", "Edit", "Glob", "Grep"] + +# Hermes runs every top-level delegate_task in the background; a headless +# single-query session exits when the orchestrator concludes, killing +# in-flight children. The kickoff must teach the wait protocol explicitly. +_DELEGATION_PROTOCOL = ( + "\n\nDelegation protocol (important): every top-level delegate_task runs " + "in the background — the call returns immediately with status " + "\"dispatched\" and the child's result is only delivered while your turn " + "is alive. After dispatching any subagent, do NOT conclude or error out " + "while children are running; keep the turn alive by periodically calling " + "delegate_task with action=\"list\" until every child shows completed, " + "then verify that phase's output files exist before continuing." +) + + +class HermesEngine(SubprocessEngine): + name = "hermes" + _binary_name = "hermes" + _install_target = "hermes" + _binary_hint = ( + "install Hermes (https://github.com/weav/hermes-agent)" + ) + + def _skill_paths(self) -> tuple[Path, ...]: + return _HERMES_SKILL_CANDIDATES + + def _build_command(self, spec: ScanSpec, binary: str, prompt: str) -> list[str]: + scan = spec.config.scan + cmd = [ + binary, + "chat", + "-Q", + "-s", + "vulnhunt", + "-t", + _TOOLSETS_BASH if spec.enable_bash else _TOOLSETS_READ_ONLY, + ] + if scan.engine_provider: + cmd += ["--provider", scan.engine_provider] + if spec.model: + cmd += ["-m", spec.model] + cmd += list(scan.engine_extra_args) + cmd += ["-q", prompt] + return cmd + + def _build_kickoff( + self, spec: ScanSpec, *, results_dir: Path, git_ctx: dict[str, str] + ) -> str: + prompt = _runner._build_vulnhunt_prompt( + spec.clone_dir, + spec.model, + read_only=spec.read_only, + results_dir=results_dir, + branch_label=git_ctx["branch_label"], + repo_url=git_ctx["repo_url"], + enable_bash=spec.enable_bash, + effective_tools=list(_EFFECTIVE_TOOLS), + ) + return prompt + _DELEGATION_PROTOCOL + + def _log_launch(self, cmd: list[str]) -> None: + logger.info("hermes engine: %s", " ".join(cmd[:8]) + " … -q ") diff --git a/vulnhunter-agent/agent/runner.py b/vulnhunter-agent/agent/runner.py index ea252fe..c75bcf9 100644 --- a/vulnhunter-agent/agent/runner.py +++ b/vulnhunter-agent/agent/runner.py @@ -244,6 +244,29 @@ def _find_results_dir(clone_dir: Path) -> Path | None: return max(candidates, key=lambda p: p.stat().st_mtime) +def _results_dir_is_complete(results_dir: Path | None) -> bool: + """True when ``results_dir`` holds a real scan report, not an empty shell. + + The subprocess engines pre-create the ``*_VULNHUNT_RESULTS_*`` directory + before launching the harness, so its mere *existence* proves nothing — a + crashed, OOM-killed, or non-zero-exiting engine leaves the empty shell + behind, and keying success on the directory alone reports "died before + looking" as a clean "found nothing". The skill's completion contract + writes ``README.md`` as its final report (downstream stages — + issues_extract, issues_remote_report — require it), so success keys on + that file existing with real content. The ``> 100`` byte floor matches + the harness's ``has_valid_results`` threshold, rejecting a stub README a + partially-run scan may have touched. + """ + if results_dir is None or not results_dir.is_dir(): + return False + readme = results_dir / "README.md" + try: + return readme.is_file() and readme.stat().st_size > 100 + except OSError: + return False + + class PriorResultsError(RuntimeError): """Raised when an existing ``*_VULNHUNT_RESULTS_*`` dir would shadow this scan. diff --git a/vulnhunter-agent/tests/test_engines.py b/vulnhunter-agent/tests/test_engines.py new file mode 100644 index 0000000..397aa2a --- /dev/null +++ b/vulnhunter-agent/tests/test_engines.py @@ -0,0 +1,493 @@ +"""Tests for the pluggable scan engines (agent/engines/). + +All subprocess execution is faked — these tests verify command +construction, pre-staging (results dir, prior-results guard), the +*contents-based* results-directory success contract (an empty results dir +is a failure, not a clean scan), timeout handling, and config wiring. No +real hermes / copilot / codex / claude invocation happens. + +The three subprocess engines share ``SubprocessEngine``; the shared +behaviors (success contract, non-zero exit, timeout, extra-args) are +covered once, parametrized across all three, so no engine's copy can +regress independently. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from agent import engines +from agent.engines import _subprocess as _subprocess_mod +from agent.engines import ENGINE_NAMES, EngineError, ScanSpec, get_engine +from agent.engines.claude_code import ClaudeCodeEngine +from agent.engines.codex import CodexEngine +from agent.engines.copilot import CopilotCliEngine +from agent.engines.hermes import EngineError as HermesEngineError +from agent.engines.hermes import HermesEngine + +# A README long enough to clear the >100-byte completion floor. +_VALID_README = "# VulnHunter Results\n\n" + ("finding detail. " * 20) + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + + +class TestGetEngine: + @pytest.mark.parametrize("name,cls", [ + ("claude-code", ClaudeCodeEngine), + ("hermes", HermesEngine), + ("copilot", CopilotCliEngine), + ("codex", CodexEngine), + ]) + def test_returns_engine_for_known_names(self, name: str, cls: type) -> None: + cfg = SimpleNamespace(scan=SimpleNamespace(engine=name)) + engine = get_engine(cfg) + assert isinstance(engine, cls) + assert engine.name == name + + def test_unknown_engine_raises(self) -> None: + cfg = SimpleNamespace(scan=SimpleNamespace(engine="skynet")) + with pytest.raises(ValueError, match="unknown scan engine"): + get_engine(cfg) + + def test_engine_names_cover_registry(self) -> None: + assert ENGINE_NAMES == ("claude-code", "hermes", "copilot", "codex") + + def test_engine_error_is_shared_symbol(self) -> None: + # EngineError's home is engines/__init__; the hermes re-export must + # be the very same class so `except EngineError` catches all engines. + assert HermesEngineError is EngineError + + +# --------------------------------------------------------------------------- +# Config wiring +# --------------------------------------------------------------------------- + + +class TestEngineConfig: + def _toml(self, engine_block: str) -> str: + return ( + """ +[anthropic] +model = "claude-opus-4-8" + +[oauth] +token_endpoint = "https://oauth.example.com/token" +client_id = "cid" +client_secret = "csec" +""" + + engine_block + ) + + def test_default_engine_is_claude_code(self, tmp_path: Path) -> None: + from agent.config import load_config + + path = tmp_path / "cfg.toml" + path.write_text(self._toml("")) + cfg = load_config(path) + assert cfg.scan.engine == "claude-code" + assert cfg.scan.engine_timeout_seconds == 21_600 + assert cfg.scan.engine_extra_args == [] + + def test_hermes_engine_fields_from_toml(self, tmp_path: Path) -> None: + from agent.config import load_config + + path = tmp_path / "cfg.toml" + path.write_text( + self._toml( + """ +[scan] +engine = "hermes" +engine_command = "/opt/hermes/bin/hermes" +engine_provider = "anthropic" +engine_timeout_seconds = 3600 +engine_extra_args = ["--accept-hooks", "--reasoning=high"] +""" + ) + ) + cfg = load_config(path) + assert cfg.scan.engine == "hermes" + assert cfg.scan.engine_command == "/opt/hermes/bin/hermes" + assert cfg.scan.engine_provider == "anthropic" + assert cfg.scan.engine_timeout_seconds == 3600 + assert cfg.scan.engine_extra_args == ["--accept-hooks", "--reasoning=high"] + + def test_extra_args_string_is_shlex_split(self, tmp_path: Path) -> None: + # A bare string is accepted for convenience and shlex-split, so a + # value with commas/parens survives intact (Copilot --allow-tool). + from agent.config import load_config + + path = tmp_path / "cfg.toml" + path.write_text( + self._toml( + """ +[scan] +engine = "copilot" +engine_extra_args = "--allow-tool 'shell(ls,cat)'" +""" + ) + ) + cfg = load_config(path) + assert cfg.scan.engine_extra_args == ["--allow-tool", "shell(ls,cat)"] + + def test_invalid_engine_rejected(self, tmp_path: Path) -> None: + from agent.config import load_config + + path = tmp_path / "cfg.toml" + path.write_text(self._toml('[scan]\nengine = "nope"\n')) + with pytest.raises(ValueError, match=r"\[scan\] engine"): + load_config(path) + + +# --------------------------------------------------------------------------- +# Fakes / helpers shared across the subprocess engines +# --------------------------------------------------------------------------- + + +class _FakeProc: + def __init__(self, returncode: int = 0, stdout: bytes = b"ok", stderr: bytes = b""): + self.returncode = returncode + self._stdout = stdout + self._stderr = stderr + self.killed = False + + async def communicate(self): + return (self._stdout, self._stderr) + + def kill(self): + self.killed = True + + async def wait(self): + return self.returncode + + +def _find_results_dir(clone_dir: Path) -> Path | None: + if not clone_dir.is_dir(): + return None + for entry in clone_dir.iterdir(): + if entry.is_dir() and "_VULNHUNT_RESULTS_" in entry.name: + return entry + return None + + +def _exec_writing_readme(returncode: int = 0, stdout: bytes = b"ok", stderr: bytes = b""): + """Fake create_subprocess_exec that simulates a *completed* scan. + + The engine pre-creates the results dir before launching, so the fake + finds it (via cwd) and drops a valid README.md — mirroring what a real + engine does on success. This is what makes the contents-based contract + return the dir. + """ + async def fake_exec(*cmd, **kw): + clone_dir = Path(kw["cwd"]) + results = _find_results_dir(clone_dir) + if results is not None: + (results / "README.md").write_text(_VALID_README) + return _FakeProc(returncode, stdout, stderr) + + return fake_exec + + +def _exec_leaving_empty(returncode: int = 0, stdout: bytes = b"", stderr: bytes = b"boom"): + """Fake exec that leaves the pre-created results dir EMPTY (crash/OOM).""" + async def fake_exec(*cmd, **kw): + return _FakeProc(returncode, stdout, stderr) + + return fake_exec + + +def _capturing_exec(captured: list, returncode: int = 0): + async def fake_exec(*cmd, **kw): + captured.append((cmd, kw)) + clone_dir = Path(kw["cwd"]) + results = _find_results_dir(clone_dir) + if results is not None: + (results / "README.md").write_text(_VALID_README) + return _FakeProc(returncode) + + return fake_exec + + +def _spec( + tmp_path: Path, + engine_name: str = "hermes", + *, + read_only: bool = True, + enable_bash: bool = False, + **scan_overrides, +) -> ScanSpec: + scan_fields = dict( + engine=engine_name, + engine_command=f"/fake/{engine_name}", + engine_provider="", + engine_timeout_seconds=60, + engine_extra_args=[], + ) + scan_fields.update(scan_overrides) + scan = SimpleNamespace(**scan_fields) + audit = SimpleNamespace(app_id="app", actor="tester") + return ScanSpec( + clone_dir=tmp_path / "clone", + config=SimpleNamespace(scan=scan, audit=audit, anthropic=SimpleNamespace()), + model="claude-opus-4-8", + read_only=read_only, + enable_bash=enable_bash, + ) + + +# Per-engine wiring: the engine class and how to point its skill lookup at a +# temp SKILL.md. Subprocess launch lives in the shared base, so all exec +# patching targets ``agent.engines._subprocess`` regardless of engine. +_SUBPROC = _subprocess_mod +_ENGINE_TABLE = [ + ("hermes", HermesEngine, "agent.engines.hermes._HERMES_SKILL_CANDIDATES", True), + ("copilot", CopilotCliEngine, "agent.engines.copilot._COPILOT_SKILL", False), + ("codex", CodexEngine, "agent.engines.codex._CODEX_SKILL", False), +] + + +def _install_skill(monkeypatch, tmp_path: Path, attr: str, as_tuple: bool) -> Path: + skill = tmp_path / "SKILL.md" + skill.write_text("# vulnhunt\n") + monkeypatch.setattr(attr, (skill,) if as_tuple else skill) + return skill + + +# --------------------------------------------------------------------------- +# Shared subprocess-engine contract (parametrized across all three) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name,cls,skill_attr,as_tuple", _ENGINE_TABLE) +class TestSubprocessEngineContract: + async def test_missing_skill_raises( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + missing = tmp_path / "nope" / "SKILL.md" + monkeypatch.setattr(skill_attr, (missing,) if as_tuple else missing) + spec = _spec(tmp_path, name) + with pytest.raises(EngineError, match=f"install.sh --target {name}"): + await cls().run_scan(spec) + + async def test_missing_binary_raises( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + monkeypatch.setattr("agent.engines._subprocess.shutil.which", lambda n: None) + spec = _spec(tmp_path, name, engine_command="") + spec.clone_dir.mkdir(parents=True) + with pytest.raises(EngineError, match="binary not found"): + await cls().run_scan(spec) + + async def test_complete_results_dir_returned( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _exec_writing_readme(0)) + spec = _spec(tmp_path, name) + spec.clone_dir.mkdir(parents=True) + results = await cls().run_scan(spec) + assert results is not None + assert "_VULNHUNT_RESULTS_" in results.name + assert (results / "README.md").is_file() + + async def test_zero_exit_empty_results_is_failure( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + """The core fix: exit 0 but an empty results dir is NOT a clean scan. + + A crashed/OOM-killed engine that returns 0 while writing nothing + must raise, not report the empty pre-created dir as success. + """ + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _exec_leaving_empty(0)) + spec = _spec(tmp_path, name) + spec.clone_dir.mkdir(parents=True) + with pytest.raises(EngineError, match="did not complete"): + await cls().run_scan(spec) + + async def test_nonzero_exit_empty_results_raises( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _exec_leaving_empty(3)) + spec = _spec(tmp_path, name) + spec.clone_dir.mkdir(parents=True) + with pytest.raises(EngineError, match="exited 3"): + await cls().run_scan(spec) + + async def test_nonzero_exit_even_with_results_raises( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + """A non-zero exit is a failure even if a README got written.""" + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _exec_writing_readme(1)) + spec = _spec(tmp_path, name) + spec.clone_dir.mkdir(parents=True) + with pytest.raises(EngineError, match="exited 1"): + await cls().run_scan(spec) + + async def test_timeout_kills_process( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + proc_holder: dict = {} + + async def fake_exec(*cmd, **kw): + proc = _FakeProc(0) + + async def communicate(): + await asyncio.sleep(999) + + proc.communicate = communicate + proc_holder["proc"] = proc + return proc + + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", fake_exec) + # A tiny positive timeout fires; 0/negative means "no timeout" (below). + spec = _spec(tmp_path, name, engine_timeout_seconds=0.01) + spec.clone_dir.mkdir(parents=True) + with pytest.raises(EngineError, match="engine_timeout_seconds"): + await cls().run_scan(spec) + assert proc_holder["proc"].killed + + async def test_extra_args_appended( + self, tmp_path, monkeypatch, name, cls, skill_attr, as_tuple + ): + _install_skill(monkeypatch, tmp_path, skill_attr, as_tuple) + captured: list = [] + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _capturing_exec(captured)) + spec = _spec(tmp_path, name, engine_extra_args=["--allow-tool", "shell(ls,cat)"]) + spec.clone_dir.mkdir(parents=True) + await cls().run_scan(spec) + cmd = list(captured[0][0]) + # The exact whitespace/parens of the arg survive as a single token. + assert "--allow-tool" in cmd + assert "shell(ls,cat)" in cmd + + +class TestTimeoutDisabled: + """engine_timeout_seconds <= 0 means 'no timeout', not 'instant timeout'.""" + + async def test_zero_timeout_waits_indefinitely(self, tmp_path, monkeypatch): + captured_timeouts: list = [] + real_wait_for = asyncio.wait_for + + async def spy_wait_for(aw, timeout): + captured_timeouts.append(timeout) + return await real_wait_for(aw, timeout) + + skill = tmp_path / "SKILL.md" + skill.write_text("# vulnhunt\n") + monkeypatch.setattr("agent.engines.hermes._HERMES_SKILL_CANDIDATES", (skill,)) + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _exec_writing_readme(0)) + monkeypatch.setattr("agent.engines._subprocess.asyncio.wait_for", spy_wait_for) + spec = _spec(tmp_path, "hermes", engine_timeout_seconds=0) + spec.clone_dir.mkdir(parents=True) + await HermesEngine().run_scan(spec) + # None => asyncio.wait_for waits forever (no cap). + assert captured_timeouts == [None] + + +# --------------------------------------------------------------------------- +# Engine-specific command construction +# --------------------------------------------------------------------------- + + +class TestHermesCommand: + async def test_command_construction_read_only(self, tmp_path, monkeypatch): + skill = tmp_path / "SKILL.md" + skill.write_text("# vulnhunt\n") + monkeypatch.setattr("agent.engines.hermes._HERMES_SKILL_CANDIDATES", (skill,)) + captured: list = [] + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _capturing_exec(captured)) + spec = _spec(tmp_path, "hermes") + spec.clone_dir.mkdir(parents=True) + results = await HermesEngine().run_scan(spec) + + cmd = captured[0][0] + assert cmd[0] == "/fake/hermes" + assert cmd[1:5] == ("chat", "-Q", "-s", "vulnhunt") + assert "-t" in cmd and cmd[cmd.index("-t") + 1] == "file,delegation" + assert cmd[cmd.index("-m") + 1] == "claude-opus-4-8" + assert cmd[-2] == "-q" + assert "/vulnhunt" in cmd[-1] + assert "VULNHUNT_DIR" in cmd[-1] + assert "read-only" in cmd[-1] + assert 'action="list"' in cmd[-1] # delegation wait protocol + assert results is not None and "_VULNHUNT_RESULTS_" in results.name + + async def test_command_construction_bash_and_provider(self, tmp_path, monkeypatch): + skill = tmp_path / "SKILL.md" + skill.write_text("# vulnhunt\n") + monkeypatch.setattr("agent.engines.hermes._HERMES_SKILL_CANDIDATES", (skill,)) + captured: list = [] + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _capturing_exec(captured)) + spec = _spec( + tmp_path, + "hermes", + read_only=False, + enable_bash=True, + engine_provider="anthropic", + engine_extra_args=["--reasoning=high"], + ) + spec.clone_dir.mkdir(parents=True) + await HermesEngine().run_scan(spec) + + cmd = captured[0][0] + assert cmd[cmd.index("-t") + 1] == "file,terminal,delegation" + assert cmd[cmd.index("--provider") + 1] == "anthropic" + assert "--reasoning=high" in cmd + + +class TestCopilotCommand: + async def test_kickoff_points_at_skill_and_metadata(self, tmp_path, monkeypatch): + skill = tmp_path / "SKILL.md" + skill.write_text("# vulnhunt\n") + monkeypatch.setattr("agent.engines.copilot._COPILOT_SKILL", skill) + captured: list = [] + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _capturing_exec(captured)) + spec = _spec(tmp_path, "copilot", engine_extra_args=["--allow-tool", "write"]) + spec.clone_dir.mkdir(parents=True) + results = await CopilotCliEngine().run_scan(spec) + + cmd = captured[0][0] + assert cmd[0] == "/fake/copilot" + assert cmd[1] == "-p" + prompt = cmd[2] + assert str(skill) in prompt + assert "VULNHUNT_DIR" in prompt + assert "opus48" in prompt # model tag derived + assert "--allow-tool" in cmd and "write" in cmd + assert results is not None and "_VULNHUNT_RESULTS_" in results.name + + +class TestCodexCommand: + async def test_command_and_kickoff(self, tmp_path, monkeypatch): + skill = tmp_path / "SKILL.md" + skill.write_text("# vulnhunt\n") + monkeypatch.setattr("agent.engines.codex._CODEX_SKILL", skill) + captured: list = [] + monkeypatch.setattr(_SUBPROC.asyncio, "create_subprocess_exec", _capturing_exec(captured)) + spec = _spec(tmp_path, "codex", engine_extra_args=["--skip-git-repo-check"]) + spec.clone_dir.mkdir(parents=True) + results = await CodexEngine().run_scan(spec) + + cmd = captured[0][0] + assert cmd[0] == "/fake/codex" + assert cmd[1] == "exec" + assert cmd[cmd.index("-C") + 1] == str(spec.clone_dir) + assert cmd[cmd.index("-s") + 1] == "workspace-write" + assert cmd[cmd.index("-m") + 1] == "claude-opus-4-8" + assert "--skip-git-repo-check" in cmd + prompt = cmd[-1] + assert str(skill) in prompt + assert "VULNHUNT_DIR" in prompt + assert "read-only" in prompt + assert results is not None and "_VULNHUNT_RESULTS_" in results.name