From 10eb69503bcac1530839025bf155f730cf68f65b Mon Sep 17 00:00:00 2001 From: Amadeusz Sadowski Date: Mon, 13 Jul 2026 20:17:31 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20M3=20=E2=80=94=20executable=20bug=20rep?= =?UTF-8?q?orts,=20per-engine=20evaluation,=20fail-on-broke?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tree-copy of the M3 squash (8b728e8, PR #5) onto main — PR #5 was inadvertently based on feat/m0-m2 (the repo's then-default branch). Content-identical to the reviewed and CI-green M3 branch head 22ddd18. Closes #4. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KcYK8hmTmXi8eLWpg9LGKJ --- .github/workflows/report-check.yml | 186 ++ Muster.slnx | 1 + action.yml | 16 + docs/authoring-fixtures.md | 5 + docs/executable-bug-reports.md | 193 ++ ...-07-13-muster-m3-executable-bug-reports.md | 2804 +++++++++++++++++ ...muster-m3-executable-bug-reports-design.md | 153 + entrypoint.sh | 268 +- kit/callers/muster-report.yml | 14 + kit/issue-form/report-a-data-bug.yml | 45 + lib/wham | 2 +- src/Muster.Cli/Commands/ConvertCommand.cs | 114 + src/Muster.Cli/Commands/DiffCommand.cs | 56 +- src/Muster.Cli/Commands/PromoteCommand.cs | 159 + src/Muster.Cli/Commands/ReportCommand.cs | 273 ++ src/Muster.Cli/Commands/TestCommand.cs | 89 +- src/Muster.Cli/Converters/NrListParser.cs | 154 + src/Muster.Cli/Converters/ReplayRoster.cs | 20 + .../Converters/RosterFileConverter.cs | 147 + src/Muster.Cli/Converters/SpecEmitter.cs | 175 + src/Muster.Cli/Engines/EngineRegistry.cs | 125 + src/Muster.Cli/Engines/EngineSpec.cs | 39 + src/Muster.Cli/Muster.Cli.csproj | 7 + src/Muster.Cli/NewRecruit/NrClient.cs | 72 + src/Muster.Cli/NewRecruit/NrShareLink.cs | 19 + src/Muster.Cli/Program.cs | 3 + src/Muster.Cli/Reporting/BlastRadius.cs | 107 + src/Muster.Cli/Reporting/MultiRunReport.cs | 77 + src/Muster.Cli/Reports/AttachmentClient.cs | 63 + src/Muster.Cli/Reports/IssueBody.cs | 97 + src/Muster.Cli/Reports/ReplyRenderer.cs | 195 ++ src/Muster.Cli/Reports/SnapshotExtractor.cs | 97 + src/Muster.Cli/Reports/SpecRePinner.cs | 130 + src/Muster.Cli/Reports/Verdict.cs | 72 + tests/Muster.Cli.Tests/BlastRadiusTests.cs | 109 + tests/Muster.Cli.Tests/ConvertCommandTests.cs | 57 + .../Converters/NrListParserTests.cs | 97 + .../Converters/RosterFileConverterTests.cs | 171 + .../Converters/SpecEmitterTests.cs | 147 + tests/Muster.Cli.Tests/DiffCommandTests.cs | 95 +- .../Engines/EngineRegistryTests.cs | 64 + .../Fakes/FakeRosterEngine.cs | 76 + .../Fakes/FakeRosterEngineTests.cs | 24 + .../MultiEngineTestCommandTests.cs | 53 + .../Muster.Cli.Tests/Muster.Cli.Tests.csproj | 10 + .../NewRecruit/NrClientTests.cs | 149 + .../NewRecruit/NrShareLinkTests.cs | 24 + .../Muster.Cli.Tests/ProgramExitCodeTests.cs | 7 + tests/Muster.Cli.Tests/PromoteCommandTests.cs | 142 + .../RepoDataSourceResolverTests.cs | 62 +- tests/Muster.Cli.Tests/ReportCommandTests.cs | 286 ++ .../Reports/AttachmentClientTests.cs | 175 + .../Reports/IssueBodyTests.cs | 110 + .../Reports/PromoteChainTests.cs | 73 + .../Reports/ReplyRendererTests.cs | 49 + .../Reports/SnapshotExtractorTests.cs | 56 + .../Reports/SpecRePinnerTests.cs | 97 + .../Muster.Cli.Tests/Reports/VerdictTests.cs | 92 + tests/Muster.Cli.Tests/TestCommandTests.cs | 16 +- .../TestData/nr-list-war-horde.json | 1030 ++++++ tests/Muster.Cli.Tests/TestPaths.cs | 33 + tests/Muster.Cli.Tests/TestRepoFactory.cs | 7 +- .../Muster.TestAdapter.csproj | 11 + tests/Muster.TestAdapter/Program.cs | 12 + 64 files changed, 9173 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/report-check.yml create mode 100644 docs/executable-bug-reports.md create mode 100644 docs/superpowers/plans/2026-07-13-muster-m3-executable-bug-reports.md create mode 100644 docs/superpowers/specs/2026-07-13-muster-m3-executable-bug-reports-design.md create mode 100644 kit/callers/muster-report.yml create mode 100644 kit/issue-form/report-a-data-bug.yml create mode 100644 src/Muster.Cli/Commands/ConvertCommand.cs create mode 100644 src/Muster.Cli/Commands/PromoteCommand.cs create mode 100644 src/Muster.Cli/Commands/ReportCommand.cs create mode 100644 src/Muster.Cli/Converters/NrListParser.cs create mode 100644 src/Muster.Cli/Converters/ReplayRoster.cs create mode 100644 src/Muster.Cli/Converters/RosterFileConverter.cs create mode 100644 src/Muster.Cli/Converters/SpecEmitter.cs create mode 100644 src/Muster.Cli/Engines/EngineRegistry.cs create mode 100644 src/Muster.Cli/Engines/EngineSpec.cs create mode 100644 src/Muster.Cli/NewRecruit/NrClient.cs create mode 100644 src/Muster.Cli/NewRecruit/NrShareLink.cs create mode 100644 src/Muster.Cli/Reporting/MultiRunReport.cs create mode 100644 src/Muster.Cli/Reports/AttachmentClient.cs create mode 100644 src/Muster.Cli/Reports/IssueBody.cs create mode 100644 src/Muster.Cli/Reports/ReplyRenderer.cs create mode 100644 src/Muster.Cli/Reports/SnapshotExtractor.cs create mode 100644 src/Muster.Cli/Reports/SpecRePinner.cs create mode 100644 src/Muster.Cli/Reports/Verdict.cs create mode 100644 tests/Muster.Cli.Tests/ConvertCommandTests.cs create mode 100644 tests/Muster.Cli.Tests/Converters/NrListParserTests.cs create mode 100644 tests/Muster.Cli.Tests/Converters/RosterFileConverterTests.cs create mode 100644 tests/Muster.Cli.Tests/Converters/SpecEmitterTests.cs create mode 100644 tests/Muster.Cli.Tests/Engines/EngineRegistryTests.cs create mode 100644 tests/Muster.Cli.Tests/Fakes/FakeRosterEngine.cs create mode 100644 tests/Muster.Cli.Tests/Fakes/FakeRosterEngineTests.cs create mode 100644 tests/Muster.Cli.Tests/MultiEngineTestCommandTests.cs create mode 100644 tests/Muster.Cli.Tests/NewRecruit/NrClientTests.cs create mode 100644 tests/Muster.Cli.Tests/NewRecruit/NrShareLinkTests.cs create mode 100644 tests/Muster.Cli.Tests/PromoteCommandTests.cs create mode 100644 tests/Muster.Cli.Tests/ReportCommandTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/AttachmentClientTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/IssueBodyTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/PromoteChainTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/ReplyRendererTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/SnapshotExtractorTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/SpecRePinnerTests.cs create mode 100644 tests/Muster.Cli.Tests/Reports/VerdictTests.cs create mode 100644 tests/Muster.Cli.Tests/TestData/nr-list-war-horde.json create mode 100644 tests/Muster.Cli.Tests/TestPaths.cs create mode 100644 tests/Muster.TestAdapter/Muster.TestAdapter.csproj create mode 100644 tests/Muster.TestAdapter/Program.cs diff --git a/.github/workflows/report-check.yml b/.github/workflows/report-check.yml new file mode 100644 index 0000000..9c2c92b --- /dev/null +++ b/.github/workflows/report-check.yml @@ -0,0 +1,186 @@ +name: Muster report check + +# Reusable workflow (workflow_call). Data repos install this via the thin caller stub in +# kit/callers/muster-report.yml. +# +# IMPLEMENTATION NOTE (deviation from the original M3 plan draft): the muster image +# (ghcr.io/warhub/muster:latest, see Dockerfile) is dotnet-runtime + git ONLY — it has no +# `gh` CLI and no `python3`. An earlier draft of this workflow ran every step, including +# `gh`/label/PR steps, inside `container: ghcr.io/warhub/muster:latest`, which cannot work. +# Both jobs below instead run as plain `runs-on: ubuntu-latest` (gh CLI + jq preinstalled, +# GH_TOKEN available) and invoke the muster image explicitly via `docker run`, mounting the +# checkout the same way the docker-action (action.yml) itself does. Only the muster +# invocation itself runs inside the container; every gh/label/PR step runs on the runner. + +on: + workflow_call: + inputs: + data-path: + type: string + default: "." + data-source: + type: string + description: dataSource URI matching the repo's fixtures (e.g. github:BSData/wh40k-11e) + required: true + engines: + type: string + default: "wham" + governing: + type: string + default: "newrecruit battlescribe wham" + fixtures-out: + type: string + default: "tests/rosters" + secrets: + pr-token: + description: Token used to open the promotion PR. Pass a PAT or GitHub App installation token so the PR triggers workflows; defaults to github.token (PR checks will NOT auto-run). + required: false + +permissions: + issues: write + contents: write + pull-requests: write + +jobs: + evaluate: + if: > + github.event_name == 'issues' || + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/muster check')) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Save issue body + env: + BODY: ${{ github.event.issue.body }} + run: | + mkdir -p .muster + printf '%s' "$BODY" > .muster/issue-body.md + + - name: Find previous reply + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.issue.number }} + body-includes: "" + + - name: Save previous reply body + # Sticky comment (Post reply, below) is posted with edit-mode: replace on this single + # comment. Handing the previous body to `muster report` (via --previous-reply) lets it + # carry the durable snapshot forward when this evaluation produces none of its own — + # otherwise a rotted NR link / re-edit would silently destroy the only stored snapshot, + # making promotion permanently impossible. Only written when a previous comment exists; + # entrypoint.sh treats a missing file as "no previous reply". + if: steps.fc.outputs.comment-id != '' + env: + PREV: ${{ steps.fc.outputs.comment-body }} + run: | + mkdir -p .muster + printf '%s' "$PREV" > .muster/previous-reply.md + + - name: Evaluate report + run: | + docker run --rm \ + -v "$PWD:/workspace" -w /workspace \ + --entrypoint /entrypoint.sh \ + ghcr.io/warhub/muster:latest \ + report "${{ inputs.data-path }}" ".muster/issue-body.md" \ + "${{ inputs.data-source }}" "${{ inputs.engines }}" "${{ inputs.governing }}" . \ + .muster/previous-reply.md + + - name: Notify reporter of harness error + # Evaluate report failing means the harness crashed outright (exit 1 from + # entrypoint.sh's report mode) -- silence otherwise, since no reply.md was ever + # produced for the sticky-comment step below to post. Plain issue comment, NEVER the + # sticky comment (that would need a real verdict to replace it + # with, which is exactly what we don't have here). + if: failure() + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue comment "${{ github.event.issue.number }}" -R "$GITHUB_REPOSITORY" \ + -b "⚠ The muster harness hit an internal error evaluating this report. Maintainers have been notified — no action needed from you." + + - name: Post reply + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.issue.number }} + body-path: reply.md + edit-mode: replace + + - name: Apply labels + env: + GH_TOKEN: ${{ github.token }} + ISSUE: ${{ github.event.issue.number }} + run: | + for label in confirmed not-reproducible needs-info inconclusive engine-gap; do + color=$(case "$label" in + confirmed) echo d73a4a ;; + not-reproducible) echo 0e8a16 ;; + needs-info) echo fbca04 ;; + inconclusive) echo d4c5f9 ;; + engine-gap) echo 5319e7 ;; + esac) + gh label create "$label" --force --color "$color" -R "$GITHUB_REPOSITORY" || true + done + + labels=$(jq -r '.labels | join(",")' report.json) + current=$(gh issue view "$ISSUE" -R "$GITHUB_REPOSITORY" --json labels \ + -q '[.labels[].name] | map(select(. == "confirmed" or . == "not-reproducible" or . == "needs-info" or . == "inconclusive" or . == "engine-gap")) | join(",")') + [ -n "$current" ] && gh issue edit "$ISSUE" -R "$GITHUB_REPOSITORY" --remove-label "$current" || true + [ -n "$labels" ] && gh issue edit "$ISSUE" -R "$GITHUB_REPOSITORY" --add-label "$labels" || true + + promote: + if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/muster promote') + runs-on: ubuntu-latest + steps: + - name: Check commenter permission + env: + GH_TOKEN: ${{ github.token }} + run: | + perm=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/${{ github.event.comment.user.login }}/permission" -q .permission) + case "$perm" in + admin|write|maintain) ;; + *) + gh issue comment "${{ github.event.issue.number }}" -R "$GITHUB_REPOSITORY" \ + -b "Sorry @${{ github.event.comment.user.login }}, promotion needs write access." + exit 1 + ;; + esac + + - uses: actions/checkout@v4 + + - name: Fetch comments and issue body + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .muster + gh api "repos/$GITHUB_REPOSITORY/issues/${{ github.event.issue.number }}/comments" --paginate > .muster/comments.json + gh api "repos/$GITHUB_REPOSITORY/issues/${{ github.event.issue.number }}" -q .body > .muster/issue-body.md + + - name: Promote + run: | + docker run --rm \ + -v "$PWD:/workspace" -w /workspace \ + --entrypoint /entrypoint.sh \ + ghcr.io/warhub/muster:latest \ + promote "${{ inputs.data-path }}" ".muster/issue-body.md" ".muster/comments.json" \ + "${{ inputs.data-source }}" "${{ github.event.issue.number }}" "${{ inputs.fixtures-out }}" \ + "${{ inputs.engines }}" "${{ inputs.governing }}" + + - name: Open PR + env: + GH_TOKEN: ${{ secrets.pr-token || github.token }} + ISSUE: ${{ github.event.issue.number }} + run: | + git config user.name "muster-bot" + git config user.email "muster@users.noreply.github.com" + branch="muster/promote-issue-$ISSUE" + git checkout -b "$branch" + git add "${{ inputs.fixtures-out }}" + git commit -m "test: promote issue #$ISSUE roster to golden fixture" + git push origin "$branch" + gh pr create -R "$GITHUB_REPOSITORY" --head "$branch" \ + --title "Promote issue #$ISSUE roster to golden fixture" \ + --body "Promotes the reproducing roster from #$ISSUE to ${{ inputs.fixtures-out }}/. Review the pinned expected values — they were captured from the current (post-fix) engine evaluation. Closes #$ISSUE." diff --git a/Muster.slnx b/Muster.slnx index 2376de0..3e59f8c 100644 --- a/Muster.slnx +++ b/Muster.slnx @@ -4,5 +4,6 @@ + diff --git a/action.yml b/action.yml index da47bfc..ef21adc 100644 --- a/action.yml +++ b/action.yml @@ -10,6 +10,18 @@ inputs: base-ref: description: "Base git ref for blast-radius diff (empty = plain test)" default: "" + fail-on-broke: + description: Fail the check when the governing engine classifies any fixture as broke/verdict-changed (diff mode). + default: "true" + fail-on-inconclusive: + description: Fail the check (exit 1) instead of neutral warning when the run is inconclusive. + default: "false" + engines: + description: Space-separated engine specs (e.g. "wham newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest"). + default: "wham" + governing: + description: Space-separated governing precedence. + default: "newrecruit battlescribe wham" outputs: report-path: description: "Path to muster-report.md" @@ -20,3 +32,7 @@ runs: - "${{ inputs.data-path }}" - "${{ inputs.fixtures-path }}" - "${{ inputs.base-ref }}" + - "${{ inputs.fail-on-broke }}" + - "${{ inputs.fail-on-inconclusive }}" + - "${{ inputs.engines }}" + - "${{ inputs.governing }}" diff --git a/docs/authoring-fixtures.md b/docs/authoring-fixtures.md index f9e7a1a..f59d62d 100644 --- a/docs/authoring-fixtures.md +++ b/docs/authoring-fixtures.md @@ -1,5 +1,10 @@ # Authoring golden-roster fixtures +> Fixtures aren't only hand-authored — `/muster promote` on a confirmed bug report writes +> one for you, pinned to the engine's current values. See +> [`docs/executable-bug-reports.md`](executable-bug-reports.md) for the issue-form → report +> → promote flow that produces regression fixtures from real bug reports. + A **fixture** is a battlescribe-spec roster DSL YAML file that drives wham's roster engine against real (or inline) game data and asserts on the resulting roster state. `muster test` discovers every `*.yaml` file under a diff --git a/docs/executable-bug-reports.md b/docs/executable-bug-reports.md new file mode 100644 index 0000000..e85bca0 --- /dev/null +++ b/docs/executable-bug-reports.md @@ -0,0 +1,193 @@ +# Executable bug reports + +Muster's second install turns a data repo's bug reports into reproductions and, once fixed, +into permanent regression fixtures. A reporter files an issue (or New Recruit auto-files one +for them); muster evaluates the reported roster against current data with every configured +engine, replies with a verdict and a per-engine matrix, and labels the issue. Once a +maintainer fixes the underlying data, `/muster promote` turns the same report into a golden +fixture under `tests/rosters/` — a bug becomes a test, and the correctness review happens +exactly once, on that PR. + +This is a second, separate install from the [blast-radius CI check](authoring-fixtures.md) +(`action.yml` / `muster test` / `muster diff`) — you can use one without the other, but they +share the same fixture format, so most repos will want both. + +## Install (2 files) + +Copy these two files from [WarHub/muster](https://github.com/WarHub/muster) into your data +repo, no forking or vendoring required: + +1. **`kit/issue-form/report-a-data-bug.yml`** → `.github/ISSUE_TEMPLATE/report-a-data-bug.yml` + + The issue form reporters fill in. Its `Roster`/`Problem`/`Expected` field labels are + load-bearing — muster's body parser (`IssueBody.Parse`) matches GitHub's rendered + `### Problem` / `### Expected` headings by exact label text. If you customize the form, + keep those three field labels as-is (or don't rename `problem`/`expected`'s **label** + text; the field `id`s can change freely). + +2. **`kit/callers/muster-report.yml`** → `.github/workflows/muster-report.yml` + + A thin caller that wires GitHub's `issues`/`issue_comment` events to muster's reusable + workflow (`WarHub/muster/.github/workflows/report-check.yml@main`) and sets your repo's + `data-source`: + + ```yaml + jobs: + report: + uses: WarHub/muster/.github/workflows/report-check.yml@main + with: + data-source: "github:YourOrg/your-data-repo" # ← your org/repo[@ref] + ``` + + `data-source` must match what your fixtures declare under `setup.dataSource` (see + [authoring-fixtures.md](authoring-fixtures.md)) — it seeds the same + `github/{org}/{repo}/{ref-or-latest}/` cache layout `muster test`/`muster diff` use, built + fresh from your repo's own checkout each run (never a live clone). + +No secrets to configure: the reusable workflow uses the ambient `GITHUB_TOKEN` +(`github.token`) for comments, labels, and PRs. The five labels +(`confirmed`/`not-reproducible`/`needs-info`/`inconclusive`/`engine-gap`) are created +idempotently on first run — nothing to pre-provision. + +## Roster formats + +The issue form (and NR's own auto-filed reports) accept a roster in any of three shapes; +muster's body parser tries them in this order and uses the first match: + +| Format | How it's recognized | Notes | +|---|---|---| +| **New Recruit share link** | `https://www.newrecruit.eu/app/list/` anywhere in the body | Fetched live (undocumented NR RPC, allowlisted to this exact host/path — any failure degrades to `needs-info`, never a crash). NR only exposes roster-level `totalCosts`, not per-selection costs. Link rot is expected — once evaluated, the reply's snapshot is the durable copy. | +| **`.ros`/`.rosz` attachment** | a GitHub `user-attachments`/legacy `owner/repo/files` URL ending in `.ros`/`.rosz`/`.zip` | Richer than NR: stores per-selection costs, not just roster totals. GitHub sometimes forces a `.zip` rename on upload — the issue form's help text tells reporters to rename it back. | +| **Inline fixture-DSL YAML** | a fenced `` ```yaml `` block containing a `steps:` list | For power users / data authors. The pasted YAML *is* the spec — no conversion or value-pinning step; author it exactly like a fixture in `tests/rosters/` (see [authoring-fixtures.md](authoring-fixtures.md)), including `id`/`setup.dataSource`/assertions. | + +Whichever format is used, muster converts (or, for inline YAML, validates) it into a fixture-DSL +spec whose assertions **pin the observed values** — what the reporter's app showed — and +evaluates that spec against current data. If any node can't be mapped (unknown id, unsupported +structure), the verdict is always `needs-info`, naming the unmapped id — a partial replay is +never allowed to produce `confirmed`/`not-reproducible`. + +## Verdict and label semantics + +One label from the first group is always applied; `engine-gap` is added alongside it when the +engines that ran disagree with each other: + +| Label | Verdict | Meaning | +|---|---|---| +| `confirmed` | Confirmed | The governing engine reproduces the reported values against current data — the bug is real (or still present). | +| `not-reproducible` | Not reproducible | The governing engine computes different values than reported — often means "already fixed"; the reply shows the exact per-engine diff. | +| `needs-info` | Needs info | Unusable input: no roster found, a bad/rotted link, or a roster that couldn't be fully mapped to current data. Polite reply, never a stack trace. | +| `inconclusive` | Inconclusive | Every requested engine was unavailable (couldn't start) — never silently treated as any other verdict. | +| `engine-gap` | *(additive)* | The engines that ran disagree with each other on the asserted values, independent of the governing verdict — surfaced for the wham conformance backlog, not itself a pass/fail signal. | + +The reply also names which engines ran, which one governed the verdict, and which requested +engines were unavailable — and embeds the generated spec in a collapsed +`
Executable spec (snapshot)` block (marked internally with +``). That snapshot is what `/muster promote` reads back later. + +## Commands + +Both are issue *comments*; the reusable workflow's `evaluate`/`promote` jobs match them by +substring, so exact comment text (`/muster check`, `/muster promote`) matters, not exact +match. + +- **`/muster check`** — anyone can comment this to force a re-evaluation (e.g. after data or + the report itself changed). Same flow as the initial issue-opened/edited trigger: re-parses + the current issue body, re-runs every engine, updates the sticky reply comment + (``) and labels in place. + +- **`/muster promote`** — **write-access gated**: the workflow checks the commenter's + permission via the collaborators API (`admin`/`write`/`maintain` only) before doing + anything; anyone else gets a polite refusal comment and the job fails closed. On success it: + 1. re-reads the newest `` block from the issue's own comments, + 2. re-evaluates it against current data with the governing engine, + 3. re-pins the spec's assertions to the engine's **current** (post-fix) values, + 4. opens a PR (`muster/promote-issue-`) adding `tests/rosters/report-issue-.yaml` + (suffixed `-2`, `-3`, … on a slug collision) and linking `Closes #`. + + Human correctness review happens exactly once, on that PR — reviewing the pinned expected + values is reviewing "is this actually correct now," same as any fixture. + +## Promotion PRs and CI checks + +By default, the promotion PR is opened with `GITHUB_TOKEN` and GitHub will not run workflows on it +(a documented GitHub Actions limitation). If you want the promotion PR to automatically trigger +your CI checks, provide a PAT (personal access token) or GitHub App installation token as the +optional `pr-token` secret: + +```yaml +jobs: + report: + uses: WarHub/muster/.github/workflows/report-check.yml@main + with: + data-source: "github:YourOrg/your-data-repo" + secrets: + pr-token: ${{ secrets.MUSTER_PR_TOKEN }} +``` + +Without this token, you can still run workflows manually by closing and re-opening the PR or by +pushing to its branch. + +## Engines and governing precedence + +Both the `evaluate` and `promote` jobs accept `engines`/`governing` inputs on the reusable +workflow (same registry and syntax as `muster test`/`muster diff` — see +[authoring-fixtures.md](authoring-fixtures.md) and the CLI's `--engines`/`--governing` +help text): + +```yaml +jobs: + report: + uses: WarHub/muster/.github/workflows/report-check.yml@main + with: + data-source: "github:YourOrg/your-data-repo" + engines: "wham newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest" + governing: "newrecruit battlescribe wham" # default; first-in-precedence engine that ran governs + fixtures-out: "tests/rosters" # default; where /muster promote writes fixtures +``` + +- **`wham`** — builtin, in-process, always available; the only engine guaranteed in the + public `ghcr.io/warhub/muster` image. Registered by name alone (no `=command`). +- **`name=dotnet:`**, **`name=exe [args]`**, **`name=docker:`** — any other + engine is an out-of-process adapter speaking the NDJSON protocol over stdio, spawned per + run. `docker:` runs `docker run -i --rm ` under the hood. +- **`governing`** picks the single engine whose result is authoritative for the + `confirmed`/`not-reproducible` verdict: the first name in the `governing` list that's + actually among the engines that ran. Default precedence is `newrecruit battlescribe wham` — + New Recruit (what reporters and players actually use today) governs when available, the + legacy BattleScribe oracle is the fallback reference, and wham (not yet fully spec-compliant) + governs only as a last resort — the reply says explicitly which engine governed. +- Requested-but-unavailable engines are never silently dropped — they're named `unavailable` + in the reply, and if *no* requested engine is available the verdict is `inconclusive` + (abstain, don't lie). + +### Registering the New Recruit adapter (default governor) + +To get the default `newrecruit`-governs behavior outside WarHub-hosted workflows, register +the public adapter image as a Docker engine: + +```yaml +engines: "wham newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest" +``` + +This image ships Playwright + a browser baked in (published by battlescribe-spec's CI) — no +extra setup beyond the `engines` input above. It has no proprietary-JAR encumbrance, unlike +the legacy `battlescribe` oracle (IKVM-wrapped proprietary BattleScribe JARs), which is only +available where a private `battlescribe-spec` checkout exists (WarHub-hosted workflows via the +bot app token, or maintainers with direct access) — it can never ship as a public image, and +non-WarHub repos simply won't have it in their `engines` list. + +## Implementation note: why the reusable workflow doesn't run inside the muster container + +`ghcr.io/warhub/muster:latest` (see `Dockerfile`) is `dotnet/runtime` + `git` only — no `gh` +CLI, no `python3`/`jq`. `.github/workflows/report-check.yml`'s `evaluate` and `promote` jobs +therefore run on plain `ubuntu-latest` runners (where `gh` and `jq` are preinstalled and +`GITHUB_TOKEN` is ambient) and invoke the muster image explicitly for the `report`/`promote` +steps only, mounting the checkout the same way the container-action (`action.yml`) does: + +```bash +docker run --rm -v "$PWD:/workspace" -w /workspace \ + --entrypoint /entrypoint.sh ghcr.io/warhub/muster:latest \ + report +``` + +Every `gh`/label/PR/sticky-comment step runs on the runner itself, outside the container. diff --git a/docs/superpowers/plans/2026-07-13-muster-m3-executable-bug-reports.md b/docs/superpowers/plans/2026-07-13-muster-m3-executable-bug-reports.md new file mode 100644 index 0000000..48dda2f --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-muster-m3-executable-bug-reports.md @@ -0,0 +1,2804 @@ +# Muster M3 — Executable Bug Reports Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bug-report rosters (NR share links, `.ros`/`.rosz`, inline YAML) become executable specs evaluated per-engine with a configurable governing engine; issues get auto-labeled verdict replies with durable snapshots; maintainers promote fixed bugs to golden fixtures via `/muster promote`; `muster diff` gains `--fail-on-broke` (muster#4). + +**Architecture:** Everything becomes a spec — converters emit fixture-DSL YAML with assertions pinning the reporter's *observed* values; the existing `RosterRunner` pipeline executes them per engine via an engine registry (builtin wham in-proc; external adapters over the NDJSON protocol via `AdapterProcess`+`JsonProtocolEngine`, including `docker:` commands). One engine *governs* each verdict (default precedence `newrecruit > battlescribe > wham`); engine disagreement raises `engine-gap`. + +**Tech Stack:** .NET 10 (SDK 10.0.100), System.CommandLine 2.0.3, xunit.v3 3.2.2, System.Text.Json (BCL), TestKit (`BattleScribeSpec.TestKit` project ref), wham (`WarHub.ArmouryModel.*` project refs). No new package references in Muster.Cli. + +**Spec:** `docs/superpowers/specs/2026-07-13-muster-m3-executable-bug-reports-design.md` (approved 2026-07-13). + +## Global Constraints + +- Exit-code discipline: **0** = evaluated/reply produced, **1** = genuine fixture failure (test) or governing-engine broke with `--fail-on-broke` (diff), **2** = harness error/usage error/inconclusive. A crash must NEVER surface as exit 1. +- `TreatWarningsAsErrors=true` — **always verify `dotnet build -c Release`**, not just Debug; CA analyzers are Release-enforced. +- Hermeticity: the `RepoDataSourceResolver.IsPopulatedFor` gate stays in front of every fixture evaluation, all engines. The ONLY permitted network calls are the NR share-link fetch and GitHub attachment downloads, both behind `UrlAllowlist`. +- Hostile input: issue bodies are stranger-controlled. Caps (verbatim from spec §6): fetch timeout 30 s, response size cap 5 MB, JSON depth cap 64, node-count cap 20 000, roster selection-count cap 5 000. Exceeding any → `needs-info`, never a crash. No stack traces in replies. +- Licensing (hard): nothing in muster's public artifacts may embed or download the proprietary BattleScribe JARs. The public Docker image ships the wham engine only. +- Engine names are exactly `wham`, `battlescribe`, `newrecruit` (lowercase). Default governing precedence exactly `newrecruit > battlescribe > wham`. +- Verdict labels are exactly: `confirmed`, `not-reproducible`, `needs-info`, `inconclusive`, `engine-gap`. +- Blast-radius classification strings stay exactly: `unchanged`, `broke`, `fixed`, `still-failing`, `verdict-changed`, `inconclusive`. +- String comparisons: `StringComparer.Ordinal` / `StringComparison.Ordinal` unless matching user text. +- External repos: all battlescribe-spec changes go through the nested submodule at `lib/wham/lib/battlescribe-spec` on branch `feat/muster-support`, bumped through wham branch `feat/yaml-reader`; PRs opened at the end. Never touch `D:\repos\battlescribe-spec` (separate checkout with uncommitted user work). +- WarHub org ruleset requires PRs for ALL branch updates — never push directly to `main`; each muster CI iteration needs a fresh branch + PR. +- Markdown comment markers: report reply ``, snapshot block `` — exact strings, used for sticky-comment lookup and snapshot extraction. + +## Reference: key existing interfaces (verified 2026-07-13) + +- `TestCommand.RunFixtures(string dataDir, string fixturesDir)` → `RunReport` — shared machinery (`src/Muster.Cli/Commands/TestCommand.cs:88`); constructs `new SpecRosterEngineAdapter()` per fixture, `new RosterRunner(engine, resolver, engineName: "wham")`. +- `RosterRunner(IRosterEngine engine, DataSourceResolver? resolver = null, string? engineName = null)`; `SpecResult Run(SpecFile spec)`; public `Action>? OnStepCompleted`. Action step exceptions set `SpecResult.HarnessError`; assertion mismatches only append `Failures` (failure strings start `"Step {index}: …"`). +- `SpecResult(string SpecId, string Category, string Description, IReadOnlyList Failures)` + `string? HarnessError { get; init; }`; `Passed => Failures.Count == 0`. +- `SpecLoader.Load(string path)`, `SpecLoader.LoadFromYaml(string yaml, string? defaultId = null)` — CamelCase YamlDotNet binding; **no SpecFile→YAML serializer exists** (emit YAML text by hand, round-trip-validate via `LoadFromYaml`). +- `expectedState.costs` asserts **roster-level totals** (`RosterState.Costs`, aggregated across forces — comparer `RosterRunner.AssertExpectedState`, cost block `RosterRunner.cs:310-347`). Per-selection: `forces[].selections[].costs`. `expectedState.engines.` = per-engine override merged by `ExpectedStateDef.ForEngine(engineName)`. +- DSL step actions + YAML args: `addForce{forceEntryId, catalogueId?}`, `addChildForce{forceId, forceEntryId, catalogueId?}`, `selectEntry{forceId, entryId}` → outputs `selectionId`+`selections` map, `selectChildEntry{forceId, selectionId, entryId}`, `setSelectionCount{forceId, selectionId, count}`, `setCustomization{forceId, selectionId?, categoryEntryId?, customName?, customNotes?}`, `setCostLimit{costTypeId, value}`. Instance-ID fields accept `${{ steps..forceId }}` / `${{ steps..selectionId }}`; definition-ID fields are literal. +- Out-of-proc engines: `AdapterProcess.Start(string executable, string? arguments = null)` + `new JsonProtocolEngine(adapterProcess)` (implements `IRosterEngine`; 30 s default per-command timeout) — both in `BattleScribeSpec.TestKit/Protocol/`. +- `.ros` load: `BattleScribeXml.LoadRoster(string path)` / `(Stream)` (`WarHub.ArmouryModel.Source.BattleScribe`); `.rosz`: `stream.LoadSourceAuto(filename)` (`XmlFileExtensions`, `Workspaces.BattleScribe`) — returns `SourceNode?`, cast to `RosterNode`. +- `RosterNode`: `Name, GameSystemId, Costs (NodeList), CostLimits, Forces`. `ForceNode`: `CatalogueId, Categories, Forces (nested), EntryId, Selections`. `SelectionNode`: `EntryId (composite "linkId::targetId" when via link), EntryGroupId, Number, Type, CustomName, Costs, Selections (nested)`. `CostNode`: `Name, TypeId, Value (decimal)`. +- NR list JSON (sample: `tests/Muster.Cli.Tests/TestData/nr-list-war-horde.json`, fetched live 2026-07-13): top-level `{name, totalCost, totalCosts[{name,value,typeId}], bsid_system, bsid_book, books_revision[], nrversion, army}`. `army.options[]` tree levels: catalogue node (`option_id` = catalogue id, no `catalogue_id` key) → force node (has `catalogue_id`, `option_id` = force entry id) → descendants. Node classifier (verified on all 120 nodes of the sample): **has `amount` ⇒ selection** (count = `amount`, entry id = `link_id::option_id` if `link_id` present else `option_id`); **no `amount` ⇒ transparent container** (category/group — recurse through, don't emit a step). +- NR fetch: `POST https://www.newrecruit.eu/api/rpc`, JSON body `{"method":"open_share_link","params":[""]}`, `Content-Type: application/json` → list JSON, or `null` for missing/rotted lists. +- `DiffCommand.Run` currently ends `return 0;` (`DiffCommand.cs:72`) — the `--fail-on-broke` hook point. +- entrypoint.sh: positional ` [base-ref]`, `MUSTER_CMD` env override, `build_dataroot()` scans fixtures for `dataSource:` decls, exit-2 → `::warning::` + exit 0. + +--- + +### Task 1: Shared FakeEngine + out-of-proc TestAdapter host + +Test infrastructure everything later leans on: promote the private `FakeEngine` to a shared, configurable test double, and add a console host that serves it over the NDJSON protocol so the engine registry's out-of-proc path is testable without Docker or Playwright. + +**Files:** +- Create: `tests/Muster.Cli.Tests/Fakes/FakeRosterEngine.cs` +- Create: `tests/Muster.TestAdapter/Muster.TestAdapter.csproj` +- Create: `tests/Muster.TestAdapter/Program.cs` +- Modify: `tests/Muster.Cli.Tests/RepoDataSourceResolverTests.cs` (replace private nested `FakeEngine` with the shared one) +- Modify: `Muster.slnx` (add Muster.TestAdapter) +- Test: `tests/Muster.Cli.Tests/Fakes/FakeRosterEngineTests.cs` + +**Interfaces:** +- Consumes: `IRosterEngine`, `RosterState`, `CostState`, `ActionOutputs` (TestKit `Roster/`), `AdapterHandler.RunAsync` (TestKit `Protocol/AdapterHandler.cs:16`). +- Produces: `Muster.Cli.Tests.Fakes.FakeRosterEngine : IRosterEngine` — ctor `FakeRosterEngine(decimal ptsValue = 20m)`; tracks `SelectEntry`/`SetSelectionCount` calls; `GetRosterState()` returns a roster whose `Costs` contains `new CostState("pts", "pts", ptsValue * _totalCount)` where `_totalCount` = number of selections × their counts; `ReceivedFiles` property as before. `Muster.TestAdapter` console: env var `FAKE_PTS` (default `20`) controls `ptsValue` — a *divergent* engine is simply the same adapter launched with a different `FAKE_PTS`. + +- [ ] **Step 1: Write the failing test** + +```csharp +// tests/Muster.Cli.Tests/Fakes/FakeRosterEngineTests.cs +using BattleScribeSpec.Roster; +using Muster.Cli.Tests.Fakes; + +namespace Muster.Cli.Tests.Fakes; + +public class FakeRosterEngineTests +{ + [Fact] + public void Costs_scale_with_selection_count_and_pts_value() + { + using var engine = new FakeRosterEngine(ptsValue: 30m); + engine.SetupFromFiles([("a.gst", "")]); + var force = engine.AddForce("fe-1", "cat-1"); + var sel = engine.SelectEntry(force.ForceId!, "se-1"); + engine.SetSelectionCount(force.ForceId!, sel.SelectionId!, 2); + + var state = engine.GetRosterState(); + + var pts = Assert.Single(state.Costs); + Assert.Equal("pts", pts.TypeId); + Assert.Equal(60m, pts.Value); // 30 pts × count 2 + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `dotnet test tests/Muster.Cli.Tests --filter FakeRosterEngineTests -v minimal` +Expected: FAIL — `FakeRosterEngine` does not exist (compile error). + +- [ ] **Step 3: Implement FakeRosterEngine** + +```csharp +// tests/Muster.Cli.Tests/Fakes/FakeRosterEngine.cs +using BattleScribeSpec.Roster; + +namespace Muster.Cli.Tests.Fakes; + +/// +/// Configurable in-proc engine double. Every selection contributes +/// ptsValue × count to a single roster-level "pts" cost, so tests can +/// pin totals and simulate divergent engines by varying ptsValue. +/// +public sealed class FakeRosterEngine(decimal ptsValue = 20m) : IRosterEngine +{ + private readonly List<(string SelectionId, int Count)> _selections = []; + private int _nextId; + + public List<(string FileName, string Content)>? ReceivedFiles { get; private set; } + + public IReadOnlyList Setup(ProtocolGameSystem gameSystem, ProtocolCatalogue[] catalogues) => []; + + public IReadOnlyList SetupFromFiles(IReadOnlyList<(string FileName, string Content)> files) + { + ReceivedFiles = [.. files]; + return []; + } + + public ActionOutputs AddForce(string forceEntryId, string catalogueId) => + new() { ForceId = $"force-{_nextId++}" }; + + public ActionOutputs AddChildForce(string parentForceId, string forceEntryId, string catalogueId) => + new() { ForceId = $"force-{_nextId++}" }; + + public void RemoveForce(string forceId) { } + + public ActionOutputs SelectEntry(string forceId, string entryId) + { + var id = $"sel-{_nextId++}"; + _selections.Add((id, 1)); + return new() { SelectionId = id, Selections = [] }; + } + + public ActionOutputs SelectChildEntry(string forceId, string parentSelectionId, string entryId) => + SelectEntry(forceId, entryId); + + public void DeselectSelection(string forceId, string selectionId) => + _selections.RemoveAll(s => s.SelectionId == selectionId); + + public void SetSelectionCount(string forceId, string selectionId, int count) + { + var i = _selections.FindIndex(s => s.SelectionId == selectionId); + if (i < 0) throw new InvalidOperationException($"unknown selection: {selectionId}"); + _selections[i] = (selectionId, count); + } + + public ActionOutputs DuplicateSelection(string forceId, string selectionId) + { + var id = $"sel-{_nextId++}"; + _selections.Add((id, 1)); + return new() { SelectionId = id }; + } + + public ActionOutputs DuplicateForce(string forceId) => new() { ForceId = $"force-{_nextId++}" }; + + public void SetCostLimit(string costTypeId, decimal value) { } + + public void SetCustomization(string forceId, string? selectionId, string? categoryEntryId, string? customName, string? customNotes) { } + + public RosterState GetRosterState() + { + var total = _selections.Sum(s => s.Count) * ptsValue; + return new RosterState("roster", "gs", [], [new CostState("pts", "pts", total)], []); + } + + public IReadOnlyList GetValidationErrors() => []; + + public void Dispose() { } +} +``` + +Note: `RosterState`'s positional order is `(Name, GameSystemId, Forces, Costs, ValidationErrors)` in the existing nested fake (`RepoDataSourceResolverTests.cs:202` passes `("roster", "gs", [], [], [])`) — check the record definition in `BattleScribeSpec.TestKit/Roster/RosterTypes.cs:17-24` (it has 5+ components incl. `CostLimits`) and match it exactly; adjust the constructor call if `CostLimits` is a sixth positional or init property. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `dotnet test tests/Muster.Cli.Tests --filter FakeRosterEngineTests -v minimal` +Expected: PASS. + +- [ ] **Step 5: Create the TestAdapter console host** + +```xml + + + + Exe + net10.0 + false + + + + + + +``` + +```csharp +// tests/Muster.TestAdapter/Program.cs +// NDJSON adapter host serving FakeRosterEngine — lets tests exercise the +// out-of-proc engine path (AdapterProcess + JsonProtocolEngine) hermetically. +// FAKE_PTS env var sets the per-selection pts value (default 20). +using BattleScribeSpec.Protocol; +using Muster.Cli.Tests.Fakes; + +var pts = decimal.TryParse(Environment.GetEnvironmentVariable("FAKE_PTS"), out var v) ? v : 20m; + +await AdapterHandler.RunAsync( + engineFactory: () => new FakeRosterEngine(pts), + input: Console.In, + output: Console.Out); +``` + +Add the project to `Muster.slnx` (follow the existing `` entries). Replace the private nested `FakeEngine` in `RepoDataSourceResolverTests.cs` with `Fakes.FakeRosterEngine` (delete the nested class; the existing test only needs `ReceivedFiles` + benign action behavior, both preserved). + +- [ ] **Step 6: Verify the whole suite + Release build** + +Run: `dotnet test -v minimal && dotnet build -c Release` +Expected: all tests PASS (including `RepoDataSourceResolverTests`), Release build zero warnings. + +- [ ] **Step 7: Commit** + +```bash +git add tests Muster.slnx +git commit -m "test: shared FakeRosterEngine + NDJSON TestAdapter host" +``` + +--- + +### Task 2: Engine registry + +`EngineSpec` parsing (`name`, `name=dotnet:path`, `name=docker:image`, `name=exe args`), availability probing, and `IRosterEngine` creation for builtin/exec/docker kinds. Governing-precedence resolution. + +**Files:** +- Create: `src/Muster.Cli/Engines/EngineSpec.cs` +- Create: `src/Muster.Cli/Engines/EngineRegistry.cs` +- Test: `tests/Muster.Cli.Tests/Engines/EngineRegistryTests.cs` + +**Interfaces:** +- Consumes: `SpecRosterEngineAdapter` (wham), `AdapterProcess.Start(string, string?)`, `JsonProtocolEngine(AdapterProcess)` (TestKit). +- Produces: + - `sealed record EngineSpec(string Name, EngineKind Kind, string? Executable, string? Arguments)` with `enum EngineKind { Builtin, Exec, Docker }`; `static EngineSpec Parse(string spec)`. + - `static class EngineRegistry`: `IReadOnlyList ParseAll(IReadOnlyList specs)` (defaults to `[EngineSpec.Parse("wham")]` when empty); `bool IsAvailable(EngineSpec spec)`; `IRosterEngine CreateEngine(EngineSpec spec)`; `string? ResolveGoverning(IReadOnlyList precedence, IReadOnlyList ranEngines)` (first precedence entry present in `ranEngines`, ordinal; default precedence constant `DefaultGoverning = ["newrecruit", "battlescribe", "wham"]`). + +- [ ] **Step 1: Write the failing tests** + +```csharp +// tests/Muster.Cli.Tests/Engines/EngineRegistryTests.cs +using Muster.Cli.Engines; + +namespace Muster.Cli.Tests.Engines; + +public class EngineRegistryTests +{ + [Theory] + [InlineData("wham", "wham", EngineKind.Builtin, null, null)] + [InlineData("battlescribe=dotnet:/opt/adapter/Ref.dll", "battlescribe", EngineKind.Exec, "dotnet", "/opt/adapter/Ref.dll")] + [InlineData("newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest", "newrecruit", EngineKind.Docker, "docker", "run -i --rm ghcr.io/warhub/bsspec-adapter-newrecruit:latest")] + [InlineData("custom=/usr/bin/my-adapter --flag", "custom", EngineKind.Exec, "/usr/bin/my-adapter", "--flag")] + public void Parse_handles_all_command_forms(string input, string name, EngineKind kind, string? exe, string? args) + { + var spec = EngineSpec.Parse(input); + Assert.Equal(name, spec.Name); + Assert.Equal(kind, spec.Kind); + Assert.Equal(exe, spec.Executable); + Assert.Equal(args, spec.Arguments); + } + + [Fact] + public void Parse_rejects_builtin_command_for_unknown_name() + { + // bare name with no command is only valid for the builtin engine + Assert.Throws(() => EngineSpec.Parse("newrecruit")); + } + + [Fact] + public void ParseAll_defaults_to_builtin_wham() + { + var engines = EngineRegistry.ParseAll([]); + var e = Assert.Single(engines); + Assert.Equal("wham", e.Name); + Assert.Equal(EngineKind.Builtin, e.Kind); + } + + [Fact] + public void Builtin_wham_is_available_and_creates_adapter() + { + var spec = EngineSpec.Parse("wham"); + Assert.True(EngineRegistry.IsAvailable(spec)); + using var engine = EngineRegistry.CreateEngine(spec); + Assert.NotNull(engine); + } + + [Fact] + public void Missing_executable_is_unavailable_not_a_crash() + { + var spec = EngineSpec.Parse("ghost=/no/such/binary-xyz"); + Assert.False(EngineRegistry.IsAvailable(spec)); + } + + [Fact] + public void ResolveGoverning_picks_first_precedence_entry_that_ran() + { + Assert.Equal("battlescribe", EngineRegistry.ResolveGoverning( + ["newrecruit", "battlescribe", "wham"], ["wham", "battlescribe"])); + Assert.Equal("wham", EngineRegistry.ResolveGoverning( + ["newrecruit", "battlescribe", "wham"], ["wham"])); + Assert.Null(EngineRegistry.ResolveGoverning( + ["newrecruit"], ["wham"])); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter EngineRegistryTests -v minimal` +Expected: FAIL (types don't exist). + +- [ ] **Step 3: Implement** + +```csharp +// src/Muster.Cli/Engines/EngineSpec.cs +namespace Muster.Cli.Engines; + +public enum EngineKind { Builtin, Exec, Docker } + +/// +/// One engine registration: wham (builtin) or +/// name=dotnet:path.dll / name=docker:image / name=exe [args]. +/// +public sealed record EngineSpec(string Name, EngineKind Kind, string? Executable, string? Arguments) +{ + public const string BuiltinName = "wham"; + + public static EngineSpec Parse(string spec) + { + ArgumentException.ThrowIfNullOrWhiteSpace(spec); + var eq = spec.IndexOf('=', StringComparison.Ordinal); + if (eq < 0) + { + if (!string.Equals(spec.Trim(), BuiltinName, StringComparison.Ordinal)) + throw new ArgumentException($"engine '{spec}' has no adapter command; only '{BuiltinName}' is builtin"); + return new(BuiltinName, EngineKind.Builtin, null, null); + } + + var name = spec[..eq].Trim(); + var command = spec[(eq + 1)..].Trim(); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + + if (command.StartsWith("dotnet:", StringComparison.Ordinal)) + return new(name, EngineKind.Exec, "dotnet", command["dotnet:".Length..]); + if (command.StartsWith("docker:", StringComparison.Ordinal)) + return new(name, EngineKind.Docker, "docker", $"run -i --rm {command["docker:".Length..]}"); + + var space = command.IndexOf(' ', StringComparison.Ordinal); + return space < 0 + ? new(name, EngineKind.Exec, command, null) + : new(name, EngineKind.Exec, command[..space], command[(space + 1)..]); + } +} +``` + +```csharp +// src/Muster.Cli/Engines/EngineRegistry.cs +using BattleScribeSpec.Protocol; +using BattleScribeSpec.Roster; +using WarHub.ArmouryModel.RosterEngine.Spec; + +namespace Muster.Cli.Engines; + +public static class EngineRegistry +{ + public static readonly IReadOnlyList DefaultGoverning = ["newrecruit", "battlescribe", "wham"]; + + public static IReadOnlyList ParseAll(IReadOnlyList specs) => + specs.Count == 0 ? [EngineSpec.Parse(EngineSpec.BuiltinName)] : [.. specs.Select(EngineSpec.Parse)]; + + public static bool IsAvailable(EngineSpec spec) => spec.Kind switch + { + EngineKind.Builtin => true, + EngineKind.Docker => CanStart("docker", "--version"), + EngineKind.Exec when string.Equals(spec.Executable, "dotnet", StringComparison.Ordinal) => + spec.Arguments is { } args && File.Exists(FirstToken(args)), + EngineKind.Exec => File.Exists(spec.Executable) || CanStart(spec.Executable!, "--version"), + _ => false, + }; + + public static IRosterEngine CreateEngine(EngineSpec spec) => spec.Kind switch + { + EngineKind.Builtin => new SpecRosterEngineAdapter(), + _ => new JsonProtocolEngine(AdapterProcess.Start(spec.Executable!, spec.Arguments)), + }; + + public static string? ResolveGoverning(IReadOnlyList precedence, IReadOnlyList ranEngines) => + precedence.FirstOrDefault(p => ranEngines.Contains(p, StringComparer.Ordinal)); + + private static string FirstToken(string s) + { + var i = s.IndexOf(' ', StringComparison.Ordinal); + return i < 0 ? s : s[..i]; + } + + private static bool CanStart(string exe, string args) + { + try + { + using var p = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo(exe, args) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + p?.WaitForExit(5000); + return p is { ExitCode: 0 }; + } + catch (Exception e) when (e is System.ComponentModel.Win32Exception or InvalidOperationException or PlatformNotSupportedException) + { + return false; + } + } +} +``` + +Check `JsonProtocolEngine`'s constructor signature in `BattleScribeSpec.TestKit/Protocol/JsonProtocolEngine.cs` (takes `AdapterProcess` + optional timeout); it owns disposal of the process via its own `Dispose` — verify and, if it does not dispose the `AdapterProcess`, wrap both in a small `CompositeDisposableEngine` returned from `CreateEngine`. + +- [ ] **Step 4: Run tests + Release build** + +Run: `dotnet test tests/Muster.Cli.Tests --filter EngineRegistryTests -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 5: Commit** + +```bash +git add src/Muster.Cli/Engines tests/Muster.Cli.Tests/Engines +git commit -m "feat: engine registry — builtin/exec/docker engine specs, governing resolution" +``` + +--- + +### Task 3: Multi-engine `test` — engine-parameterized RunFixtures + MultiRunReport + +`RunFixtures` gains an engine parameter; `test` gets `--engines`/`--governing`, loops engines, renders a per-engine report, and keeps exit discipline (1 on ANY engine's assertion failure — golden fixtures resolve expectations per engine via the DSL's `engines:` blocks). + +**Files:** +- Modify: `src/Muster.Cli/Commands/TestCommand.cs` +- Create: `src/Muster.Cli/Reporting/MultiRunReport.cs` +- Modify: `src/Muster.Cli/Reporting/RunReport.cs` (no shape change; add nothing — writers for multi live in MultiRunReport) +- Test: `tests/Muster.Cli.Tests/MultiEngineTestCommandTests.cs` + +**Interfaces:** +- Consumes: `EngineRegistry`, `EngineSpec` (Task 2); `TestCommand.RunFixtures` internals; `TestRepoFactory.CreateTestRepo()` (existing test helper: green fixture `unit-costs-20.yaml` pinning pts=20, data under `github/muster-e2e/test-data/main/`). +- Produces: + - `TestCommand.RunFixtures(string dataDir, string fixturesDir, EngineSpec engine)` → `RunReport` (existing 2-arg overload delegates with builtin wham; `RunReport.Engine` carries `engine.Name`; the engine name is passed to `RosterRunner`'s `engineName` so DSL `engines:` overrides resolve). + - `sealed record MultiRunReport(string? Governing, IReadOnlyList Unavailable, IReadOnlyList Runs)` with `static MultiRunReport Run(string dataDir, string fixturesDir, IReadOnlyList engineSpecs, IReadOnlyList governing)` (parses, probes availability, runs each available engine, resolves governor), `static string ToJson(MultiRunReport)`, `static void Write(MultiRunReport, string mode, TextWriter)` (`summary` | `json` | `github-actions`; single-run degenerate output identical in spirit to today's). + - Per-fixture-per-engine: reuses `FixtureResult` unchanged inside each engine's `RunReport`. +- Out-of-proc engines: ONE `IRosterEngine` per engine per run (AdapterProcess reused across fixtures; `RosterRunner` calls `Cleanup()` between specs). Builtin wham: fresh `SpecRosterEngineAdapter` per fixture (current behavior — cheap, in-proc). + +- [ ] **Step 1: Write the failing tests** + +```csharp +// tests/Muster.Cli.Tests/MultiEngineTestCommandTests.cs +using Muster.Cli.Commands; +using Muster.Cli.Reporting; + +namespace Muster.Cli.Tests; + +public class MultiEngineTestCommandTests +{ + private static string TestAdapterDll => + // built alongside the test assembly; artifacts output layout + Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, + "..", "..", "..", "..", "Muster.TestAdapter", "debug", "Muster.TestAdapter.dll")); + + [Fact] + public void Two_engines_produce_two_runs_and_governing_resolves_by_precedence() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + + var multi = MultiRunReport.Run(dataDir, fixturesDir, + engineSpecs: ["wham", $"fake=dotnet:{TestAdapterDll}"], + governing: ["fake", "wham"]); + + Assert.Equal(2, multi.Runs.Count); + Assert.Equal("fake", multi.Governing); + Assert.Contains(multi.Runs, r => r.Engine == "wham"); + Assert.Contains(multi.Runs, r => r.Engine == "fake"); + } + + [Fact] + public void Unavailable_engine_is_named_not_dropped() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + + var multi = MultiRunReport.Run(dataDir, fixturesDir, + engineSpecs: ["wham", "ghost=/no/such/adapter-xyz"], + governing: ["ghost", "wham"]); + + Assert.Single(multi.Runs); + Assert.Equal(["ghost"], multi.Unavailable); + Assert.Equal("wham", multi.Governing); // ghost didn't run, precedence falls through + } + + [Fact] + public void Github_actions_output_renders_engine_matrix() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var multi = MultiRunReport.Run(dataDir, fixturesDir, ["wham"], ["wham"]); + + using var sw = new StringWriter(); + MultiRunReport.Write(multi, "github-actions", sw); + var text = sw.ToString(); + + Assert.Contains("wham", text, StringComparison.Ordinal); + Assert.Contains("governing", text, StringComparison.OrdinalIgnoreCase); + } +} +``` + +Note on `TestAdapterDll`: verify the actual artifacts path after building (`UseArtifactsOutput=true` → `artifacts/bin/Muster.TestAdapter/debug/Muster.TestAdapter.dll` from repo root). Compute it robustly: walk up from `AppContext.BaseDirectory` to the directory containing `Muster.slnx`, then `artifacts/bin/Muster.TestAdapter/debug/Muster.TestAdapter.dll`. Adjust the helper accordingly — the two candidate layouts must be checked in Step 3, not guessed. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter MultiEngineTestCommandTests -v minimal` +Expected: FAIL (MultiRunReport does not exist). + +- [ ] **Step 3: Implement** + +`TestCommand.cs` changes: +- Extract the per-fixture body so the engine is injected. New signature (keep old one delegating): + +```csharp +internal static RunReport RunFixtures(string dataDir, string fixturesDir) => + RunFixtures(dataDir, fixturesDir, EngineSpec.Parse(EngineSpec.BuiltinName)); + +internal static RunReport RunFixtures(string dataDir, string fixturesDir, EngineSpec engineSpec) +{ + // existing validation + fixture discovery + resolver creation unchanged + // … + // Builtin: fresh adapter per fixture (today's behavior). + // External: one engine instance for the whole run, reused across fixtures. + IRosterEngine? sharedEngine = engineSpec.Kind == EngineKind.Builtin ? null : EngineRegistry.CreateEngine(engineSpec); + try + { + var results = new List(); + foreach (var path in fixturePaths) + results.Add(RunFixture(path, dataDir, resolver, engineSpec, sharedEngine)); + return RunReport.Create(engineSpec.Name, dataDir, results); + } + finally + { + sharedEngine?.Dispose(); + } +} +``` + +`RunFixture` gains `(EngineSpec engineSpec, IRosterEngine? sharedEngine)`; inside: + +```csharp +IRosterEngine engine = sharedEngine ?? new SpecRosterEngineAdapter(); +try +{ + var runner = new RosterRunner(engine, resolver, engineName: engineSpec.Name); + var result = runner.Run(spec); + // existing HarnessError / result mapping unchanged +} +finally +{ + if (sharedEngine is null) engine.Dispose(); +} +``` + +`MultiRunReport.cs`: + +```csharp +// src/Muster.Cli/Reporting/MultiRunReport.cs +using System.Text.Json; +using Muster.Cli.Commands; +using Muster.Cli.Engines; + +namespace Muster.Cli.Reporting; + +public sealed record MultiRunReport(string? Governing, IReadOnlyList Unavailable, IReadOnlyList Runs) +{ + public static MultiRunReport Run(string dataDir, string fixturesDir, + IReadOnlyList engineSpecs, IReadOnlyList governing) + { + var specs = EngineRegistry.ParseAll(engineSpecs); + var unavailable = new List(); + var runs = new List(); + foreach (var spec in specs) + { + if (!EngineRegistry.IsAvailable(spec)) { unavailable.Add(spec.Name); continue; } + runs.Add(TestCommand.RunFixtures(dataDir, fixturesDir, spec)); + } + var precedence = governing.Count > 0 ? governing : EngineRegistry.DefaultGoverning; + var governor = EngineRegistry.ResolveGoverning(precedence, [.. runs.Select(r => r.Engine)]); + return new(governor, unavailable, runs); + } + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + }; + + public static string ToJson(MultiRunReport report) => JsonSerializer.Serialize(report, JsonOptions); + + public static void Write(MultiRunReport report, string mode, TextWriter writer) + { + if (mode == "json") { writer.WriteLine(ToJson(report)); return; } + foreach (var run in report.Runs) + { + writer.WriteLine(mode == "github-actions" + ? $"### Engine: {run.Engine}{(run.Engine == report.Governing ? " (governing)" : "")}" + : $"-- engine: {run.Engine}{(run.Engine == report.Governing ? " (governing)" : "")} --"); + RunReport.Write(run, mode, writer); + writer.WriteLine(); + } + foreach (var name in report.Unavailable) + writer.WriteLine(mode == "github-actions" + ? $"> ⚠ engine `{name}` was requested but is unavailable in this environment." + : $"[????] engine {name}: unavailable"); + } +} +``` + +`TestCommand.Create()` additions (System.CommandLine 2.0.3 object-initializer style, matching existing options): + +```csharp +var enginesOption = new Option("--engines") +{ + Description = "Engines to run: 'wham' (builtin), 'name=dotnet:path.dll', 'name=docker:image', 'name=exe args'. Default: wham.", + AllowMultipleArgumentsPerToken = true, +}; +var governingOption = new Option("--governing") +{ + Description = "Governing-engine precedence (first match that ran governs). Default: newrecruit battlescribe wham.", + AllowMultipleArgumentsPerToken = true, +}; +``` + +`Run` uses `MultiRunReport.Run(...)`; exit code: `1` if ANY run has `Failed > 0`; else `2` if any run has `Inconclusive > 0` or (`Runs.Count == 0`); else `0`. `--report` writes `MultiRunReport.ToJson`. + +- [ ] **Step 4: Run the full suite + Release build** + +Run: `dotnet test -v minimal && dotnet build -c Release` +Expected: PASS (existing `TestCommandTests`/`DiffCommandTests` unchanged — 2-arg `RunFixtures` still works; single-engine JSON report shape check in `TestCommandTests` may need updating if `--report` output changed to MultiRunReport — update that test's assertions to the new `{governing, unavailable, runs:[…]}` shape). + +- [ ] **Step 5: Commit** + +```bash +git add src tests +git commit -m "feat: multi-engine test — engine-parameterized RunFixtures, MultiRunReport, --engines/--governing" +``` + +--- + +### Task 4: Multi-engine `diff` + `--fail-on-broke` + engine-gap + Action inputs (muster#4) + +Per-engine blast radius; `--fail-on-broke` gates on the governing engine's rows; cross-engine head-status disagreement surfaces as an `engine-gap` note; `action.yml` + `entrypoint.sh` wire `fail-on-broke` (default **true**), `fail-on-inconclusive` (default false), `engines`, `governing`. + +**Files:** +- Modify: `src/Muster.Cli/Commands/DiffCommand.cs` +- Modify: `src/Muster.Cli/Reporting/BlastRadius.cs` (add multi-engine writer; `Classify` stays pure and single-engine — called per engine) +- Modify: `action.yml`, `entrypoint.sh` +- Test: `tests/Muster.Cli.Tests/DiffCommandTests.cs` (extend), `tests/Muster.Cli.Tests/BlastRadiusTests.cs` (extend) + +**Interfaces:** +- Consumes: `MultiRunReport.Run` (Task 3), `BlastRadius.Classify(RunReport, RunReport)` (existing, unchanged). +- Produces: `sealed record EngineDiff(string Engine, IReadOnlyList Rows)`; `sealed record MultiDiffReport(string? Governing, IReadOnlyList Unavailable, IReadOnlyList Diffs, IReadOnlyList EngineGaps)`; `BlastRadius.ClassifyMulti(MultiRunReport baseRuns, MultiRunReport headRuns)` → `MultiDiffReport` (pairs engines by name; engine present on only one side → listed in `Unavailable`, excluded from rows and gating); `EngineGaps` = fixture ids where head status differs across engines that ran. `DiffCommand` exit: with `--fail-on-broke`, exit 1 iff the governing engine's rows contain `broke` or `verdict-changed`; else existing semantics (0 when both runs completed). + +- [ ] **Step 1: Write the failing tests** + +```csharp +// append to tests/Muster.Cli.Tests/DiffCommandTests.cs +[Fact] +public void FailOnBroke_exits_1_when_governing_engine_broke() +{ + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var headData = TestRepoFactory.CopyWithReplacement(dataDir, "20", "25"); // breaks the pts=20 pin + + var exit = RunDiff(dataDir, headData, fixturesDir, "markdown", + failOnBroke: true, engines: ["wham"], governing: ["wham"]); + + Assert.Equal(1, exit); +} + +[Fact] +public void FailOnBroke_ignores_non_governing_break() +{ + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var headData = TestRepoFactory.CopyWithReplacement(dataDir, "20", "25"); + // fake adapter always computes pts=20 → fixture passes both sides for 'fake'; + // wham (non-governing) breaks; check must stay green but report the gap. + var exit = RunDiff(dataDir, headData, fixturesDir, "markdown", + failOnBroke: true, + engines: [$"fake=dotnet:{TestAdapterDll}", "wham"], + governing: ["fake", "wham"]); + + Assert.Equal(0, exit); +} + +[Fact] +public void Head_status_disagreement_reports_engine_gap() +{ + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var headData = TestRepoFactory.CopyWithReplacement(dataDir, "20", "25"); + + var output = CaptureDiffOutput(dataDir, headData, fixturesDir, "markdown", + engines: [$"fake=dotnet:{TestAdapterDll}", "wham"], governing: ["fake"]); + + Assert.Contains("engine-gap", output, StringComparison.Ordinal); +} +``` + +(`RunDiff`/`CaptureDiffOutput`/`TestAdapterDll` are small local helpers invoking `DiffCommand.Run` with the new parameters — write them in the test file; `TestAdapterDll` shared via a tiny `TestPaths` helper class if Task 3 already created one.) + +**Caveat for the fake-engine scenario:** the green fixture pins `pts=20` and `FakeRosterEngine` computes `20 × count` — confirm the fixture in `TestRepoFactory` executes exactly one `selectEntry` with count 1 (read `TestRepoFactory.cs` first; adjust `FAKE_PTS` via an env-var pass-through on the engine spec if the fixture shape differs: `AdapterProcess.Start` inherits the parent environment, so set `Environment.SetEnvironmentVariable("FAKE_PTS", "20")` in the test before running). + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter DiffCommandTests -v minimal` +Expected: FAIL (new parameters don't exist). + +- [ ] **Step 3: Implement** + +`BlastRadius.cs` — add (below the existing single-engine members, which stay untouched): + +```csharp +public sealed record EngineDiff(string Engine, IReadOnlyList Rows); + +public sealed record MultiDiffReport( + string? Governing, + IReadOnlyList Unavailable, + IReadOnlyList Diffs, + IReadOnlyList EngineGaps); + +public static MultiDiffReport ClassifyMulti(MultiRunReport baseRuns, MultiRunReport headRuns) +{ + var baseByEngine = baseRuns.Runs.ToDictionary(r => r.Engine, StringComparer.Ordinal); + var headByEngine = headRuns.Runs.ToDictionary(r => r.Engine, StringComparer.Ordinal); + + var unavailable = new List(baseRuns.Unavailable.Union(headRuns.Unavailable, StringComparer.Ordinal)); + var diffs = new List(); + foreach (var (engine, baseRun) in baseByEngine) + { + if (!headByEngine.TryGetValue(engine, out var headRun)) { unavailable.Add(engine); continue; } + diffs.Add(new(engine, Classify(baseRun, headRun))); + } + foreach (var engine in headByEngine.Keys.Except(baseByEngine.Keys, StringComparer.Ordinal)) + unavailable.Add(engine); + + // engine-gap: fixtures whose HEAD status differs across engines that ran both sides + var gaps = new List(); + if (diffs.Count > 1) + { + var byFixture = diffs + .SelectMany(d => d.Rows.Select(r => (r.FixtureId, r.HeadStatus))) + .GroupBy(x => x.FixtureId, StringComparer.Ordinal); + foreach (var g in byFixture) + if (g.Select(x => x.HeadStatus).Distinct(StringComparer.Ordinal).Count() > 1) + gaps.Add(g.Key); + } + + return new(headRuns.Governing, unavailable, diffs, gaps); +} +``` + +Add `WriteMulti(MultiDiffReport report, string mode, TextWriter writer)`: `json` → serialize the record (reuse `JsonOptions`); `markdown` → per-engine `### Engine: {name}` sections reusing the existing single-engine markdown body, then, when `EngineGaps.Count > 0`, a section: + +```markdown +### ⚠ engine-gap +Engines disagree on the head state of: `fixture-a`, `fixture-b`. The governing +engine's verdict stands; divergence should be triaged as an engine defect. +``` + +`DiffCommand.cs`: add `--engines`, `--governing`, `--fail-on-broke` (`Option`, default false); `Run` signature becomes +`internal static int Run(DirectoryInfo baseDir, DirectoryInfo headDir, DirectoryInfo fixtures, string output, bool failOnBroke, string[] engines, string[] governing)`; +body: two `MultiRunReport.Run` calls (base/head), `BlastRadius.ClassifyMulti`, `WriteMulti`; then: + +```csharp +if (failOnBroke && report.Governing is { } gov) +{ + var governingRows = report.Diffs.FirstOrDefault(d => d.Engine == gov)?.Rows ?? []; + if (governingRows.Any(r => r.Classification is "broke" or "verdict-changed")) + return 1; +} +return 0; +``` + +`action.yml` — add inputs (after `base-ref`): + +```yaml + fail-on-broke: + description: Fail the check when the governing engine classifies any fixture as broke/verdict-changed (diff mode). + default: "true" + fail-on-inconclusive: + description: Fail the check (exit 1) instead of neutral warning when the run is inconclusive. + default: "false" + engines: + description: Space-separated engine specs (e.g. "wham newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest"). + default: "wham" + governing: + description: Space-separated governing precedence. + default: "newrecruit battlescribe wham" +``` + +and append them to `runs.args` (positional 4–7). + +`entrypoint.sh` — accept the new positionals with defaults, build flag arrays: + +```bash +FAIL_ON_BROKE="${4:-true}" +FAIL_ON_INCONCLUSIVE="${5:-false}" +ENGINES_INPUT="${6:-wham}" +GOVERNING_INPUT="${7:-newrecruit battlescribe wham}" + +ENGINE_ARGS=(--engines) +read -r -a ENGINE_LIST <<< "$ENGINES_INPUT" +ENGINE_ARGS+=("${ENGINE_LIST[@]}") +GOVERNING_ARGS=(--governing) +read -r -a GOVERNING_LIST <<< "$GOVERNING_INPUT" +GOVERNING_ARGS+=("${GOVERNING_LIST[@]}") +``` + +pass `"${ENGINE_ARGS[@]}" "${GOVERNING_ARGS[@]}"` to both `muster test` and `muster diff`; add `--fail-on-broke` to the diff invocation when `FAIL_ON_BROKE == "true"`; change the exit-2 mapping: + +```bash +if [[ "$rc" -eq 2 ]]; then + if [[ "$FAIL_ON_INCONCLUSIVE" == "true" ]]; then + echo "::error::muster run was inconclusive (exit 2) and fail-on-inconclusive is set" + exit 1 + fi + echo "::warning::muster run was inconclusive (exit 2) -- treating as neutral" + exit 0 +fi +``` + +- [ ] **Step 4: Run the full suite + Release build** + +Run: `dotnet test -v minimal && dotnet build -c Release` +Expected: PASS. The pre-existing `Diff_reports_broken_fixture_between_trees` (exit 0 without the flag) must STILL pass — CLI default is unchanged; only the Action default flips. + +- [ ] **Step 5: Commit and close the loop on muster#4** + +```bash +git add src tests action.yml entrypoint.sh +git commit -m "feat: per-engine diff, --fail-on-broke governing gate, engine-gap surfacing (closes #4)" +``` + +--- + +### Task 5: NR share-link fetcher + +`NrShareLink.TryParse` (strict allowlist) + `NrClient` (POST `open_share_link`, caps, graceful failure). Pure HTTP layer — no conversion. + +**Files:** +- Create: `src/Muster.Cli/NewRecruit/NrShareLink.cs` +- Create: `src/Muster.Cli/NewRecruit/NrClient.cs` +- Test: `tests/Muster.Cli.Tests/NewRecruit/NrShareLinkTests.cs`, `tests/Muster.Cli.Tests/NewRecruit/NrClientTests.cs` + +**Interfaces:** +- Produces: + - `static class NrShareLink`: `bool TryParse(string url, out string key)` — accepts EXACTLY `https://www.newrecruit.eu/app/list/` (optional single trailing slash) where `` matches `^[A-Za-z0-9]{1,32}$`; https only, exact host `www.newrecruit.eu`, no query strings. + - `sealed class NrClient(HttpMessageHandler? handler = null) : IDisposable`: `Task FetchListAsync(string key, CancellationToken ct = default)`; `sealed record NrFetchResult(string? Json, string? Error)` — `Json` null on any failure with a human-readable `Error`. Never throws for remote/protocol problems. RPC returning literal `null`/`[]`/empty ⇒ "list not found or no longer shared" error. + - Caps: timeout 30 s, max response 5 MB (streamed read with cap check). + +- [ ] **Step 1: Write the failing tests** + +```csharp +// tests/Muster.Cli.Tests/NewRecruit/NrShareLinkTests.cs +using Muster.Cli.NewRecruit; + +namespace Muster.Cli.Tests.NewRecruit; + +public class NrShareLinkTests +{ + [Theory] + [InlineData("https://www.newrecruit.eu/app/list/3Pbpd", true, "3Pbpd")] + [InlineData("https://www.newrecruit.eu/app/list/3Pbpd/", true, "3Pbpd")] + [InlineData("https://www.newrecruit.eu/app/list/tr5BL", true, "tr5BL")] + [InlineData("http://www.newrecruit.eu/app/list/3Pbpd", false, "")] + [InlineData("https://newrecruit.eu/app/list/3Pbpd", false, "")] + [InlineData("https://evil.example/app/list/3Pbpd", false, "")] + [InlineData("https://www.newrecruit.eu/app/list/../secret", false, "")] + [InlineData("https://www.newrecruit.eu/app/list/3Pbpd?x=1", false, "")] + [InlineData("https://www.newrecruit.eu/api/rpc", false, "")] + [InlineData("not a url", false, "")] + public void TryParse_allowlist(string url, bool ok, string key) + { + Assert.Equal(ok, NrShareLink.TryParse(url, out var k)); + if (ok) Assert.Equal(key, k); + } +} +``` + +```csharp +// tests/Muster.Cli.Tests/NewRecruit/NrClientTests.cs +using System.Net; +using System.Text; +using Muster.Cli.NewRecruit; + +namespace Muster.Cli.Tests.NewRecruit; + +public class NrClientTests +{ + private sealed class StubHandler(Func respond) : HttpMessageHandler + { + public HttpRequestMessage? LastRequest { get; private set; } + public string? LastBody { get; private set; } + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + LastRequest = request; + LastBody = request.Content is null ? null : await request.Content.ReadAsStringAsync(ct); + return respond(request); + } + } + + [Fact] + public async Task Fetch_posts_open_share_link_rpc_and_returns_json() + { + var handler = new StubHandler(_ => new(HttpStatusCode.OK) + { + Content = new StringContent("""{"name":"war horde","army":{}}""", Encoding.UTF8, "application/json"), + }); + using var client = new NrClient(handler); + + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + + Assert.Null(result.Error); + Assert.Contains("war horde", result.Json, StringComparison.Ordinal); + Assert.Equal("https://www.newrecruit.eu/api/rpc", handler.LastRequest!.RequestUri!.ToString()); + Assert.Contains("\"open_share_link\"", handler.LastBody, StringComparison.Ordinal); + Assert.Contains("\"3Pbpd\"", handler.LastBody, StringComparison.Ordinal); + } + + [Fact] + public async Task Null_rpc_response_is_not_found() + { + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new StringContent("null") }); + using var client = new NrClient(handler); + var result = await client.FetchListAsync("gone1", TestContext.Current.CancellationToken); + Assert.Null(result.Json); + Assert.Contains("no longer shared", result.Error, StringComparison.Ordinal); + } + + [Fact] + public async Task Http_error_is_graceful() + { + var handler = new StubHandler(_ => new(HttpStatusCode.InternalServerError)); + using var client = new NrClient(handler); + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + Assert.Null(result.Json); + Assert.NotNull(result.Error); + } + + [Fact] + public async Task Oversized_response_is_rejected() + { + var big = new string('x', 6 * 1024 * 1024); + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new StringContent($"{{\"pad\":\"{big}\"}}") }); + using var client = new NrClient(handler); + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + Assert.Null(result.Json); + Assert.Contains("too large", result.Error, StringComparison.Ordinal); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "NrShareLinkTests|NrClientTests" -v minimal` +Expected: FAIL (types missing). + +- [ ] **Step 3: Implement** + +```csharp +// src/Muster.Cli/NewRecruit/NrShareLink.cs +using System.Text.RegularExpressions; + +namespace Muster.Cli.NewRecruit; + +public static partial class NrShareLink +{ + [GeneratedRegex(@"^https://www\.newrecruit\.eu/app/list/([A-Za-z0-9]{1,32})/?$")] + private static partial Regex Pattern(); + + public static bool TryParse(string url, out string key) + { + key = ""; + if (string.IsNullOrWhiteSpace(url)) return false; + var m = Pattern().Match(url.Trim()); + if (!m.Success) return false; + key = m.Groups[1].Value; + return true; + } +} +``` + +```csharp +// src/Muster.Cli/NewRecruit/NrClient.cs +using System.Text; +using System.Text.Json; + +namespace Muster.Cli.NewRecruit; + +public sealed record NrFetchResult(string? Json, string? Error); + +/// +/// Fetches a shared New Recruit list via the (undocumented) open_share_link RPC. +/// All remote failures degrade to — callers +/// map them to needs-info, never a crash. +/// +public sealed class NrClient(HttpMessageHandler? handler = null) : IDisposable +{ + private const long MaxResponseBytes = 5 * 1024 * 1024; + private static readonly Uri RpcUri = new("https://www.newrecruit.eu/api/rpc"); + + private readonly HttpClient _http = new(handler ?? new HttpClientHandler()) + { + Timeout = TimeSpan.FromSeconds(30), + }; + + public async Task FetchListAsync(string key, CancellationToken ct = default) + { + try + { + var body = JsonSerializer.Serialize(new Dictionary + { + ["method"] = "open_share_link", + ["params"] = new[] { key }, + }); + using var request = new HttpRequestMessage(HttpMethod.Post, RpcUri) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + using var response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct); + if (!response.IsSuccessStatusCode) + return new(null, $"New Recruit returned HTTP {(int)response.StatusCode}"); + + using var stream = await response.Content.ReadAsStreamAsync(ct); + using var limited = new MemoryStream(); + var buffer = new byte[81920]; + int read; + while ((read = await stream.ReadAsync(buffer, ct)) > 0) + { + limited.Write(buffer, 0, read); + if (limited.Length > MaxResponseBytes) + return new(null, "response too large (>5 MB)"); + } + var text = Encoding.UTF8.GetString(limited.ToArray()).Trim(); + if (text is "null" or "" or "[]") + return new(null, "list not found or no longer shared (New Recruit share links expire)"); + if (!text.StartsWith('{')) + return new(null, "New Recruit returned an unexpected response"); + return new(text, null); + } + catch (TaskCanceledException) + { + return new(null, "New Recruit did not respond within 30 seconds"); + } + catch (HttpRequestException e) + { + return new(null, $"could not reach New Recruit: {e.Message}"); + } + } + + public void Dispose() => _http.Dispose(); +} +``` + +- [ ] **Step 4: Run tests + Release build** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "NrShareLinkTests|NrClientTests" -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 5: Commit** + +```bash +git add src/Muster.Cli/NewRecruit tests/Muster.Cli.Tests/NewRecruit +git commit -m "feat: NR share-link parser (strict allowlist) + open_share_link client with caps" +``` + +--- + +### Task 6: NR list parser + spec emitter (the "everything becomes a spec" core) + +`NrListParser` (System.Text.Json, hostile-input caps) → intermediate `ReplayRoster` tree → `SpecEmitter` renders fixture-DSL YAML (hand-emitted text — no TestKit serializer exists; every path is round-trip-validated with `SpecLoader.LoadFromYaml`). Real sample is already committed at `tests/Muster.Cli.Tests/TestData/nr-list-war-horde.json`. + +**Files:** +- Create: `src/Muster.Cli/Converters/ReplayRoster.cs` +- Create: `src/Muster.Cli/Converters/NrListParser.cs` +- Create: `src/Muster.Cli/Converters/SpecEmitter.cs` +- Modify: `tests/Muster.Cli.Tests/Muster.Cli.Tests.csproj` (add ``) +- Test: `tests/Muster.Cli.Tests/Converters/NrListParserTests.cs`, `tests/Muster.Cli.Tests/Converters/SpecEmitterTests.cs` + +**Interfaces:** +- Produces (shared intermediate model, also consumed by Task 7's `.ros` converter and Task 9/10's report flow): + - `sealed record ReplayCost(string Name, string TypeId, decimal Value)` + - `sealed record ReplaySelection(string EntryId, int Count, string? CustomName, IReadOnlyList ObservedCosts, IReadOnlyList Children)` — `EntryId` already composite (`linkId::targetId`) where applicable + - `sealed record ReplayForce(string ForceEntryId, string CatalogueId, IReadOnlyList Selections, IReadOnlyList ChildForces)` + - `sealed record ReplayRoster(string Name, string? GameSystemId, IReadOnlyList ObservedTotals, IReadOnlyList BooksRevisions, IReadOnlyList Forces, IReadOnlyList Unmapped)` — non-empty `Unmapped` ⇒ report verdict `needs-info` (spec rule: partial replay never yields confirmed/not-reproducible) + - `static class NrListParser`: `ReplayRoster Parse(string json)` — throws `FormatException` (safe message) on malformed/hostile input. Caps: JSON depth 64, nodes 20 000, selections 5 000. + - `static class SpecEmitter`: `string Emit(ReplayRoster roster, string specId, string dataSource, bool pinObserved)`. Emitted spec: `category: report`; steps use ids `force-1…`/`sel-1…`; selections reference `${{ steps.force-N.forceId }}` / parent `${{ steps.sel-N.selectionId }}`; `count != 1` → `setSelectionCount`; `CustomName` → `setCustomization`; final step `expectedState.costs` pins `ObservedTotals` when `pinObserved`. +- NR node classification (verified on all 120 nodes of the committed sample): `army.options[]` level 1 = catalogue node (`option_id` = catalogue id) → children carrying `catalogue_id` key = force nodes (`option_id` = force entry id) → descendants: **has `amount` ⇒ selection** (entry id = `link_id::option_id` when `link_id` present, else `option_id`; count = `amount`); **no `amount` ⇒ transparent container** (category/group — recurse through, emit nothing). + +- [ ] **Step 1: Write the failing parser tests** + +```csharp +// tests/Muster.Cli.Tests/Converters/NrListParserTests.cs +using Muster.Cli.Converters; + +namespace Muster.Cli.Tests.Converters; + +public class NrListParserTests +{ + private static string SampleJson => File.ReadAllText( + Path.Combine(AppContext.BaseDirectory, "TestData", "nr-list-war-horde.json")); + + [Fact] + public void Parses_war_horde_sample() + { + var roster = NrListParser.Parse(SampleJson); + + Assert.Equal("war horde", roster.Name); + var pts = Assert.Single(roster.ObservedTotals); + Assert.Equal(950m, pts.Value); + Assert.Equal("51b2-306e-1021-d207", pts.TypeId); + var force = Assert.Single(roster.Forces); + Assert.Equal("bb9d-299a-ed60-2d8a", force.ForceEntryId); + Assert.Equal("a55f-b7b3-6c65-a05f", force.CatalogueId); + Assert.NotEmpty(force.Selections); + Assert.Empty(roster.Unmapped); + Assert.Contains("Xenos - Orks: 2", roster.BooksRevisions); + } + + [Fact] + public void Linked_selection_gets_composite_entry_id() + { + var roster = NrListParser.Parse(SampleJson); + var all = Flatten(roster); + // "Battle Size" selection: option_id 564e-fbc6-5266-3ea4 selected via link 7380-3e40-6ed6-b7cc + Assert.Contains(all, s => s.EntryId == "7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4"); + } + + [Fact] + public void Container_nodes_are_transparent() + { + var roster = NrListParser.Parse(SampleJson); + var all = Flatten(roster); + // "Configuration" (category node, no amount) must not appear as a selection… + Assert.DoesNotContain(all, s => s.EntryId.Contains("4ac9-fd30-1e3d-b249", StringComparison.Ordinal)); + // …but "Incursion", nested category → selection → group → child, must + Assert.Contains(all, s => s.EntryId.EndsWith("d62d-db22-4893-4bc0", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("not json at all")] + [InlineData("{}")] + [InlineData("""{"army":{"options":[]},"name":"x"}""")] + public void Hostile_or_empty_input_throws_FormatException(string json) => + Assert.Throws(() => NrListParser.Parse(json)); + + private static List Flatten(ReplayRoster r) + { + var result = new List(); + void Walk(IEnumerable sels) + { + foreach (var s in sels) { result.Add(s); Walk(s.Children); } + } + foreach (var f in r.Forces) Walk(f.Selections); + return result; + } +} +``` + +Add a depth-cap test as well: build a pathologically nested `options` chain programmatically (a loop wrapping `{"name":"n","option_id":"x","amount":1,"options":[…]}` 100 times inside a minimal valid catalogue/force envelope) and assert `FormatException`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter NrListParserTests -v minimal` +Expected: FAIL (types missing). + +- [ ] **Step 3: Implement ReplayRoster + NrListParser** + +```csharp +// src/Muster.Cli/Converters/ReplayRoster.cs +namespace Muster.Cli.Converters; + +public sealed record ReplayCost(string Name, string TypeId, decimal Value); + +public sealed record ReplaySelection( + string EntryId, int Count, string? CustomName, + IReadOnlyList ObservedCosts, + IReadOnlyList Children); + +public sealed record ReplayForce( + string ForceEntryId, string CatalogueId, + IReadOnlyList Selections, + IReadOnlyList ChildForces); + +public sealed record ReplayRoster( + string Name, string? GameSystemId, + IReadOnlyList ObservedTotals, + IReadOnlyList BooksRevisions, + IReadOnlyList Forces, + IReadOnlyList Unmapped); +``` + +```csharp +// src/Muster.Cli/Converters/NrListParser.cs +using System.Text.Json; + +namespace Muster.Cli.Converters; + +/// +/// Parses a New Recruit shared-list JSON payload into a ReplayRoster. +/// Node classification (verified against real NR data 2026-07-13): +/// army.options[] level 1 = catalogue; its children carrying "catalogue_id" +/// are forces; below that, nodes with "amount" are selections (entry id = +/// "link_id::option_id" when linked), nodes without are transparent +/// containers (categories / selection-entry-groups). +/// +public static class NrListParser +{ + private const int MaxDepth = 64; + private const int MaxNodes = 20_000; + private const int MaxSelections = 5_000; + + public static ReplayRoster Parse(string json) + { + JsonDocument doc; + try + { + doc = JsonDocument.Parse(json, new JsonDocumentOptions { MaxDepth = MaxDepth }); + } + catch (JsonException e) + { + throw new FormatException($"not a valid New Recruit list: {e.Message}"); + } + + using (doc) + { + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object || !root.TryGetProperty("army", out var army)) + throw new FormatException("not a New Recruit list: missing 'army'"); + + var name = GetString(root, "name") ?? "unnamed roster"; + var totals = ParseCosts(root, "totalCosts"); + var revisions = root.TryGetProperty("books_revision", out var revs) && revs.ValueKind == JsonValueKind.Array + ? [.. revs.EnumerateArray().Where(r => r.ValueKind == JsonValueKind.String).Select(r => r.GetString()!)] + : (List)[]; + var gameSystemId = GetString(root, "bsid_system"); + + var state = new ParseState(); + var forces = new List(); + foreach (var catNode in Options(army)) + { + state.CountNode(); + var catalogueId = GetString(catNode, "option_id") + ?? throw new FormatException("catalogue node missing option_id"); + foreach (var child in Options(catNode)) + { + state.CountNode(); + if (child.TryGetProperty("catalogue_id", out _)) + forces.Add(ParseForce(child, catalogueId, state)); + else + state.Unmapped.Add($"unexpected non-force node '{GetString(child, "name")}' under catalogue"); + } + } + + if (forces.Count == 0) + throw new FormatException("no forces found in the list"); + + return new(name, gameSystemId, totals, revisions, forces, state.Unmapped); + } + } + + private sealed class ParseState + { + public List Unmapped { get; } = []; + public int Selections; + private int _nodes; + + public void CountNode() + { + if (++_nodes > MaxNodes) throw new FormatException($"list too large (over {MaxNodes} nodes)"); + } + } + + private static ReplayForce ParseForce(JsonElement node, string catalogueId, ParseState state) + { + var forceEntryId = GetString(node, "option_id") ?? throw new FormatException("force node missing option_id"); + var selections = new List(); + CollectSelections(node, selections, state, depth: 0); + return new(forceEntryId, catalogueId, selections, ChildForces: []); + } + + private static void CollectSelections(JsonElement node, List into, ParseState state, int depth) + { + if (depth > MaxDepth) throw new FormatException("list too deeply nested"); + foreach (var child in Options(node)) + { + state.CountNode(); + if (child.TryGetProperty("amount", out var amountEl)) + { + if (++state.Selections > MaxSelections) + throw new FormatException($"list too large (over {MaxSelections} selections)"); + var optionId = GetString(child, "option_id"); + if (optionId is null) + { + state.Unmapped.Add($"selection '{GetString(child, "name")}' missing option_id"); + continue; + } + var linkId = GetString(child, "link_id"); + var entryId = linkId is null ? optionId : $"{linkId}::{optionId}"; + var count = amountEl.ValueKind == JsonValueKind.Number ? amountEl.GetInt32() : 1; + var children = new List(); + CollectSelections(child, children, state, depth + 1); + into.Add(new(entryId, count, GetString(child, "customName"), ObservedCosts: [], children)); + } + else + { + CollectSelections(child, into, state, depth + 1); + } + } + } + + private static IEnumerable Options(JsonElement node) => + node.ValueKind == JsonValueKind.Object + && node.TryGetProperty("options", out var options) + && options.ValueKind == JsonValueKind.Array + ? options.EnumerateArray() : []; + + private static string? GetString(JsonElement el, string name) => + el.ValueKind == JsonValueKind.Object && el.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String + ? v.GetString() : null; + + private static List ParseCosts(JsonElement el, string property) + { + var result = new List(); + if (el.TryGetProperty(property, out var costs) && costs.ValueKind == JsonValueKind.Array) + { + foreach (var c in costs.EnumerateArray()) + { + var typeId = GetString(c, "typeId"); + if (typeId is null) continue; + var value = c.TryGetProperty("value", out var v) && v.ValueKind == JsonValueKind.Number ? v.GetDecimal() : 0m; + result.Add(new(GetString(c, "name") ?? typeId, typeId, value)); + } + } + return result; + } +} +``` + +Run: `dotnet test tests/Muster.Cli.Tests --filter NrListParserTests -v minimal` — iterate until PASS. + +- [ ] **Step 4: Write the failing emitter tests** + +```csharp +// tests/Muster.Cli.Tests/Converters/SpecEmitterTests.cs +using BattleScribeSpec; +using Muster.Cli.Converters; + +namespace Muster.Cli.Tests.Converters; + +public class SpecEmitterTests +{ + private static ReplayRoster Sample() => new( + Name: "war horde", + GameSystemId: "sys-1", + ObservedTotals: [new("pts", "51b2-306e-1021-d207", 950m)], + BooksRevisions: ["Xenos - Orks: 2"], + Forces: + [ + new("bb9d-299a-ed60-2d8a", "a55f-b7b3-6c65-a05f", + Selections: + [ + new("7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4", 1, null, [], + Children: [new("d62d-db22-4893-4bc0", 1, null, [], [])]), + new("boy-entry", 3, "Da Ladz", [new("pts", "pts-id", 60m)], []), + ], + ChildForces: []), + ], + Unmapped: []); + + [Fact] + public void Emitted_yaml_round_trips_through_SpecLoader() + { + var yaml = SpecEmitter.Emit(Sample(), "issue-42", "github:test/repo", pinObserved: true); + var spec = SpecLoader.LoadFromYaml(yaml); // throws on invalid spec + + Assert.Equal("issue-42", spec.Id); + Assert.Equal("github:test/repo", spec.Setup.DataSource); + Assert.Contains(spec.Steps, s => s.Action == "addForce" && s.ForceEntryId == "bb9d-299a-ed60-2d8a"); + Assert.Contains(spec.Steps, s => s.Action == "selectEntry" && s.EntryId == "7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4"); + Assert.Contains(spec.Steps, s => s.Action == "selectChildEntry" && s.EntryId == "d62d-db22-4893-4bc0"); + Assert.Contains(spec.Steps, s => s.Action == "setSelectionCount" && s.Count == 3); + Assert.Contains(spec.Steps, s => s.Action == "setCustomization" && s.CustomName == "Da Ladz"); + var assertStep = spec.Steps.Last(); + Assert.NotNull(assertStep.ExpectedState); + var cost = Assert.Single(assertStep.ExpectedState!.Costs!); + Assert.Equal(950m, cost.Value); + Assert.Equal("51b2-306e-1021-d207", cost.TypeId); + } + + [Fact] + public void Without_pins_no_expectedState_is_emitted() + { + var yaml = SpecEmitter.Emit(Sample(), "issue-42", "github:test/repo", pinObserved: false); + var spec = SpecLoader.LoadFromYaml(yaml); + Assert.DoesNotContain(spec.Steps, s => s.ExpectedState is not null); + } + + [Fact] + public void Yaml_special_characters_are_escaped() + { + var roster = Sample() with { Name = "list: with \"quotes\" & #hash" }; + var yaml = SpecEmitter.Emit(roster, "x", "local:.", pinObserved: true); + Assert.NotNull(SpecLoader.LoadFromYaml(yaml)); // must not throw + } + + [Fact] + public void Selection_steps_reference_parent_outputs() + { + var yaml = SpecEmitter.Emit(Sample(), "x", "local:.", pinObserved: false); + Assert.Contains("${{ steps.sel-1.selectionId }}", yaml, StringComparison.Ordinal); + Assert.Contains("${{ steps.force-1.forceId }}", yaml, StringComparison.Ordinal); + } +} +``` + +- [ ] **Step 5: Implement SpecEmitter** + +```csharp +// src/Muster.Cli/Converters/SpecEmitter.cs +using System.Globalization; +using System.Text; + +namespace Muster.Cli.Converters; + +/// +/// Emits fixture-DSL YAML from a ReplayRoster. TestKit has no SpecFile→YAML +/// serializer, so YAML is emitted as text; every caller path is validated by +/// round-tripping through SpecLoader.LoadFromYaml in tests. +/// Step ids: force-1, force-2, … / sel-1, sel-2, … in document order. +/// All string scalars are double-quoted — sidesteps YAML coercion traps. +/// +public static class SpecEmitter +{ + public static string Emit(ReplayRoster roster, string specId, string dataSource, bool pinObserved) + { + var sb = new StringBuilder(); + sb.AppendLine($"id: {Quote(specId)}"); + sb.AppendLine("category: report"); + sb.AppendLine($"description: {Quote($"Converted from bug report roster '{roster.Name}'")}"); + sb.AppendLine("setup:"); + sb.AppendLine($" dataSource: {Quote(dataSource)}"); + sb.AppendLine(); + sb.AppendLine("steps:"); + + var forceIndex = 0; + var selIndex = 0; + foreach (var force in roster.Forces) + EmitForce(sb, force, parentForceStep: null, ref forceIndex, ref selIndex); + + if (pinObserved && roster.ObservedTotals.Count > 0) + { + sb.AppendLine(); + sb.AppendLine(" # observed values from the report — PASS means the reported state reproduces"); + sb.AppendLine(" - expectedState:"); + sb.AppendLine(" costs:"); + foreach (var cost in roster.ObservedTotals) + { + sb.AppendLine($" - typeId: {Quote(cost.TypeId)}"); + sb.AppendLine($" value: {cost.Value.ToString(CultureInfo.InvariantCulture)}"); + } + } + return sb.ToString(); + } + + private static void EmitForce(StringBuilder sb, ReplayForce force, string? parentForceStep, ref int forceIndex, ref int selIndex) + { + forceIndex++; + var stepId = $"force-{forceIndex}"; + if (parentForceStep is null) + { + sb.AppendLine(" - action: addForce"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceEntryId: {Quote(force.ForceEntryId)}"); + sb.AppendLine($" catalogueId: {Quote(force.CatalogueId)}"); + } + else + { + sb.AppendLine(" - action: addChildForce"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceId: {Quote($"${{{{ steps.{parentForceStep}.forceId }}}}")}"); + sb.AppendLine($" forceEntryId: {Quote(force.ForceEntryId)}"); + sb.AppendLine($" catalogueId: {Quote(force.CatalogueId)}"); + } + + foreach (var sel in force.Selections) + EmitSelection(sb, sel, forceStep: stepId, parentSelStep: null, ref selIndex); + + foreach (var child in force.ChildForces) + EmitForce(sb, child, stepId, ref forceIndex, ref selIndex); + } + + private static void EmitSelection(StringBuilder sb, ReplaySelection sel, string forceStep, string? parentSelStep, ref int selIndex) + { + selIndex++; + var stepId = $"sel-{selIndex}"; + var forceRef = Quote($"${{{{ steps.{forceStep}.forceId }}}}"); + if (parentSelStep is null) + { + sb.AppendLine(" - action: selectEntry"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" entryId: {Quote(sel.EntryId)}"); + } + else + { + sb.AppendLine(" - action: selectChildEntry"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" selectionId: {Quote($"${{{{ steps.{parentSelStep}.selectionId }}}}")}"); + sb.AppendLine($" entryId: {Quote(sel.EntryId)}"); + } + + if (sel.Count != 1) + { + sb.AppendLine(" - action: setSelectionCount"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" selectionId: {Quote($"${{{{ steps.{stepId}.selectionId }}}}")}"); + sb.AppendLine($" count: {sel.Count}"); + } + + if (!string.IsNullOrEmpty(sel.CustomName)) + { + sb.AppendLine(" - action: setCustomization"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" selectionId: {Quote($"${{{{ steps.{stepId}.selectionId }}}}")}"); + sb.AppendLine($" customName: {Quote(sel.CustomName!)}"); + } + + foreach (var child in sel.Children) + EmitSelection(sb, child, forceStep, stepId, ref selIndex); + } + + private static string Quote(string value) => + "\"" + value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal) + "\""; +} +``` + +Check while implementing: whether the ExpressionResolver resolves `${{ … }}` inside a double-quoted YAML scalar (it operates on the deserialized string value, so quoting is transparent — but verify with the round-trip test asserting `s.ForceId == "${{ steps.force-1.forceId }}"` after load). If `StepDef.Id` is not a quoted-string-friendly field, drop `Quote` around step ids (they are `[a-z0-9-]+`, safe bare). + +Per-selection observed costs (`ObservedCosts` on `ReplaySelection`) are carried in the model but NOT emitted as assertions in this task — Task 7 decides after reading `RosterRunner.AssertSelections` matching semantics (name vs index). Roster totals are the v1 pin. + +- [ ] **Step 6: Run tests + Release build** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "NrListParserTests|SpecEmitterTests" -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 7: Commit** + +```bash +git add src/Muster.Cli/Converters tests/Muster.Cli.Tests/Converters tests/Muster.Cli.Tests/Muster.Cli.Tests.csproj +git commit -m "feat: NR list parser + fixture-DSL spec emitter with observed-value pins" +``` + +--- + +### Task 7: `.ros`/`.rosz` → ReplayRoster converter + +Walks a wham `RosterNode` into the same `ReplayRoster` intermediate, with per-selection observed costs (richer than NR). Decides per-selection pin emission after reading the comparer. + +**Files:** +- Create: `src/Muster.Cli/Converters/RosterFileConverter.cs` +- Modify: `src/Muster.Cli/Muster.Cli.csproj` (add ProjectReferences: `WarHub.ArmouryModel.Source.BattleScribe`, `WarHub.ArmouryModel.Workspaces.BattleScribe` — check first whether they arrive transitively via `RosterEngine.Spec`; add only what's missing) +- Modify: `src/Muster.Cli/Converters/SpecEmitter.cs` (per-selection pins IF viable — see Step 4) +- Test: `tests/Muster.Cli.Tests/Converters/RosterFileConverterTests.cs` + +**Interfaces:** +- Consumes: `BattleScribeXml.LoadRoster(Stream)` (`WarHub.ArmouryModel.Source.BattleScribe/BattleScribeXml.cs:70`), `XmlFileExtensions.LoadSourceAuto(this Stream, string filename, CancellationToken)` (`Workspaces.BattleScribe/XmlFileExtensions.cs:225`, handles `.rosz` zip with exactly-one-entry rule), `RosterNode` model (Costs/CostLimits/Forces; ForceNode.CatalogueId/EntryId/Selections/Forces; SelectionNode.EntryId (already composite `link::target`)/Number/CustomName/Costs/Selections). +- Produces: `static class RosterFileConverter`: `ReplayRoster Convert(Stream stream, string fileName)` — dispatches `.ros` (plain XML) vs `.rosz` (zipped) by extension; throws `FormatException` (safe message) on unparseable input; selection-count cap 5 000 applies here too. + +- [ ] **Step 1: Write the failing test** + +The test builds a minimal `.ros` XML inline (BattleScribe roster namespace) so no binary asset is needed: + +```csharp +// tests/Muster.Cli.Tests/Converters/RosterFileConverterTests.cs +using System.IO.Compression; +using System.Text; +using Muster.Cli.Converters; + +namespace Muster.Cli.Tests.Converters; + +public class RosterFileConverterTests +{ + // Minimal valid BattleScribe roster XML. Namespace/version: match what + // BattleScribeXml.LoadRoster expects — copy the exact xmlns and + // battleScribeVersion from an existing wham test asset + // (lib/wham/tests/**/ *.ros or the RosterCore XmlRoot: RootElementNames.Roster, + // Namespaces.RosterXmlns). Adjust the constant if LoadRoster returns null. + private const string RosterXml = """ + + + + + + + + + + + + + + + + + + + + + """; + + [Fact] + public void Converts_ros_xml_to_replay_roster() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(RosterXml)); + var roster = RosterFileConverter.Convert(stream, "test.ros"); + + Assert.Equal("Test Roster", roster.Name); + Assert.Equal("gs-1", roster.GameSystemId); + var total = Assert.Single(roster.ObservedTotals); + Assert.Equal(65.0m, total.Value); + var force = Assert.Single(roster.Forces); + Assert.Equal("fe-1", force.ForceEntryId); + Assert.Equal("cat-1", force.CatalogueId); + var unit = Assert.Single(force.Selections); + Assert.Equal("link-1::se-1", unit.EntryId); // composite id preserved verbatim + var unitCost = Assert.Single(unit.ObservedCosts); + Assert.Equal(65.0m, unitCost.Value); + var child = Assert.Single(unit.Children); + Assert.Equal("se-2", child.EntryId); + Assert.Equal(5, child.Count); + Assert.Equal("Fancy guns", child.CustomName); + } + + [Fact] + public void Converts_rosz_zip() + { + using var zipStream = new MemoryStream(); + using (var zip = new ZipArchive(zipStream, ZipArchiveMode.Create, leaveOpen: true)) + { + using var entry = zip.CreateEntry("test.ros").Open(); + entry.Write(Encoding.UTF8.GetBytes(RosterXml)); + } + zipStream.Position = 0; + + var roster = RosterFileConverter.Convert(zipStream, "test.rosz"); + Assert.Equal("Test Roster", roster.Name); + } + + [Fact] + public void Garbage_input_throws_FormatException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("")); + Assert.Throws(() => RosterFileConverter.Convert(stream, "x.ros")); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter RosterFileConverterTests -v minimal` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +```csharp +// src/Muster.Cli/Converters/RosterFileConverter.cs +using WarHub.ArmouryModel.Source; +using WarHub.ArmouryModel.Source.BattleScribe; +using WarHub.ArmouryModel.Workspaces.BattleScribe; + +namespace Muster.Cli.Converters; + +/// +/// Converts a BattleScribe .ros/.rosz roster file into a ReplayRoster. +/// SelectionNode.EntryId is already the composite "linkId::targetId" form the +/// engine's by-id selection expects, so ids pass through verbatim. +/// +public static class RosterFileConverter +{ + private const int MaxSelections = 5_000; + + public static ReplayRoster Convert(Stream stream, string fileName) + { + RosterNode roster; + try + { + roster = fileName.EndsWith(".rosz", StringComparison.OrdinalIgnoreCase) + ? stream.LoadSourceAuto(fileName) as RosterNode + ?? throw new FormatException("archive did not contain a roster") + : BattleScribeXml.LoadRoster(stream) + ?? throw new FormatException("file is not a BattleScribe roster"); + } + catch (Exception e) when (e is not FormatException) + { + throw new FormatException($"could not read roster file: {e.Message}"); + } + + var count = 0; + var forces = roster.Forces.Select(f => ConvertForce(f, ref count)).ToList(); + if (forces.Count == 0) + throw new FormatException("roster has no forces"); + + return new( + Name: roster.Name ?? "unnamed roster", + GameSystemId: roster.GameSystemId, + ObservedTotals: [.. roster.Costs.Select(c => new ReplayCost(c.Name ?? "", c.TypeId ?? "", c.Value))], + BooksRevisions: [], + Forces: forces, + Unmapped: []); + } + + private static ReplayForce ConvertForce(ForceNode force, ref int count) => new( + ForceEntryId: force.EntryId ?? throw new FormatException($"force '{force.Name}' has no entryId"), + CatalogueId: force.CatalogueId ?? "", + Selections: ConvertSelections(force.Selections, ref count), + ChildForces: ConvertForces(force.Forces, ref count)); + + private static List ConvertForces(IEnumerable forces, ref int count) + { + var result = new List(); + foreach (var f in forces) result.Add(ConvertForce(f, ref count)); + return result; + } + + private static List ConvertSelections(IEnumerable selections, ref int count) + { + var result = new List(); + foreach (var s in selections) + { + if (++count > MaxSelections) + throw new FormatException($"roster too large (over {MaxSelections} selections)"); + if (s.EntryId is null) continue; // no way to replay — surfaced via structural drift + result.Add(new( + EntryId: s.EntryId, + Count: s.Number, + CustomName: s.CustomName, + ObservedCosts: [.. s.Costs.Select(c => new ReplayCost(c.Name ?? "", c.TypeId ?? "", c.Value))], + Children: ConvertSelections(s.Selections, ref count))); + } + return result; + } +} +``` + +Note the `ref int count` across lambdas is illegal in C# — use the explicit loop style shown (no LINQ where `ref` is captured); for the top-level `roster.Forces.Select(...)` call, replace with `ConvertForces(roster.Forces, ref count)`. `NodeList` enumerates as `IEnumerable` — verify member access names against the generated node API while implementing (properties confirmed: `RosterNode.Forces/Costs/Name/GameSystemId`, `ForceNode.EntryId/CatalogueId/Selections/Forces`, `SelectionNode.EntryId/Number/CustomName/Costs/Selections`, `CostNode.Name/TypeId/Value`). If a selection with `EntryId == null` is skipped, append a note to `Unmapped` instead of silent skip (spec rule) — adjust the code accordingly: collect notes in a `List` threaded like `count`. + +- [ ] **Step 4: Decide per-selection pins (read the comparer first)** + +Read `RosterRunner.AssertSelections` (`lib/wham/lib/battlescribe-spec/src/BattleScribeSpec.TestKit/Roster/RosterRunner.cs:757-782`): +- If expected selections match actual by **name** (or entryId): extend `SpecEmitter.Emit` to append, when `pinObserved` and any selection has `ObservedCosts`, a second `expectedState` block with `forces: [ { selections: [ { entryId/name…, costs: […] } ] } ]` pinning per-selection costs. Add an emitter test mirroring `Emitted_yaml_round_trips_through_SpecLoader` asserting the selection-level pin appears and round-trips. +- If matching is **positional/index-based** (fragile against engine auto-adds): do NOT emit per-selection pins; instead add a YAML comment in the emitted spec (`# per-selection costs observed but not pinned (comparer is order-sensitive)`) and record the decision in the commit message. + +- [ ] **Step 5: Run tests + Release build** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "RosterFileConverterTests|SpecEmitterTests" -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 6: Commit** + +```bash +git add src tests +git commit -m "feat: .ros/.rosz to ReplayRoster converter with per-selection observed costs" +``` + +--- + +### Task 8: `muster convert` command + +Wires fetcher + converters into the CLI seam the M0–M2 spec named. Input: file path (`.ros`, `.rosz`, `.json`) or NR share URL. + +**Files:** +- Create: `src/Muster.Cli/Commands/ConvertCommand.cs` +- Modify: `src/Muster.Cli/Program.cs` (register) +- Test: `tests/Muster.Cli.Tests/ConvertCommandTests.cs` + +**Interfaces:** +- Consumes: `NrShareLink`, `NrClient` (Task 5), `NrListParser`, `SpecEmitter` (Task 6), `RosterFileConverter` (Task 7). +- Produces: `muster convert [--id ] [--data-source ] [--pin-observed] [-o ]`. + - `` `Argument`: existing file path → dispatch by extension (`.ros`/`.rosz` → RosterFileConverter; `.json` → NrListParser); else if `NrShareLink.TryParse` → fetch via NrClient; else usage error (exit 2). + - `--id` default: input file name stem or `nr-`. + - `--data-source` default `"local:."` (documented: replace with the repo's `github:` uri when committing as a fixture). + - `--pin-observed` default **true** for convert (flag `--no-pin` alternative is NOT added; use `--pin-observed false` — `Option` with explicit default). + - Output: YAML to stdout or `-o` file. Exit 0 on success; exit 2 on any failure (fetch error, FormatException) with the human-readable message on stderr — a convert failure is never "fixture failure", so never exit 1. + - `internal static async Task Run(string input, string? id, string dataSource, bool pinObserved, FileInfo? output)` for testability. + +- [ ] **Step 1: Write the failing tests** + +```csharp +// tests/Muster.Cli.Tests/ConvertCommandTests.cs +using Muster.Cli.Commands; + +namespace Muster.Cli.Tests; + +public class ConvertCommandTests +{ + private static string SamplePath => Path.Combine(AppContext.BaseDirectory, "TestData", "nr-list-war-horde.json"); + + [Fact] + public async Task Converts_nr_json_file_to_spec_yaml() + { + var outFile = new FileInfo(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".yaml")); + try + { + var exit = await ConvertCommand.Run(SamplePath, id: null, dataSource: "local:.", pinObserved: true, output: outFile); + + Assert.Equal(0, exit); + var yaml = File.ReadAllText(outFile.FullName); + var spec = BattleScribeSpec.SpecLoader.LoadFromYaml(yaml); + Assert.Equal("nr-list-war-horde", spec.Id); + Assert.Contains(spec.Steps, s => s.Action == "addForce"); + Assert.Contains(spec.Steps, s => s.ExpectedState is not null); + } + finally { outFile.Delete(); } + } + + [Fact] + public async Task Missing_file_exits_2() + { + var exit = await ConvertCommand.Run("no-such-file.ros", null, "local:.", true, null); + Assert.Equal(2, exit); + } + + [Fact] + public async Task Non_allowlisted_url_exits_2() + { + var exit = await ConvertCommand.Run("https://evil.example/app/list/x", null, "local:.", true, null); + Assert.Equal(2, exit); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter ConvertCommandTests -v minimal` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +`ConvertCommand.Create()` mirrors the existing command style (`TestCommand.cs`). Core of `Run`: + +```csharp +internal static async Task Run(string input, string? id, string dataSource, bool pinObserved, FileInfo? output) +{ + try + { + ReplayRoster roster; + string defaultId; + if (File.Exists(input)) + { + defaultId = Path.GetFileNameWithoutExtension(input); + var ext = Path.GetExtension(input); + if (ext.Equals(".json", StringComparison.OrdinalIgnoreCase)) + { + roster = NrListParser.Parse(await File.ReadAllTextAsync(input)); + } + else if (ext.Equals(".ros", StringComparison.OrdinalIgnoreCase) || ext.Equals(".rosz", StringComparison.OrdinalIgnoreCase)) + { + await using var stream = File.OpenRead(input); + roster = RosterFileConverter.Convert(stream, input); + } + else + { + Console.Error.WriteLine($"unsupported input extension: {ext}"); + return 2; + } + } + else if (NrShareLink.TryParse(input, out var key)) + { + defaultId = $"nr-{key}"; + using var client = new NrClient(); + var fetched = await client.FetchListAsync(key); + if (fetched.Json is null) + { + Console.Error.WriteLine(fetched.Error); + return 2; + } + roster = NrListParser.Parse(fetched.Json); + } + else + { + Console.Error.WriteLine($"input is neither an existing file nor a New Recruit share link: {input}"); + return 2; + } + + var yaml = SpecEmitter.Emit(roster, id ?? defaultId, dataSource, pinObserved); + if (output is null) Console.Out.Write(yaml); + else await File.WriteAllTextAsync(output.FullName, yaml); + return 0; + } + catch (FormatException e) + { + Console.Error.WriteLine(e.Message); + return 2; + } +} +``` + +Register in `Program.CreateRootCommand`: `root.Subcommands.Add(ConvertCommand.Create());`. System.CommandLine async action: `command.SetAction((parse, ct) => Run(...))` — follow the async overload pattern; check how `InvokeAsync` maps `Task` (existing commands are sync — mirror `TestCommand` but with the async `SetAction` overload, which 2.0.3 supports). + +- [ ] **Step 4: Run the full suite + Release build** + +Run: `dotnet test -v minimal && dotnet build -c Release` +Expected: PASS (includes `ProgramExitCodeTests` — `convert` with no args must exit 2 via parse-error remap). + +- [ ] **Step 5: Commit** + +```bash +git add src tests +git commit -m "feat: muster convert — roster file or NR link to fixture-DSL spec" +``` + +--- + +### Task 9: Issue body parsing + attachment fetching + +Extract the roster source and reporter text from a hostile issue body: muster's own form layout, NR's auto-report layout, attachment URLs, inline YAML blocks. + +**Files:** +- Create: `src/Muster.Cli/Reports/IssueBody.cs` +- Create: `src/Muster.Cli/Reports/AttachmentClient.cs` +- Test: `tests/Muster.Cli.Tests/Reports/IssueBodyTests.cs`, `tests/Muster.Cli.Tests/Reports/AttachmentClientTests.cs` + +**Interfaces:** +- Produces: + - `sealed record RosterSource(RosterSourceKind Kind, string Value)`; `enum RosterSourceKind { NrLink, Attachment, InlineYaml }` + - `sealed record IssueBody(RosterSource? Roster, string? Problem, string? Expected)`; `static IssueBody Parse(string body)`: + - NR link: first match of the `NrShareLink` pattern anywhere in the body (covers both muster's form and NR auto-reports' `**List:** ` line) → `NrLink` with the URL. + - Attachment: first URL matching `^https://github\.com/user-attachments/files/\d+/[A-Za-z0-9._-]+\.(ros|rosz|zip)$` (also accept legacy `https://github.com///files/\d+/…`) → `Attachment`. + - Inline YAML: first fenced code block ` ```yaml … ``` ` whose content contains `steps:` → `InlineYaml` with the block content. + - Precedence: NrLink > Attachment > InlineYaml (a body with several candidates uses the first by that ranking). No match → `Roster = null`. + - `Problem`/`Expected`: text following `**Problem:**` / `**Expected:**` markers up to the next `**` marker or blank-line paragraph break (both muster's form and NR auto-reports use these); null when absent. Body size cap: parse at most the first 65 536 characters. + - `sealed class AttachmentClient(HttpMessageHandler? handler = null) : IDisposable`: `Task<(byte[]? Data, string? Error)> DownloadAsync(string url, CancellationToken ct = default)` — re-validates the URL against the attachment pattern (defense in depth), 30 s timeout, 5 MB cap, follows redirects (GitHub attachment URLs redirect to storage), same graceful-error contract as `NrClient`. + +- [ ] **Step 1: Write the failing tests** + +```csharp +// tests/Muster.Cli.Tests/Reports/IssueBodyTests.cs +using Muster.Cli.Reports; + +namespace Muster.Cli.Tests.Reports; + +public class IssueBodyTests +{ + [Fact] + public void Parses_nr_auto_report() + { + // Verbatim shape of New Recruit auto-filed issues (BSData/wh40k-11e#234) + var body = """ + **Problem:** + Outriders Squad 6 members - 145 points + + **Expected:** + Outriders Squad 6 members - 140 points + + **List:** https://www.newrecruit.eu/app/list/tr5BL + **NewRecruit Version:** 34.99 + """; + var parsed = IssueBody.Parse(body); + + Assert.Equal(RosterSourceKind.NrLink, parsed.Roster!.Kind); + Assert.Equal("https://www.newrecruit.eu/app/list/tr5BL", parsed.Roster.Value); + Assert.Contains("145 points", parsed.Problem, StringComparison.Ordinal); + Assert.Contains("140 points", parsed.Expected, StringComparison.Ordinal); + } + + [Fact] + public void Parses_attachment_url() + { + var body = "My roster: https://github.com/user-attachments/files/12345/my-list.rosz broken!"; + var parsed = IssueBody.Parse(body); + Assert.Equal(RosterSourceKind.Attachment, parsed.Roster!.Kind); + Assert.EndsWith("my-list.rosz", parsed.Roster.Value, StringComparison.Ordinal); + } + + [Fact] + public void Parses_inline_yaml_block() + { + var body = """ + Points look wrong: + + ```yaml + steps: + - action: addForce + forceEntryId: "fe-1" + ``` + """; + var parsed = IssueBody.Parse(body); + Assert.Equal(RosterSourceKind.InlineYaml, parsed.Roster!.Kind); + Assert.Contains("addForce", parsed.Roster.Value, StringComparison.Ordinal); + } + + [Fact] + public void Nr_link_wins_over_attachment_and_yaml() + { + var body = """ + https://github.com/user-attachments/files/1/x.rosz + **List:** https://www.newrecruit.eu/app/list/abc12 + ```yaml + steps: [] + ``` + """; + Assert.Equal(RosterSourceKind.NrLink, IssueBody.Parse(body).Roster!.Kind); + } + + [Theory] + [InlineData("")] + [InlineData("just words, no roster anywhere")] + [InlineData("https://evil.example/app/list/abc12")] + [InlineData("```yaml\nname: no steps here\n```")] + public void No_roster_source_yields_null(string body) => + Assert.Null(IssueBody.Parse(body).Roster); + + [Fact] + public void Huge_body_is_truncated_not_hung() + { + var body = new string('a', 10_000_000) + " **List:** https://www.newrecruit.eu/app/list/abc12"; + var parsed = IssueBody.Parse(body); // must return fast + Assert.Null(parsed.Roster); // link is beyond the 64 KiB parse window + } +} +``` + +`AttachmentClientTests`: mirror `NrClientTests` structure — stub handler; assert URL re-validation rejects `https://evil.example/x.rosz` without any HTTP call (`StubHandler` asserts it was never invoked); assert 5 MB cap; assert success path returns bytes. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "IssueBodyTests|AttachmentClientTests" -v minimal` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +`IssueBody.Parse` outline (implement with `GeneratedRegex`, all patterns anchored/bounded, input truncated to 65 536 chars first): + +```csharp +[GeneratedRegex(@"https://www\.newrecruit\.eu/app/list/[A-Za-z0-9]{1,32}")] +private static partial Regex NrLinkPattern(); + +[GeneratedRegex(@"https://github\.com/(?:user-attachments/files|[\w.-]+/[\w.-]+/files)/\d+/[A-Za-z0-9._-]+\.(?:ros|rosz|zip)")] +private static partial Regex AttachmentPattern(); + +[GeneratedRegex(@"```ya?ml\s*\n(.*?)```", RegexOptions.Singleline)] +private static partial Regex YamlBlockPattern(); + +[GeneratedRegex(@"\*\*Problem:\*\*\s*\n?(.*?)(?=\n\s*\*\*|\z)", RegexOptions.Singleline)] +private static partial Regex ProblemPattern(); + +[GeneratedRegex(@"\*\*Expected:\*\*\s*\n?(.*?)(?=\n\s*\*\*|\z)", RegexOptions.Singleline)] +private static partial Regex ExpectedPattern(); +``` + +Precedence logic: check NR link first, then attachment, then the first yaml block containing `steps:`. Trim captured Problem/Expected to 2 000 chars. + +`AttachmentClient`: copy the `NrClient` streaming-cap body; `HttpClientHandler { AllowAutoRedirect = true }`; re-validate the URL with `AttachmentPattern` before any request. + +- [ ] **Step 4: Run tests + Release build** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "IssueBodyTests|AttachmentClientTests" -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 5: Commit** + +```bash +git add src/Muster.Cli/Reports tests/Muster.Cli.Tests/Reports +git commit -m "feat: hostile-input issue body parser + allowlisted attachment client" +``` + +--- + +### Task 10: Verdict mapping + reply rendering + `muster report` + +The report pipeline: body → roster → spec → per-engine evaluation → verdict + labels + markdown reply with per-engine matrix and `` details block. + +**Files:** +- Create: `src/Muster.Cli/Reports/Verdict.cs` +- Create: `src/Muster.Cli/Reports/ReplyRenderer.cs` +- Create: `src/Muster.Cli/Commands/ReportCommand.cs` +- Modify: `src/Muster.Cli/Program.cs` (register) +- Test: `tests/Muster.Cli.Tests/Reports/VerdictTests.cs`, `tests/Muster.Cli.Tests/ReportCommandTests.cs` + +**Interfaces:** +- Consumes: `IssueBody` (Task 9), `NrClient`/`NrShareLink` (5), `NrListParser`/`SpecEmitter`/`ReplayRoster` (6), `RosterFileConverter` (7), `MultiRunReport` machinery (3) — report runs the ONE generated spec by writing it to a temp fixtures dir and calling `MultiRunReport.Run(dataDir, tempFixturesDir, engines, governing)`. +- Produces: + - `enum VerdictKind { Confirmed, NotReproducible, NeedsInfo, Inconclusive }` + - `sealed record Verdict(VerdictKind Kind, bool EngineGap, IReadOnlyList Labels, string? Detail)` + - `static class VerdictMapper`: `Verdict Map(ReplayRoster? roster, string? conversionError, MultiRunReport? runs)`: + 1. `conversionError != null` OR `roster == null` OR `roster.Unmapped.Count > 0` → `NeedsInfo` (detail = error / unmapped notes). + 2. `runs` has no successful runs (all unavailable) → `Inconclusive`. + 3. Governing engine's single fixture result: `HarnessError`/inconclusive → `NeedsInfo` (detail: "the roster could not be replayed against current data" + first failure line) — replay failures are NOT "not-reproducible". + 4. Governing passed (pins matched) → `Confirmed`. + 5. Governing failed assertions → `NotReproducible` (detail = failure lines, which carry expected-vs-actual). + 6. `EngineGap = true` when ≥2 engines ran and their fixture pass/fail status differs; adds `engine-gap` to `Labels`. + - `Labels` always includes the kind's label (`confirmed`, `not-reproducible`, `needs-info`, `inconclusive`). + - `static class ReplyRenderer`: `string Render(Verdict verdict, ReplayRoster? roster, MultiRunReport? runs, string specYaml, string? problem, string? expected)` — markdown reply: + - first line: `` (sticky-comment marker) + - verdict heading + one-paragraph explanation (which engine governed; engines that ran/unavailable) + - value × engine matrix: rows = pinned observed values (name + observed value from `roster.ObservedTotals`), columns = `reported` + each engine that ran (engine cell: `reproduced ✔` when that engine passed pins / actual value parsed from its failure line when not — if the actual value can't be parsed from the failure string, render `differs ✖`) + - reporter's Problem/Expected quoted back (truncated 500 chars each) + - `books_revision` note: "reported against: Xenos - Orks: 2 — evaluated against current data" + - collapsed snapshot: `
Executable spec (snapshot)` + `` + fenced yaml block with `specYaml` + `
` + - needs-info variant: polite ask naming exactly what was missing/invalid, plus the three accepted roster formats. + - `ReportCommand`: `muster report --issue-body --data [--engines …] [--governing …] [--out-dir ]` — writes `reply.md`, `report.json` (`{verdict, labels, engineGap, governing}` camelCase), `snapshot.yaml` into `--out-dir` (default `.`); prints reply to stdout too. Exit 0 whenever a reply was produced (ALL verdicts including needs-info); exit 2 only on harness errors (missing --data dir, unwritable out-dir). + - Data-source rule: the generated spec's `dataSource` must match what the data repo's fixtures use so the hermeticity gate passes — `ReportCommand` takes `--data-source ` (default: detect from an existing fixture in `--fixtures`? NO — explicit option, default `local:.`; entrypoint wires the real value in Task 12). + +- [ ] **Step 1: Write the failing verdict tests** + +```csharp +// tests/Muster.Cli.Tests/Reports/VerdictTests.cs +using Muster.Cli.Reports; +using Muster.Cli.Reporting; + +namespace Muster.Cli.Tests.Reports; + +public class VerdictTests +{ + private static ReplayRoster Roster(params string[] unmapped) => new( + "r", "gs", [new("pts", "pt-1", 100m)], [], + [new("fe-1", "cat-1", [], [])], [.. unmapped]); + + private static MultiRunReport Runs(params (string Engine, bool Passed, bool Inconclusive)[] engines) => new( + Governing: engines.Length > 0 ? engines[0].Engine : null, + Unavailable: [], + Runs: [.. engines.Select(e => RunReport.Create(e.Engine, "data", + [new FixtureResult("spec", "p", e.Passed, e.Passed ? [] : ["Step 3: cost pts: expected 100 but got 95"], 1, e.Inconclusive)]))]); + + [Fact] + public void Conversion_error_is_needs_info() + { + var v = VerdictMapper.Map(null, "list not found", null); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + Assert.Contains("needs-info", v.Labels); + } + + [Fact] + public void Unmapped_nodes_force_needs_info() + { + var v = VerdictMapper.Map(Roster("selection 'X' missing option_id"), null, Runs(("wham", true, false))); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + } + + [Fact] + public void Governing_pass_is_confirmed() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("wham", true, false))); + Assert.Equal(VerdictKind.Confirmed, v.Kind); + Assert.Contains("confirmed", v.Labels); + Assert.False(v.EngineGap); + } + + [Fact] + public void Governing_assertion_failure_is_not_reproducible() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("wham", false, false))); + Assert.Equal(VerdictKind.NotReproducible, v.Kind); + Assert.Contains("expected 100 but got 95", v.Detail, StringComparison.Ordinal); + } + + [Fact] + public void Replay_crash_is_needs_info_not_notreproducible() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("wham", false, true))); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + } + + [Fact] + public void Engine_disagreement_raises_engine_gap() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("newrecruit", true, false), ("wham", false, false))); + Assert.Equal(VerdictKind.Confirmed, v.Kind); // newrecruit governs + Assert.True(v.EngineGap); + Assert.Contains("engine-gap", v.Labels); + } + + [Fact] + public void No_engines_ran_is_inconclusive() + { + var runs = new MultiRunReport(null, ["newrecruit"], []); + var v = VerdictMapper.Map(Roster(), null, runs); + Assert.Equal(VerdictKind.Inconclusive, v.Kind); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter VerdictTests -v minimal` +Expected: FAIL. + +- [ ] **Step 3: Implement Verdict + VerdictMapper + ReplyRenderer** + +`Verdict.cs`: + +```csharp +// src/Muster.Cli/Reports/Verdict.cs +using Muster.Cli.Converters; +using Muster.Cli.Reporting; + +namespace Muster.Cli.Reports; + +public enum VerdictKind { Confirmed, NotReproducible, NeedsInfo, Inconclusive } + +public sealed record Verdict(VerdictKind Kind, bool EngineGap, IReadOnlyList Labels, string? Detail); + +public static class VerdictMapper +{ + public static Verdict Map(ReplayRoster? roster, string? conversionError, MultiRunReport? runs) + { + if (conversionError is not null || roster is null) + return Make(VerdictKind.NeedsInfo, false, conversionError ?? "no roster found in the report"); + if (roster.Unmapped.Count > 0) + return Make(VerdictKind.NeedsInfo, false, string.Join("\n", roster.Unmapped)); + if (runs is null || runs.Runs.Count == 0 || runs.Governing is null) + return Make(VerdictKind.Inconclusive, false, "no engine was available to evaluate the roster"); + + var governing = runs.Runs.First(r => r.Engine == runs.Governing); + var fixture = governing.Fixtures[0]; + + var gap = runs.Runs.Count > 1 && runs.Runs + .Select(r => (r.Fixtures[0].Passed, r.Fixtures[0].Inconclusive)) + .Distinct().Count() > 1; + + if (fixture.Inconclusive) + return Make(VerdictKind.NeedsInfo, gap, + "the roster could not be replayed against current data: " + fixture.Failures.FirstOrDefault()); + return fixture.Passed + ? Make(VerdictKind.Confirmed, gap, null) + : Make(VerdictKind.NotReproducible, gap, string.Join("\n", fixture.Failures)); + } + + private static Verdict Make(VerdictKind kind, bool gap, string? detail) + { + var labels = new List + { + kind switch + { + VerdictKind.Confirmed => "confirmed", + VerdictKind.NotReproducible => "not-reproducible", + VerdictKind.NeedsInfo => "needs-info", + _ => "inconclusive", + }, + }; + if (gap) labels.Add("engine-gap"); + return new(kind, gap, labels, detail); + } +} +``` + +`ReplyRenderer.Render`: build the markdown per the interface contract above (plain `StringBuilder`; escape `|` in user text with `\|`; wrap Problem/Expected in `> ` blockquotes). Write focused tests inline in `ReportCommandTests` (marker presence, matrix row for the pinned pts value, snapshot block contains the YAML, needs-info variant names the reason). + +`ReportCommand.Run` outline: + +```csharp +internal static async Task Run(FileInfo issueBody, DirectoryInfo data, string dataSource, + string[] engines, string[] governing, DirectoryInfo? outDir) +{ + // harness-level validation → exit 2 + if (!issueBody.Exists || !data.Exists) { Console.Error.WriteLine("…"); return 2; } + var body = IssueBody.Parse(await File.ReadAllTextAsync(issueBody.FullName)); + + ReplayRoster? roster = null; + string? specYaml = null; + string? error = null; + switch (body.Roster) + { + case null: + error = "no roster found: accepted formats are a New Recruit share link, a .ros/.rosz attachment, or an inline ```yaml steps block"; + break; + case { Kind: RosterSourceKind.NrLink } src: + // NrShareLink.TryParse (already matched), NrClient fetch, NrListParser.Parse — FormatException → error + break; + case { Kind: RosterSourceKind.Attachment } src: + // AttachmentClient download → RosterFileConverter.Convert + break; + case { Kind: RosterSourceKind.InlineYaml } src: + // inline: the YAML *is* the spec — validate via SpecLoader.LoadFromYaml (catch → error), + // specYaml = src.Value; roster stays null but conversion is fine: + // handle via a separate bool inlineMode — VerdictMapper needs runs only. + break; + } + if (roster is not null) + specYaml = SpecEmitter.Emit(roster, specId: "report", dataSource, pinObserved: true); + + MultiRunReport? runs = null; + if (specYaml is not null && error is null) + { + var tempFixtures = Directory.CreateTempSubdirectory("muster-report-"); + try + { + await File.WriteAllTextAsync(Path.Combine(tempFixtures.FullName, "report.yaml"), specYaml); + runs = MultiRunReport.Run(data.FullName, tempFixtures.FullName, engines, governing); + } + finally { tempFixtures.Delete(recursive: true); } + } + + var verdict = VerdictMapper.Map(roster, error, runs); + // inlineMode adjustment: when inline spec ran, roster is null but error is null too — + // pass a minimal ReplayRoster (name from spec id, no pins) so Map doesn't call it needs-info. + var reply = ReplyRenderer.Render(verdict, roster, runs, specYaml ?? "", body.Problem, body.Expected); + + var dir = outDir?.FullName ?? "."; + await File.WriteAllTextAsync(Path.Combine(dir, "reply.md"), reply); + await File.WriteAllTextAsync(Path.Combine(dir, "report.json"), ToJson(verdict, runs)); + if (specYaml is not null) await File.WriteAllTextAsync(Path.Combine(dir, "snapshot.yaml"), specYaml); + Console.Out.WriteLine(reply); + return 0; +} +``` + +Resolve the `inlineMode` wrinkle cleanly: change `VerdictMapper.Map`'s first parameter to `bool hasRoster, IReadOnlyList unmapped` OR overload for inline mode — implementer's choice, tests pin the behavior: inline spec + passing run = `Confirmed`. + +`ReportCommandTests`: end-to-end with `TestRepoFactory` data + an inline-yaml issue body (reuses the green fixture's YAML with pins matching → Confirmed; with pins mismatching → NotReproducible; garbage body → NeedsInfo with exit 0; missing data dir → exit 2). Assert `reply.md` contains `` and `snapshot.yaml` written. + +- [ ] **Step 4: Run the full suite + Release build** + +Run: `dotnet test -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 5: Commit** + +```bash +git add src tests +git commit -m "feat: muster report — issue body to verdict, labels, per-engine reply, snapshot" +``` + +--- + +### Task 11: `muster promote` + +Extract the newest `` block from the harness's own issue comments, strip observed pins, re-run capturing final state via `RosterRunner.OnStepCompleted`, re-pin to current values, write `tests/rosters/.yaml`. + +**Files:** +- Create: `src/Muster.Cli/Reports/SnapshotExtractor.cs` +- Create: `src/Muster.Cli/Reports/SpecRePinner.cs` +- Create: `src/Muster.Cli/Commands/PromoteCommand.cs` +- Modify: `src/Muster.Cli/Program.cs` (register) +- Test: `tests/Muster.Cli.Tests/Reports/SnapshotExtractorTests.cs`, `tests/Muster.Cli.Tests/PromoteCommandTests.cs` + +**Interfaces:** +- Consumes: `SpecLoader.LoadFromYaml`, `RosterRunner` + `OnStepCompleted` callback (signature `Action>`), `EngineRegistry` (Task 2), `RepoDataSourceResolver`. +- Produces: + - `static class SnapshotExtractor`: `string? ExtractLatest(string commentsJson)` — input is `gh api /repos/{o}/{r}/issues/{n}/comments` output (JSON array of objects with `body`); returns the fenced-yaml content of the LAST comment containing ``, or null. Parse with System.Text.Json + regex for the fenced block; body cap 256 KiB per comment. + - `static class SpecRePinner`: `string RePin(string specYaml, string dataDir, EngineSpec engine, string newSpecId)` — loads the spec, removes all `expectedState`-only steps, runs the remaining steps via `RosterRunner` (with `RepoDataSourceResolver.Create(dataDir)` + `IsPopulatedFor` gate → throws `HarnessInconclusiveException` if unpopulated), captures the final `RosterState` from `OnStepCompleted`, and re-emits the spec (via a small extension to `SpecEmitter` or direct text surgery on the step list: keep the original step lines verbatim, replace the trailing expectedState block) with `expectedState.costs` pinned to the captured state's `Costs` and the new id. Throws `InvalidOperationException` when the run itself fails (a fixture that can't run can't be promoted). + - `PromoteCommand`: `muster promote --issue-body --comments --data --issue-number [--engines …] [--governing …] [--out ]` — out default `tests/rosters`; slug = kebab-cased issue title from `--issue-body` first line if it looks like a title file? NO — title is not in the body; slug = `report-issue-`; collision → `report-issue--2`, `-3`, …. Prints the written file path on stdout. Exit 0 written / 2 anything else. + +- [ ] **Step 1: Write the failing tests** + +```csharp +// tests/Muster.Cli.Tests/Reports/SnapshotExtractorTests.cs +using Muster.Cli.Reports; + +namespace Muster.Cli.Tests.Reports; + +public class SnapshotExtractorTests +{ + [Fact] + public void Extracts_yaml_from_latest_snapshot_comment() + { + var comments = """ + [ + {"body": "just a human comment"}, + {"body": "\nold reply\n
Executable spec (snapshot)\n\n\n```yaml\nid: \"old\"\n```\n\n
"}, + {"body": "\nnew reply\n
Executable spec (snapshot)\n\n\n```yaml\nid: \"new\"\nsteps: []\n```\n\n
"} + ] + """; + var yaml = SnapshotExtractor.ExtractLatest(comments); + Assert.NotNull(yaml); + Assert.Contains("id: \"new\"", yaml, StringComparison.Ordinal); + Assert.DoesNotContain("old", yaml, StringComparison.Ordinal); + } + + [Theory] + [InlineData("[]")] + [InlineData("""[{"body": "no snapshot here"}]""")] + [InlineData("not json")] + public void Missing_snapshot_returns_null(string comments) => + Assert.Null(SnapshotExtractor.ExtractLatest(comments)); +} +``` + +`PromoteCommandTests` (uses `TestRepoFactory` + the builtin wham engine over the existing green test-repo data): +- comments file containing a snapshot whose steps replay against the test repo (reuse the green fixture's steps with WRONG pinned costs, e.g. `value: 999`) → promote writes `tests/rosters/report-issue-7.yaml` whose `expectedState.costs` pin the CURRENT engine value (20), not 999 — assert by `SpecLoader.Load` on the written file. +- name collision: pre-create `report-issue-7.yaml` → new file is `report-issue-7-2.yaml`. +- comments without snapshot → exit 2, stderr message. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `dotnet test tests/Muster.Cli.Tests --filter "SnapshotExtractorTests|PromoteCommandTests" -v minimal +Expected: FAIL. + +- [ ] **Step 3: Implement** + +`SnapshotExtractor`: JSON-parse the array defensively (bad JSON → null); iterate in order keeping the last match of + +```csharp +[GeneratedRegex(@"\s*```ya?ml\s*\n(.*?)```", RegexOptions.Singleline)] +``` + +`SpecRePinner` core: + +```csharp +public static string RePin(string specYaml, string dataDir, EngineSpec engineSpec, string newSpecId) +{ + var spec = SpecLoader.LoadFromYaml(specYaml, defaultId: newSpecId); + if (spec.Setup.DataSource is { Length: > 0 } ds && !RepoDataSourceResolver.IsPopulatedFor(dataDir, ds)) + throw new InvalidOperationException($"data source not populated locally: {ds}"); + + // strip assertion-only steps (they may pin stale/buggy values) + var replaySteps = spec.Steps.Where(s => s.Action is not null).ToList(); + var replaySpec = /* SpecFile with Steps = replaySteps — records: spec with { Steps = replaySteps } + (check SpecFile is a record / has settable Steps; if init-only list, + construct a new SpecFile copying Id/Category/Description/Setup) */; + + RosterState? finalState = null; + using var engine = EngineRegistry.CreateEngine(engineSpec); + var runner = new RosterRunner(engine, RepoDataSourceResolver.Create(dataDir), engineName: engineSpec.Name) + { + OnStepCompleted = (_, _, state, _) => finalState = state, + }; + var result = runner.Run(replaySpec); + if (result.HarnessError is not null || !result.Passed || finalState is null) + throw new InvalidOperationException( + "cannot promote: replay failed against current data: " + + (result.HarnessError ?? result.Failures.FirstOrDefault() ?? "no state captured")); + + // re-emit: original replay step YAML text preserved, new trailing pin block + return RewriteWithPins(specYaml, newSpecId, finalState.Costs); +} +``` + +`RewriteWithPins`: text-level — replace the `id:` line value with `newSpecId`; drop any existing `- expectedState:`-only trailing block (recognize by the emitted comment marker `# observed values from the report` OR by locating steps that have `expectedState` and no `action` — since SpecEmitter emits pins as the final step, dropping from the marker line to EOF is deterministic for muster-emitted snapshots; fall back to append-only when the marker is absent); append a fresh pin block with the captured costs and comment `# expected values pinned at promotion (engine: )`. Validate output with `SpecLoader.LoadFromYaml` before returning (throw otherwise). + +`PromoteCommand.Run`: read comments file → `ExtractLatest` (null → exit 2) → slug/collision loop → `SpecRePinner.RePin` (catch `InvalidOperationException`/`HarnessInconclusiveException` → stderr + exit 2) → write file, print path, exit 0. + +- [ ] **Step 4: Run the full suite + Release build** + +Run: `dotnet test -v minimal && dotnet build -c Release` +Expected: PASS, zero warnings. + +- [ ] **Step 5: Commit** + +```bash +git add src tests +git commit -m "feat: muster promote — snapshot extraction, re-pin to current values, fixture file" +``` + +--- + +### Task 12: GitHub kit — issue form, reusable report workflow, promote flow, docs + +Everything a data repo installs. Lives in WarHub/muster; the pilot fork copies the caller stubs. + +**Files:** +- Create: `kit/issue-form/report-a-data-bug.yml` (template data repos copy to `.github/ISSUE_TEMPLATE/`) +- Create: `.github/workflows/report-check.yml` (reusable, `workflow_call`) +- Create: `kit/callers/muster-report.yml` (thin caller data repos copy to `.github/workflows/`) +- Modify: `entrypoint.sh` (report/promote modes) +- Modify: `docs/authoring-fixtures.md` (link) + Create: `docs/executable-bug-reports.md` +- Test: manual `act`-free validation — `python -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))"` per file + entrypoint bash gates below + +**Interfaces:** +- Consumes: `muster report` (Task 10), `muster promote` (Task 11) via the container image; sticky comments via `peter-evans/find-comment@v3` + `peter-evans/create-or-update-comment@v4` keyed on ``; labels + PR via `gh` CLI with `GITHUB_TOKEN`. +- Produces: a data repo needs exactly two files (issue template + caller workflow) and the labels get created idempotently on first run. + +- [ ] **Step 1: Issue form template** + +```yaml +# kit/issue-form/report-a-data-bug.yml +name: Report a data bug +description: Report wrong points, missing options, or broken constraints. Your roster makes the bug reproducible. +title: "[data bug]: " +labels: [] +body: + - type: markdown + attributes: + value: | + Muster will automatically evaluate your roster against the latest data + and reply with what the engines actually compute. + - type: input + id: roster + attributes: + label: Roster + description: > + A New Recruit share link (https://www.newrecruit.eu/app/list/…), OR drag + a .rosz file into the field below, OR paste fixture YAML in the details field. + placeholder: "https://www.newrecruit.eu/app/list/abc12" + validations: + required: false + - type: textarea + id: problem + attributes: + label: Problem + description: What does the app show? (Muster echoes this back — markers must stay) + placeholder: "Outriders Squad 6 members - 145 points" + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected + description: What should it show, per the printed rules? + placeholder: "Outriders Squad 6 members - 140 points" + validations: + required: true + - type: textarea + id: details + attributes: + label: Attachments / inline spec + description: "Drop a .rosz here (GitHub may require zipping it), or paste a ```yaml steps block." + validations: + required: false +``` + +GitHub renders form fields into the body as `### Roster\n\n\n\n### Problem\n\n` — **verify `IssueBody.Parse` handles the `### Problem` heading form too**; if it only matches `**Problem:**`, extend the Task 9 regexes with the alternative `(?:\*\*Problem:\*\*|### Problem)` and add a form-shaped test case (do this as part of this task; it is one regex + one test). + +- [ ] **Step 2: Reusable workflow** + +```yaml +# .github/workflows/report-check.yml +name: Muster report check +on: + workflow_call: + inputs: + data-path: + type: string + default: "." + data-source: + type: string + description: dataSource URI matching the repo's fixtures (e.g. github:BSData/wh40k-11e) + required: true + engines: + type: string + default: "wham" + governing: + type: string + default: "newrecruit battlescribe wham" + fixtures-out: + type: string + default: "tests/rosters" + +permissions: + issues: write + contents: write + pull-requests: write + +jobs: + evaluate: + if: > + github.event_name == 'issues' || + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/muster check')) + runs-on: ubuntu-latest + container: ghcr.io/warhub/muster:latest + steps: + - uses: actions/checkout@v4 + - name: Save issue body + env: + BODY: ${{ github.event.issue.body }} + run: printf '%s' "$BODY" > /tmp/issue-body.md + - name: Evaluate report + run: | + /entrypoint.sh report "${{ inputs.data-path }}" /tmp/issue-body.md \ + "${{ inputs.data-source }}" "${{ inputs.engines }}" "${{ inputs.governing }}" + - name: Find previous reply + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.issue.number }} + body-includes: "" + - name: Post reply + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.issue.number }} + body-path: reply.md + edit-mode: replace + - name: Apply labels + env: + GH_TOKEN: ${{ github.token }} + ISSUE: ${{ github.event.issue.number }} + run: | + for label in confirmed not-reproducible needs-info inconclusive engine-gap; do + gh label create "$label" --force --color "$(case $label in + confirmed) echo d73a4a;; not-reproducible) echo 0e8a16;; + needs-info) echo fbca04;; inconclusive) echo d4c5f9;; + engine-gap) echo 5319e7;; esac)" -R "$GITHUB_REPOSITORY" || true + done + labels=$(python3 -c "import json;print(','.join(json.load(open('report.json'))['labels']))") + current=$(gh issue view "$ISSUE" -R "$GITHUB_REPOSITORY" --json labels -q '[.labels[].name] | map(select(. == "confirmed" or . == "not-reproducible" or . == "needs-info" or . == "inconclusive" or . == "engine-gap")) | join(",")') + [ -n "$current" ] && gh issue edit "$ISSUE" -R "$GITHUB_REPOSITORY" --remove-label "$current" || true + gh issue edit "$ISSUE" -R "$GITHUB_REPOSITORY" --add-label "$labels" + + promote: + if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/muster promote') + runs-on: ubuntu-latest + container: ghcr.io/warhub/muster:latest + steps: + - name: Check commenter permission + env: + GH_TOKEN: ${{ github.token }} + run: | + perm=$(gh api "repos/$GITHUB_REPOSITORY/collaborators/${{ github.event.comment.user.login }}/permission" -q .permission) + case "$perm" in admin|write|maintain) ;; *) + gh issue comment "${{ github.event.issue.number }}" -R "$GITHUB_REPOSITORY" \ + -b "Sorry @${{ github.event.comment.user.login }}, promotion needs write access." + exit 1;; esac + - uses: actions/checkout@v4 + - name: Fetch comments and issue body + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api "repos/$GITHUB_REPOSITORY/issues/${{ github.event.issue.number }}/comments" --paginate > /tmp/comments.json + gh api "repos/$GITHUB_REPOSITORY/issues/${{ github.event.issue.number }}" -q .body > /tmp/issue-body.md + - name: Promote + run: | + /entrypoint.sh promote "${{ inputs.data-path }}" /tmp/issue-body.md /tmp/comments.json \ + "${{ github.event.issue.number }}" "${{ inputs.fixtures-out }}" \ + "${{ inputs.engines }}" "${{ inputs.governing }}" + - name: Open PR + env: + GH_TOKEN: ${{ github.token }} + ISSUE: ${{ github.event.issue.number }} + run: | + git config user.name "muster-bot" + git config user.email "muster@users.noreply.github.com" + branch="muster/promote-issue-$ISSUE" + git checkout -b "$branch" + git add "${{ inputs.fixtures-out }}" + git commit -m "test: promote issue #$ISSUE roster to golden fixture" + git push origin "$branch" + gh pr create -R "$GITHUB_REPOSITORY" --head "$branch" \ + --title "Promote issue #$ISSUE roster to golden fixture" \ + --body "Promotes the reproducing roster from #$ISSUE to tests/rosters/. Review the pinned expected values — they were captured from the current (post-fix) engine evaluation. Closes #$ISSUE." +``` + +- [ ] **Step 3: Caller stub + entrypoint modes** + +```yaml +# kit/callers/muster-report.yml — data repos copy to .github/workflows/ +name: Muster bug reports +on: + issues: + types: [opened, edited] + issue_comment: + types: [created] +jobs: + report: + uses: WarHub/muster/.github/workflows/report-check.yml@main + with: + data-source: "github:BSData/wh40k-11e" # ← repo-specific +``` + +`entrypoint.sh`: prepend mode dispatch — first arg `report` / `promote` selects new flows; anything else falls through to the existing test/diff flow (backward compatible: the Action passes data-path first, never a mode word): + +```bash +MODE="$1" +if [[ "$MODE" == "report" ]]; then + shift + DATA_PATH="$1"; BODY_FILE="$2"; DATA_SOURCE="$3"; ENGINES_INPUT="$4"; GOVERNING_INPUT="$5" + # build dataroot exactly like test mode, but seeded from DATA_SOURCE instead of + # scanning fixtures (extract org/repo/ref → same cache layout) + # then: muster report --issue-body "$BODY_FILE" --data "$DATAROOT" \ + # --data-source "$DATA_SOURCE" --engines … --governing … --out-dir . + # exit code passthrough: report exits 0 for all verdicts; 2 → ::error:: + exit 1 + # (a report harness error must be visible, not neutral) + … +elif [[ "$MODE" == "promote" ]]; then + shift + # analogous: muster promote --issue-body … --comments … --data "$DATAROOT" --issue-number … --out … + … +fi +``` + +Write the two blocks fully in the implementation (mirror `build_dataroot`'s cache-layout logic with the dataSource string as the seed — factor the layout code into a `build_dataroot_for_source()` function reused by both paths). + +- [ ] **Step 4: Validate + docs + commit** + +- YAML-validate all three workflow/template files (`python -c "import yaml; yaml.safe_load(open('…'))"`). +- Bash-gate entrypoint: `bash -n entrypoint.sh` plus a dry-run of `report` mode against the test repo fixture data with `MUSTER_CMD` pointing at a local `dotnet publish` output (same gate style as M0–M2 Task 11). +- `docs/executable-bug-reports.md`: install steps (2 files), the three roster formats, verdict/label semantics table, `/muster check` + `/muster promote` commands, engines/governing configuration, the NR-adapter docker registration for repos that want the default governor. + +```bash +git add kit .github/workflows/report-check.yml entrypoint.sh docs +git commit -m "feat: issue-form kit + reusable report/promote workflows + docs" +``` + +--- + +### Task 13: NR adapter console host + public Docker image (battlescribe-spec submodule) + +The NR adapter is a library today — give it an NDJSON console host and a public image. Work happens in the NESTED submodule `lib/wham/lib/battlescribe-spec` on branch `feat/muster-support` (continue the existing branch; commit + PR at the end; never touch `D:\repos\battlescribe-spec`). + +**Files (inside `lib/wham/lib/battlescribe-spec/`):** +- Create: `src/BattleScribeSpec.NewRecruit.Adapter/BattleScribeSpec.NewRecruit.Adapter.csproj` +- Create: `src/BattleScribeSpec.NewRecruit.Adapter/Program.cs` +- Create: `docker/newrecruit-adapter/Dockerfile` +- Create: `.github/workflows/publish-nr-adapter.yml` +- Modify: `BattleScribeSpec.slnx` (add project) + +**Interfaces:** +- Consumes: `AdapterHandler.RunAsync(Func, TextReader, TextWriter, CancellationToken)` (`TestKit/Protocol/AdapterHandler.cs:16`), `NewRecruitRosterEngine` (`src/BattleScribeSpec.NewRecruit/NewRecruitRosterEngine.cs` — READ its constructor/lifecycle first: it likely needs `NewRecruitBrowser`/`NewRecruitEnginePool` setup; mirror however `BattleScribeSpec.Runner` or the Debugger constructs it today — find the construction site with `grep -rn "new NewRecruitRosterEngine" src tools tests`). +- Produces: console adapter runnable as `dotnet BattleScribeSpec.NewRecruit.Adapter.dll` (stdio NDJSON), and image `ghcr.io/warhub/bsspec-adapter-newrecruit:latest`. + +- [ ] **Step 1: Console host** + +`Program.cs` mirrors `src/BattleScribeSpec.ReferenceAdapter/Program.cs` (line 17 shape): + +```csharp +using BattleScribeSpec.NewRecruit; +using BattleScribeSpec.Protocol; + +// NDJSON adapter host for the New Recruit engine (Playwright-driven). +// Speaks the same stdio protocol as ReferenceAdapter; ships as a public +// Docker image with the browser baked in (no proprietary JARs involved). +await AdapterHandler.RunAsync( + engineFactory: () => /* construct NewRecruitRosterEngine the way the Runner does — copy the construction found via grep */, + input: Console.In, + output: Console.Out); +``` + +Copy any required Playwright bootstrap (browser install path env vars) from the existing NR test/runner setup — check `docker/docker-compose.yaml` and CI workflows in the spec repo for how NR runs headless today. + +- [ ] **Step 2: Dockerfile** + +```dockerfile +# docker/newrecruit-adapter/Dockerfile +# Public image: adapter binaries + Playwright browser. Publishes NO repo sources +# and NO BattleScribe JARs (this adapter has no JAR dependency). +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src +COPY . . +RUN dotnet publish src/BattleScribeSpec.NewRecruit.Adapter -c Release -o /app + +FROM mcr.microsoft.com/playwright/dotnet:v1.49.0-noble +COPY --from=build /app /app +ENTRYPOINT ["dotnet", "/app/BattleScribeSpec.NewRecruit.Adapter.dll"] +``` + +(Pin the Playwright image tag to whatever `Microsoft.Playwright` version the NR project references — check `Directory.Packages.props` in the spec repo and match major.minor.) + +- [ ] **Step 3: Publish workflow** + +`.github/workflows/publish-nr-adapter.yml`: mirrors muster's own docker job — build on PRs, push `latest` + `sha-…` tags to `ghcr.io/warhub/bsspec-adapter-newrecruit` on push to the default branch, `docker/login-action@v3` with `GITHUB_TOKEN`, `permissions: packages: write`. The user makes the GHCR package public after first publish (same as muster's image). + +- [ ] **Step 4: Build gate + commit + submodule bumps** + +```bash +cd lib/wham/lib/battlescribe-spec +dotnet build -c Release src/BattleScribeSpec.NewRecruit.Adapter +git add -A && git commit -m "feat: NR adapter console host + public Docker image publishing" +cd ../.. # wham +git add lib/battlescribe-spec && git commit -m "chore: bump battlescribe-spec (NR adapter host)" +cd ../.. # muster +git add lib/wham && git commit -m "chore: bump wham (battlescribe-spec NR adapter host)" +``` + +(Full docker build can't run locally — no Docker on this machine; the PR's CI validates the image. The `dotnet build` gate catches everything else.) + +--- + +### Task 14: E2E pilot demo — three beats (controller-led, NOT a subagent task) + +Requires live GitHub + the merged muster image. Executed by the session controller after Tasks 1–13 merge (muster PR → main → image republished; battlescribe-spec + wham PRs opened). + +- [ ] **Beat 0 — install the kit on the fork:** on `amis92/wh40k-11e` branch `muster-fixtures`: copy `kit/issue-form/report-a-data-bug.yml` → `.github/ISSUE_TEMPLATE/`, `kit/callers/muster-report.yml` → `.github/workflows/` with `data-source: "github:BSData/wh40k-11e"`. Push, merge to the fork's default branch (workflows for `issues` events only run from the default branch). +- [ ] **Beat 1 — confirmed:** file an issue on the fork using the form with a live NR share link for a Necrons/Orks list (grab a fresh link from a recent BSData/wh40k-11e issue, verify it still resolves via the RPC first) OR an inline YAML spec reproducing the wham#310-adjacent Deathmarks pin (65 observed). Expect: reply comment with per-engine matrix + `confirmed` label + snapshot block. Screenshot. +- [ ] **Beat 2 — not-reproducible after fix:** push a data change to the fork's default branch that alters the pinned value (or use the muster-demo branch's existing 60→65 edit inverted), comment `/muster check`. Expect: reply flips to `not-reproducible` with the value diff. Screenshot. +- [ ] **Beat 3 — promote:** comment `/muster promote`. Expect: PR opens adding `tests/rosters/report-issue-.yaml` with re-pinned values; the muster PR-check workflow on that PR runs the new fixture green. Screenshot; merge. +- [ ] Measure and record: report-workflow wall time with `engines: wham` vs (if the NR adapter image is published by then) `wham newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest` — docker-in-container caveat: the reusable workflow runs muster IN a container; `docker:` engines need the host socket. If unavailable, run the NR engine measurement in a follow-up plain-runner job instead; record findings in the ledger + an issue for M3.5 if socket access blocks NR-in-CI. +- [ ] Update `docs/superpowers/specs/2026-07-13-muster-m3-executable-bug-reports-design.md` status → SHIPPED, memory file, progress ledger. + +--- + +## Execution notes + +- Tasks 1→11 are strictly ordered by interface dependency except: 5 ∥ (2,3,4) and 7 ∥ 5 — but run them sequentially anyway (single implementer at a time; subagent-driven-development forbids parallel implementers). +- Task 12 needs 10+11; Task 13 is independent of 5–12 (can slot anywhere after 1); Task 14 needs everything merged. +- Branch: `feat/m3-executable-bug-reports` (already exists, carries the spec). PR to `WarHub/muster` main at the end; org ruleset = PR-only, expect fresh-branch iterations if CI needs fixes. +- The Muster.Cli.Tests JSON report-shape tests from M2 will need updating exactly once (Task 3) — `--report` output becomes the MultiRunReport envelope. That is the only intentional breaking change to existing outputs; `summary`/`markdown` human outputs stay recognizable, and the Action's report-path contract is unchanged. diff --git a/docs/superpowers/specs/2026-07-13-muster-m3-executable-bug-reports-design.md b/docs/superpowers/specs/2026-07-13-muster-m3-executable-bug-reports-design.md new file mode 100644 index 0000000..f8ad60c --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-muster-m3-executable-bug-reports-design.md @@ -0,0 +1,153 @@ +# Muster M3 — Executable Bug Reports: Design + +Date: 2026-07-13. Status: approved. Prior art: [M0–M2 design](2026-07-13-muster-ci-harness-design.md) (shipped: WarHub/muster live, blast-radius Action working on the pilot fork). + +## 0. Decisions log + +| Decision | Choice | Rejected | +|---|---|---| +| Scope | Spec-M3 (issue-form evaluation + fixture promotion), muster#4 folded in | Hardening-only cycle; combined mega-cycle | +| Roster inputs (v1) | YAML DSL steps, `.ros`/`.rosz`, **and New Recruit share links (fetched)** | YAML-only; ignoring the NR stream | +| Confirm logic | Replay vs **observed** values (assertions pin what the reporter saw) | Structured expected-value form field; report-only with manual labels | +| Promotion | `/muster promote` slash command → PR (write-permission gated) | Label-driven; local-CLI-only | +| Architecture | **Everything becomes a spec** — all inputs convert to fixture-DSL YAML, executed by the existing RosterRunner pipeline | Bespoke replay comparator in wham; report-only | +| Engines | **Per-engine evaluation everywhere** (report, test, diff) via an engine registry; verdicts gain `engine-gap`; **governing engine configurable, default New Recruit** (the ecosystem's current main app) | wham-only; reports-only matrix; wham or legacy BS as default governor | + +## 1. Product contract + +A data repo that already uses the muster Action gains a second install: an issue form ("Report a data bug") plus a reusable workflow. The flow: + +1. **Report.** A reporter files the issue form (or New Recruit auto-files its own report — recognized too). The body carries a roster as one of: NR share link, `.ros`/`.rosz` attachment URL, or inline fixture-DSL YAML steps. +2. **Evaluate.** The workflow runs the muster container. Muster parses the body, obtains the roster, converts it to a fixture-DSL spec whose assertions **pin the observed values** (what the reporter's app showed), and evaluates it against latest data with every available engine (§5). +3. **Label + reply.** Sticky comment with a **per-engine result matrix** (see §5) and the verdict; label applied: + - `confirmed` — the governing engine reproduces what the reporter saw. The bug reproduces. + - `not-reproducible` — the governing engine computes different values (often "already fixed"); reply shows the exact per-engine diffs. + - `engine-gap` — engines that ran disagree with *each other*; added alongside the governing verdict and auto-flagged for the wham backlog. + - `needs-info` — unusable input (bad link, rotted list, unparseable roster). Polite reply, never a stack trace. + - `inconclusive` — every available engine abstained (M2 discipline unchanged): visible, honest, never wrong. +4. **Snapshot.** The reply embeds the generated spec in a collapsed `
` block. This is the durable copy — NR links rot (BSData/wh40k-11e#234's list is already gone), so after first evaluation the issue thread is self-contained. +5. **Promote.** After a fix lands, a maintainer comments `/muster promote`. The workflow re-reads the snapshot from its own comment, re-evaluates, re-pins assertions to the current (post-fix, correct) engine values, and opens a PR adding the fixture to `tests/rosters/` with the issue linked. Human correctness review happens exactly once — on that PR. Bugs become regression tests. + +## 2. Why "everything becomes a spec" + +All three inputs normalize to the battlescribe-spec roster DSL, and the existing pipeline (SpecLoader → RosterRunner → SpecRosterEngineAdapter → wham) executes them. No second execution path, no bespoke comparison engine: + +- **Confirmation is assertion semantics inverted at the label layer.** Assertions pin observed (buggy) values; PASS means "reproduced" ⇒ `confirmed`. FAIL lists per-value diffs ⇒ `not-reproducible`. +- **The snapshot IS the promotable artifact.** One file format from report to regression test; promotion just re-pins expected values. +- **`muster convert` finally exists** — the CLI seam the M0–M2 spec named but didn't build. + +## 3. Input formats (exploration findings, verified 2026-07-13) + +### New Recruit share links + +- NR auto-files bug reports on BSData repos itself; the body pattern is `**Problem:** … **Expected:** … **List:** https://www.newrecruit.eu/app/list/` (see BSData/wh40k-11e#234). This is the dominant existing report stream — supporting it makes real reports executable with zero reporter effort. +- Fetch: `POST https://www.newrecruit.eu/api/rpc` body `{"method":"open_share_link","params":[""]}` — answers anonymously (verified live; discovered via the NR app bundle, confirmed by instrumenting the page's fetch). **Undocumented internal API**: any failure/shape drift degrades to `needs-info`, never a crash. Reach out to NR maintainers about a stable contract when pitching BSData (M5). +- Payload: `{name, totalCost, totalCosts[{name,value,typeId}], bsid_system, bsid_book, books_revision[], army{options[…]}}`. The `options` tree is BattleScribe-shaped: `option_id`/`link_id` are BS entry/link ids, `amount` is selection count, `catalogue_id`/`typeId` where relevant. **No per-selection costs** — only roster-level `totalCosts`, so NR-sourced specs pin roster totals only. +- A real sample (`war horde`, 950 pts, Orks) is committed as converter test data. +- Link rot is real: lists get deleted/expire. Hence the snapshot-on-first-evaluation rule. + +### `.ros` / `.rosz` + +Parsed by wham's existing `Workspaces.BattleScribe` into `RosterNode`. Stores per-selection costs → richer pins than NR (roster totals + per-selection costs). Attachment URLs must match GitHub's user-attachments host patterns. + +### Inline YAML steps + +Fixture-DSL steps pasted in the form (power users / data authors). Already assertion-bearing; no conversion, no pinning step. + +## 4. Components + +### muster CLI + +- **`muster convert [--pin-observed] [-o ]`** — input: `.ros`/`.rosz` path, NR list JSON path, or NR share URL (triggers fetch). Output: fixture-DSL YAML. `--pin-observed` adds assertions for the stored/observed values (default for the report flow). Without it: steps only. +- **`muster report --issue-body --data `** — full report pipeline: parse form/NR body → obtain roster → convert → evaluate → emit (a) markdown reply, (b) verdict + suggested labels, (c) machine JSON, (d) generated spec. Exit 0 = reply produced (any verdict, including needs-info); exit 2 = harness error. Verdict is data, not exit code. +- **`muster promote --issue-body --comments --data `** — locate the newest snapshot block in the harness's own comments, re-evaluate against current data, re-pin assertions to current values, write `tests/rosters/.yaml`. Branch/PR mechanics are workflow-side. +- **`muster diff --fail-on-broke`** (muster#4) — exit 1 when any row classifies `broke` or `verdict-changed` **in the governing engine** (§5); non-governing breaks surface as `engine-gap`, not check failures. CLI default remains report-don't-judge (flag opt-in). +- **`--engines `** on `test`, `diff`, and `report` — engine names from the registry (default: all available). See §5. + +New sources: `Muster.Cli/Reports/` (form parsing, verdict mapping, markdown rendering), `Muster.Cli/Converters/` (NR JSON → spec, RosterNode → spec, shared step-emitter), `Muster.Cli/NewRecruit/` (fetcher). Converters are pure model traversal — **no new wham work identified**. Possible small additive gap in battlescribe-spec if `expectedState` can't assert roster-level total costs (verify at plan time; if missing, extend the DSL additively in the spec repo, as with `HarnessError`). + +### Conversion mapping + +NR `army.options` tree / `RosterNode` forces+selections → + +| Source | DSL step | +|---|---| +| force node | `addForce` (`forceEntryId`), nested via `addChildForce` | +| selection node | `selectEntry` / `selectChildEntry` (by `option_id`; `link_id` when present) | +| `amount` ≠ 1 | `setSelectionCount` | +| custom name | `setCustomization` | +| observed roster totals (`totalCosts`) | `expectedState` roster cost assertions | +| observed per-selection costs (`.ros` only) | `expectedState` selection cost assertions | + +Unmappable nodes (unknown ids, unsupported structures) are reported **loudly in the reply** ("selection X could not be mapped — id not found in current data"). Rule: if any step or assertion cannot be emitted, the verdict is `needs-info` — a partially replayed roster must never produce `confirmed`/`not-reproducible`. Note: an id missing from *current* data can itself be the bug (entry deleted); the reply names the missing ids so a maintainer can judge. + +### GitHub kit (lives in WarHub/muster) + +- **Issue form template** `report-a-data-bug.yml`: roster field (link, attachment, or inline YAML), free-text expected/actual, system/book dropdown optional. Copyable into data repos; docs page explains install. +- **Reusable workflow** `report-check.yml` (`workflow_call`, wired to `issues: [opened, edited]` and `issue_comment: [created]` in the data repo's thin caller): + - issue events → `muster report` → sticky comment + labels (`issues: write`). + - `engines` input (default: all available) and `governing` input (default `newrecruit > battlescribe > wham`); WarHub-hosted callers reach the private adapters via the bot app token. Also `fail-on-broke` (default true) and `fail-on-inconclusive` (default false) on the PR-check workflow. + - `/muster check` comment → re-run evaluation (anyone). + - `/muster promote` comment → permission check (commenter has write access, via collaborators API) → `muster promote` → branch `muster/promote-issue-` → PR opened linking the issue. + - NR auto-report recognition: body regex for the `**List:** ` pattern, no form required. +- Labels are created idempotently if missing. + +## 5. Per-engine evaluation + +The battlescribe-spec machinery already supports this — the DSL has per-engine `engines:` expectation blocks, the Runner takes `--assertion-engine`, adapters speak the NDJSON protocol, and `--matrix` merges per-engine conformance reports. Muster orchestrates; it invents no comparison machinery. + +### Engine registry + +Muster config (`muster.yml` in the data repo, or CLI/Action input) maps engine names to launch commands: + +- `newrecruit` — the ecosystem's **current main app** and the default governing reference: what reporters and players actually see. Live Playwright adapter (battlescribe-spec's `BattleScribeSpec.NewRecruit`); heavier than in-proc engines but evaluating one generated spec is a single browser session — acceptable for issue replies, costly for full fixture sweeps (repos tune the engine set per command). +- `battlescribe` — the **legacy oracle**: battlescribe-spec's `ReferenceAdapter` (IKVM-wrapped proprietary BattleScribe JARs). Historical reference semantics; second in default precedence. +- `wham` — builtin, in-proc via `SpecRosterEngineAdapter`; always available and the only engine guaranteed in the public image. **Not yet compliant enough to be the assertion reference** — it informs, catches regressions cheaply, and its divergences from the governor feed its own conformance backlog. + +Adapter commands are registered as `dotnet:`, any executable, or `docker:` (runs `docker run -i --rm `; the NDJSON protocol flows over stdio unchanged), spawned per run via TestKit's `AdapterProcess`. + +**NR adapter distribution (decided):** `BattleScribeSpec.NewRecruit` ships as a **public Docker image** (Playwright + browser baked in), published by battlescribe-spec's CI the same way muster publishes its own image — binaries only, no repo sources, and no JAR encumbrance exists for this adapter. CI callers register it as `docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest`; GitHub's container-action Docker-socket mount makes this reachable from the muster Action (verify the socket path during implementation; fall back to running the adapter as a workflow service container if not). The `battlescribe` oracle can never ship this way — its image would redistribute the proprietary JARs. + +Availability detection: an engine is *available* when builtin or its adapter command resolves and starts. Requested-but-unavailable engines appear in output as `unavailable` — named, never silently dropped. Default engine set: all available. + +**Licensing boundary (hard):** nothing in muster's public artifacts may embed or download the proprietary BattleScribe JARs — the `battlescribe` adapter exists only where the private battlescribe-spec checkout is present (WarHub-hosted workflows via the bot app token; maintainers with access, locally). The NR adapter has no JAR encumbrance, only private-repo residency, and ships as a public Docker image (above); the muster image itself still guarantees wham only. + +### Governing-engine rule + +One engine **governs** each verdict; the rest inform. Precedence is **configurable** (`engines.governing` in `muster.yml` / Action input), default `newrecruit > battlescribe > wham`: the governor is the first engine in precedence that actually ran. Rationale: NR is what the community currently plays with, so "confirmed" should mean "the main app still shows this today"; legacy BS carries reference semantics when NR can't run; wham governs only as a last resort and the reply says so explicitly. Whenever engines that ran disagree with each other on any asserted value, `engine-gap` is raised alongside the governing verdict — abstain-don't-lie extended to multi-engine: wham never silently masquerades as the reference, and every wham-vs-governor divergence auto-feeds the wham conformance backlog (the harness keeps finishing wham). + +### Command semantics + +- **`report`:** the reply matrix is *value × engine* — the **NR-reported** column (the roster's stored values from report time; an observation, never the governor) then one column per engine that ran now. Verdict per the governing-engine rule; with NR governing, `confirmed` literally means "New Recruit still shows this against latest data". +- **`test`:** per-fixture, per-engine pass/fail/inconclusive; a fixture passes when every ran engine matches its engine-resolved expectations (DSL `engines:` blocks apply). Exit 1 on any engine's assertion failure; abstention stays per-engine inconclusive. +- **`diff`:** blast radius classified **per engine** — each fixture row carries base→head classification for every engine that ran in both states (e.g. `wham: pass→fail broke ❌ · newrecruit: pass→pass unchanged` — that combination signals a wham gap, not a data break). The comment table shows one column per engine; `--fail-on-broke` trips on `broke`/`verdict-changed` in the **governing engine**; non-governing breaks are surfaced but gate only the `engine-gap` signal, not the check. Engines available in only one of base/head are reported `unavailable`, excluded from gating. +- **JSON schema:** `RunReport` gains an engine dimension on every result (additive; single-engine output remains the degenerate case). + +### Performance note + +Adapter engines run out-of-proc per fixture over NDJSON; total work scales fixtures × engines (× 2 for diff). The per-fixture time budget applies per engine, and the report names any engine that blew it (that engine goes inconclusive; others still count). + +## 6. Error handling (hostile input by default) + +- Issue bodies are stranger-controlled. Fetch allowlist: exactly `https://www.newrecruit.eu/app/list/<[A-Za-z0-9]+>` and GitHub user-attachment URLs; nothing else is ever fetched. +- Caps: fetch timeout, response size cap, JSON depth/node-count caps, roster selection-count cap. Exceeding any → `needs-info` reply naming the limit. +- No stack traces or raw exceptions in replies — harness errors exit 2 and the workflow posts a generic "harness error, maintainers notified" comment (`::error::` in the log carries details). +- Every reply names which engines ran, which governed the verdict, and which were unavailable. Reply also shows `books_revision` the reporter used vs. data evaluated. + +## 7. Testing + +- **Unit:** form/body parser (adversarial: injection attempts, malformed markdown, huge bodies, wrong URLs); NR JSON converter against the committed `war horde` sample; RosterNode converter; verdict mapping incl. governing-engine rule and `engine-gap`; URL allowlist; engine-registry availability detection (missing adapter → `unavailable`, not crash). +- **Integration:** generated specs execute through the FakeEngine end-to-end; a second, deliberately-divergent FakeEngine exercises multi-engine runs (matrix rendering, `engine-gap`, per-engine diff classification); `--fail-on-broke` exit codes across engines; promote round-trip (snapshot → re-pin → fixture file). +- **E2E on the pilot fork (milestone exit):** hand-file an NR-style issue (clone of BSData#234 content with a live list) → `confirmed`; fix the data → `/muster check` → `not-reproducible`; `/muster promote` → fixture PR opened. Three beats, screenshot each. + +## 8. Out of scope (this cycle) + +Remaining hardening backlog: per-fixture setup perf (shared compilation cache), wham#310 fix, engine evaluation of `ModifierKind.Replace/Ceil/Floor` + `ConditionGroupKind.Count`, associations/alias round-trip — next cycle. M4 narrows: per-engine evaluation lands the shadow-compare *mechanism* now, so M4 keeps only the capability manifest and the nightly schedule/trend reporting. NR stable-API collaboration is an M5 conversation, not code. + +## Open questions (tracked, not blocking) + +1. Does battlescribe-spec `expectedState` support roster-level total-cost assertions today? Verify at plan time; extend additively if not. +2. `.rosz` attachment friction: GitHub issue attachments may reject the extension (zip rename workaround) — document in the form's help text; verify while building the template. +3. Promotion slug collisions (`tests/rosters/.yaml` exists) — suffix with issue number; confirm during implementation. +4. ~~NR adapter distribution~~ — resolved: public Docker image published from battlescribe-spec CI (see §5); non-WarHub repos get the default governor by registering `docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest`. +5. NR adapter runtime cost in CI — the prebuilt image removes Playwright install cost; browser-session cost per spec remains. Measure during implementation; full-sweep `test`/`diff` with NR may need an opt-in or fixture-count guard. diff --git a/entrypoint.sh b/entrypoint.sh index 7260a6b..d8274e9 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,12 +3,65 @@ set -euo pipefail # muster GitHub Action entrypoint. # -# Usage: entrypoint.sh [base-ref] +# Usage: +# entrypoint.sh [base-ref] [fail-on-broke] [fail-on-inconclusive] [engines] [governing] +# entrypoint.sh report [engines] [governing] [out-dir] +# entrypoint.sh promote [out-dir] [engines] [governing] # -# data-path Data repo root (checkout-relative or absolute). -# fixtures-path Golden fixtures directory (checkout-relative or absolute). -# base-ref Git ref to diff against. Empty (or omitted) -> `muster test`. -# Non-empty -> `muster diff --base --head `. +# --- positional-args form (no mode word) --- +# +# Unchanged, backward compatible with the published `docker://ghcr.io/warhub/muster:latest` +# Action (action.yml), which never passes a mode word as its first argument: +# +# data-path Data repo root (checkout-relative or absolute). +# fixtures-path Golden fixtures directory (checkout-relative or absolute). +# base-ref Git ref to diff against. Empty (or omitted) -> `muster test`. +# Non-empty -> `muster diff --base --head `. +# fail-on-broke "true"/"false" (default: true). Passes `--fail-on-broke` to +# `muster diff` when true; ignored in `muster test` mode. +# fail-on-inconclusive "true"/"false" (default: false). When true, an inconclusive +# (exit 2) muster run fails the check instead of warning neutrally. +# engines Space-separated engine specs, forwarded as `--engines ...` to +# both `muster test` and `muster diff` (default: "wham"). +# governing Space-separated governing precedence, forwarded as +# `--governing ...` (default: "newrecruit battlescribe wham"). +# +# --- `report` mode --- +# +# Used by the reusable .github/workflows/report-check.yml workflow's `evaluate` job (M3): +# an issue is opened/edited, or someone comments `/muster check`. +# +# data-path Data repo root. +# issue-body-file Path to a file containing the raw GitHub issue body text. +# data-source dataSource URI the generated spec should target, e.g. +# "github:BSData/wh40k-11e[@ref]" — also seeds the dataroot cache +# layout (see build_dataroot_for_source below). +# engines (optional, default "wham") +# governing (optional, default "newrecruit battlescribe wham") +# out-dir (optional, default ".") — reply.md/report.json/snapshot.yaml land here. +# previous-reply-file (optional) Path to the previous sticky-comment reply body, if any. +# Forwarded as `--previous-reply` only when the path is non-empty AND +# the file exists, so a missing/never-written file (no prior comment) +# is silently treated as "no previous reply" rather than an error. +# +# Exit code: 0 always, EXCEPT a genuine harness error (muster report exit 2, meaning no +# usable reply was produced at all) is surfaced as `::error::` + exit 1 — unlike test/diff +# mode's exit-2 handling below, report mode never treats exit 2 as a neutral pass, because a +# reporter is waiting on a reply that never got written. +# +# --- `promote` mode --- +# +# Used by the reusable .github/workflows/report-check.yml workflow's `promote` job (M3): +# a write-permission commenter comments `/muster promote`. +# +# data-path Data repo root. +# issue-body-file Path to a file containing the raw GitHub issue body text. +# comments-file Path to `gh api .../issues//comments --paginate` JSON output. +# data-source Same dataSource URI as report mode (seeds the dataroot layout). +# issue-number GitHub issue number. +# out-dir (optional, default "tests/rosters") +# engines (optional, default "wham") +# governing (optional, default "newrecruit battlescribe wham") # # MUSTER_CMD (default: "dotnet /app/muster.dll") is the muster invocation, # split on whitespace. This lets the same script run inside the prebuilt @@ -20,41 +73,186 @@ set -euo pipefail # /github/{org}/{repo}/{ref-or-latest}/. In a fork's CI run, # GITHUB_REPOSITORY names the fork/consumer repo, not the org/repo baked # into the fixture -- so this script derives the dataroot layout from the -# fixtures themselves (grep for `dataSource: github:...` declarations) -# rather than from GITHUB_REPOSITORY, and populates each derived +# fixtures themselves (grep for `dataSource: github:...` declarations, test/diff +# modes) or from an explicit `data-source` argument (report/promote modes), rather +# than from GITHUB_REPOSITORY, and populates each derived # github/{org}/{repo}/{ref}/ directory with only the TOP-LEVEL data files # (*.yaml/*.yml/*.cat/*.gst) from data-path. It never recurses into # data-path, so a fixtures/tests subdirectory living under data-path is # never swept into the dataroot (muster's fixture-discovery file walk is # recursive and would choke on fixture YAML mixed into game data). +MUSTER_CMD="${MUSTER_CMD:-dotnet /app/muster.dll}" +read -ra MUSTER_CMD_ARR <<< "$MUSTER_CMD" + +# Docker container actions run as root against a mounted checkout owned by +# a different uid; `git worktree`/`git -C` refuse to operate on such repos +# without this. Harmless (and idempotent) when running locally too. +git config --global --add safe.directory '*' + +# Populates dataroot "$1" from data directory "$2" for a SINGLE +# `github:{org}/{repo}[@{ref}]` dataSource URI "$3": creates +# $1/github/{org}/{repo}/{ref-or-latest}/ and copies the top-level +# *.yaml/*.yml/*.cat/*.gst files from "$2" into it. Shared cache-layout logic used by both +# `build_dataroot` below (test/diff modes: source(s) scanned out of fixture declarations) +# and report/promote modes (source given directly as a CLI argument, no fixtures dir to scan). +build_dataroot_for_source() { + local dataroot="$1" + local data_path="$2" + local uri="$3" + + if [[ "$uri" != github:* ]]; then + echo "::error::muster: unsupported data-source '$uri' (only 'github:{org}/{repo}[@{ref}]' is supported)" >&2 + return 1 + fi + + local rest org repo_and_ref repo ref dest + rest="${uri#github:}" + org="${rest%%/*}" + repo_and_ref="${rest#*/}" + if [[ "$repo_and_ref" == *@* ]]; then + repo="${repo_and_ref%@*}" + ref="${repo_and_ref##*@}" + else + repo="$repo_and_ref" + ref="latest" + fi + + dest="$dataroot/github/$org/$repo/$ref" + mkdir -p "$dest" + find "$data_path" -maxdepth 1 -type f \ + \( -name '*.yaml' -o -name '*.yml' -o -name '*.cat' -o -name '*.gst' \) \ + -exec cp -t "$dest" {} + +} + +MODE="${1:-}" + +if [[ "$MODE" == "report" ]]; then + shift + if [[ $# -lt 3 ]]; then + echo "usage: entrypoint.sh report [engines] [governing] [out-dir] [previous-reply-file]" >&2 + exit 2 + fi + + DATA_PATH="$1" + BODY_FILE="$2" + DATA_SOURCE="$3" + ENGINES_INPUT="${4:-wham}" + GOVERNING_INPUT="${5:-newrecruit battlescribe wham}" + OUT_DIR="${6:-.}" + PREVIOUS_REPLY_FILE="${7:-}" + + ENGINE_ARGS=(--engines) + read -r -a ENGINE_LIST <<< "$ENGINES_INPUT" + ENGINE_ARGS+=("${ENGINE_LIST[@]}") + GOVERNING_ARGS=(--governing) + read -r -a GOVERNING_LIST <<< "$GOVERNING_INPUT" + GOVERNING_ARGS+=("${GOVERNING_LIST[@]}") + + PREVIOUS_REPLY_ARGS=() + if [[ -n "$PREVIOUS_REPLY_FILE" && -f "$PREVIOUS_REPLY_FILE" ]]; then + PREVIOUS_REPLY_ARGS=(--previous-reply "$PREVIOUS_REPLY_FILE") + fi + + DATA_PATH_ABS="$(cd "$DATA_PATH" && pwd)" + REPORT_DATAROOT="$(mktemp -d)" + trap 'rm -rf "$REPORT_DATAROOT"' EXIT + + build_dataroot_for_source "$REPORT_DATAROOT" "$DATA_PATH_ABS" "$DATA_SOURCE" + + rc=0 + "${MUSTER_CMD_ARR[@]}" report \ + --issue-body "$BODY_FILE" \ + --data "$REPORT_DATAROOT" \ + --data-source "$DATA_SOURCE" \ + --out-dir "$OUT_DIR" \ + "${ENGINE_ARGS[@]}" "${GOVERNING_ARGS[@]}" "${PREVIOUS_REPLY_ARGS[@]}" || rc=$? + + if [[ "$rc" -ne 0 ]]; then + echo "::error::muster report harness error (exit $rc) -- see log above" >&2 + exit 1 + fi + exit 0 +fi + +if [[ "$MODE" == "promote" ]]; then + shift + if [[ $# -lt 5 ]]; then + echo "usage: entrypoint.sh promote [out-dir] [engines] [governing]" >&2 + exit 2 + fi + + DATA_PATH="$1" + BODY_FILE="$2" + COMMENTS_FILE="$3" + DATA_SOURCE="$4" + ISSUE_NUMBER="$5" + OUT_DIR="${6:-tests/rosters}" + ENGINES_INPUT="${7:-wham}" + GOVERNING_INPUT="${8:-newrecruit battlescribe wham}" + + ENGINE_ARGS=(--engines) + read -r -a ENGINE_LIST <<< "$ENGINES_INPUT" + ENGINE_ARGS+=("${ENGINE_LIST[@]}") + GOVERNING_ARGS=(--governing) + read -r -a GOVERNING_LIST <<< "$GOVERNING_INPUT" + GOVERNING_ARGS+=("${GOVERNING_LIST[@]}") + + DATA_PATH_ABS="$(cd "$DATA_PATH" && pwd)" + PROMOTE_DATAROOT="$(mktemp -d)" + trap 'rm -rf "$PROMOTE_DATAROOT"' EXIT + + build_dataroot_for_source "$PROMOTE_DATAROOT" "$DATA_PATH_ABS" "$DATA_SOURCE" + + rc=0 + "${MUSTER_CMD_ARR[@]}" promote \ + --issue-body "$BODY_FILE" \ + --comments "$COMMENTS_FILE" \ + --data "$PROMOTE_DATAROOT" \ + --issue-number "$ISSUE_NUMBER" \ + --out "$OUT_DIR" \ + "${ENGINE_ARGS[@]}" "${GOVERNING_ARGS[@]}" || rc=$? + + if [[ "$rc" -ne 0 ]]; then + echo "::error::muster promote failed (exit $rc) -- see log above" >&2 + exit 1 + fi + exit 0 +fi + +# --- existing test/diff flow (backward compatible: $1 is always data-path here, never a +# mode word -- the published Action never passes one) --- + if [[ $# -lt 2 ]]; then echo "usage: entrypoint.sh [base-ref]" >&2 exit 2 fi -MUSTER_CMD="${MUSTER_CMD:-dotnet /app/muster.dll}" -read -ra MUSTER_CMD_ARR <<< "$MUSTER_CMD" - DATA_PATH="$1" FIXTURES_PATH="$2" BASE_REF="${3:-}" +FAIL_ON_BROKE="${4:-true}" +FAIL_ON_INCONCLUSIVE="${5:-false}" +ENGINES_INPUT="${6:-wham}" +GOVERNING_INPUT="${7:-newrecruit battlescribe wham}" + +ENGINE_ARGS=(--engines) +read -r -a ENGINE_LIST <<< "$ENGINES_INPUT" +ENGINE_ARGS+=("${ENGINE_LIST[@]}") +GOVERNING_ARGS=(--governing) +read -r -a GOVERNING_LIST <<< "$GOVERNING_INPUT" +GOVERNING_ARGS+=("${GOVERNING_LIST[@]}") REPORT_MD="muster-report.md" REPORT_JSON="muster-report.json" -# Docker container actions run as root against a mounted checkout owned by -# a different uid; `git worktree`/`git -C` refuse to operate on such repos -# without this. Harmless (and idempotent) when running locally too. -git config --global --add safe.directory '*' - DATA_PATH_ABS="$(cd "$DATA_PATH" && pwd)" FIXTURES_PATH_ABS="$(cd "$FIXTURES_PATH" && pwd)" # Populates dataroot "$1" from data directory "$2": for every distinct # `dataSource: github:{org}/{repo}[@{ref}]` declaration found under -# $FIXTURES_PATH_ABS, create $1/github/{org}/{repo}/{ref-or-latest}/ and -# copy the top-level *.yaml/*.yml/*.cat/*.gst files from "$2" into it. +# $FIXTURES_PATH_ABS, delegate to build_dataroot_for_source for the actual +# github/{org}/{repo}/{ref}/ layout + copy. build_dataroot() { local dataroot="$1" local data_path="$2" @@ -71,24 +269,9 @@ build_dataroot() { return 0 fi - local uri rest org repo_and_ref repo ref dest + local uri for uri in "${sources[@]}"; do - rest="${uri#github:}" - org="${rest%%/*}" - repo_and_ref="${rest#*/}" - if [[ "$repo_and_ref" == *@* ]]; then - repo="${repo_and_ref%@*}" - ref="${repo_and_ref##*@}" - else - repo="$repo_and_ref" - ref="latest" - fi - - dest="$dataroot/github/$org/$repo/$ref" - mkdir -p "$dest" - find "$data_path" -maxdepth 1 -type f \ - \( -name '*.yaml' -o -name '*.yml' -o -name '*.cat' -o -name '*.gst' \) \ - -exec cp -t "$dest" {} + + build_dataroot_for_source "$dataroot" "$data_path" "$uri" done } @@ -116,7 +299,8 @@ if [[ -z "$BASE_REF" ]]; then --data "$HEAD_DATAROOT" \ --fixtures "$FIXTURES_PATH_ABS" \ --output github-actions \ - --report "$REPORT_JSON" > "$REPORT_MD"; then + --report "$REPORT_JSON" \ + "${ENGINE_ARGS[@]}" "${GOVERNING_ARGS[@]}" > "$REPORT_MD"; then rc=0 else rc=$? @@ -138,11 +322,17 @@ else BASE_DATAROOT="$(mktemp -d)" build_dataroot "$BASE_DATAROOT" "$WORKTREE_DIR/$REL_DATA_PATH" - if "${MUSTER_CMD_ARR[@]}" diff \ + DIFF_ARGS=(diff \ --base "$BASE_DATAROOT" \ --head "$HEAD_DATAROOT" \ --fixtures "$FIXTURES_PATH_ABS" \ - --output markdown > "$REPORT_MD"; then + --output markdown \ + "${ENGINE_ARGS[@]}" "${GOVERNING_ARGS[@]}") + if [[ "$FAIL_ON_BROKE" == "true" ]]; then + DIFF_ARGS+=(--fail-on-broke) + fi + + if "${MUSTER_CMD_ARR[@]}" "${DIFF_ARGS[@]}" > "$REPORT_MD"; then rc=0 else rc=$? @@ -156,6 +346,10 @@ if [[ -n "${GITHUB_OUTPUT:-}" ]]; then fi if [[ "$rc" -eq 2 ]]; then + if [[ "$FAIL_ON_INCONCLUSIVE" == "true" ]]; then + echo "::error::muster run was inconclusive (exit 2) and fail-on-inconclusive is set" + exit 1 + fi echo "::warning::muster run was inconclusive (exit 2) -- treating as neutral" exit 0 fi diff --git a/kit/callers/muster-report.yml b/kit/callers/muster-report.yml new file mode 100644 index 0000000..4d4fc35 --- /dev/null +++ b/kit/callers/muster-report.yml @@ -0,0 +1,14 @@ +# kit/callers/muster-report.yml — data repos copy this to .github/workflows/muster-report.yml +name: Muster bug reports +on: + issues: + types: [opened, edited] + issue_comment: + types: [created] +jobs: + report: + uses: WarHub/muster/.github/workflows/report-check.yml@main + with: + data-source: "github:BSData/wh40k-11e" # ← repo-specific: your org/repo[@ref] + # secrets: + # pr-token: ${{ secrets.MUSTER_PR_TOKEN }} # optional: lets the promotion PR trigger CI checks diff --git a/kit/issue-form/report-a-data-bug.yml b/kit/issue-form/report-a-data-bug.yml new file mode 100644 index 0000000..dfbcc76 --- /dev/null +++ b/kit/issue-form/report-a-data-bug.yml @@ -0,0 +1,45 @@ +name: Report a data bug +description: Report wrong points, missing options, or broken constraints. Your roster makes the bug reproducible. +title: "[data bug]: " +labels: [] +body: + - type: markdown + attributes: + value: | + Muster will automatically evaluate your roster against the latest data + and reply with what the engines actually compute. + - type: input + id: roster + attributes: + label: Roster + description: > + A New Recruit share link (https://www.newrecruit.eu/app/list/…), OR drag + a .rosz file into the field below, OR paste fixture YAML in the details field. + (GitHub may require zipping a .ros/.rosz file to accept it as an attachment — + rename the zip's extension back to .rosz/.ros if it gets renamed to .zip.) + placeholder: "https://www.newrecruit.eu/app/list/abc12" + validations: + required: false + - type: textarea + id: problem + attributes: + label: Problem + description: What does the app show? (Muster echoes this back — markers must stay) + placeholder: "Outriders Squad 6 members - 145 points" + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected + description: What should it show, per the printed rules? + placeholder: "Outriders Squad 6 members - 140 points" + validations: + required: true + - type: textarea + id: details + attributes: + label: Attachments / inline spec + description: "Drop a .rosz here (GitHub may require zipping it), or paste a ```yaml steps block." + validations: + required: false diff --git a/lib/wham b/lib/wham index c980b14..98e416d 160000 --- a/lib/wham +++ b/lib/wham @@ -1 +1 @@ -Subproject commit c980b14dc2839bc2335401a7a5d8d732d8a591ba +Subproject commit 98e416dd6397bfcd8ca5f73b6d8348ccf1b37430 diff --git a/src/Muster.Cli/Commands/ConvertCommand.cs b/src/Muster.Cli/Commands/ConvertCommand.cs new file mode 100644 index 0000000..68cfdba --- /dev/null +++ b/src/Muster.Cli/Commands/ConvertCommand.cs @@ -0,0 +1,114 @@ +using System.CommandLine; +using Muster.Cli.Converters; +using Muster.Cli.NewRecruit; + +namespace Muster.Cli.Commands; + +/// +/// muster convert <input> [--id <spec-id>] [--data-source <uri>] [--pin-observed <bool>] [-o <file>] +/// — converts a roster file (.ros, .rosz, .json) or New Recruit share link to a fixture-DSL spec. +/// +public static class ConvertCommand +{ + public static Command Create() + { + var inputArg = new Argument("input") + { + Description = "File path (.ros, .rosz, .json) or New Recruit share URL.", + }; + var idOption = new Option("--id") + { + Description = "Spec ID (defaults to input file stem or nr-).", + }; + var dataSourceOption = new Option("--data-source") + { + Description = "Data source URI.", + DefaultValueFactory = _ => "local:.", + }; + var pinObservedOption = new Option("--pin-observed") + { + Description = "Include observed totals as expectedState (default: true).", + DefaultValueFactory = _ => true, + }; + var outputOption = new Option("--output", ["-o"]) + { + Description = "Output file (default: stdout).", + }; + + var command = new Command("convert", "Convert roster file or New Recruit link to fixture-DSL spec."); + command.Arguments.Add(inputArg); + command.Options.Add(idOption); + command.Options.Add(dataSourceOption); + command.Options.Add(pinObservedOption); + command.Options.Add(outputOption); + command.SetAction((parse, ct) => Run( + parse.GetValue(inputArg)!, + parse.GetValue(idOption), + parse.GetValue(dataSourceOption)!, + parse.GetValue(pinObservedOption), + parse.GetValue(outputOption), + ct)); + + return command; + } + + internal static async Task Run( + string input, string? id, string dataSource, bool pinObserved, FileInfo? output, + CancellationToken ct = default) + { + try + { + ReplayRoster roster; + string defaultId; + if (File.Exists(input)) + { + defaultId = Path.GetFileNameWithoutExtension(input); + var ext = Path.GetExtension(input); + if (ext.Equals(".json", StringComparison.OrdinalIgnoreCase)) + { + roster = NrListParser.Parse(await File.ReadAllTextAsync(input, ct)); + } + else if (ext.Equals(".ros", StringComparison.OrdinalIgnoreCase) || + ext.Equals(".rosz", StringComparison.OrdinalIgnoreCase)) + { + await using var stream = File.OpenRead(input); + roster = RosterFileConverter.Convert(stream, input); + } + else + { + Console.Error.WriteLine($"unsupported input extension: {ext}"); + return 2; + } + } + else if (NrShareLink.TryParse(input, out var key)) + { + defaultId = $"nr-{key}"; + using var client = new NrClient(); + var fetched = await client.FetchListAsync(key, ct); + if (fetched.Json is null) + { + Console.Error.WriteLine(fetched.Error); + return 2; + } + roster = NrListParser.Parse(fetched.Json); + } + else + { + Console.Error.WriteLine($"input is neither an existing file nor a New Recruit share link: {input}"); + return 2; + } + + var yaml = SpecEmitter.Emit(roster, id ?? defaultId, dataSource, pinObserved); + if (output is null) + Console.Out.Write(yaml); + else + await File.WriteAllTextAsync(output.FullName, yaml, ct); + return 0; + } + catch (Exception e) when (e is not OperationCanceledException) + { + Console.Error.WriteLine(e.Message); + return 2; + } + } +} diff --git a/src/Muster.Cli/Commands/DiffCommand.cs b/src/Muster.Cli/Commands/DiffCommand.cs index 9cedf06..9bf2c03 100644 --- a/src/Muster.Cli/Commands/DiffCommand.cs +++ b/src/Muster.Cli/Commands/DiffCommand.cs @@ -5,8 +5,8 @@ namespace Muster.Cli.Commands; /// /// muster diff --base <dir> --head <dir> --fixtures <dir> — runs the same golden-roster -/// fixtures against two data trees and reports the blast radius: which fixtures changed -/// outcome between base and head. +/// fixtures against two data trees, for one or more roster engines, and reports the blast +/// radius: which fixtures changed outcome between base and head. /// public static class DiffCommand { @@ -32,29 +32,52 @@ public static Command Create() Description = "markdown|json", DefaultValueFactory = _ => "markdown", }; + var failOnBrokeOption = new Option("--fail-on-broke") + { + Description = "Exit 1 when the governing engine classifies any fixture as broke or verdict-changed.", + DefaultValueFactory = _ => false, + }; + var enginesOption = new Option("--engines") + { + Description = "Engines to run: 'wham' (builtin), 'name=dotnet:path.dll', 'name=docker:image', 'name=exe args'. Default: wham.", + AllowMultipleArgumentsPerToken = true, + }; + var governingOption = new Option("--governing") + { + Description = "Governing-engine precedence (first match that ran governs). Default: newrecruit battlescribe wham.", + AllowMultipleArgumentsPerToken = true, + }; var command = new Command("diff", "Report blast radius of fixture outcomes between two data trees."); command.Options.Add(baseOption); command.Options.Add(headOption); command.Options.Add(fixturesOption); command.Options.Add(outputOption); + command.Options.Add(failOnBrokeOption); + command.Options.Add(enginesOption); + command.Options.Add(governingOption); command.SetAction(parse => Run( parse.GetValue(baseOption)!, parse.GetValue(headOption)!, parse.GetValue(fixturesOption)!, - parse.GetValue(outputOption)!)); + parse.GetValue(outputOption)!, + parse.GetValue(failOnBrokeOption), + parse.GetValue(enginesOption) ?? [], + parse.GetValue(governingOption) ?? [])); return command; } - internal static int Run(DirectoryInfo baseDir, DirectoryInfo headDir, DirectoryInfo fixtures, string output) + internal static int Run( + DirectoryInfo baseDir, DirectoryInfo headDir, DirectoryInfo fixtures, string output, + bool failOnBroke, IReadOnlyList engines, IReadOnlyList governing) { - RunReport baseRun; - RunReport headRun; + MultiRunReport baseRuns; + MultiRunReport headRuns; try { - baseRun = TestCommand.RunFixtures(baseDir.FullName, fixtures.FullName); - headRun = TestCommand.RunFixtures(headDir.FullName, fixtures.FullName); + baseRuns = MultiRunReport.Run(baseDir.FullName, fixtures.FullName, engines, governing); + headRuns = MultiRunReport.Run(headDir.FullName, fixtures.FullName, engines, governing); } catch (HarnessInconclusiveException ex) { @@ -66,9 +89,20 @@ internal static int Run(DirectoryInfo baseDir, DirectoryInfo headDir, DirectoryI } // Diff reports what changed, it doesn't judge — exit 0 even when fixtures broke, - // as long as both runs completed at the harness level. - var rows = BlastRadius.Classify(baseRun, headRun); - BlastRadius.Write(baseRun, headRun, rows, output, Console.Out); + // as long as both runs completed at the harness level, unless --fail-on-broke gates + // on the governing engine's own rows. + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + BlastRadius.WriteMulti(report, output, Console.Out); + + if (failOnBroke && report.Governing is { } gov) + { + var governingRows = report.Diffs.FirstOrDefault(d => d.Engine == gov)?.Rows ?? []; + if (governingRows.Any(r => r.Classification is "broke" or "verdict-changed")) + { + return 1; + } + } + return 0; } diff --git a/src/Muster.Cli/Commands/PromoteCommand.cs b/src/Muster.Cli/Commands/PromoteCommand.cs new file mode 100644 index 0000000..2ba7925 --- /dev/null +++ b/src/Muster.Cli/Commands/PromoteCommand.cs @@ -0,0 +1,159 @@ +using System.CommandLine; +using Muster.Cli.Engines; +using Muster.Cli.Reports; + +namespace Muster.Cli.Commands; + +/// +/// muster promote --issue-body <file> --comments <file> --data <root> +/// --issue-number <n> [--engines …] [--governing …] [--out <dir>] — extracts the +/// newest executable-spec snapshot from a muster report issue's comments, re-runs it +/// against current data to re-pin its assertions, and writes the result as a golden fixture +/// under tests/rosters. +/// +public static class PromoteCommand +{ + public static Command Create() + { + var issueBodyOption = new Option("--issue-body") + { + Description = "Path to a file containing the GitHub issue body text.", + Required = true, + }; + var commentsOption = new Option("--comments") + { + Description = "Path to a file containing `gh api /repos/{o}/{r}/issues/{n}/comments` JSON output.", + Required = true, + }; + var dataOption = new Option("--data") + { + Description = "Data repo root.", + Required = true, + }; + var issueNumberOption = new Option("--issue-number") + { + Description = "GitHub issue number — used to slug the fixture file (report-issue-.yaml).", + Required = true, + }; + var enginesOption = new Option("--engines") + { + Description = "Engines to run: 'wham' (builtin), 'name=dotnet:path.dll', 'name=docker:image', 'name=exe args'. Default: wham.", + AllowMultipleArgumentsPerToken = true, + }; + var governingOption = new Option("--governing") + { + Description = "Governing-engine precedence (first match that's available governs). Default: newrecruit battlescribe wham.", + AllowMultipleArgumentsPerToken = true, + }; + var outOption = new Option("--out") + { + Description = "Directory to write the promoted fixture into.", + DefaultValueFactory = _ => new DirectoryInfo(Path.Combine("tests", "rosters")), + }; + + var command = new Command("promote", "Promote a bug report's executable spec snapshot into a pinned golden fixture."); + command.Options.Add(issueBodyOption); + command.Options.Add(commentsOption); + command.Options.Add(dataOption); + command.Options.Add(issueNumberOption); + command.Options.Add(enginesOption); + command.Options.Add(governingOption); + command.Options.Add(outOption); + command.SetAction((parse, ct) => Run( + parse.GetValue(issueBodyOption)!, + parse.GetValue(commentsOption)!, + parse.GetValue(dataOption)!, + parse.GetValue(issueNumberOption), + parse.GetValue(enginesOption) ?? [], + parse.GetValue(governingOption) ?? [], + parse.GetValue(outOption)!, + ct)); + + return command; + } + + internal static async Task Run( + FileInfo issueBody, FileInfo comments, DirectoryInfo data, int issueNumber, + string[] engines, string[] governing, DirectoryInfo outDir, CancellationToken ct = default) + { + try + { + if (!issueBody.Exists) + { + Console.Error.WriteLine($"issue body file not found: {issueBody.FullName}"); + return 2; + } + + if (!comments.Exists) + { + Console.Error.WriteLine($"comments file not found: {comments.FullName}"); + return 2; + } + + if (!data.Exists) + { + Console.Error.WriteLine($"data directory not found: {data.FullName}"); + return 2; + } + + var commentsJson = await File.ReadAllTextAsync(comments.FullName, ct); + var snapshotYaml = SnapshotExtractor.ExtractLatest(commentsJson); + if (snapshotYaml is null) + { + Console.Error.WriteLine("no executable spec snapshot found in issue comments"); + return 2; + } + + var engineSpec = ResolveGoverningEngine(engines, governing); + if (engineSpec is null) + { + Console.Error.WriteLine("no governing engine is available to replay the snapshot"); + return 2; + } + + outDir.Create(); + var (slug, path) = ResolveSlug(outDir.FullName, issueNumber); + + var rePinned = SpecRePinner.RePin(snapshotYaml, data.FullName, engineSpec, slug); + + await File.WriteAllTextAsync(path, rePinned, ct); + Console.Out.WriteLine(path); + return 0; + } + catch (Exception e) when (e is not OperationCanceledException) + { + Console.Error.WriteLine(e.Message); + return 2; + } + } + + /// + /// Resolves the single governing engine among those available: parses + /// (default: builtin wham), filters to the ones actually + /// available in this environment, and picks the first match in + /// precedence (default: ). + /// Promote runs exactly one engine — the one whose evaluation would govern a + /// muster report/muster test verdict — never a multi-engine matrix. + /// + private static EngineSpec? ResolveGoverningEngine(string[] engines, string[] governing) + { + var available = EngineRegistry.ParseAll(engines).Where(EngineRegistry.IsAvailable).ToList(); + var precedence = governing.Length > 0 ? governing : EngineRegistry.DefaultGoverning; + var governingName = EngineRegistry.ResolveGoverning(precedence, [.. available.Select(s => s.Name)]); + return available.FirstOrDefault(s => s.Name == governingName); + } + + private static (string Slug, string Path) ResolveSlug(string outDir, int issueNumber) + { + var baseSlug = $"report-issue-{issueNumber}"; + var slug = baseSlug; + var path = Path.Combine(outDir, $"{slug}.yaml"); + for (var n = 2; File.Exists(path); n++) + { + slug = $"{baseSlug}-{n}"; + path = Path.Combine(outDir, $"{slug}.yaml"); + } + + return (slug, path); + } +} diff --git a/src/Muster.Cli/Commands/ReportCommand.cs b/src/Muster.Cli/Commands/ReportCommand.cs new file mode 100644 index 0000000..3ca8c39 --- /dev/null +++ b/src/Muster.Cli/Commands/ReportCommand.cs @@ -0,0 +1,273 @@ +using System.CommandLine; +using System.Text.Json; +using BattleScribeSpec; +using Muster.Cli.Converters; +using Muster.Cli.NewRecruit; +using Muster.Cli.Reporting; +using Muster.Cli.Reports; + +namespace Muster.Cli.Commands; + +/// +/// muster report --issue-body <file> --data <root> [--engines …] [--governing …] +/// [--data-source <uri>] [--out-dir <dir>] — turns a GitHub issue body (New Recruit +/// share link, roster attachment, or inline fixture-DSL YAML) into a verdict, labels, and a +/// markdown reply with a snapshot of the generated spec. +/// +public static class ReportCommand +{ + public static Command Create() + { + var issueBodyOption = new Option("--issue-body") + { + Description = "Path to a file containing the GitHub issue body text.", + Required = true, + }; + var dataOption = new Option("--data") + { + Description = "Data repo root.", + Required = true, + }; + var dataSourceOption = new Option("--data-source") + { + Description = "Data source URI the generated spec should target.", + DefaultValueFactory = _ => "local:.", + }; + var enginesOption = new Option("--engines") + { + Description = "Engines to run: 'wham' (builtin), 'name=dotnet:path.dll', 'name=docker:image', 'name=exe args'. Default: wham.", + AllowMultipleArgumentsPerToken = true, + }; + var governingOption = new Option("--governing") + { + Description = "Governing-engine precedence (first match that ran governs). Default: newrecruit battlescribe wham.", + AllowMultipleArgumentsPerToken = true, + }; + var outDirOption = new Option("--out-dir") + { + Description = "Directory to write reply.md, report.json, and snapshot.yaml into.", + DefaultValueFactory = _ => new DirectoryInfo("."), + }; + var previousReplyOption = new Option("--previous-reply") + { + Description = "Path to the previous sticky-comment reply body (if any). When this " + + "evaluation produces no spec of its own, the durable snapshot embedded in the " + + "previous reply is carried forward instead of being replaced with an empty one.", + }; + + var command = new Command("report", "Evaluate a bug-report issue body and render a verdict reply."); + command.Options.Add(issueBodyOption); + command.Options.Add(dataOption); + command.Options.Add(dataSourceOption); + command.Options.Add(enginesOption); + command.Options.Add(governingOption); + command.Options.Add(outDirOption); + command.Options.Add(previousReplyOption); + command.SetAction((parse, ct) => Run( + parse.GetValue(issueBodyOption)!, + parse.GetValue(dataOption)!, + parse.GetValue(dataSourceOption)!, + parse.GetValue(enginesOption) ?? [], + parse.GetValue(governingOption) ?? [], + parse.GetValue(outDirOption), + parse.GetValue(previousReplyOption), + ct)); + + return command; + } + + internal static async Task Run( + FileInfo issueBody, DirectoryInfo data, string dataSource, + string[] engines, string[] governing, DirectoryInfo? outDir, FileInfo? previousReply = null, + CancellationToken ct = default) + { + try + { + if (!issueBody.Exists) + { + Console.Error.WriteLine($"issue body file not found: {issueBody.FullName}"); + return 2; + } + + if (!data.Exists) + { + Console.Error.WriteLine($"data directory not found: {data.FullName}"); + return 2; + } + + var outDirPath = outDir?.FullName ?? "."; + Directory.CreateDirectory(outDirPath); + + var body = IssueBody.Parse(await File.ReadAllTextAsync(issueBody.FullName, ct)); + + ReplayRoster? roster = null; + string? specYaml = null; + string? error = null; + var inlineSpec = false; + + switch (body.Roster) + { + case null: + error = "no roster found: accepted formats are a New Recruit share link, a .ros/.rosz attachment, or an inline fenced yaml code block containing a steps: list"; + break; + + case { Kind: RosterSourceKind.NrLink } src: + { + // Reviewer-mandated: re-validate before fetching. IssueBody's discovery + // regex is kept in lockstep with NrShareLink's, but a caller-supplied + // RosterSource must never be trusted to have gone through IssueBody.Parse. + if (!NrShareLink.TryParse(src.Value, out var key)) + { + error = "the New Recruit link in this report is not a valid share link"; + break; + } + + using var client = new NrClient(); + var fetched = await client.FetchListAsync(key, ct); + if (fetched.Json is null) + { + error = fetched.Error; + break; + } + + try + { + roster = NrListParser.Parse(fetched.Json); + } + catch (FormatException e) + { + error = e.Message; + } + + break; + } + + case { Kind: RosterSourceKind.Attachment } src: + { + using var client = new AttachmentClient(); + var (bytes, downloadError) = await client.DownloadAsync(src.Value, ct); + if (bytes is null) + { + error = downloadError; + break; + } + + try + { + using var stream = new MemoryStream(bytes); + var fileName = src.Value[(src.Value.LastIndexOf('/') + 1)..]; + roster = RosterFileConverter.Convert(stream, fileName); + } + catch (FormatException e) + { + error = e.Message; + } + + break; + } + + case { Kind: RosterSourceKind.InlineYaml } src: + { + // The pasted YAML *is* the spec: validate it via SpecLoader, but write it + // through verbatim (no re-emit) — roster stays null and VerdictMapper is + // told via inlineSpec so it doesn't treat that as "no roster found". + inlineSpec = true; + try + { + var inline = SpecLoader.LoadFromYaml(src.Value, defaultId: "report"); + + // Hermeticity: a hostile inline spec could declare its own + // setup.dataSource (e.g. "local:/etc" or another repo's path) to make + // the engine read arbitrary container-reachable paths instead of this + // repository's data. A dataSource that doesn't match the one this + // workflow is running against is rejected outright; an absent/empty + // dataSource (a fully self-contained inline setup) is unaffected. + if (inline.Setup.DataSource is { Length: > 0 } inlineDataSource + && !string.Equals(inlineDataSource, dataSource, StringComparison.Ordinal)) + { + error = "inline spec declares a dataSource that does not match this repository's data source"; + } + else + { + specYaml = src.Value; + } + } + catch (Exception e) + { + error = $"the inline spec is not valid: {e.Message}"; + } + + break; + } + } + + if (roster is not null) + { + specYaml = SpecEmitter.Emit(roster, specId: "report", dataSource, pinObserved: true); + } + + MultiRunReport? runs = null; + if (specYaml is not null && error is null) + { + var tempFixtures = Directory.CreateTempSubdirectory("muster-report-"); + try + { + await File.WriteAllTextAsync(Path.Combine(tempFixtures.FullName, "report.yaml"), specYaml, ct); + runs = MultiRunReport.Run(data.FullName, tempFixtures.FullName, engines, governing); + } + finally + { + tempFixtures.Delete(recursive: true); + } + } + + // This evaluation produced no spec of its own (roster/spec conversion failed, or no + // roster was found at all). Sticky-comment replies are posted with edit-mode: + // replace on a SINGLE comment, so if we render an empty snapshot here it destroys + // the only durable copy of a spec captured by an earlier, successful evaluation — + // making promotion permanently impossible even after the report is fixed. Carry the + // previous reply's snapshot forward instead (never re-run engines against it here — + // it is stale by construction). + string? carriedYaml = null; + if (specYaml is null && previousReply is { Exists: true }) + { + var previousBody = await File.ReadAllTextAsync(previousReply.FullName, ct); + carriedYaml = SnapshotExtractor.ExtractFromBody(previousBody); + } + + var carriedForward = carriedYaml is not null; + var effectiveSpecYaml = specYaml ?? carriedYaml ?? ""; + + var verdict = VerdictMapper.Map(roster, error, runs, inlineSpec); + var reply = ReplyRenderer.Render( + verdict, roster, runs, effectiveSpecYaml, body.Problem, body.Expected, carriedForward); + + await File.WriteAllTextAsync(Path.Combine(outDirPath, "reply.md"), reply, ct); + await File.WriteAllTextAsync(Path.Combine(outDirPath, "report.json"), ToJson(verdict, runs), ct); + if (effectiveSpecYaml.Length > 0) + { + await File.WriteAllTextAsync(Path.Combine(outDirPath, "snapshot.yaml"), effectiveSpecYaml, ct); + } + + Console.Out.WriteLine(reply); + return 0; + } + catch (Exception e) when (e is not OperationCanceledException) + { + Console.Error.WriteLine(e.Message); + return 2; + } + } + + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true }; + + private static string ToJson(Verdict verdict, MultiRunReport? runs) => JsonSerializer.Serialize( + new + { + verdict = verdict.Labels[0], + labels = verdict.Labels, + engineGap = verdict.EngineGap, + governing = runs?.Governing, + }, + JsonOptions); +} diff --git a/src/Muster.Cli/Commands/TestCommand.cs b/src/Muster.Cli/Commands/TestCommand.cs index 47dc2e4..85452fc 100644 --- a/src/Muster.Cli/Commands/TestCommand.cs +++ b/src/Muster.Cli/Commands/TestCommand.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using BattleScribeSpec; using BattleScribeSpec.Roster; +using Muster.Cli.Engines; using Muster.Cli.Fixtures; using Muster.Cli.Reporting; using WarHub.ArmouryModel.RosterEngine.Spec; @@ -11,12 +12,11 @@ namespace Muster.Cli.Commands; /// /// muster test --data <dir> --fixtures <dir> — evaluates golden-roster /// fixtures (battlescribe-spec roster DSL YAML files) against a wargame data repo, -/// using wham's roster engine. +/// using one or more roster engines (builtin wham by default, or external adapters via +/// --engines). /// public static class TestCommand { - private const string EngineName = "wham"; - public static Command Create() { var dataOption = new Option("--data") @@ -38,33 +38,53 @@ public static Command Create() { Description = "Write JSON run report to path.", }; + var enginesOption = new Option("--engines") + { + Description = "Engines to run: 'wham' (builtin), 'name=dotnet:path.dll', 'name=docker:image', 'name=exe args'. Default: wham.", + AllowMultipleArgumentsPerToken = true, + }; + var governingOption = new Option("--governing") + { + Description = "Governing-engine precedence (first match that ran governs). Default: newrecruit battlescribe wham.", + AllowMultipleArgumentsPerToken = true, + }; var command = new Command("test", "Evaluate golden roster fixtures against the data repo."); command.Options.Add(dataOption); command.Options.Add(fixturesOption); command.Options.Add(outputOption); command.Options.Add(reportOption); + command.Options.Add(enginesOption); + command.Options.Add(governingOption); command.SetAction(parse => Run( parse.GetValue(dataOption)!, parse.GetValue(fixturesOption)!, parse.GetValue(outputOption)!, - parse.GetValue(reportOption))); + parse.GetValue(reportOption), + parse.GetValue(enginesOption) ?? [], + parse.GetValue(governingOption) ?? [])); return command; } - internal static int Run(DirectoryInfo data, DirectoryInfo fixtures, string output, FileInfo? report) + internal static int Run( + DirectoryInfo data, DirectoryInfo fixtures, string output, FileInfo? report, + IReadOnlyList engines, IReadOnlyList governing) { try { - var runReport = RunFixtures(data.FullName, fixtures.FullName); - RunReport.Write(runReport, output, Console.Out); + var multiReport = MultiRunReport.Run(data.FullName, fixtures.FullName, engines, governing); + MultiRunReport.Write(multiReport, output, Console.Out); if (report is not null) { - File.WriteAllText(report.FullName, RunReport.ToJson(runReport)); + File.WriteAllText(report.FullName, MultiRunReport.ToJson(multiReport)); } - return runReport.Failed > 0 ? 1 : runReport.Inconclusive > 0 ? 2 : 0; + return multiReport.Runs.Any(r => r.Failed > 0) + ? 1 + : multiReport.Runs.Count == 0 || multiReport.Runs.Any(r => r.Inconclusive > 0) + ? 2 + : 0; } catch (HarnessInconclusiveException ex) { @@ -85,7 +105,18 @@ internal static int Run(DirectoryInfo data, DirectoryInfo fixtures, string outpu /// Thrown for harness-level (not per-fixture) problems: a missing data or fixtures /// directory, or a fixtures directory with no *.yaml fixtures in it. /// - internal static RunReport RunFixtures(string dataDir, string fixturesDir) + internal static RunReport RunFixtures(string dataDir, string fixturesDir) => + RunFixtures(dataDir, fixturesDir, EngineSpec.Parse(EngineSpec.BuiltinName)); + + /// + /// Runs every fixture under against + /// using and returns the aggregate . + /// + /// + /// Thrown for harness-level (not per-fixture) problems: a missing data or fixtures + /// directory, or a fixtures directory with no *.yaml fixtures in it. + /// + internal static RunReport RunFixtures(string dataDir, string fixturesDir, EngineSpec engineSpec) { if (!Directory.Exists(dataDir)) { @@ -107,16 +138,31 @@ internal static RunReport RunFixtures(string dataDir, string fixturesDir) } var resolver = RepoDataSourceResolver.Create(dataDir); - var results = new List(fixturePaths.Count); - foreach (var path in fixturePaths) + + // Builtin wham: fresh in-proc adapter per fixture (cheap, current behavior). + // External engines: one shared engine instance for the whole run — RosterRunner + // calls Cleanup() between specs so a single adapter process is reused across fixtures. + IRosterEngine? sharedEngine = engineSpec.Kind == EngineKind.Builtin + ? null + : EngineRegistry.CreateEngine(engineSpec); + try { - results.Add(RunFixture(path, dataDir, resolver)); - } + var results = new List(fixturePaths.Count); + foreach (var path in fixturePaths) + { + results.Add(RunFixture(path, dataDir, resolver, engineSpec, sharedEngine)); + } - return RunReport.Create(EngineName, dataDir, results); + return RunReport.Create(engineSpec.Name, dataDir, results); + } + finally + { + sharedEngine?.Dispose(); + } } - private static FixtureResult RunFixture(string path, string dataDir, DataSourceResolver resolver) + private static FixtureResult RunFixture( + string path, string dataDir, DataSourceResolver resolver, EngineSpec engineSpec, IRosterEngine? sharedEngine) { var sw = Stopwatch.StartNew(); @@ -146,10 +192,10 @@ private static FixtureResult RunFixture(string path, string dataDir, DataSourceR sw.ElapsedMilliseconds, Inconclusive: true); } + IRosterEngine engine = sharedEngine ?? new SpecRosterEngineAdapter(); try { - using IRosterEngine engine = new SpecRosterEngineAdapter(); - var runner = new RosterRunner(engine, resolver, engineName: EngineName); + var runner = new RosterRunner(engine, resolver, engineName: engineSpec.Name); var result = runner.Run(spec); if (result.HarnessError is { } harnessError) { @@ -168,6 +214,13 @@ private static FixtureResult RunFixture(string path, string dataDir, DataSourceR spec.Id, path, Passed: false, [$"engine crash: {ex.GetType().Name}: {ex.Message}"], sw.ElapsedMilliseconds, Inconclusive: true); } + finally + { + if (sharedEngine is null) + { + engine.Dispose(); + } + } } private static int Inconclusive(string message) diff --git a/src/Muster.Cli/Converters/NrListParser.cs b/src/Muster.Cli/Converters/NrListParser.cs new file mode 100644 index 0000000..cd04ecd --- /dev/null +++ b/src/Muster.Cli/Converters/NrListParser.cs @@ -0,0 +1,154 @@ +using System.Text.Json; + +namespace Muster.Cli.Converters; + +/// +/// Parses a New Recruit shared-list JSON payload into a ReplayRoster. +/// Node classification (verified against real NR data 2026-07-13): +/// army.options[] level 1 = catalogue; its children carrying "catalogue_id" +/// are forces; below that, nodes with "amount" are selections (entry id = +/// "link_id::option_id" when linked), nodes without are transparent +/// containers (categories / selection-entry-groups). +/// +public static class NrListParser +{ + private const int MaxDepth = 64; + private const int MaxNodes = 20_000; + private const int MaxSelections = 5_000; + + public static ReplayRoster Parse(string json) + { + JsonDocument doc; + try + { + doc = JsonDocument.Parse(json, new JsonDocumentOptions { MaxDepth = MaxDepth }); + } + catch (JsonException e) + { + throw new FormatException($"not a valid New Recruit list: {e.Message}"); + } + + using (doc) + { + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object || !root.TryGetProperty("army", out var army)) + throw new FormatException("not a New Recruit list: missing 'army'"); + + var name = GetString(root, "name") ?? "unnamed roster"; + var totals = ParseCosts(root, "totalCosts"); + + List revisions; + if (root.TryGetProperty("books_revision", out var revs) && revs.ValueKind == JsonValueKind.Array) + { + revisions = revs.EnumerateArray() + .Where(r => r.ValueKind == JsonValueKind.String) + .Select(r => r.GetString()!) + .ToList(); + } + else + { + revisions = []; + } + + var gameSystemId = GetString(root, "bsid_system"); + + var state = new ParseState(); + var forces = new List(); + foreach (var catNode in Options(army)) + { + state.CountNode(); + var catalogueId = GetString(catNode, "option_id") + ?? throw new FormatException("catalogue node missing option_id"); + foreach (var child in Options(catNode)) + { + state.CountNode(); + if (child.TryGetProperty("catalogue_id", out _)) + forces.Add(ParseForce(child, catalogueId, state)); + else + state.Unmapped.Add($"unexpected non-force node '{GetString(child, "name")}' under catalogue"); + } + } + + if (forces.Count == 0) + throw new FormatException("no forces found in the list"); + + return new(name, gameSystemId, totals, revisions, forces, state.Unmapped); + } + } + + private sealed class ParseState + { + public List Unmapped { get; } = []; + public int Selections; + private int _nodes; + + public void CountNode() + { + if (++_nodes > MaxNodes) throw new FormatException($"list too large (over {MaxNodes} nodes)"); + } + } + + private static ReplayForce ParseForce(JsonElement node, string catalogueId, ParseState state) + { + var forceEntryId = GetString(node, "option_id") ?? throw new FormatException("force node missing option_id"); + var selections = new List(); + CollectSelections(node, selections, state, depth: 0); + return new(forceEntryId, catalogueId, selections, ChildForces: []); + } + + private static void CollectSelections(JsonElement node, List into, ParseState state, int depth) + { + if (depth > MaxDepth) throw new FormatException("list too deeply nested"); + foreach (var child in Options(node)) + { + state.CountNode(); + if (child.TryGetProperty("amount", out var amountEl)) + { + if (++state.Selections > MaxSelections) + throw new FormatException($"list too large (over {MaxSelections} selections)"); + var optionId = GetString(child, "option_id"); + if (optionId is null) + { + state.Unmapped.Add($"selection '{GetString(child, "name")}' missing option_id"); + continue; + } + var linkId = GetString(child, "link_id"); + var entryId = linkId is null ? optionId : $"{linkId}::{optionId}"; + var count = amountEl.ValueKind == JsonValueKind.Number ? amountEl.GetInt32() : 1; + var children = new List(); + CollectSelections(child, children, state, depth + 1); + into.Add(new(entryId, count, GetString(child, "customName"), ObservedCosts: [], children)); + } + else + { + CollectSelections(child, into, state, depth + 1); + } + } + } + + private static IEnumerable Options(JsonElement node) => + node.ValueKind == JsonValueKind.Object + && node.TryGetProperty("options", out var options) + && options.ValueKind == JsonValueKind.Array + ? options.EnumerateArray() : []; + + private static string? GetString(JsonElement el, string name) => + el.ValueKind == JsonValueKind.Object && el.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String + ? v.GetString() : null; + + private static List ParseCosts(JsonElement el, string property) + { + var result = new List(); + if (el.TryGetProperty(property, out var costs) && costs.ValueKind == JsonValueKind.Array) + { + foreach (var c in costs.EnumerateArray()) + { + var typeId = GetString(c, "typeId"); + if (typeId is null) continue; + var value = c.TryGetProperty("value", out var v) && v.ValueKind == JsonValueKind.Number ? v.GetDecimal() : 0m; + result.Add(new(GetString(c, "name") ?? typeId, typeId, value)); + } + } + return result; + } +} diff --git a/src/Muster.Cli/Converters/ReplayRoster.cs b/src/Muster.Cli/Converters/ReplayRoster.cs new file mode 100644 index 0000000..f478852 --- /dev/null +++ b/src/Muster.Cli/Converters/ReplayRoster.cs @@ -0,0 +1,20 @@ +namespace Muster.Cli.Converters; + +public sealed record ReplayCost(string Name, string TypeId, decimal Value); + +public sealed record ReplaySelection( + string EntryId, int Count, string? CustomName, + IReadOnlyList ObservedCosts, + IReadOnlyList Children); + +public sealed record ReplayForce( + string ForceEntryId, string CatalogueId, + IReadOnlyList Selections, + IReadOnlyList ChildForces); + +public sealed record ReplayRoster( + string Name, string? GameSystemId, + IReadOnlyList ObservedTotals, + IReadOnlyList BooksRevisions, + IReadOnlyList Forces, + IReadOnlyList Unmapped); diff --git a/src/Muster.Cli/Converters/RosterFileConverter.cs b/src/Muster.Cli/Converters/RosterFileConverter.cs new file mode 100644 index 0000000..36913aa --- /dev/null +++ b/src/Muster.Cli/Converters/RosterFileConverter.cs @@ -0,0 +1,147 @@ +using System.IO.Compression; +using WarHub.ArmouryModel.Source; +using WarHub.ArmouryModel.Source.BattleScribe; +using WarHub.ArmouryModel.Workspaces.BattleScribe; + +namespace Muster.Cli.Converters; + +/// +/// Converts a BattleScribe .ros/.rosz roster file into a ReplayRoster. +/// SelectionNode.EntryId is already the composite "linkId::targetId" form the +/// engine's by-id selection expects, so ids pass through verbatim. +/// +/// +/// .rosz handling is done here with a plain rather than +/// wham's Stream.LoadSourceAuto (Workspaces.BattleScribe/XmlFileExtensions.cs). +/// That helper dispatches zipped vs. plain XML via +/// XmlDocumentKind.IsXmlZipped(), which resolves the extension through +/// ExtensionsByKinds[kind][0] — always the *unzipped* extension for the kind, +/// regardless of what the actual file/entry name was. In practice IsXmlZipped() +/// is always false, so LoadSourceAuto("x.rosz") silently takes the plain-XML +/// path and throws "Data at the root level is invalid" on real .rosz input (confirmed +/// by a throwaway repro test against the submodule as vendored). We only use +/// from that assembly (a plain constant, +/// unaffected by the bug) and do the exactly-one-entry zip rule ourselves. +/// +public static class RosterFileConverter +{ + private const int MaxSelections = 5_000; + + public static ReplayRoster Convert(Stream stream, string fileName) + { + RosterNode roster; + try + { + roster = fileName.EndsWith(XmlFileExtensions.RosterZipped, StringComparison.OrdinalIgnoreCase) + ? LoadZipped(stream) + : BattleScribeXml.LoadRoster(stream) + ?? throw new FormatException("file is not a BattleScribe roster"); + } + catch (Exception e) when (e is not FormatException) + { + throw new FormatException($"could not read roster file: {e.Message}"); + } + + var count = 0; + var unmapped = new List(); + var forces = ConvertForces(roster.Forces, ref count, unmapped); + if (forces.Count == 0) + throw new FormatException("roster has no forces"); + + return new( + Name: roster.Name ?? "unnamed roster", + GameSystemId: roster.GameSystemId, + ObservedTotals: [.. roster.Costs.Select(c => new ReplayCost(c.Name ?? "", c.TypeId ?? "", c.Value))], + BooksRevisions: [], + Forces: forces, + Unmapped: unmapped); + } + + private const int MaxDecompressedBytes = 50 * 1024 * 1024; + + private static RosterNode LoadZipped(Stream stream) + { + using var archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); + if (archive.Entries.Count != 1) + { + throw new FormatException( + $".rosz archive must contain exactly one entry, found {archive.Entries.Count}"); + } + using var entryStream = archive.Entries[0].Open(); + using var bounded = ReadBounded(entryStream, MaxDecompressedBytes); + return BattleScribeXml.LoadRoster(bounded) + ?? throw new FormatException("archive entry is not a BattleScribe roster"); + } + + /// + /// Copies into an in-memory buffer, aborting as soon as + /// more than have been read. Guards against zip-bomb + /// .rosz entries that would otherwise decompress unbounded into memory. + /// + private static MemoryStream ReadBounded(Stream source, int maxBytes) + { + var buffer = new byte[81_920]; + var result = new MemoryStream(); + long total = 0; + int read; + while ((read = source.Read(buffer, 0, buffer.Length)) > 0) + { + total += read; + if (total > maxBytes) + { + throw new FormatException( + $"roster archive decompresses too large (over {maxBytes / (1024 * 1024)} MB)"); + } + result.Write(buffer, 0, read); + } + result.Position = 0; + return result; + } + + private static ReplayForce ConvertForce(ForceNode force, ref int count, List unmapped) => new( + ForceEntryId: force.EntryId!, + CatalogueId: force.CatalogueId ?? "", + Selections: ConvertSelections(force.Selections, ref count, unmapped), + ChildForces: ConvertForces(force.Forces, ref count, unmapped)); + + private static List ConvertForces(IEnumerable forces, ref int count, List unmapped) + { + var result = new List(); + foreach (var f in forces) + { + if (f.EntryId is null) + { + // No way to replay a force without an entryId — record structural + // drift and skip it rather than aborting the whole conversion. + unmapped.Add($"force '{f.Name}' has no entryId — skipped"); + continue; + } + result.Add(ConvertForce(f, ref count, unmapped)); + } + return result; + } + + private static List ConvertSelections(IEnumerable selections, ref int count, List unmapped) + { + var result = new List(); + foreach (var s in selections) + { + if (++count > MaxSelections) + throw new FormatException($"roster too large (over {MaxSelections} selections)"); + if (s.EntryId is null) + { + // No way to replay a selection without an entryId — surface as structural + // drift rather than silently dropping it from the converted roster. + unmapped.Add($"selection '{s.Name}' (id={s.Id}) has no entryId — cannot be replayed"); + continue; + } + result.Add(new( + EntryId: s.EntryId, + Count: s.Number, + CustomName: s.CustomName, + ObservedCosts: [.. s.Costs.Select(c => new ReplayCost(c.Name ?? "", c.TypeId ?? "", c.Value))], + Children: ConvertSelections(s.Selections, ref count, unmapped))); + } + return result; + } +} diff --git a/src/Muster.Cli/Converters/SpecEmitter.cs b/src/Muster.Cli/Converters/SpecEmitter.cs new file mode 100644 index 0000000..60cfc09 --- /dev/null +++ b/src/Muster.Cli/Converters/SpecEmitter.cs @@ -0,0 +1,175 @@ +using System.Globalization; +using System.Text; + +namespace Muster.Cli.Converters; + +/// +/// Emits fixture-DSL YAML from a ReplayRoster. TestKit has no SpecFile→YAML +/// serializer, so YAML is emitted as text; every caller path is validated by +/// round-tripping through SpecLoader.LoadFromYaml in tests. +/// Step ids: force-1, force-2, … / sel-1, sel-2, … in document order. +/// All string scalars are double-quoted — sidesteps YAML coercion traps. +/// Step-reference expressions (${{ steps.x.y }}) are quoted too: YamlDotNet's +/// double-quoted scalar parsing yields the literal text unchanged (no special +/// handling of "${{" / "}}"), and expression resolution happens later, at run +/// time, against the deserialized string — so quoting is transparent to it. +/// +public static class SpecEmitter +{ + public static string Emit(ReplayRoster roster, string specId, string dataSource, bool pinObserved) + { + var sb = new StringBuilder(); + sb.AppendLine($"id: {Quote(specId)}"); + sb.AppendLine("category: report"); + sb.AppendLine($"description: {Quote($"Converted from bug report roster '{roster.Name}'")}"); + sb.AppendLine("setup:"); + sb.AppendLine($" dataSource: {Quote(dataSource)}"); + sb.AppendLine(); + sb.AppendLine("steps:"); + + var forceIndex = 0; + var selIndex = 0; + foreach (var force in roster.Forces) + EmitForce(sb, force, parentForceStep: null, ref forceIndex, ref selIndex); + + if (pinObserved && roster.ObservedTotals.Count > 0) + { + sb.AppendLine(); + sb.AppendLine(" # observed values from the report — PASS means the reported state reproduces"); + sb.AppendLine(" - expectedState:"); + sb.AppendLine(" costs:"); + foreach (var cost in roster.ObservedTotals) + { + sb.AppendLine($" - typeId: {Quote(cost.TypeId)}"); + sb.AppendLine($" value: {cost.Value.ToString(CultureInfo.InvariantCulture)}"); + } + } + + if (pinObserved && HasAnyObservedCosts(roster)) + { + sb.AppendLine(); + sb.AppendLine(" # per-selection costs observed but not pinned (comparer is order-sensitive)"); + } + + return sb.ToString(); + } + + // Step 4 decision (see RosterRunner.AssertSelections in + // BattleScribeSpec.TestKit/Roster/RosterRunner.cs ~line 691-808): expected vs. + // actual selections are matched by *position* (expected[si] against actual[si]), + // not by name or entryId — costs within a matched pair are then looked up by + // TypeId/Name, but getting to that pair at all depends on index alignment. If the + // engine auto-adds/reorders selections (e.g. default child selections), a + // per-selection cost pin emitted at the observed index would silently compare + // against the wrong selection instead of failing loudly. So per-selection + // observed costs are surfaced only as a YAML comment, never as expectedState + // pins; only the roster-level totals (order-independent) are pinned. + private static bool HasAnyObservedCosts(ReplayRoster roster) => roster.Forces.Any(HasAnyObservedCosts); + + private static bool HasAnyObservedCosts(ReplayForce force) => + force.Selections.Any(HasAnyObservedCosts) || force.ChildForces.Any(HasAnyObservedCosts); + + private static bool HasAnyObservedCosts(ReplaySelection sel) => + sel.ObservedCosts.Count > 0 || sel.Children.Any(HasAnyObservedCosts); + + private static void EmitForce(StringBuilder sb, ReplayForce force, string? parentForceStep, ref int forceIndex, ref int selIndex) + { + forceIndex++; + var stepId = $"force-{forceIndex}"; + if (parentForceStep is null) + { + sb.AppendLine(" - action: addForce"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceEntryId: {Quote(force.ForceEntryId)}"); + sb.AppendLine($" catalogueId: {Quote(force.CatalogueId)}"); + } + else + { + sb.AppendLine(" - action: addChildForce"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceId: {Quote($"${{{{ steps.{parentForceStep}.forceId }}}}")}"); + sb.AppendLine($" forceEntryId: {Quote(force.ForceEntryId)}"); + sb.AppendLine($" catalogueId: {Quote(force.CatalogueId)}"); + } + + foreach (var sel in force.Selections) + EmitSelection(sb, sel, forceStep: stepId, parentSelStep: null, ref selIndex); + + foreach (var child in force.ChildForces) + EmitForce(sb, child, stepId, ref forceIndex, ref selIndex); + } + + private static void EmitSelection(StringBuilder sb, ReplaySelection sel, string forceStep, string? parentSelStep, ref int selIndex) + { + selIndex++; + var stepId = $"sel-{selIndex}"; + var forceRef = Quote($"${{{{ steps.{forceStep}.forceId }}}}"); + if (parentSelStep is null) + { + sb.AppendLine(" - action: selectEntry"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" entryId: {Quote(sel.EntryId)}"); + } + else + { + sb.AppendLine(" - action: selectChildEntry"); + sb.AppendLine($" id: {Quote(stepId)}"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" selectionId: {Quote($"${{{{ steps.{parentSelStep}.selectionId }}}}")}"); + sb.AppendLine($" entryId: {Quote(sel.EntryId)}"); + } + + if (sel.Count != 1) + { + sb.AppendLine(" - action: setSelectionCount"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" selectionId: {Quote($"${{{{ steps.{stepId}.selectionId }}}}")}"); + sb.AppendLine($" count: {sel.Count}"); + } + + if (!string.IsNullOrEmpty(sel.CustomName)) + { + sb.AppendLine(" - action: setCustomization"); + sb.AppendLine($" forceId: {forceRef}"); + sb.AppendLine($" selectionId: {Quote($"${{{{ steps.{stepId}.selectionId }}}}")}"); + sb.AppendLine($" customName: {Quote(sel.CustomName!)}"); + } + + foreach (var child in sel.Children) + EmitSelection(sb, child, forceStep, stepId, ref selIndex); + } + + private static string Quote(string value) + { + var escaped = value + .Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal); + + var sb = new StringBuilder(escaped.Length + 2); + sb.Append('"'); + foreach (var c in escaped) + { + switch (c) + { + case '\n': + sb.Append("\\n"); + break; + case '\r': + sb.Append("\\r"); + break; + case '\t': + sb.Append("\\t"); + break; + default: + if (c < ' ') + sb.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + else + sb.Append(c); + break; + } + } + sb.Append('"'); + return sb.ToString(); + } +} diff --git a/src/Muster.Cli/Engines/EngineRegistry.cs b/src/Muster.Cli/Engines/EngineRegistry.cs new file mode 100644 index 0000000..b0215d2 --- /dev/null +++ b/src/Muster.Cli/Engines/EngineRegistry.cs @@ -0,0 +1,125 @@ +using BattleScribeSpec.Protocol; +using BattleScribeSpec.Roster; +using WarHub.ArmouryModel.RosterEngine.Spec; + +namespace Muster.Cli.Engines; + +public static class EngineRegistry +{ + public static readonly IReadOnlyList DefaultGoverning = ["newrecruit", "battlescribe", "wham"]; + + public static IReadOnlyList ParseAll(IReadOnlyList specs) => + specs.Count == 0 ? [EngineSpec.Parse(EngineSpec.BuiltinName)] : [.. specs.Select(EngineSpec.Parse)]; + + public static bool IsAvailable(EngineSpec spec) => spec.Kind switch + { + EngineKind.Builtin => true, + EngineKind.Docker => CanStart("docker", "--version"), + EngineKind.Exec when string.Equals(spec.Executable, "dotnet", StringComparison.Ordinal) => + spec.Arguments is { } args && File.Exists(FirstToken(args)), + EngineKind.Exec => File.Exists(spec.Executable) || CanStart(spec.Executable!, "--version"), + _ => false, + }; + + public static IRosterEngine CreateEngine(EngineSpec spec) => spec.Kind switch + { + EngineKind.Builtin => new SpecRosterEngineAdapter(), + _ => new CompositeDisposableEngine(AdapterProcess.Start(spec.Executable!, spec.Arguments)), + }; + + public static string? ResolveGoverning(IReadOnlyList precedence, IReadOnlyList ranEngines) => + precedence.FirstOrDefault(p => ranEngines.Contains(p, StringComparer.Ordinal)); + + private static string FirstToken(string s) + { + var i = s.IndexOf(' ', StringComparison.Ordinal); + return i < 0 ? s : s[..i]; + } + + private static bool CanStart(string exe, string args) + { + try + { + using var p = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo(exe, args) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + p?.WaitForExit(5000); + return p is { ExitCode: 0 }; + } + catch (Exception e) when (e is System.ComponentModel.Win32Exception or InvalidOperationException or PlatformNotSupportedException) + { + return false; + } + } + + /// + /// Wraps JsonProtocolEngine and AdapterProcess to ensure both are disposed. + /// JsonProtocolEngine does not dispose the AdapterProcess it owns, so we must do it here. + /// + private sealed class CompositeDisposableEngine : IRosterEngine + { + private readonly AdapterProcess _process; + private readonly JsonProtocolEngine _engine; + + public CompositeDisposableEngine(AdapterProcess process) + { + _process = process; + _engine = new JsonProtocolEngine(process); + } + + public void SetTestContext(string specId) => _engine.SetTestContext(specId); + + public IReadOnlyList Setup(ProtocolGameSystem gameSystem, ProtocolCatalogue[] catalogues) => + _engine.Setup(gameSystem, catalogues); + + public IReadOnlyList SetupFromFiles(IReadOnlyList<(string FileName, string Content)> files) => + _engine.SetupFromFiles(files); + + public ActionOutputs AddForce(string forceEntryId, string catalogueId) => + _engine.AddForce(forceEntryId, catalogueId); + + public ActionOutputs AddChildForce(string parentForceId, string forceEntryId, string catalogueId) => + _engine.AddChildForce(parentForceId, forceEntryId, catalogueId); + + public void RemoveForce(string forceId) => _engine.RemoveForce(forceId); + + public ActionOutputs SelectEntry(string forceId, string entryId) => + _engine.SelectEntry(forceId, entryId); + + public ActionOutputs SelectChildEntry(string forceId, string parentSelectionId, string entryId) => + _engine.SelectChildEntry(forceId, parentSelectionId, entryId); + + public void DeselectSelection(string forceId, string selectionId) => + _engine.DeselectSelection(forceId, selectionId); + + public void SetSelectionCount(string forceId, string selectionId, int count) => + _engine.SetSelectionCount(forceId, selectionId, count); + + public ActionOutputs DuplicateSelection(string forceId, string selectionId) => + _engine.DuplicateSelection(forceId, selectionId); + + public ActionOutputs DuplicateForce(string forceId) => _engine.DuplicateForce(forceId); + + public void SetCostLimit(string costTypeId, decimal value) => + _engine.SetCostLimit(costTypeId, value); + + public void SetCustomization(string forceId, string? selectionId, string? categoryEntryId, string? customName, string? customNotes) => + _engine.SetCustomization(forceId, selectionId, categoryEntryId, customName, customNotes); + + public RosterState GetRosterState() => _engine.GetRosterState(); + + public IReadOnlyList GetValidationErrors() => + _engine.GetValidationErrors(); + + public void Cleanup() => ((IRosterEngine)_engine).Cleanup(); + + public void Dispose() + { + _engine.Dispose(); + _process.Dispose(); + } + } +} diff --git a/src/Muster.Cli/Engines/EngineSpec.cs b/src/Muster.Cli/Engines/EngineSpec.cs new file mode 100644 index 0000000..12a3cae --- /dev/null +++ b/src/Muster.Cli/Engines/EngineSpec.cs @@ -0,0 +1,39 @@ +namespace Muster.Cli.Engines; + +public enum EngineKind { Builtin, Exec, Docker } + +/// +/// One engine registration: wham (builtin) or +/// name=dotnet:path.dll / name=docker:image / name=exe [args]. +/// +public sealed record EngineSpec(string Name, EngineKind Kind, string? Executable, string? Arguments) +{ + public const string BuiltinName = "wham"; + + public static EngineSpec Parse(string spec) + { + ArgumentException.ThrowIfNullOrWhiteSpace(spec); + var eq = spec.IndexOf('=', StringComparison.Ordinal); + if (eq < 0) + { + if (!string.Equals(spec.Trim(), BuiltinName, StringComparison.Ordinal)) + throw new ArgumentException($"engine '{spec}' has no adapter command; only '{BuiltinName}' is builtin"); + return new(BuiltinName, EngineKind.Builtin, null, null); + } + + var name = spec[..eq].Trim(); + var command = spec[(eq + 1)..].Trim(); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + + if (command.StartsWith("dotnet:", StringComparison.Ordinal)) + return new(name, EngineKind.Exec, "dotnet", command["dotnet:".Length..]); + if (command.StartsWith("docker:", StringComparison.Ordinal)) + return new(name, EngineKind.Docker, "docker", $"run -i --rm {command["docker:".Length..]}"); + + var space = command.IndexOf(' ', StringComparison.Ordinal); + return space < 0 + ? new(name, EngineKind.Exec, command, null) + : new(name, EngineKind.Exec, command[..space], command[(space + 1)..]); + } +} diff --git a/src/Muster.Cli/Muster.Cli.csproj b/src/Muster.Cli/Muster.Cli.csproj index ff73622..5a5ca5e 100644 --- a/src/Muster.Cli/Muster.Cli.csproj +++ b/src/Muster.Cli/Muster.Cli.csproj @@ -13,6 +13,13 @@ + + + + + + diff --git a/src/Muster.Cli/NewRecruit/NrClient.cs b/src/Muster.Cli/NewRecruit/NrClient.cs new file mode 100644 index 0000000..b3c06f4 --- /dev/null +++ b/src/Muster.Cli/NewRecruit/NrClient.cs @@ -0,0 +1,72 @@ +using System.Text; +using System.Text.Json; + +namespace Muster.Cli.NewRecruit; + +public sealed record NrFetchResult(string? Json, string? Error); + +/// +/// Fetches a shared New Recruit list via the (undocumented) open_share_link RPC. +/// All remote failures degrade to — callers +/// map them to needs-info, never a crash. +/// +public sealed class NrClient(HttpMessageHandler? handler = null) : IDisposable +{ + private const long MaxResponseBytes = 5 * 1024 * 1024; + private static readonly Uri RpcUri = new("https://www.newrecruit.eu/api/rpc"); + + private readonly HttpClient _http = new(handler ?? new HttpClientHandler()) + { + Timeout = TimeSpan.FromSeconds(30), + }; + + public async Task FetchListAsync(string key, CancellationToken ct = default) + { + try + { + var body = JsonSerializer.Serialize(new Dictionary + { + ["method"] = "open_share_link", + ["params"] = new[] { key }, + }); + using var request = new HttpRequestMessage(HttpMethod.Post, RpcUri) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + using var response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct); + if (!response.IsSuccessStatusCode) + return new(null, $"New Recruit returned HTTP {(int)response.StatusCode}"); + + using var stream = await response.Content.ReadAsStreamAsync(ct); + using var limited = new MemoryStream(); + var buffer = new byte[81920]; + int read; + while ((read = await stream.ReadAsync(buffer, ct)) > 0) + { + limited.Write(buffer, 0, read); + if (limited.Length > MaxResponseBytes) + return new(null, "response too large (>5 MB)"); + } + var text = Encoding.UTF8.GetString(limited.ToArray()).Trim(); + if (text is "null" or "" or "[]") + return new(null, "list not found or no longer shared (New Recruit share links expire)"); + if (!text.StartsWith('{')) + return new(null, "New Recruit returned an unexpected response"); + return new(text, null); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return new(null, "New Recruit did not respond within 30 seconds"); + } + catch (HttpRequestException e) + { + return new(null, $"could not reach New Recruit: {e.Message}"); + } + catch (IOException e) + { + return new(null, $"connection to New Recruit failed: {e.Message}"); + } + } + + public void Dispose() => _http.Dispose(); +} diff --git a/src/Muster.Cli/NewRecruit/NrShareLink.cs b/src/Muster.Cli/NewRecruit/NrShareLink.cs new file mode 100644 index 0000000..70758ee --- /dev/null +++ b/src/Muster.Cli/NewRecruit/NrShareLink.cs @@ -0,0 +1,19 @@ +using System.Text.RegularExpressions; + +namespace Muster.Cli.NewRecruit; + +public static partial class NrShareLink +{ + [GeneratedRegex(@"^https://www\.newrecruit\.eu/app/list/([A-Za-z0-9]{1,32})/?$")] + private static partial Regex Pattern(); + + public static bool TryParse(string url, out string key) + { + key = ""; + if (string.IsNullOrWhiteSpace(url)) return false; + var m = Pattern().Match(url.Trim()); + if (!m.Success) return false; + key = m.Groups[1].Value; + return true; + } +} diff --git a/src/Muster.Cli/Program.cs b/src/Muster.Cli/Program.cs index f570caa..aab451f 100644 --- a/src/Muster.Cli/Program.cs +++ b/src/Muster.Cli/Program.cs @@ -24,6 +24,9 @@ public static RootCommand CreateRootCommand() var root = new RootCommand("muster — your data passes muster. CI toolchain for wargame data repos."); root.Subcommands.Add(TestCommand.Create()); root.Subcommands.Add(DiffCommand.Create()); + root.Subcommands.Add(ConvertCommand.Create()); + root.Subcommands.Add(ReportCommand.Create()); + root.Subcommands.Add(PromoteCommand.Create()); return root; } } diff --git a/src/Muster.Cli/Reporting/BlastRadius.cs b/src/Muster.Cli/Reporting/BlastRadius.cs index 2aba562..9c7dff6 100644 --- a/src/Muster.Cli/Reporting/BlastRadius.cs +++ b/src/Muster.Cli/Reporting/BlastRadius.cs @@ -29,6 +29,29 @@ public sealed record BlastRow( /// public sealed record DiffReport(RunReport Base, RunReport Head, IReadOnlyList Rows); +/// +/// Per-engine blast radius rows for one engine in a . +/// +/// Engine name. +/// This engine's classified fixture rows (base run vs head run). +public sealed record EngineDiff(string Engine, IReadOnlyList Rows); + +/// +/// Combined payload for muster diff across every engine that ran on both base and head. +/// +/// Name of the governing engine (see ), or null. +/// Names of engines that did not run on both sides (unavailable on either, or only present on one side). +/// Per-engine classified rows, one per engine that ran on both sides. +/// +/// Fixture ids where engines that ran on both sides disagree on head status — a signal of +/// engine divergence that the governing engine's verdict does not resolve. +/// +public sealed record MultiDiffReport( + string? Governing, + IReadOnlyList Unavailable, + IReadOnlyList Diffs, + IReadOnlyList EngineGaps); + /// /// Compares two s (same fixtures, different data trees) and classifies /// each fixture's change in outcome — the blast radius of the base-to-head diff. @@ -70,6 +93,46 @@ public static IReadOnlyList Classify(RunReport baseRun, RunReport head return rows; } + /// + /// Pairs engines that ran on both and + /// by name and classifies each pair's fixtures via . Engines present on + /// only one side (unavailable, or ran on only base/head) are reported in + /// and excluded from rows and gating. + /// + public static MultiDiffReport ClassifyMulti(MultiRunReport baseRuns, MultiRunReport headRuns) + { + ArgumentNullException.ThrowIfNull(baseRuns); + ArgumentNullException.ThrowIfNull(headRuns); + + var baseByEngine = baseRuns.Runs.ToDictionary(r => r.Engine, StringComparer.Ordinal); + var headByEngine = headRuns.Runs.ToDictionary(r => r.Engine, StringComparer.Ordinal); + + var unavailable = new List(baseRuns.Unavailable.Union(headRuns.Unavailable, StringComparer.Ordinal)); + var diffs = new List(); + foreach (var (engine, baseRun) in baseByEngine) + { + if (!headByEngine.TryGetValue(engine, out var headRun)) { unavailable.Add(engine); continue; } + diffs.Add(new(engine, Classify(baseRun, headRun))); + } + + foreach (var engine in headByEngine.Keys.Except(baseByEngine.Keys, StringComparer.Ordinal)) + unavailable.Add(engine); + + // engine-gap: fixtures whose HEAD status differs across engines that ran both sides + var gaps = new List(); + if (diffs.Count > 1) + { + var byFixture = diffs + .SelectMany(d => d.Rows.Select(r => (r.FixtureId, r.HeadStatus))) + .GroupBy(x => x.FixtureId, StringComparer.Ordinal); + foreach (var g in byFixture) + if (g.Select(x => x.HeadStatus).Distinct(StringComparer.Ordinal).Count() > 1) + gaps.Add(g.Key); + } + + return new(headRuns.Governing, unavailable, diffs, gaps); + } + private static BlastRow ClassifyPair(FixtureResult baseFixture, FixtureResult headFixture) { var baseStatus = Status(baseFixture); @@ -104,6 +167,8 @@ private static BlastRow ClassifyPair(FixtureResult baseFixture, FixtureResult he public static string ToJson(RunReport baseRun, RunReport headRun, IReadOnlyList rows) => JsonSerializer.Serialize(new DiffReport(baseRun, headRun, rows), JsonOptions); + public static string ToJson(MultiDiffReport report) => JsonSerializer.Serialize(report, JsonOptions); + public static void Write(RunReport baseRun, RunReport headRun, IReadOnlyList rows, string mode, TextWriter writer) { switch (mode) @@ -117,6 +182,43 @@ public static void Write(RunReport baseRun, RunReport headRun, IReadOnlyList + /// Writes a : json serializes the whole record; markdown + /// renders one ### Engine: {name} section per engine (reusing the single-engine + /// markdown row/detail rendering), followed by unavailable-engine notices and, when + /// is non-empty, an engine-gap section. + /// + public static void WriteMulti(MultiDiffReport report, string mode, TextWriter writer) + { + if (mode == "json") + { + writer.WriteLine(ToJson(report)); + return; + } + + foreach (var diff in report.Diffs) + { + writer.WriteLine($"### Engine: {diff.Engine}{(diff.Engine == report.Governing ? " (governing)" : "")}"); + writer.WriteLine(); + WriteMarkdownBody(diff.Rows, writer); + writer.WriteLine(); + } + + foreach (var name in report.Unavailable) + { + writer.WriteLine($"> ⚠ engine `{name}` was requested but is unavailable, or only ran on one side."); + } + + if (report.EngineGaps.Count > 0) + { + writer.WriteLine(); + writer.WriteLine("### ⚠ engine-gap"); + writer.WriteLine( + $"Engines disagree on the head state of: {string.Join(", ", report.EngineGaps.Select(g => $"`{g}`"))}. " + + "The governing engine's verdict stands; divergence should be triaged as an engine defect."); + } + } + private static void WriteMarkdown(RunReport baseRun, RunReport headRun, IReadOnlyList rows, TextWriter writer) { writer.WriteLine("## Muster diff — blast radius"); @@ -128,6 +230,11 @@ private static void WriteMarkdown(RunReport baseRun, RunReport headRun, IReadOnl CultureInfo.InvariantCulture, $"Head: `{headRun.DataDir}` — {headRun.Passed} passed, {headRun.Failed} failed, {headRun.Inconclusive} inconclusive")); writer.WriteLine(); + WriteMarkdownBody(rows, writer); + } + + private static void WriteMarkdownBody(IReadOnlyList rows, TextWriter writer) + { writer.WriteLine("| Fixture | Base | Head | Change |"); writer.WriteLine("| --- | --- | --- | --- |"); foreach (var row in rows) diff --git a/src/Muster.Cli/Reporting/MultiRunReport.cs b/src/Muster.Cli/Reporting/MultiRunReport.cs new file mode 100644 index 0000000..6fb5875 --- /dev/null +++ b/src/Muster.Cli/Reporting/MultiRunReport.cs @@ -0,0 +1,77 @@ +using System.Text.Json; +using Muster.Cli.Commands; +using Muster.Cli.Engines; + +namespace Muster.Cli.Reporting; + +/// +/// Aggregate result of running golden-roster fixtures across one or more engines. +/// +/// +/// Name of the engine whose results are authoritative (first match, in --governing +/// precedence order, among the engines that actually ran) — or null if none ran. +/// +/// Names of requested engines that could not be probed as available. +/// Per-engine s, one per engine that ran. +public sealed record MultiRunReport(string? Governing, IReadOnlyList Unavailable, IReadOnlyList Runs) +{ + /// + /// Parses , probes each for availability, runs fixtures for + /// every available engine, and resolves the governing engine from + /// precedence (falling back to when empty). + /// + public static MultiRunReport Run(string dataDir, string fixturesDir, + IReadOnlyList engineSpecs, IReadOnlyList governing) + { + var specs = EngineRegistry.ParseAll(engineSpecs); + var unavailable = new List(); + var runs = new List(); + foreach (var spec in specs) + { + if (!EngineRegistry.IsAvailable(spec)) + { + unavailable.Add(spec.Name); + continue; + } + + runs.Add(TestCommand.RunFixtures(dataDir, fixturesDir, spec)); + } + + var precedence = governing.Count > 0 ? governing : EngineRegistry.DefaultGoverning; + var governor = EngineRegistry.ResolveGoverning(precedence, [.. runs.Select(r => r.Engine)]); + return new(governor, unavailable, runs); + } + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + }; + + public static string ToJson(MultiRunReport report) => JsonSerializer.Serialize(report, JsonOptions); + + public static void Write(MultiRunReport report, string mode, TextWriter writer) + { + if (mode == "json") + { + writer.WriteLine(ToJson(report)); + return; + } + + foreach (var run in report.Runs) + { + writer.WriteLine(mode == "github-actions" + ? $"### Engine: {run.Engine}{(run.Engine == report.Governing ? " (governing)" : "")}" + : $"-- engine: {run.Engine}{(run.Engine == report.Governing ? " (governing)" : "")} --"); + RunReport.Write(run, mode, writer); + writer.WriteLine(); + } + + foreach (var name in report.Unavailable) + { + writer.WriteLine(mode == "github-actions" + ? $"> ⚠ engine `{name}` was requested but is unavailable in this environment." + : $"[????] engine {name}: unavailable"); + } + } +} diff --git a/src/Muster.Cli/Reports/AttachmentClient.cs b/src/Muster.Cli/Reports/AttachmentClient.cs new file mode 100644 index 0000000..5c7187f --- /dev/null +++ b/src/Muster.Cli/Reports/AttachmentClient.cs @@ -0,0 +1,63 @@ +using System.Text.RegularExpressions; + +namespace Muster.Cli.Reports; + +/// +/// Downloads an allowlisted GitHub issue attachment. Mirrors 's +/// streaming 5 MB cap and graceful-error contract. All remote failures degrade to an +/// error result — callers map them to needs-info, never a crash. +/// +public sealed partial class AttachmentClient(HttpMessageHandler? handler = null) : IDisposable +{ + private const long MaxResponseBytes = 5 * 1024 * 1024; + + // Anchored full-match re-validation, independent of IssueBody's search pattern: + // defense in depth so a caller-supplied URL that doesn't match the allowlisted + // shape is rejected before any network activity occurs. + [GeneratedRegex(@"^https://github\.com/(?:user-attachments/files|[\w.-]+/[\w.-]+/files)/\d+/[A-Za-z0-9._-]+\.(?:rosz|ros|zip)$")] + private static partial Regex AttachmentUrlPattern(); + + private readonly HttpClient _http = new(handler ?? new HttpClientHandler { AllowAutoRedirect = true }) + { + Timeout = TimeSpan.FromSeconds(30), + }; + + public async Task<(byte[]? Data, string? Error)> DownloadAsync(string url, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(url) || !AttachmentUrlPattern().IsMatch(url.Trim())) + return (null, "attachment URL is not an allowlisted GitHub attachment link"); + + try + { + using var response = await _http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct); + if (!response.IsSuccessStatusCode) + return (null, $"GitHub returned HTTP {(int)response.StatusCode}"); + + using var stream = await response.Content.ReadAsStreamAsync(ct); + using var limited = new MemoryStream(); + var buffer = new byte[81920]; + int read; + while ((read = await stream.ReadAsync(buffer, ct)) > 0) + { + limited.Write(buffer, 0, read); + if (limited.Length > MaxResponseBytes) + return (null, "attachment too large (>5 MB)"); + } + return (limited.ToArray(), null); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return (null, "GitHub did not respond within 30 seconds"); + } + catch (HttpRequestException e) + { + return (null, $"could not reach GitHub: {e.Message}"); + } + catch (IOException e) + { + return (null, $"connection to GitHub failed: {e.Message}"); + } + } + + public void Dispose() => _http.Dispose(); +} diff --git a/src/Muster.Cli/Reports/IssueBody.cs b/src/Muster.Cli/Reports/IssueBody.cs new file mode 100644 index 0000000..abdc423 --- /dev/null +++ b/src/Muster.Cli/Reports/IssueBody.cs @@ -0,0 +1,97 @@ +using System.Text.RegularExpressions; + +namespace Muster.Cli.Reports; + +public enum RosterSourceKind +{ + NrLink, + Attachment, + InlineYaml, +} + +public sealed record RosterSource(RosterSourceKind Kind, string Value); + +/// +/// Parses a (potentially hostile) GitHub issue body into a roster source and +/// reporter-supplied problem/expected text. Covers both muster's own issue-form +/// layout (GitHub renders form fields as `### Label` headings) and New Recruit's +/// auto-filed report layout (`**Label:**` markers). +/// +public sealed partial record IssueBody(RosterSource? Roster, string? Problem, string? Expected) +{ + /// Hard cap on how much of the body is ever handed to a regex. + private const int MaxBodyLength = 65_536; + + /// Hard cap on captured Problem/Expected text. + private const int MaxCaptureLength = 2_000; + + // Unanchored: finds a candidate NR-link substring anywhere in free text. The + // extracted URL is later re-validated by NrShareLink.TryParse before use, so the + // charset/host/scheme here must stay in lockstep with NrShareLink's pattern. + [GeneratedRegex(@"https://www\.newrecruit\.eu/app/list/[A-Za-z0-9]{1,32}")] + private static partial Regex NrLinkPattern(); + + // Unanchored search variant; AttachmentClient re-validates with an anchored + // full-match version of the same shape before making any HTTP call. + [GeneratedRegex(@"https://github\.com/(?:user-attachments/files|[\w.-]+/[\w.-]+/files)/\d+/[A-Za-z0-9._-]+\.(?:rosz|ros|zip)")] + private static partial Regex AttachmentPattern(); + + [GeneratedRegex(@"```ya?ml\s*\n(.*?)```", RegexOptions.Singleline)] + private static partial Regex YamlBlockPattern(); + + // Matches both NR auto-report style (**Problem:**) and GitHub issue-form style + // (### Problem heading), stopping at the next ** marker, the next ### heading, or + // end of input. + [GeneratedRegex(@"(?:\*\*Problem:\*\*|###\s*Problem\s*\n)\s*\n?(.*?)(?=\n\s*\*\*|\n\s*###|\z)", RegexOptions.Singleline)] + private static partial Regex ProblemPattern(); + + [GeneratedRegex(@"(?:\*\*Expected:\*\*|###\s*Expected\s*\n)\s*\n?(.*?)(?=\n\s*\*\*|\n\s*###|\z)", RegexOptions.Singleline)] + private static partial Regex ExpectedPattern(); + + public static IssueBody Parse(string body) + { + if (string.IsNullOrEmpty(body)) + return new IssueBody(null, null, null); + + var input = body.Length > MaxBodyLength ? body[..MaxBodyLength] : body; + + var roster = FindRoster(input); + var problem = ExtractCapture(ProblemPattern(), input); + var expected = ExtractCapture(ExpectedPattern(), input); + + return new IssueBody(roster, problem, expected); + } + + private static RosterSource? FindRoster(string input) + { + var nrMatch = NrLinkPattern().Match(input); + if (nrMatch.Success) + return new RosterSource(RosterSourceKind.NrLink, nrMatch.Value); + + var attachmentMatch = AttachmentPattern().Match(input); + if (attachmentMatch.Success) + return new RosterSource(RosterSourceKind.Attachment, attachmentMatch.Value); + + foreach (Match yamlMatch in YamlBlockPattern().Matches(input)) + { + var content = yamlMatch.Groups[1].Value; + if (content.Contains("steps:", StringComparison.Ordinal)) + return new RosterSource(RosterSourceKind.InlineYaml, content); + } + + return null; + } + + private static string? ExtractCapture(Regex pattern, string input) + { + var match = pattern.Match(input); + if (!match.Success) + return null; + + var value = match.Groups[1].Value.Trim(); + if (value.Length == 0) + return null; + + return value.Length > MaxCaptureLength ? value[..MaxCaptureLength] : value; + } +} diff --git a/src/Muster.Cli/Reports/ReplyRenderer.cs b/src/Muster.Cli/Reports/ReplyRenderer.cs new file mode 100644 index 0000000..714a102 --- /dev/null +++ b/src/Muster.Cli/Reports/ReplyRenderer.cs @@ -0,0 +1,195 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using Muster.Cli.Converters; +using Muster.Cli.Reporting; + +namespace Muster.Cli.Reports; + +/// +/// Renders a plus its evidence (roster, per-engine runs, the generated +/// spec) as a markdown GitHub issue-comment reply. +/// +/// +/// The first line and the snapshot markers are consumed downstream by Task 11's +/// SnapshotExtractor and Task 12's find-comment step — their exact text is load-bearing and +/// must not be reformatted. +/// +public static partial class ReplyRenderer +{ + private const int MaxQuoteLength = 500; + + // Matches RosterRunner.AssertEqual's roster-level cost failure wording: + // "Step {n}: cost[{typeId-or-name}].value: expected {x} but got {y}". + [GeneratedRegex(@"cost\[(?[^\]]+)\]\.value:\s*expected\s*(?-?[\d.]+)\s*but got\s*(?-?[\d.]+)")] + private static partial Regex CostFailurePattern(); + + public static string Render( + Verdict verdict, ReplayRoster? roster, MultiRunReport? runs, string specYaml, string? problem, string? expected, + bool carriedForwardSnapshot = false) + { + var sb = new StringBuilder(); + sb.Append("\n"); + sb.Append('\n'); + sb.Append($"## Verdict: {Heading(verdict.Kind)}\n"); + sb.Append('\n'); + sb.Append(Explanation(verdict, runs)); + sb.Append('\n'); + sb.Append('\n'); + + if (verdict.Kind == VerdictKind.NeedsInfo) + { + AppendNeedsInfo(sb, verdict); + } + + if (roster is not null && roster.ObservedTotals.Count > 0 && runs is { Runs.Count: > 0 }) + { + AppendMatrix(sb, roster, runs); + } + + AppendQuoted(sb, "Problem", problem); + AppendQuoted(sb, "Expected", expected); + + if (roster is { BooksRevisions.Count: > 0 }) + { + sb.Append($"reported against: {string.Join(", ", roster.BooksRevisions)} — evaluated against current data\n"); + sb.Append('\n'); + } + + // Never emit an empty ```yaml block: when there is no spec at all (neither freshly + // derived nor carried forward from a previous evaluation), omit the whole snapshot + // section rather than posting an empty, misleading
block. + if (specYaml.Length > 0) + { + if (carriedForwardSnapshot) + { + sb.Append("> ⚠ The roster source is no longer reachable — snapshot preserved from a previous evaluation.\n"); + sb.Append('\n'); + } + + sb.Append("
Executable spec (snapshot)\n"); + sb.Append("\n"); + sb.Append('\n'); + sb.Append("```yaml\n"); + sb.Append(specYaml); + if (!specYaml.EndsWith('\n')) + { + sb.Append('\n'); + } + sb.Append("```\n"); + sb.Append("
\n"); + } + + return sb.ToString(); + } + + private static string Heading(VerdictKind kind) => kind switch + { + VerdictKind.Confirmed => "Confirmed", + VerdictKind.NotReproducible => "Not reproducible", + VerdictKind.NeedsInfo => "Needs info", + _ => "Inconclusive", + }; + + private static string Explanation(Verdict verdict, MultiRunReport? runs) + { + var ran = runs?.Runs.Select(r => r.Engine).ToList() ?? []; + var unavailable = runs?.Unavailable ?? []; + + var parts = new List + { + verdict.Kind switch + { + VerdictKind.Confirmed => "The reported values reproduce against current data.", + VerdictKind.NotReproducible => "The reported values do not reproduce against current data.", + VerdictKind.NeedsInfo => "More information is needed to evaluate this report.", + _ => "No engine was available to evaluate this report.", + }, + }; + if (runs?.Governing is { } governing) + parts.Add($"Governing engine: `{governing}`."); + if (ran.Count > 0) + parts.Add($"Engines run: {string.Join(", ", ran.Select(e => $"`{e}`"))}."); + if (unavailable.Count > 0) + parts.Add($"Unavailable: {string.Join(", ", unavailable.Select(e => $"`{e}`"))}."); + + return string.Join(" ", parts); + } + + private static void AppendNeedsInfo(StringBuilder sb, Verdict verdict) + { + sb.Append("I couldn't fully evaluate this report"); + if (!string.IsNullOrEmpty(verdict.Detail)) + { + sb.Append(": "); + sb.Append('\n'); + sb.Append('\n'); + sb.Append($"> {Truncate(verdict.Detail).Replace("\n", "\n> ", StringComparison.Ordinal)}\n"); + } + else + { + sb.Append(".\n"); + } + sb.Append('\n'); + sb.Append("Please provide the roster in one of these formats:\n"); + sb.Append('\n'); + sb.Append("- a New Recruit share link (`https://www.newrecruit.eu/app/list/...`)\n"); + sb.Append("- a `.ros`/`.rosz` roster file attachment\n"); + sb.Append("- an inline fenced `yaml` code block containing a `steps:` list\n"); + sb.Append('\n'); + } + + private static void AppendMatrix(StringBuilder sb, ReplayRoster roster, MultiRunReport runs) + { + var ranEngines = runs.Runs.Select(r => r.Engine).ToList(); + + sb.Append($"| Value | reported | {string.Join(" | ", ranEngines)} |\n"); + sb.Append($"| --- | --- | {string.Join(" | ", ranEngines.Select(_ => "---"))} |\n"); + foreach (var cost in roster.ObservedTotals) + { + var cells = runs.Runs.Select(r => EngineCell(r, cost)); + sb.Append($"| {Escape(cost.Name)} | {cost.Value.ToString(CultureInfo.InvariantCulture)} | {string.Join(" | ", cells)} |\n"); + } + sb.Append('\n'); + } + + private static string EngineCell(RunReport run, ReplayCost cost) + { + if (run.Fixtures.Count == 0) + return "differs ✖"; + + var fixture = run.Fixtures[0]; + if (fixture.Passed) + return "reproduced ✔"; + + foreach (var failure in fixture.Failures) + { + var m = CostFailurePattern().Match(failure); + if (m.Success && string.Equals(m.Groups["key"].Value, cost.TypeId, StringComparison.Ordinal)) + return Escape(m.Groups["act"].Value); + } + + return "differs ✖"; + } + + private static void AppendQuoted(StringBuilder sb, string label, string? text) + { + if (string.IsNullOrEmpty(text)) + { + return; + } + + sb.Append($"**{label}:**\n"); + sb.Append('\n'); + var truncated = Truncate(text); + foreach (var line in truncated.Split('\n')) + { + sb.Append($"> {line.TrimEnd('\r')}\n"); + } + sb.Append('\n'); + } + + private static string Truncate(string s) => s.Length > MaxQuoteLength ? s[..MaxQuoteLength] : s; + + private static string Escape(string s) => s.Replace("|", "\\|", StringComparison.Ordinal); +} diff --git a/src/Muster.Cli/Reports/SnapshotExtractor.cs b/src/Muster.Cli/Reports/SnapshotExtractor.cs new file mode 100644 index 0000000..28bb77d --- /dev/null +++ b/src/Muster.Cli/Reports/SnapshotExtractor.cs @@ -0,0 +1,97 @@ +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace Muster.Cli.Reports; + +/// +/// Extracts the executable-spec snapshot from a GitHub issue's comments, as rendered by +/// (Task 10). Input is the raw JSON array produced by +/// gh api /repos/{o}/{r}/issues/{n}/comments — a list of objects each with a +/// body string. +/// +public static partial class SnapshotExtractor +{ + /// Defensive cap on how much of a single comment body is scanned. + private const int MaxBodyLength = 256 * 1024; + + // Byte-compatible with ReplyRenderer's emitted snapshot block — verified against Task 10's + // output by its reviewer. Do not reformat without re-verifying against ReplyRenderer.Render. + [GeneratedRegex(@"\s*```ya?ml\s*\n(.*?)```", RegexOptions.Singleline)] + private static partial Regex SnapshotPattern(); + + /// + /// Returns the fenced-yaml content of the <!-- muster:snapshot --> block in a + /// single comment body (e.g. the sticky report comment's current or previous text), or + /// if the marker is absent. + /// + public static string? ExtractFromBody(string body) + { + if (string.IsNullOrEmpty(body)) + { + return null; + } + + if (body.Length > MaxBodyLength) + { + body = body[..MaxBodyLength]; + } + + var match = SnapshotPattern().Match(body); + return match.Success ? match.Groups[1].Value : null; + } + + /// + /// Returns the fenced-yaml content of the LAST comment (in array order) containing a + /// <!-- muster:snapshot --> marker, or if none is + /// found or is not a well-formed JSON array. + /// + public static string? ExtractLatest(string commentsJson) + { + JsonDocument doc; + try + { + doc = JsonDocument.Parse(commentsJson); + } + catch (JsonException) + { + return null; + } + + using (doc) + { + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Array) + { + return null; + } + + string? latest = null; + foreach (var item in root.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (!item.TryGetProperty("body", out var bodyProp) || bodyProp.ValueKind != JsonValueKind.String) + { + continue; + } + + var body = bodyProp.GetString(); + if (string.IsNullOrEmpty(body)) + { + continue; + } + + var fromBody = ExtractFromBody(body); + if (fromBody is not null) + { + latest = fromBody; + } + } + + return latest; + } + } +} diff --git a/src/Muster.Cli/Reports/SpecRePinner.cs b/src/Muster.Cli/Reports/SpecRePinner.cs new file mode 100644 index 0000000..c66c841 --- /dev/null +++ b/src/Muster.Cli/Reports/SpecRePinner.cs @@ -0,0 +1,130 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using BattleScribeSpec; +using BattleScribeSpec.Roster; +using Muster.Cli.Engines; +using Muster.Cli.Fixtures; + +namespace Muster.Cli.Reports; + +/// +/// Re-pins a promoted bug-report snapshot spec to the roster engine's current values: drops any +/// stale expectedState-only assertion steps, replays the remaining action steps against +/// the local data repo, and re-emits the spec with a fresh expectedState block pinned to +/// the observed final . +/// +public static partial class SpecRePinner +{ + // The comment SpecEmitter (Task 9/10) writes immediately above its observed-value pin step — + // load-bearing for RewriteWithPins' text-surgery: everything from this line to EOF in a + // muster-emitted snapshot is exactly the trailing expectedState-only step, so dropping from + // here is a deterministic way to strip stale pins without a full YAML re-emit. + // + // Line-anchored (^\s*...$ with Multiline) so that a quoted scalar field (e.g. an + // attacker-controlled roster customName) containing this text mid-line can never match: a + // quoted step field sits inline within a `key: "..."` line, never as a standalone comment + // line, and SpecEmitter.Quote escapes embedded newlines so it can't fake one either. + [GeneratedRegex(@"^\s*# observed values from the report.*$", RegexOptions.Multiline)] + private static partial Regex ObservedValuesMarkerPattern(); + + [GeneratedRegex(@"^id:.*$", RegexOptions.Multiline)] + private static partial Regex IdLinePattern(); + + /// + /// Loads , replays its action steps (assertion-only steps are + /// dropped — they may pin stale/buggy values) against using + /// , and returns the spec's YAML text re-pinned to the engine's + /// current final under . + /// + /// + /// The data source is not populated locally, or the replay itself fails (a fixture that + /// can't run can't be promoted) — never returns a spec built from a partial/failed replay. + /// + public static string RePin(string specYaml, string dataDir, EngineSpec engineSpec, string newSpecId) + { + var spec = SpecLoader.LoadFromYaml(specYaml, defaultId: newSpecId); + if (spec.Setup.DataSource is { Length: > 0 } ds && !RepoDataSourceResolver.IsPopulatedFor(dataDir, ds)) + { + throw new InvalidOperationException($"data source not populated locally: {ds}"); + } + + // Strip assertion-only steps — they may pin stale/buggy values from the original report. + var replaySteps = spec.Steps.Where(s => s.Action is not null).ToList(); + var replaySpec = new SpecFile + { + Id = spec.Id, + Category = spec.Category, + Description = spec.Description, + Tags = spec.Tags, + Engines = spec.Engines, + Setup = spec.Setup, + Steps = replaySteps, + }; + + RosterState? finalState = null; + using var engine = EngineRegistry.CreateEngine(engineSpec); + var runner = new RosterRunner(engine, RepoDataSourceResolver.Create(dataDir), engineName: engineSpec.Name) + { + OnStepCompleted = (_, _, state, _) => finalState = state, + }; + var result = runner.Run(replaySpec); + if (result.HarnessError is not null || !result.Passed || finalState is null) + { + throw new InvalidOperationException( + "cannot promote: replay failed against current data: " + + (result.HarnessError ?? result.Failures.FirstOrDefault() ?? "no state captured")); + } + + return RewriteWithPins(specYaml, newSpecId, finalState.Costs, engineSpec.Name); + } + + /// + /// Text-level rewrite of : replaces the top-level id: line + /// with , drops any existing observed-value pin block (recognized + /// by ; falls back to append-only when absent), and appends + /// a fresh expectedState.costs block pinned to . The result is + /// validated with before being + /// returned — a rewrite that fails to parse is a harness error, never a silently written + /// broken fixture. + /// + internal static string RewriteWithPins( + string specYaml, string newSpecId, IReadOnlyList costs, string engineName) + { + var text = IdLinePattern().Replace(specYaml, $"id: {Quote(newSpecId)}", count: 1); + + var markerMatch = ObservedValuesMarkerPattern().Match(text); + if (markerMatch.Success) + { + text = text[..markerMatch.Index]; + } + + text = text.TrimEnd('\n', '\r') + "\n"; + + var sb = new StringBuilder(text); + sb.AppendLine(); + sb.AppendLine($" # expected values pinned at promotion (engine: {engineName})"); + sb.AppendLine(" - expectedState:"); + sb.AppendLine(" costs:"); + foreach (var cost in costs) + { + sb.AppendLine($" - typeId: {Quote(cost.TypeId)}"); + sb.AppendLine($" value: {cost.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + var rewritten = sb.ToString(); + + // Never write a broken fixture: validate the rewrite round-trips before returning it. + SpecLoader.LoadFromYaml(rewritten); + + return rewritten; + } + + private static string Quote(string value) + { + var escaped = value + .Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal); + return $"\"{escaped}\""; + } +} diff --git a/src/Muster.Cli/Reports/Verdict.cs b/src/Muster.Cli/Reports/Verdict.cs new file mode 100644 index 0000000..8bf43f6 --- /dev/null +++ b/src/Muster.Cli/Reports/Verdict.cs @@ -0,0 +1,72 @@ +using Muster.Cli.Converters; +using Muster.Cli.Reporting; + +namespace Muster.Cli.Reports; + +public enum VerdictKind { Confirmed, NotReproducible, NeedsInfo, Inconclusive } + +public sealed record Verdict(VerdictKind Kind, bool EngineGap, IReadOnlyList Labels, string? Detail); + +/// +/// Maps roster conversion + per-engine fixture evaluation results to a . +/// +public static class VerdictMapper +{ + /// + /// The converted roster, or when conversion failed or (in + /// mode) never produced one — an inline spec pasted by the + /// reporter has no of its own. + /// + /// Non-null when the roster/spec could not be converted or validated. + /// Per-engine fixture evaluation results for the one generated/pasted spec. + /// + /// when the report's roster source was an inline + /// ```yaml steps block — the YAML *is* the spec, so there is no + /// to check for a null value or unmapped nodes; the verdict + /// is derived purely from . + /// + public static Verdict Map(ReplayRoster? roster, string? conversionError, MultiRunReport? runs, bool inlineSpec = false) + { + if (conversionError is not null) + return Make(VerdictKind.NeedsInfo, false, conversionError); + if (!inlineSpec) + { + if (roster is null) + return Make(VerdictKind.NeedsInfo, false, "no roster found in the report"); + if (roster.Unmapped.Count > 0) + return Make(VerdictKind.NeedsInfo, false, string.Join("\n", roster.Unmapped)); + } + if (runs is null || runs.Runs.Count == 0 || runs.Governing is null) + return Make(VerdictKind.Inconclusive, false, "no engine was available to evaluate the roster"); + + var governing = runs.Runs.First(r => r.Engine == runs.Governing); + var fixture = governing.Fixtures[0]; + + var gap = runs.Runs.Count > 1 && runs.Runs + .Select(r => (r.Fixtures[0].Passed, r.Fixtures[0].Inconclusive)) + .Distinct().Count() > 1; + + if (fixture.Inconclusive) + return Make(VerdictKind.NeedsInfo, gap, + "the roster could not be replayed against current data: " + fixture.Failures.FirstOrDefault()); + return fixture.Passed + ? Make(VerdictKind.Confirmed, gap, null) + : Make(VerdictKind.NotReproducible, gap, string.Join("\n", fixture.Failures)); + } + + private static Verdict Make(VerdictKind kind, bool gap, string? detail) + { + var labels = new List + { + kind switch + { + VerdictKind.Confirmed => "confirmed", + VerdictKind.NotReproducible => "not-reproducible", + VerdictKind.NeedsInfo => "needs-info", + _ => "inconclusive", + }, + }; + if (gap) labels.Add("engine-gap"); + return new(kind, gap, labels, detail); + } +} diff --git a/tests/Muster.Cli.Tests/BlastRadiusTests.cs b/tests/Muster.Cli.Tests/BlastRadiusTests.cs index 15126c1..67fda65 100644 --- a/tests/Muster.Cli.Tests/BlastRadiusTests.cs +++ b/tests/Muster.Cli.Tests/BlastRadiusTests.cs @@ -132,4 +132,113 @@ public void Classify_preserves_base_fixture_order_and_appends_head_only_fixtures Assert.Equal(["f2", "f1", "f3"], rows.Select(r => r.FixtureId)); } + + private static RunReport Report(string engine, params FixtureResult[] fixtures) => + RunReport.Create(engine, "/data", fixtures); + + private static MultiRunReport Multi(string? governing, params RunReport[] runs) => + new(governing, [], runs); + + [Fact] + public void ClassifyMulti_pairs_engines_by_name_and_classifies_each_independently() + { + var baseRuns = Multi("wham", + Report("wham", Result("f1", passed: true)), + Report("fake", Result("f1", passed: true))); + var headRuns = Multi("wham", + Report("wham", Result("f1", passed: false, ["broke"])), + Report("fake", Result("f1", passed: true))); + + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + + Assert.Equal("wham", report.Governing); + Assert.Equal(2, report.Diffs.Count); + var wham = Assert.Single(report.Diffs, d => d.Engine == "wham"); + Assert.Equal("broke", Assert.Single(wham.Rows).Classification); + var fake = Assert.Single(report.Diffs, d => d.Engine == "fake"); + Assert.Equal("unchanged", Assert.Single(fake.Rows).Classification); + } + + [Fact] + public void ClassifyMulti_engine_present_on_only_one_side_is_unavailable_and_excluded() + { + var baseRuns = Multi("wham", + Report("wham", Result("f1", passed: true)), + Report("fake", Result("f1", passed: true))); + var headRuns = Multi("wham", + Report("wham", Result("f1", passed: true))); + + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + + Assert.Equal(["fake"], report.Unavailable); + Assert.Single(report.Diffs); + Assert.Equal("wham", Assert.Single(report.Diffs).Engine); + } + + [Fact] + public void ClassifyMulti_surfaces_engine_gap_when_head_status_disagrees() + { + var baseRuns = Multi("wham", + Report("wham", Result("f1", passed: true)), + Report("fake", Result("f1", passed: true))); + var headRuns = Multi("wham", + Report("wham", Result("f1", passed: false, ["broke"])), + Report("fake", Result("f1", passed: true))); + + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + + Assert.Equal(["f1"], report.EngineGaps); + } + + [Fact] + public void ClassifyMulti_no_engine_gap_when_engines_agree_on_head_status() + { + var baseRuns = Multi("wham", + Report("wham", Result("f1", passed: true)), + Report("fake", Result("f1", passed: true))); + var headRuns = Multi("wham", + Report("wham", Result("f1", passed: false, ["broke"])), + Report("fake", Result("f1", passed: false, ["also broke"]))); + + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + + Assert.Empty(report.EngineGaps); + } + + [Fact] + public void WriteMulti_markdown_renders_per_engine_sections_and_engine_gap_note() + { + var baseRuns = Multi("wham", + Report("wham", Result("f1", passed: true)), + Report("fake", Result("f1", passed: true))); + var headRuns = Multi("wham", + Report("wham", Result("f1", passed: false, ["broke"])), + Report("fake", Result("f1", passed: true))); + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + + using var sw = new StringWriter(); + BlastRadius.WriteMulti(report, "markdown", sw); + var text = sw.ToString(); + + Assert.Contains("### Engine: wham (governing)", text, StringComparison.Ordinal); + Assert.Contains("### Engine: fake", text, StringComparison.Ordinal); + Assert.Contains("engine-gap", text, StringComparison.Ordinal); + Assert.Contains("`f1`", text, StringComparison.Ordinal); + } + + [Fact] + public void WriteMulti_json_round_trips_report_shape() + { + var baseRuns = Multi("wham", Report("wham", Result("f1", passed: true))); + var headRuns = Multi("wham", Report("wham", Result("f1", passed: true))); + var report = BlastRadius.ClassifyMulti(baseRuns, headRuns); + + using var sw = new StringWriter(); + BlastRadius.WriteMulti(report, "json", sw); + var text = sw.ToString(); + + Assert.Contains("\"governing\"", text, StringComparison.Ordinal); + Assert.Contains("\"diffs\"", text, StringComparison.Ordinal); + Assert.Contains("\"engineGaps\"", text, StringComparison.Ordinal); + } } diff --git a/tests/Muster.Cli.Tests/ConvertCommandTests.cs b/tests/Muster.Cli.Tests/ConvertCommandTests.cs new file mode 100644 index 0000000..154887d --- /dev/null +++ b/tests/Muster.Cli.Tests/ConvertCommandTests.cs @@ -0,0 +1,57 @@ +using BattleScribeSpec; +using Muster.Cli.Commands; +using Xunit; + +namespace Muster.Cli.Tests; + +public class ConvertCommandTests +{ + private static string SamplePath => Path.Combine(AppContext.BaseDirectory, "TestData", "nr-list-war-horde.json"); + + [Fact] + public async Task Converts_nr_json_file_to_spec_yaml() + { + var outFile = new FileInfo(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".yaml")); + try + { + var exit = await ConvertCommand.Run(SamplePath, id: null, dataSource: "local:.", pinObserved: true, output: outFile, ct: TestContext.Current.CancellationToken); + + Assert.Equal(0, exit); + var yaml = File.ReadAllText(outFile.FullName); + var spec = SpecLoader.LoadFromYaml(yaml); + Assert.Equal("nr-list-war-horde", spec.Id); + Assert.Contains(spec.Steps, s => s.Action == "addForce"); + Assert.Contains(spec.Steps, s => s.ExpectedState is not null); + } + finally { outFile.Delete(); } + } + + [Fact] + public async Task Missing_file_exits_2() + { + var exit = await ConvertCommand.Run("no-such-file.ros", null, "local:.", true, null, TestContext.Current.CancellationToken); + Assert.Equal(2, exit); + } + + [Fact] + public async Task Non_allowlisted_url_exits_2() + { + var exit = await ConvertCommand.Run("https://evil.example/app/list/x", null, "local:.", true, null, TestContext.Current.CancellationToken); + Assert.Equal(2, exit); + } + + [Fact] + public async Task Output_to_directory_exits_2_without_throwing() + { + var tempDir = new DirectoryInfo(Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())); + tempDir.Create(); + try + { + var outputAsDir = new FileInfo(tempDir.FullName); + var exit = await ConvertCommand.Run(SamplePath, id: null, dataSource: "local:.", pinObserved: true, output: outputAsDir, ct: TestContext.Current.CancellationToken); + + Assert.Equal(2, exit); + } + finally { tempDir.Delete(recursive: true); } + } +} diff --git a/tests/Muster.Cli.Tests/Converters/NrListParserTests.cs b/tests/Muster.Cli.Tests/Converters/NrListParserTests.cs new file mode 100644 index 0000000..0a18184 --- /dev/null +++ b/tests/Muster.Cli.Tests/Converters/NrListParserTests.cs @@ -0,0 +1,97 @@ +using Muster.Cli.Converters; +using Xunit; + +namespace Muster.Cli.Tests.Converters; + +public class NrListParserTests +{ + private static string SampleJson => File.ReadAllText( + Path.Combine(AppContext.BaseDirectory, "TestData", "nr-list-war-horde.json")); + + [Fact] + public void Parses_war_horde_sample() + { + var roster = NrListParser.Parse(SampleJson); + + Assert.Equal("war horde", roster.Name); + var pts = Assert.Single(roster.ObservedTotals); + Assert.Equal(950m, pts.Value); + Assert.Equal("51b2-306e-1021-d207", pts.TypeId); + var force = Assert.Single(roster.Forces); + Assert.Equal("bb9d-299a-ed60-2d8a", force.ForceEntryId); + Assert.Equal("a55f-b7b3-6c65-a05f", force.CatalogueId); + Assert.NotEmpty(force.Selections); + Assert.Empty(roster.Unmapped); + Assert.Contains("Xenos - Orks: 2", roster.BooksRevisions); + } + + [Fact] + public void Linked_selection_gets_composite_entry_id() + { + var roster = NrListParser.Parse(SampleJson); + var all = Flatten(roster); + // "Battle Size" selection: option_id 564e-fbc6-5266-3ea4 selected via link 7380-3e40-6ed6-b7cc + Assert.Contains(all, s => s.EntryId == "7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4"); + } + + [Fact] + public void Container_nodes_are_transparent() + { + var roster = NrListParser.Parse(SampleJson); + var all = Flatten(roster); + // "Configuration" (category node, no amount) must not appear as a selection… + Assert.DoesNotContain(all, s => s.EntryId.Contains("4ac9-fd30-1e3d-b249", StringComparison.Ordinal)); + // …but "Incursion", nested category → selection → group → child, must + Assert.Contains(all, s => s.EntryId.EndsWith("d62d-db22-4893-4bc0", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("not json at all")] + [InlineData("{}")] + [InlineData("""{"army":{"options":[]},"name":"x"}""")] + public void Hostile_or_empty_input_throws_FormatException(string json) => + Assert.Throws(() => NrListParser.Parse(json)); + + [Fact] + public void Pathologically_nested_options_throws_FormatException() + { + var nested = """{"name":"n","option_id":"x","amount":1,"options":[]}"""; + for (var i = 0; i < 100; i++) + { + nested = $$"""{"name":"n","option_id":"x","amount":1,"options":[{{nested}}]}"""; + } + + var json = $$""" + { + "name": "deep", + "army": { + "options": [ + { + "option_id": "cat-1", + "options": [ + { + "option_id": "force-1", + "catalogue_id": "cat-1", + "options": [{{nested}}] + } + ] + } + ] + } + } + """; + + Assert.Throws(() => NrListParser.Parse(json)); + } + + private static List Flatten(ReplayRoster r) + { + var result = new List(); + void Walk(IEnumerable sels) + { + foreach (var s in sels) { result.Add(s); Walk(s.Children); } + } + foreach (var f in r.Forces) Walk(f.Selections); + return result; + } +} diff --git a/tests/Muster.Cli.Tests/Converters/RosterFileConverterTests.cs b/tests/Muster.Cli.Tests/Converters/RosterFileConverterTests.cs new file mode 100644 index 0000000..9d0cb39 --- /dev/null +++ b/tests/Muster.Cli.Tests/Converters/RosterFileConverterTests.cs @@ -0,0 +1,171 @@ +using System.IO.Compression; +using System.Text; +using Muster.Cli.Converters; +using Xunit; + +namespace Muster.Cli.Tests.Converters; + +public class RosterFileConverterTests +{ + // Minimal valid BattleScribe roster XML. xmlns/battleScribeVersion match + // Namespaces.RosterXmlns ("http://www.battlescribe.net/schema/rosterSchema") + // and the wham sample asset lib/wham/src/Phalanx.SampleDataset/Test Roster 1.ros. + private const string RosterXml = """ + + + + + + + + + + + + + + + + + + + + + """; + + [Fact] + public void Converts_ros_xml_to_replay_roster() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(RosterXml)); + var roster = RosterFileConverter.Convert(stream, "test.ros"); + + Assert.Equal("Test Roster", roster.Name); + Assert.Equal("gs-1", roster.GameSystemId); + var total = Assert.Single(roster.ObservedTotals); + Assert.Equal(65.0m, total.Value); + var force = Assert.Single(roster.Forces); + Assert.Equal("fe-1", force.ForceEntryId); + Assert.Equal("cat-1", force.CatalogueId); + var unit = Assert.Single(force.Selections); + Assert.Equal("link-1::se-1", unit.EntryId); // composite id preserved verbatim + var unitCost = Assert.Single(unit.ObservedCosts); + Assert.Equal(65.0m, unitCost.Value); + var child = Assert.Single(unit.Children); + Assert.Equal("se-2", child.EntryId); + Assert.Equal(5, child.Count); + Assert.Equal("Fancy guns", child.CustomName); + } + + [Fact] + public void Converts_rosz_zip() + { + using var zipStream = new MemoryStream(); + using (var zip = new ZipArchive(zipStream, ZipArchiveMode.Create, leaveOpen: true)) + { + using var entry = zip.CreateEntry("test.ros").Open(); + entry.Write(Encoding.UTF8.GetBytes(RosterXml)); + } + zipStream.Position = 0; + + var roster = RosterFileConverter.Convert(zipStream, "test.rosz"); + Assert.Equal("Test Roster", roster.Name); + } + + [Fact] + public void Garbage_input_throws_FormatException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("")); + Assert.Throws(() => RosterFileConverter.Convert(stream, "x.ros")); + } + + // Second force ("Second Force") deliberately omits entryId. + private const string RosterXmlWithOneBadForce = """ + + + + + + + + + + + + + + + + """; + + private const string RosterXmlWithOnlyBadForce = """ + + + + + + + + + + + """; + + [Fact] + public void Force_with_no_entryId_is_skipped_with_Unmapped_note() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(RosterXmlWithOneBadForce)); + var roster = RosterFileConverter.Convert(stream, "test.ros"); + + var force = Assert.Single(roster.Forces); + Assert.Equal("fe-1", force.ForceEntryId); + var note = Assert.Single(roster.Unmapped); + Assert.Contains("Second Force", note); + } + + [Fact] + public void Roster_whose_only_force_has_no_entryId_throws_FormatException() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(RosterXmlWithOnlyBadForce)); + var ex = Assert.Throws(() => RosterFileConverter.Convert(stream, "test.ros")); + Assert.Contains("no forces", ex.Message); + } + + [Fact] + public void Rosz_that_decompresses_over_50MB_throws_FormatException_fast() + { + // A large run of compressible whitespace crushes down to a tiny compressed + // entry, but decompresses well past the 50 MB cap. + var padding = new string(' ', 60 * 1024 * 1024); + var xml = $"{padding}"; + + using var zipStream = new MemoryStream(); + using (var zip = new ZipArchive(zipStream, ZipArchiveMode.Create, leaveOpen: true)) + { + using var entry = zip.CreateEntry("test.ros", CompressionLevel.Optimal).Open(); + entry.Write(Encoding.UTF8.GetBytes(xml)); + } + zipStream.Position = 0; + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var ex = Assert.Throws(() => RosterFileConverter.Convert(zipStream, "test.rosz")); + sw.Stop(); + + Assert.Contains("too large", ex.Message); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"took {sw.Elapsed}"); + } +} diff --git a/tests/Muster.Cli.Tests/Converters/SpecEmitterTests.cs b/tests/Muster.Cli.Tests/Converters/SpecEmitterTests.cs new file mode 100644 index 0000000..8277202 --- /dev/null +++ b/tests/Muster.Cli.Tests/Converters/SpecEmitterTests.cs @@ -0,0 +1,147 @@ +using BattleScribeSpec; +using Muster.Cli.Converters; +using Xunit; + +namespace Muster.Cli.Tests.Converters; + +public class SpecEmitterTests +{ + private static ReplayRoster Sample() => new( + Name: "war horde", + GameSystemId: "sys-1", + ObservedTotals: [new("pts", "51b2-306e-1021-d207", 950m)], + BooksRevisions: ["Xenos - Orks: 2"], + Forces: + [ + new("bb9d-299a-ed60-2d8a", "a55f-b7b3-6c65-a05f", + Selections: + [ + new("7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4", 1, null, [], + Children: [new("d62d-db22-4893-4bc0", 1, null, [], [])]), + new("boy-entry", 3, "Da Ladz", [new("pts", "pts-id", 60m)], []), + ], + ChildForces: []), + ], + Unmapped: []); + + [Fact] + public void Emitted_yaml_round_trips_through_SpecLoader() + { + var yaml = SpecEmitter.Emit(Sample(), "issue-42", "github:test/repo", pinObserved: true); + var spec = SpecLoader.LoadFromYaml(yaml); // throws on invalid spec + + Assert.Equal("issue-42", spec.Id); + Assert.Equal("github:test/repo", spec.Setup.DataSource); + Assert.Contains(spec.Steps, s => s.Action == "addForce" && s.ForceEntryId == "bb9d-299a-ed60-2d8a"); + Assert.Contains(spec.Steps, s => s.Action == "selectEntry" && s.EntryId == "7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4"); + Assert.Contains(spec.Steps, s => s.Action == "selectChildEntry" && s.EntryId == "d62d-db22-4893-4bc0"); + Assert.Contains(spec.Steps, s => s.Action == "setSelectionCount" && s.Count == 3); + Assert.Contains(spec.Steps, s => s.Action == "setCustomization" && s.CustomName == "Da Ladz"); + var assertStep = spec.Steps.Last(); + Assert.NotNull(assertStep.ExpectedState); + var cost = Assert.Single(assertStep.ExpectedState!.Costs!); + Assert.Equal(950m, cost.Value); + Assert.Equal("51b2-306e-1021-d207", cost.TypeId); + } + + [Fact] + public void Per_selection_observed_costs_are_commented_not_pinned() + { + // Decision (Task 7 Step 4): RosterRunner.AssertSelections matches expected vs. + // actual selections positionally (expected[si] vs actual[si]), so a per-selection + // expectedState pin keyed on the converter's emit order could silently drift onto + // the wrong selection if the engine reorders/auto-adds selections. Observed + // per-selection costs are therefore surfaced as a comment only, never pinned. + var yaml = SpecEmitter.Emit(Sample(), "issue-42", "github:test/repo", pinObserved: true); + + Assert.Contains( + "# per-selection costs observed but not pinned (comparer is order-sensitive)", + yaml, StringComparison.Ordinal); + + var spec = SpecLoader.LoadFromYaml(yaml); // still must round-trip cleanly + Assert.DoesNotContain(spec.Steps, s => + s.ExpectedState?.Forces is { Count: > 0 }); + } + + [Fact] + public void No_comment_when_no_selection_has_observed_costs() + { + var roster = Sample() with + { + Forces = + [ + new("bb9d-299a-ed60-2d8a", "a55f-b7b3-6c65-a05f", + Selections: [new("entry-1", 1, null, [], [])], + ChildForces: []), + ], + }; + var yaml = SpecEmitter.Emit(roster, "issue-42", "github:test/repo", pinObserved: true); + Assert.DoesNotContain("per-selection costs observed", yaml, StringComparison.Ordinal); + } + + [Fact] + public void Without_pins_no_expectedState_is_emitted() + { + var yaml = SpecEmitter.Emit(Sample(), "issue-42", "github:test/repo", pinObserved: false); + var spec = SpecLoader.LoadFromYaml(yaml); + Assert.DoesNotContain(spec.Steps, s => s.ExpectedState is not null); + } + + [Fact] + public void Yaml_special_characters_are_escaped() + { + var roster = Sample() with { Name = "list: with \"quotes\" & #hash" }; + var yaml = SpecEmitter.Emit(roster, "x", "local:.", pinObserved: true); + Assert.NotNull(SpecLoader.LoadFromYaml(yaml)); // must not throw + } + + [Fact] + public void Names_with_control_characters_round_trip_through_SpecLoader() + { + const string tricky = "line1\nline2: evil"; + var roster = new ReplayRoster( + Name: tricky, + GameSystemId: "sys-1", + ObservedTotals: [], + BooksRevisions: [], + Forces: + [ + new ReplayForce("force-entry", "cat-1", + Selections: + [ + new ReplaySelection("entry-1", 1, tricky + "\t", [], []), + ], + ChildForces: []), + ], + Unmapped: []); + + var yaml = SpecEmitter.Emit(roster, "x", "local:.", pinObserved: false); + var spec = SpecLoader.LoadFromYaml(yaml); // must not throw (hard round-trip guarantee) + + var customStep = Assert.Single(spec.Steps, s => s.Action == "setCustomization"); + Assert.Equal(tricky + "\t", customStep.CustomName); + } + + [Fact] + public void Selection_steps_reference_parent_outputs() + { + var yaml = SpecEmitter.Emit(Sample(), "x", "local:.", pinObserved: false); + Assert.Contains("${{ steps.sel-1.selectionId }}", yaml, StringComparison.Ordinal); + Assert.Contains("${{ steps.force-1.forceId }}", yaml, StringComparison.Ordinal); + } + + [Fact] + public void Force_and_selection_expressions_survive_round_trip_verbatim() + { + var yaml = SpecEmitter.Emit(Sample(), "x", "local:.", pinObserved: false); + var spec = SpecLoader.LoadFromYaml(yaml); + + var selectEntryStep = Assert.Single( + spec.Steps, s => s.Action == "selectEntry" && s.EntryId == "7380-3e40-6ed6-b7cc::564e-fbc6-5266-3ea4"); + Assert.Equal("${{ steps.force-1.forceId }}", selectEntryStep.ForceId); + + var childStep = Assert.Single(spec.Steps, s => s.Action == "selectChildEntry"); + Assert.Equal("${{ steps.force-1.forceId }}", childStep.ForceId); + Assert.Equal("${{ steps.sel-1.selectionId }}", childStep.SelectionId); + } +} diff --git a/tests/Muster.Cli.Tests/DiffCommandTests.cs b/tests/Muster.Cli.Tests/DiffCommandTests.cs index aa4ce5e..bbf726e 100644 --- a/tests/Muster.Cli.Tests/DiffCommandTests.cs +++ b/tests/Muster.Cli.Tests/DiffCommandTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Muster.Cli.Commands; using Xunit; namespace Muster.Cli.Tests; @@ -6,6 +7,51 @@ namespace Muster.Cli.Tests; [Collection("Console output tests")] public class DiffCommandTests { + private static string TestAdapterDll => TestPaths.TestAdapterDll; + + private static int RunDiff( + string baseDir, string headDir, string fixturesDir, string output, + bool failOnBroke = false, string[]? engines = null, string[]? governing = null) + { + var originalOut = Console.Out; + var stdout = new StringWriter(); + int exit; + try + { + Console.SetOut(stdout); + exit = DiffCommand.Run( + new DirectoryInfo(baseDir), new DirectoryInfo(headDir), new DirectoryInfo(fixturesDir), + output, failOnBroke, engines ?? [], governing ?? []); + } + finally + { + Console.SetOut(originalOut); + } + + return exit; + } + + private static string CaptureDiffOutput( + string baseDir, string headDir, string fixturesDir, string output, + bool failOnBroke = false, string[]? engines = null, string[]? governing = null) + { + var originalOut = Console.Out; + var stdout = new StringWriter(); + try + { + Console.SetOut(stdout); + DiffCommand.Run( + new DirectoryInfo(baseDir), new DirectoryInfo(headDir), new DirectoryInfo(fixturesDir), + output, failOnBroke, engines ?? [], governing ?? []); + } + finally + { + Console.SetOut(originalOut); + } + + return stdout.ToString(); + } + [Fact] public async Task Diff_reports_broken_fixture_between_trees() { @@ -80,18 +126,57 @@ public async Task Diff_json_output_contains_both_run_reports_and_classified_rows using var doc = JsonDocument.Parse(stdout.ToString()); var root = doc.RootElement; - Assert.True(root.TryGetProperty("base", out var baseReport)); - Assert.True(root.TryGetProperty("head", out var headReport)); - Assert.Equal(1, baseReport.GetProperty("passed").GetInt32()); - Assert.Equal(1, headReport.GetProperty("failed").GetInt32()); + var diffs = root.GetProperty("diffs"); + Assert.Equal(1, diffs.GetArrayLength()); + var wham = diffs[0]; + Assert.Equal("wham", wham.GetProperty("engine").GetString()); - var rows = root.GetProperty("rows"); + var rows = wham.GetProperty("rows"); Assert.Equal(1, rows.GetArrayLength()); var row = rows[0]; Assert.Equal("unit-costs-20", row.GetProperty("fixtureId").GetString()); Assert.Equal("broke", row.GetProperty("classification").GetString()); } + [Fact] + public void FailOnBroke_exits_1_when_governing_engine_broke() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var headData = TestRepoFactory.CopyWithReplacement(dataDir, "20", "25"); // breaks the pts=20 pin + + var exit = RunDiff(dataDir, headData, fixturesDir, "markdown", + failOnBroke: true, engines: ["wham"], governing: ["wham"]); + + Assert.Equal(1, exit); + } + + [Fact] + public void FailOnBroke_ignores_non_governing_break() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var headData = TestRepoFactory.CopyWithReplacement(dataDir, "20", "25"); + // fake adapter always computes pts=20 → fixture passes both sides for 'fake'; + // wham (non-governing) breaks; check must stay green but report the gap. + var exit = RunDiff(dataDir, headData, fixturesDir, "markdown", + failOnBroke: true, + engines: [$"fake=dotnet:{TestAdapterDll}", "wham"], + governing: ["fake", "wham"]); + + Assert.Equal(0, exit); + } + + [Fact] + public void Head_status_disagreement_reports_engine_gap() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var headData = TestRepoFactory.CopyWithReplacement(dataDir, "20", "25"); + + var output = CaptureDiffOutput(dataDir, headData, fixturesDir, "markdown", + engines: [$"fake=dotnet:{TestAdapterDll}", "wham"], governing: ["fake"]); + + Assert.Contains("engine-gap", output, StringComparison.Ordinal); + } + [Fact] public async Task Diff_exits_2_when_base_dir_is_missing() { diff --git a/tests/Muster.Cli.Tests/Engines/EngineRegistryTests.cs b/tests/Muster.Cli.Tests/Engines/EngineRegistryTests.cs new file mode 100644 index 0000000..120a58c --- /dev/null +++ b/tests/Muster.Cli.Tests/Engines/EngineRegistryTests.cs @@ -0,0 +1,64 @@ +using Muster.Cli.Engines; +using Xunit; + +namespace Muster.Cli.Tests.Engines; + +public class EngineRegistryTests +{ + [Theory] + [InlineData("wham", "wham", EngineKind.Builtin, null, null)] + [InlineData("battlescribe=dotnet:/opt/adapter/Ref.dll", "battlescribe", EngineKind.Exec, "dotnet", "/opt/adapter/Ref.dll")] + [InlineData("newrecruit=docker:ghcr.io/warhub/bsspec-adapter-newrecruit:latest", "newrecruit", EngineKind.Docker, "docker", "run -i --rm ghcr.io/warhub/bsspec-adapter-newrecruit:latest")] + [InlineData("custom=/usr/bin/my-adapter --flag", "custom", EngineKind.Exec, "/usr/bin/my-adapter", "--flag")] + public void Parse_handles_all_command_forms(string input, string name, EngineKind kind, string? exe, string? args) + { + var spec = EngineSpec.Parse(input); + Assert.Equal(name, spec.Name); + Assert.Equal(kind, spec.Kind); + Assert.Equal(exe, spec.Executable); + Assert.Equal(args, spec.Arguments); + } + + [Fact] + public void Parse_rejects_builtin_command_for_unknown_name() + { + // bare name with no command is only valid for the builtin engine + Assert.Throws(() => EngineSpec.Parse("newrecruit")); + } + + [Fact] + public void ParseAll_defaults_to_builtin_wham() + { + var engines = EngineRegistry.ParseAll([]); + var e = Assert.Single(engines); + Assert.Equal("wham", e.Name); + Assert.Equal(EngineKind.Builtin, e.Kind); + } + + [Fact] + public void Builtin_wham_is_available_and_creates_adapter() + { + var spec = EngineSpec.Parse("wham"); + Assert.True(EngineRegistry.IsAvailable(spec)); + using var engine = EngineRegistry.CreateEngine(spec); + Assert.NotNull(engine); + } + + [Fact] + public void Missing_executable_is_unavailable_not_a_crash() + { + var spec = EngineSpec.Parse("ghost=/no/such/binary-xyz"); + Assert.False(EngineRegistry.IsAvailable(spec)); + } + + [Fact] + public void ResolveGoverning_picks_first_precedence_entry_that_ran() + { + Assert.Equal("battlescribe", EngineRegistry.ResolveGoverning( + ["newrecruit", "battlescribe", "wham"], ["wham", "battlescribe"])); + Assert.Equal("wham", EngineRegistry.ResolveGoverning( + ["newrecruit", "battlescribe", "wham"], ["wham"])); + Assert.Null(EngineRegistry.ResolveGoverning( + ["newrecruit"], ["wham"])); + } +} diff --git a/tests/Muster.Cli.Tests/Fakes/FakeRosterEngine.cs b/tests/Muster.Cli.Tests/Fakes/FakeRosterEngine.cs new file mode 100644 index 0000000..c718dcc --- /dev/null +++ b/tests/Muster.Cli.Tests/Fakes/FakeRosterEngine.cs @@ -0,0 +1,76 @@ +using BattleScribeSpec.Protocol; +using BattleScribeSpec.Roster; + +namespace Muster.Cli.Tests.Fakes; + +/// +/// Configurable in-proc engine double. Every selection contributes +/// ptsValue × count to a single roster-level "pts" cost, so tests can +/// pin totals and simulate divergent engines by varying ptsValue. +/// +public sealed class FakeRosterEngine(decimal ptsValue = 20m) : IRosterEngine +{ + private readonly List<(string SelectionId, int Count)> _selections = []; + private int _nextId; + + public List<(string FileName, string Content)>? ReceivedFiles { get; private set; } + + public IReadOnlyList Setup(ProtocolGameSystem gameSystem, ProtocolCatalogue[] catalogues) => []; + + public IReadOnlyList SetupFromFiles(IReadOnlyList<(string FileName, string Content)> files) + { + ReceivedFiles = [.. files]; + return []; + } + + public ActionOutputs AddForce(string forceEntryId, string catalogueId) => + new() { ForceId = $"force-{_nextId++}" }; + + public ActionOutputs AddChildForce(string parentForceId, string forceEntryId, string catalogueId) => + new() { ForceId = $"force-{_nextId++}" }; + + public void RemoveForce(string forceId) { } + + public ActionOutputs SelectEntry(string forceId, string entryId) + { + var id = $"sel-{_nextId++}"; + _selections.Add((id, 1)); + return new() { SelectionId = id, Selections = [] }; + } + + public ActionOutputs SelectChildEntry(string forceId, string parentSelectionId, string entryId) => + SelectEntry(forceId, entryId); + + public void DeselectSelection(string forceId, string selectionId) => + _selections.RemoveAll(s => s.SelectionId == selectionId); + + public void SetSelectionCount(string forceId, string selectionId, int count) + { + var i = _selections.FindIndex(s => s.SelectionId == selectionId); + if (i < 0) throw new InvalidOperationException($"unknown selection: {selectionId}"); + _selections[i] = (selectionId, count); + } + + public ActionOutputs DuplicateSelection(string forceId, string selectionId) + { + var id = $"sel-{_nextId++}"; + _selections.Add((id, 1)); + return new() { SelectionId = id }; + } + + public ActionOutputs DuplicateForce(string forceId) => new() { ForceId = $"force-{_nextId++}" }; + + public void SetCostLimit(string costTypeId, decimal value) { } + + public void SetCustomization(string forceId, string? selectionId, string? categoryEntryId, string? customName, string? customNotes) { } + + public RosterState GetRosterState() + { + var total = _selections.Sum(s => s.Count) * ptsValue; + return new RosterState("roster", "gs", [], [new CostState("pts", "pts", total)], []); + } + + public IReadOnlyList GetValidationErrors() => []; + + public void Dispose() { } +} diff --git a/tests/Muster.Cli.Tests/Fakes/FakeRosterEngineTests.cs b/tests/Muster.Cli.Tests/Fakes/FakeRosterEngineTests.cs new file mode 100644 index 0000000..441934a --- /dev/null +++ b/tests/Muster.Cli.Tests/Fakes/FakeRosterEngineTests.cs @@ -0,0 +1,24 @@ +using BattleScribeSpec.Roster; +using Muster.Cli.Tests.Fakes; +using Xunit; + +namespace Muster.Cli.Tests.Fakes; + +public class FakeRosterEngineTests +{ + [Fact] + public void Costs_scale_with_selection_count_and_pts_value() + { + using var engine = new FakeRosterEngine(ptsValue: 30m); + engine.SetupFromFiles([("a.gst", "")]); + var force = engine.AddForce("fe-1", "cat-1"); + var sel = engine.SelectEntry(force.ForceId!, "se-1"); + engine.SetSelectionCount(force.ForceId!, sel.SelectionId!, 2); + + var state = engine.GetRosterState(); + + var pts = Assert.Single(state.Costs); + Assert.Equal("pts", pts.TypeId); + Assert.Equal(60m, pts.Value); // 30 pts × count 2 + } +} diff --git a/tests/Muster.Cli.Tests/MultiEngineTestCommandTests.cs b/tests/Muster.Cli.Tests/MultiEngineTestCommandTests.cs new file mode 100644 index 0000000..ca15fe6 --- /dev/null +++ b/tests/Muster.Cli.Tests/MultiEngineTestCommandTests.cs @@ -0,0 +1,53 @@ +using Muster.Cli.Commands; +using Muster.Cli.Reporting; +using Xunit; + +namespace Muster.Cli.Tests; + +public class MultiEngineTestCommandTests +{ + private static string TestAdapterDll => TestPaths.TestAdapterDll; + + [Fact] + public void Two_engines_produce_two_runs_and_governing_resolves_by_precedence() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + + var multi = MultiRunReport.Run(dataDir, fixturesDir, + engineSpecs: ["wham", $"fake=dotnet:{TestAdapterDll}"], + governing: ["fake", "wham"]); + + Assert.Equal(2, multi.Runs.Count); + Assert.Equal("fake", multi.Governing); + Assert.Contains(multi.Runs, r => r.Engine == "wham"); + Assert.Contains(multi.Runs, r => r.Engine == "fake"); + } + + [Fact] + public void Unavailable_engine_is_named_not_dropped() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + + var multi = MultiRunReport.Run(dataDir, fixturesDir, + engineSpecs: ["wham", "ghost=/no/such/adapter-xyz"], + governing: ["ghost", "wham"]); + + Assert.Single(multi.Runs); + Assert.Equal(["ghost"], multi.Unavailable); + Assert.Equal("wham", multi.Governing); // ghost didn't run, precedence falls through + } + + [Fact] + public void Github_actions_output_renders_engine_matrix() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var multi = MultiRunReport.Run(dataDir, fixturesDir, ["wham"], ["wham"]); + + using var sw = new StringWriter(); + MultiRunReport.Write(multi, "github-actions", sw); + var text = sw.ToString(); + + Assert.Contains("wham", text, StringComparison.Ordinal); + Assert.Contains("governing", text, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/Muster.Cli.Tests/Muster.Cli.Tests.csproj b/tests/Muster.Cli.Tests/Muster.Cli.Tests.csproj index 9d6c9f9..2279b49 100644 --- a/tests/Muster.Cli.Tests/Muster.Cli.Tests.csproj +++ b/tests/Muster.Cli.Tests/Muster.Cli.Tests.csproj @@ -13,6 +13,16 @@ + + + + + + diff --git a/tests/Muster.Cli.Tests/NewRecruit/NrClientTests.cs b/tests/Muster.Cli.Tests/NewRecruit/NrClientTests.cs new file mode 100644 index 0000000..50ce6d9 --- /dev/null +++ b/tests/Muster.Cli.Tests/NewRecruit/NrClientTests.cs @@ -0,0 +1,149 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using Muster.Cli.NewRecruit; +using Xunit; + +namespace Muster.Cli.Tests.NewRecruit; + +public class NrClientTests +{ + private sealed class StubHandler(Func respond) : HttpMessageHandler + { + public HttpRequestMessage? LastRequest { get; private set; } + public string? LastBody { get; private set; } + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + LastRequest = request; + LastBody = request.Content is null ? null : await request.Content.ReadAsStringAsync(ct); + return respond(request); + } + } + + /// Stream that returns one chunk of bytes, then throws IOException on the next read — + /// simulates a connection reset / truncated body after headers have already arrived. + private sealed class ResetAfterFirstReadStream(byte[] firstChunk) : Stream + { + private int _readCount; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (_readCount++ == 0) + { + var n = Math.Min(buffer.Length, firstChunk.Length); + firstChunk.AsSpan(0, n).CopyTo(buffer.Span); + return ValueTask.FromResult(n); + } + throw new IOException("connection reset by peer"); + } + } + + /// HttpContent whose read stream throws mid-body, after headers/status have already been observed. + private sealed class ResetMidBodyContent : HttpContent + { + protected override Task CreateContentReadStreamAsync() => CreateStream(); + + protected override Task CreateContentReadStreamAsync(CancellationToken cancellationToken) => CreateStream(); + + private static Task CreateStream() => + Task.FromResult(new ResetAfterFirstReadStream(Encoding.UTF8.GetBytes("{\"partial"))); + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) => + throw new NotSupportedException(); + + protected override bool TryComputeLength(out long length) + { + length = 0; + return false; + } + } + + [Fact] + public async Task Fetch_posts_open_share_link_rpc_and_returns_json() + { + var handler = new StubHandler(_ => new(HttpStatusCode.OK) + { + Content = new StringContent("""{"name":"war horde","army":{}}""", Encoding.UTF8, "application/json"), + }); + using var client = new NrClient(handler); + + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + + Assert.Null(result.Error); + Assert.Contains("war horde", result.Json, StringComparison.Ordinal); + Assert.Equal("https://www.newrecruit.eu/api/rpc", handler.LastRequest!.RequestUri!.ToString()); + Assert.Contains("\"open_share_link\"", handler.LastBody, StringComparison.Ordinal); + Assert.Contains("\"3Pbpd\"", handler.LastBody, StringComparison.Ordinal); + } + + [Fact] + public async Task Null_rpc_response_is_not_found() + { + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new StringContent("null") }); + using var client = new NrClient(handler); + var result = await client.FetchListAsync("gone1", TestContext.Current.CancellationToken); + Assert.Null(result.Json); + Assert.Contains("no longer shared", result.Error, StringComparison.Ordinal); + } + + [Fact] + public async Task Http_error_is_graceful() + { + var handler = new StubHandler(_ => new(HttpStatusCode.InternalServerError)); + using var client = new NrClient(handler); + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + Assert.Null(result.Json); + Assert.NotNull(result.Error); + } + + [Fact] + public async Task Oversized_response_is_rejected() + { + var big = new string('x', 6 * 1024 * 1024); + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new StringContent($"{{\"pad\":\"{big}\"}}") }); + using var client = new NrClient(handler); + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + Assert.Null(result.Json); + Assert.Contains("too large", result.Error, StringComparison.Ordinal); + } + + [Fact] + public async Task Connection_reset_mid_body_is_graceful() + { + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ResetMidBodyContent() }); + using var client = new NrClient(handler); + + var result = await client.FetchListAsync("3Pbpd", TestContext.Current.CancellationToken); + + Assert.Null(result.Json); + Assert.NotNull(result.Error); + Assert.Contains("connection", result.Error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Real_caller_cancellation_propagates() + { + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"name":"war horde","army":{}}"""), + }); + using var client = new NrClient(handler); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync( + () => client.FetchListAsync("3Pbpd", cts.Token)); + } +} diff --git a/tests/Muster.Cli.Tests/NewRecruit/NrShareLinkTests.cs b/tests/Muster.Cli.Tests/NewRecruit/NrShareLinkTests.cs new file mode 100644 index 0000000..8402022 --- /dev/null +++ b/tests/Muster.Cli.Tests/NewRecruit/NrShareLinkTests.cs @@ -0,0 +1,24 @@ +using Muster.Cli.NewRecruit; +using Xunit; + +namespace Muster.Cli.Tests.NewRecruit; + +public class NrShareLinkTests +{ + [Theory] + [InlineData("https://www.newrecruit.eu/app/list/3Pbpd", true, "3Pbpd")] + [InlineData("https://www.newrecruit.eu/app/list/3Pbpd/", true, "3Pbpd")] + [InlineData("https://www.newrecruit.eu/app/list/tr5BL", true, "tr5BL")] + [InlineData("http://www.newrecruit.eu/app/list/3Pbpd", false, "")] + [InlineData("https://newrecruit.eu/app/list/3Pbpd", false, "")] + [InlineData("https://evil.example/app/list/3Pbpd", false, "")] + [InlineData("https://www.newrecruit.eu/app/list/../secret", false, "")] + [InlineData("https://www.newrecruit.eu/app/list/3Pbpd?x=1", false, "")] + [InlineData("https://www.newrecruit.eu/api/rpc", false, "")] + [InlineData("not a url", false, "")] + public void TryParse_allowlist(string url, bool ok, string key) + { + Assert.Equal(ok, NrShareLink.TryParse(url, out var k)); + if (ok) Assert.Equal(key, k); + } +} diff --git a/tests/Muster.Cli.Tests/ProgramExitCodeTests.cs b/tests/Muster.Cli.Tests/ProgramExitCodeTests.cs index f10033c..eca1dff 100644 --- a/tests/Muster.Cli.Tests/ProgramExitCodeTests.cs +++ b/tests/Muster.Cli.Tests/ProgramExitCodeTests.cs @@ -40,6 +40,13 @@ public async Task Help_option_exits_zero() Assert.Equal(0, exit); } + [Fact] + public async Task Convert_with_no_args_exits_inconclusive_not_failed() + { + var exit = await RunCaptured(["convert"]); + Assert.Equal(2, exit); + } + private static async Task RunCaptured(string[] args) { var originalOut = Console.Out; diff --git a/tests/Muster.Cli.Tests/PromoteCommandTests.cs b/tests/Muster.Cli.Tests/PromoteCommandTests.cs new file mode 100644 index 0000000..b8c8b8d --- /dev/null +++ b/tests/Muster.Cli.Tests/PromoteCommandTests.cs @@ -0,0 +1,142 @@ +using System.Text.Json; +using BattleScribeSpec; +using Muster.Cli.Commands; +using Xunit; + +namespace Muster.Cli.Tests; + +[Collection("Console output tests")] +public class PromoteCommandTests +{ + // Same shape as the green `unit-costs-20` fixture (TestRepoFactory), but with the + // roster-level cost pinned to an obviously wrong value (999) — as if the original bug + // report's reply snapshot had captured a stale/buggy observation. Promotion must replay + // the action steps and re-pin to whatever the CURRENT engine actually returns (20), not + // carry the stale 999 forward. + private const string SnapshotWithWrongPin = """ + id: "report" + category: "report" + description: "Converted from bug report roster 'Test'" + setup: + dataSource: "github:muster-e2e/test-data@main" + + steps: + - action: addForce + id: "force-1" + forceEntryId: "fe-army" + catalogueId: "cat-test" + - action: selectEntry + id: "sel-1" + forceId: "${{ steps.force-1.forceId }}" + entryId: "se-unit" + + # observed values from the report — PASS means the reported state reproduces + - expectedState: + costs: + - typeId: "ct-pts" + value: 999 + """; + + private static string WriteTemp(string content, string ext = ".txt") + { + var path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ext); + File.WriteAllText(path, content); + return path; + } + + private static string CommentsJsonWithSnapshot(string snapshotYaml) + { + var body = $""" + + + ## Verdict: Confirmed + +
Executable spec (snapshot) + + + ```yaml + {snapshotYaml} + ``` + +
+ """; + return JsonSerializer.Serialize(new[] { new { body } }); + } + + private static async Task<(int Exit, string Err)> RunCaptured( + string issueBodyPath, string commentsPath, string dataDir, int issueNumber, string outDir) + { + var originalOut = Console.Out; + var originalErr = Console.Error; + var errWriter = new StringWriter(); + try + { + Console.SetOut(new StringWriter()); + Console.SetError(errWriter); + var exit = await PromoteCommand.Run( + new FileInfo(issueBodyPath), new FileInfo(commentsPath), new DirectoryInfo(dataDir), + issueNumber, [], [], new DirectoryInfo(outDir), TestContext.Current.CancellationToken); + return (exit, errWriter.ToString()); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalErr); + } + } + + [Fact] + public async Task Promotes_snapshot_repinning_to_current_engine_values() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var outDir = Directory.CreateTempSubdirectory("muster-promote-out-").FullName; + var issueBodyPath = WriteTemp("### Roster\n\nirrelevant for promote", ".md"); + var commentsPath = WriteTemp(CommentsJsonWithSnapshot(SnapshotWithWrongPin), ".json"); + + var (exit, _) = await RunCaptured(issueBodyPath, commentsPath, dataDir, 7, outDir); + + Assert.Equal(0, exit); + var written = Path.Combine(outDir, "report-issue-7.yaml"); + Assert.True(File.Exists(written)); + + var spec = SpecLoader.Load(written); + Assert.Equal("report-issue-7", spec.Id); + var pinStep = Assert.Single(spec.Steps, s => s.ExpectedState is not null); + var costs = pinStep.ExpectedState!.Costs; + Assert.NotNull(costs); + var pts = Assert.Single(costs!, c => c.TypeId == "ct-pts"); + Assert.Equal(20m, pts.Value); + } + + [Fact] + public async Task Name_collision_appends_numeric_suffix() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var outDir = Directory.CreateTempSubdirectory("muster-promote-out-").FullName; + File.WriteAllText(Path.Combine(outDir, "report-issue-7.yaml"), "pre-existing: true"); + var issueBodyPath = WriteTemp("irrelevant", ".md"); + var commentsPath = WriteTemp(CommentsJsonWithSnapshot(SnapshotWithWrongPin), ".json"); + + var (exit, _) = await RunCaptured(issueBodyPath, commentsPath, dataDir, 7, outDir); + + Assert.Equal(0, exit); + Assert.True(File.Exists(Path.Combine(outDir, "report-issue-7-2.yaml"))); + // Original file (a non-spec placeholder) must not have been overwritten. + Assert.Equal("pre-existing: true", File.ReadAllText(Path.Combine(outDir, "report-issue-7.yaml"))); + } + + [Fact] + public async Task Comments_without_snapshot_exit_2_with_stderr_message() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var outDir = Directory.CreateTempSubdirectory("muster-promote-out-").FullName; + var issueBodyPath = WriteTemp("irrelevant", ".md"); + var commentsPath = WriteTemp("""[{"body": "just a human comment, no spec here"}]""", ".json"); + + var (exit, err) = await RunCaptured(issueBodyPath, commentsPath, dataDir, 7, outDir); + + Assert.Equal(2, exit); + Assert.Contains("snapshot", err, StringComparison.OrdinalIgnoreCase); + Assert.False(File.Exists(Path.Combine(outDir, "report-issue-7.yaml"))); + } +} diff --git a/tests/Muster.Cli.Tests/RepoDataSourceResolverTests.cs b/tests/Muster.Cli.Tests/RepoDataSourceResolverTests.cs index 52bc9ff..d0db582 100644 --- a/tests/Muster.Cli.Tests/RepoDataSourceResolverTests.cs +++ b/tests/Muster.Cli.Tests/RepoDataSourceResolverTests.cs @@ -1,6 +1,7 @@ using BattleScribeSpec; using BattleScribeSpec.Roster; using Muster.Cli.Fixtures; +using Muster.Cli.Tests.Fakes; using Xunit; namespace Muster.Cli.Tests; @@ -96,7 +97,7 @@ public void Create_resolver_drives_RosterRunner_which_calls_engine_SetupFromFile - expectedState: {} """); - var engine = new FakeEngine(); + var engine = new FakeRosterEngine(); var runner = new RosterRunner(engine, RepoDataSourceResolver.Create(dataDir), engineName: "wham"); runner.Run(spec); @@ -154,63 +155,4 @@ public void IsPopulatedFor_returns_false_for_a_garbage_dataSource_uri_without_th Assert.False(RepoDataSourceResolver.IsPopulatedFor(dataDir, "ftp:some/unsupported/scheme")); Assert.False(RepoDataSourceResolver.IsPopulatedFor(dataDir, "")); } - - /// - /// Minimal fake, mirroring the pattern used by - /// RunnerAndProtocolRegressionTests.FakeEngine in the wham battlescribe-spec - /// TestKit's own regression tests. Only and the state - /// getters matter for this test; every other action member either isn't reached - /// (no action steps in the fixture) or would throw if it were. - /// - private sealed class FakeEngine : IRosterEngine - { - public List<(string FileName, string Content)>? ReceivedFiles { get; private set; } - - public IReadOnlyList Setup(BattleScribeSpec.Protocol.ProtocolGameSystem gameSystem, BattleScribeSpec.Protocol.ProtocolCatalogue[] catalogues) => - throw new NotSupportedException("Not exercised: this test uses dataSource setup, not inline setup."); - - public ActionOutputs AddForce(string forceEntryId, string catalogueId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public ActionOutputs AddChildForce(string parentForceId, string forceEntryId, string catalogueId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public void RemoveForce(string forceId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public ActionOutputs SelectEntry(string forceId, string entryId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public ActionOutputs SelectChildEntry(string forceId, string parentSelectionId, string entryId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public void DeselectSelection(string forceId, string selectionId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public void SetSelectionCount(string forceId, string selectionId, int count) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public ActionOutputs DuplicateSelection(string forceId, string selectionId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public ActionOutputs DuplicateForce(string forceId) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public void SetCostLimit(string costTypeId, decimal value) => - throw new NotSupportedException("Not exercised: this test's fixture has no action steps."); - - public RosterState GetRosterState() => new("roster", "gs", [], [], []); - - public IReadOnlyList GetValidationErrors() => []; - - public IReadOnlyList SetupFromFiles(IReadOnlyList<(string FileName, string Content)> files) - { - ReceivedFiles = [.. files]; - return []; - } - - public void Dispose() - { - } - } } diff --git a/tests/Muster.Cli.Tests/ReportCommandTests.cs b/tests/Muster.Cli.Tests/ReportCommandTests.cs new file mode 100644 index 0000000..f18b623 --- /dev/null +++ b/tests/Muster.Cli.Tests/ReportCommandTests.cs @@ -0,0 +1,286 @@ +using Muster.Cli.Commands; +using Xunit; + +namespace Muster.Cli.Tests; + +[Collection("Console output tests")] +public class ReportCommandTests +{ + private static string GreenFixtureYaml(string fixturesDir) => + File.ReadAllText(Path.Combine(fixturesDir, "unit-costs-20.yaml")); + + private static string WriteIssueBody(string content) + { + var path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".md"); + File.WriteAllText(path, content); + return path; + } + + private static string NewOutDir() => Directory.CreateTempSubdirectory("muster-report-out-").FullName; + + private static async Task RunCaptured( + string issueBodyPath, string dataDir, string outDir, + string dataSource = "local:.", string[]? engines = null, string[]? governing = null, + string? previousReplyPath = null) + { + var originalOut = Console.Out; + try + { + Console.SetOut(new StringWriter()); + return await ReportCommand.Run( + new FileInfo(issueBodyPath), new DirectoryInfo(dataDir), dataSource, + engines ?? [], governing ?? [], new DirectoryInfo(outDir), + previousReplyPath is null ? null : new FileInfo(previousReplyPath), + TestContext.Current.CancellationToken); + } + finally + { + Console.SetOut(originalOut); + } + } + + [Fact] + public async Task Inline_spec_with_matching_pins_is_confirmed() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var yaml = GreenFixtureYaml(fixturesDir); + var issueBody = $""" + ### Roster + + ```yaml + {yaml} + ``` + + ### Problem + + Test Unit seems to cost more than it should. + + ### Expected + + Test Unit should cost less. + """; + var bodyPath = WriteIssueBody(issueBody); + var outDir = NewOutDir(); + + // The fixture's inline spec declares setup.dataSource: "github:muster-e2e/test-data@main" + // itself (TestRepoFactory) — matching --data-source here so F3's hermeticity check + // (inline dataSource must match the repo's own) doesn't reject it as a mismatch. + var exit = await RunCaptured(bodyPath, dataDir, outDir, dataSource: "github:muster-e2e/test-data@main"); + + Assert.Equal(0, exit); + var reply = await File.ReadAllTextAsync(Path.Combine(outDir, "reply.md"), TestContext.Current.CancellationToken); + Assert.StartsWith("", reply, StringComparison.Ordinal); + Assert.Contains("Verdict: Confirmed", reply, StringComparison.Ordinal); + Assert.Contains("", reply, StringComparison.Ordinal); + Assert.Contains("```yaml", reply, StringComparison.Ordinal); + Assert.Contains("Test Unit seems to cost more", reply, StringComparison.Ordinal); + + var json = await File.ReadAllTextAsync(Path.Combine(outDir, "report.json"), TestContext.Current.CancellationToken); + Assert.Contains("\"verdict\": \"confirmed\"", json, StringComparison.Ordinal); + Assert.Contains("\"engineGap\": false", json, StringComparison.Ordinal); + Assert.Contains("\"governing\": \"wham\"", json, StringComparison.Ordinal); + + Assert.True(File.Exists(Path.Combine(outDir, "snapshot.yaml"))); + } + + [Fact] + public async Task Inline_spec_with_mismatched_pins_is_not_reproducible() + { + var (dataDir, fixturesDir) = TestRepoFactory.CreateTestRepo(); + var yaml = GreenFixtureYaml(fixturesDir).Replace("value: 20", "value: 999", StringComparison.Ordinal); + var issueBody = $""" + ```yaml + {yaml} + ``` + """; + var bodyPath = WriteIssueBody(issueBody); + var outDir = NewOutDir(); + + var exit = await RunCaptured(bodyPath, dataDir, outDir, dataSource: "github:muster-e2e/test-data@main"); + + Assert.Equal(0, exit); + var reply = await File.ReadAllTextAsync(Path.Combine(outDir, "reply.md"), TestContext.Current.CancellationToken); + Assert.Contains("Verdict: Not reproducible", reply, StringComparison.Ordinal); + + var json = await File.ReadAllTextAsync(Path.Combine(outDir, "report.json"), TestContext.Current.CancellationToken); + Assert.Contains("\"verdict\": \"not-reproducible\"", json, StringComparison.Ordinal); + } + + [Fact] + public async Task Garbage_body_is_needs_info_and_exits_zero() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var bodyPath = WriteIssueBody("this issue has no roster, no yaml, nothing usable at all."); + var outDir = NewOutDir(); + + var exit = await RunCaptured(bodyPath, dataDir, outDir); + + Assert.Equal(0, exit); + var reply = await File.ReadAllTextAsync(Path.Combine(outDir, "reply.md"), TestContext.Current.CancellationToken); + Assert.StartsWith("", reply, StringComparison.Ordinal); + Assert.Contains("Verdict: Needs info", reply, StringComparison.Ordinal); + Assert.Contains("New Recruit share link", reply, StringComparison.Ordinal); + Assert.Contains("no roster found", reply, StringComparison.Ordinal); + + var json = await File.ReadAllTextAsync(Path.Combine(outDir, "report.json"), TestContext.Current.CancellationToken); + Assert.Contains("\"verdict\": \"needs-info\"", json, StringComparison.Ordinal); + Assert.False(File.Exists(Path.Combine(outDir, "snapshot.yaml"))); + + // F1c: no spec at all (neither fresh nor carried forward) must never render an empty + // yaml block — the whole snapshot
section is omitted outright. + Assert.DoesNotContain("", reply, StringComparison.Ordinal); + Assert.DoesNotContain("```yaml", reply, StringComparison.Ordinal); + Assert.DoesNotContain("
", reply, StringComparison.Ordinal); + } + + [Fact] + public async Task Snapshot_is_carried_forward_from_previous_reply_when_this_evaluation_finds_no_roster() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + + // A previous reply, exactly as `muster report` would have rendered and posted it, with + // a durable snapshot embedded (e.g. captured back when the report's New Recruit link + // still resolved). + var previousReply = """ + + + ## Verdict: Confirmed + + The reported values reproduce against current data. + +
Executable spec (snapshot) + + + ```yaml + id: "report" + setup: + dataSource: "local:." + steps: + - expectedState: {} + ``` + +
+ """; + var previousReplyPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".md"); + await File.WriteAllTextAsync(previousReplyPath, previousReply, TestContext.Current.CancellationToken); + + // This time around, the NR link rotted (or was edited away): conversion produces no + // spec/roster of its own. + var bodyPath = WriteIssueBody("the link stopped working, sorry — no roster info here now."); + var outDir = NewOutDir(); + + var exit = await RunCaptured(bodyPath, dataDir, outDir, previousReplyPath: previousReplyPath); + + Assert.Equal(0, exit); + + var reply = await File.ReadAllTextAsync(Path.Combine(outDir, "reply.md"), TestContext.Current.CancellationToken); + Assert.Contains("Verdict: Needs info", reply, StringComparison.Ordinal); + Assert.Contains("snapshot preserved from a previous evaluation", reply, StringComparison.Ordinal); + Assert.Contains("", reply, StringComparison.Ordinal); + Assert.Contains("id: \"report\"", reply, StringComparison.Ordinal); + + var json = await File.ReadAllTextAsync(Path.Combine(outDir, "report.json"), TestContext.Current.CancellationToken); + Assert.Contains("\"verdict\": \"needs-info\"", json, StringComparison.Ordinal); + + var snapshotPath = Path.Combine(outDir, "snapshot.yaml"); + Assert.True(File.Exists(snapshotPath)); + var snapshot = await File.ReadAllTextAsync(snapshotPath, TestContext.Current.CancellationToken); + Assert.Contains("id: \"report\"", snapshot, StringComparison.Ordinal); + } + + [Fact] + public async Task Inline_spec_with_mismatched_dataSource_is_needs_info_and_runs_no_engine() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + + // A hostile inline spec that declares its own dataSource instead of leaving it to the + // repository's — if honored, this would let the runner read arbitrary + // container-reachable paths (F3: inline-spec dataSource hermeticity hole). + var issueBody = """ + ```yaml + id: "report" + setup: + dataSource: "local:C:/" + steps: + - expectedState: {} + ``` + """; + var bodyPath = WriteIssueBody(issueBody); + var outDir = NewOutDir(); + + var exit = await RunCaptured(bodyPath, dataDir, outDir, dataSource: "github:x/y"); + + Assert.Equal(0, exit); + var reply = await File.ReadAllTextAsync(Path.Combine(outDir, "reply.md"), TestContext.Current.CancellationToken); + Assert.Contains("Verdict: Needs info", reply, StringComparison.Ordinal); + Assert.Contains( + "inline spec declares a dataSource that does not match this repository's data source", + reply, StringComparison.Ordinal); + + var json = await File.ReadAllTextAsync(Path.Combine(outDir, "report.json"), TestContext.Current.CancellationToken); + Assert.Contains("\"verdict\": \"needs-info\"", json, StringComparison.Ordinal); + + // No spec ever made it to disk, and (implicitly) no engine ever ran against it: exit 0, + // needs-info, and no snapshot.yaml written. + Assert.False(File.Exists(Path.Combine(outDir, "snapshot.yaml"))); + } + + [Fact] + public async Task Invalid_inline_yaml_is_needs_info_and_exits_zero() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var issueBody = """ + ```yaml + steps: + - action: notAnAction + thisIsNot: validSpecYaml: [ + ``` + """; + var bodyPath = WriteIssueBody(issueBody); + var outDir = NewOutDir(); + + var exit = await RunCaptured(bodyPath, dataDir, outDir); + + Assert.Equal(0, exit); + var json = await File.ReadAllTextAsync(Path.Combine(outDir, "report.json"), TestContext.Current.CancellationToken); + Assert.Contains("\"verdict\": \"needs-info\"", json, StringComparison.Ordinal); + } + + [Fact] + public async Task Missing_data_dir_exits_2() + { + var bodyPath = WriteIssueBody("no roster here"); + var outDir = NewOutDir(); + + var exit = await RunCaptured(bodyPath, Path.Combine(Path.GetTempPath(), "no-such-muster-data-dir"), outDir); + + Assert.Equal(2, exit); + } + + [Fact] + public async Task Missing_issue_body_file_exits_2() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var outDir = NewOutDir(); + + var exit = await RunCaptured(Path.Combine(Path.GetTempPath(), "no-such-issue-body.md"), dataDir, outDir); + + Assert.Equal(2, exit); + } + + [Fact] + public async Task Unwritable_out_dir_exits_2_without_throwing() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + var bodyPath = WriteIssueBody("no roster here"); + + // A file already occupies the path we'll ask ReportCommand to create as a directory — + // Directory.CreateDirectory must fail, and that failure must be caught (exit 2), not thrown. + var collidingPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + File.WriteAllText(collidingPath, "occupied"); + + var exit = await RunCaptured(bodyPath, dataDir, collidingPath); + + Assert.Equal(2, exit); + } +} diff --git a/tests/Muster.Cli.Tests/Reports/AttachmentClientTests.cs b/tests/Muster.Cli.Tests/Reports/AttachmentClientTests.cs new file mode 100644 index 0000000..4a42a53 --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/AttachmentClientTests.cs @@ -0,0 +1,175 @@ +using System.Net; +using System.Net.Http; +using System.Text; +using Muster.Cli.Reports; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +public class AttachmentClientTests +{ + private sealed class StubHandler(Func respond) : HttpMessageHandler + { + public HttpRequestMessage? LastRequest { get; private set; } + public int InvocationCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + InvocationCount++; + LastRequest = request; + return Task.FromResult(respond(request)); + } + } + + /// Stream that returns one chunk of bytes, then throws IOException on the next read — + /// simulates a connection reset / truncated body after headers have already arrived. + private sealed class ResetAfterFirstReadStream(byte[] firstChunk) : Stream + { + private int _readCount; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (_readCount++ == 0) + { + var n = Math.Min(buffer.Length, firstChunk.Length); + firstChunk.AsSpan(0, n).CopyTo(buffer.Span); + return ValueTask.FromResult(n); + } + throw new IOException("connection reset by peer"); + } + } + + /// HttpContent whose read stream throws mid-body, after headers/status have already been observed. + private sealed class ResetMidBodyContent : HttpContent + { + protected override Task CreateContentReadStreamAsync() => CreateStream(); + + protected override Task CreateContentReadStreamAsync(CancellationToken cancellationToken) => CreateStream(); + + private static Task CreateStream() => + Task.FromResult(new ResetAfterFirstReadStream(Encoding.UTF8.GetBytes("PK\x03\x04partial"))); + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) => + throw new NotSupportedException(); + + protected override bool TryComputeLength(out long length) + { + length = 0; + return false; + } + } + + private const string ValidUrl = "https://github.com/user-attachments/files/12345/my-list.rosz"; + + [Fact] + public async Task Download_returns_bytes_on_success() + { + var payload = Encoding.UTF8.GetBytes("roster bytes"); + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(payload) }); + using var client = new AttachmentClient(handler); + + var (data, error) = await client.DownloadAsync(ValidUrl, TestContext.Current.CancellationToken); + + Assert.Null(error); + Assert.Equal(payload, data); + Assert.Equal(1, handler.InvocationCount); + Assert.Equal(ValidUrl, handler.LastRequest!.RequestUri!.ToString()); + } + + [Theory] + [InlineData("https://evil.example/x.rosz")] + [InlineData("https://github.com/user-attachments/files/12345/../../etc/passwd")] + [InlineData("http://github.com/user-attachments/files/12345/my-list.rosz")] + [InlineData("not a url")] + [InlineData("")] + public async Task Url_revalidation_rejects_non_allowlisted_urls_without_any_http_call(string url) + { + var handler = new StubHandler(_ => new(HttpStatusCode.OK)); + using var client = new AttachmentClient(handler); + + var (data, error) = await client.DownloadAsync(url, TestContext.Current.CancellationToken); + + Assert.Null(data); + Assert.NotNull(error); + Assert.Equal(0, handler.InvocationCount); + } + + [Fact] + public async Task Legacy_owner_repo_attachment_url_is_accepted() + { + var payload = Encoding.UTF8.GetBytes("roster bytes"); + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(payload) }); + using var client = new AttachmentClient(handler); + + var (data, error) = await client.DownloadAsync( + "https://github.com/BSData/wh40k-11e/files/9999/army.ros", TestContext.Current.CancellationToken); + + Assert.Null(error); + Assert.Equal(payload, data); + } + + [Fact] + public async Task Http_error_is_graceful() + { + var handler = new StubHandler(_ => new(HttpStatusCode.NotFound)); + using var client = new AttachmentClient(handler); + + var (data, error) = await client.DownloadAsync(ValidUrl, TestContext.Current.CancellationToken); + + Assert.Null(data); + Assert.NotNull(error); + } + + [Fact] + public async Task Oversized_response_is_rejected() + { + var big = new byte[6 * 1024 * 1024]; + var handler = new StubHandler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(big) }); + using var client = new AttachmentClient(handler); + + var (data, error) = await client.DownloadAsync(ValidUrl, TestContext.Current.CancellationToken); + + Assert.Null(data); + Assert.Contains("too large", error, StringComparison.Ordinal); + } + + [Fact] + public async Task Connection_reset_mid_body_is_graceful() + { + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ResetMidBodyContent() }); + using var client = new AttachmentClient(handler); + + var (data, error) = await client.DownloadAsync(ValidUrl, TestContext.Current.CancellationToken); + + Assert.Null(data); + Assert.NotNull(error); + Assert.Contains("connection", error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Real_caller_cancellation_propagates() + { + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(Encoding.UTF8.GetBytes("data")), + }); + using var client = new AttachmentClient(handler); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync( + () => client.DownloadAsync(ValidUrl, cts.Token)); + } +} diff --git a/tests/Muster.Cli.Tests/Reports/IssueBodyTests.cs b/tests/Muster.Cli.Tests/Reports/IssueBodyTests.cs new file mode 100644 index 0000000..d18b060 --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/IssueBodyTests.cs @@ -0,0 +1,110 @@ +using Muster.Cli.Reports; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +public class IssueBodyTests +{ + [Fact] + public void Parses_nr_auto_report() + { + // Verbatim shape of New Recruit auto-filed issues (BSData/wh40k-11e#234) + var body = """ + **Problem:** + Outriders Squad 6 members - 145 points + + **Expected:** + Outriders Squad 6 members - 140 points + + **List:** https://www.newrecruit.eu/app/list/tr5BL + **NewRecruit Version:** 34.99 + """; + var parsed = IssueBody.Parse(body); + + Assert.Equal(RosterSourceKind.NrLink, parsed.Roster!.Kind); + Assert.Equal("https://www.newrecruit.eu/app/list/tr5BL", parsed.Roster.Value); + Assert.Contains("145 points", parsed.Problem, StringComparison.Ordinal); + Assert.Contains("140 points", parsed.Expected, StringComparison.Ordinal); + } + + [Fact] + public void Parses_attachment_url() + { + var body = "My roster: https://github.com/user-attachments/files/12345/my-list.rosz broken!"; + var parsed = IssueBody.Parse(body); + Assert.Equal(RosterSourceKind.Attachment, parsed.Roster!.Kind); + Assert.EndsWith("my-list.rosz", parsed.Roster.Value, StringComparison.Ordinal); + } + + [Fact] + public void Parses_inline_yaml_block() + { + var body = """ + Points look wrong: + + ```yaml + steps: + - action: addForce + forceEntryId: "fe-1" + ``` + """; + var parsed = IssueBody.Parse(body); + Assert.Equal(RosterSourceKind.InlineYaml, parsed.Roster!.Kind); + Assert.Contains("addForce", parsed.Roster.Value, StringComparison.Ordinal); + } + + [Fact] + public void Nr_link_wins_over_attachment_and_yaml() + { + var body = """ + https://github.com/user-attachments/files/1/x.rosz + **List:** https://www.newrecruit.eu/app/list/abc12 + ```yaml + steps: [] + ``` + """; + Assert.Equal(RosterSourceKind.NrLink, IssueBody.Parse(body).Roster!.Kind); + } + + [Theory] + [InlineData("")] + [InlineData("just words, no roster anywhere")] + [InlineData("https://evil.example/app/list/abc12")] + [InlineData("```yaml\nname: no steps here\n```")] + public void No_roster_source_yields_null(string body) => + Assert.Null(IssueBody.Parse(body).Roster); + + [Fact] + public void Huge_body_is_truncated_not_hung() + { + var body = new string('a', 10_000_000) + " **List:** https://www.newrecruit.eu/app/list/abc12"; + var parsed = IssueBody.Parse(body); // must return fast + Assert.Null(parsed.Roster); // link is beyond the 64 KiB parse window + } + + [Fact] + public void Parses_issue_form_style_headings() + { + // GitHub renders issue-form fields as `### Label` headings rather than + // NR auto-report's `**Label:**` markers. + var body = """ + ### Roster + + https://www.newrecruit.eu/app/list/abc12 + + ### Problem + + Unit costs 65 + + ### Expected + + Unit costs 60 + """; + var parsed = IssueBody.Parse(body); + + Assert.Equal(RosterSourceKind.NrLink, parsed.Roster!.Kind); + Assert.Equal("https://www.newrecruit.eu/app/list/abc12", parsed.Roster.Value); + Assert.Contains("Unit costs 65", parsed.Problem, StringComparison.Ordinal); + Assert.Contains("Unit costs 60", parsed.Expected, StringComparison.Ordinal); + } +} diff --git a/tests/Muster.Cli.Tests/Reports/PromoteChainTests.cs b/tests/Muster.Cli.Tests/Reports/PromoteChainTests.cs new file mode 100644 index 0000000..2bb7b0f --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/PromoteChainTests.cs @@ -0,0 +1,73 @@ +using System.Text.Json; +using BattleScribeSpec; +using Muster.Cli.Converters; +using Muster.Cli.Engines; +using Muster.Cli.Reports; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +/// +/// Integration coverage for the real promotion chain: emits a +/// GitHub-comment reply with an embedded snapshot → +/// pulls the snapshot back out of raw gh api comments JSON → +/// replays it against the local data repo and re-pins it to the engine's current values. Nothing +/// exercised this end-to-end chain before — unit tests covered each stage against hand-written +/// fixtures of the *next* stage's expected input, which would not have caught a format drift +/// between what one stage emits and what the next expects. +/// +public class PromoteChainTests +{ + [Fact] + public void Render_then_extract_then_repin_round_trips_and_pins_the_engines_actual_value() + { + var (dataDir, _) = TestRepoFactory.CreateTestRepo(); + + // Build a roster that replays cleanly against TestRepoFactory's data (fe-army / + // cat-test / se-unit), with an observed roster-level cost pinned WRONG (999) — as if + // captured from a stale/buggy original report — via the real SpecEmitter, exactly as + // `muster report` would produce it. + var roster = new ReplayRoster( + Name: "Test", + GameSystemId: "gs-test", + ObservedTotals: [new ReplayCost(Name: "pts", TypeId: "ct-pts", Value: 999m)], + BooksRevisions: [], + Forces: + [ + new ReplayForce( + ForceEntryId: "fe-army", + CatalogueId: "cat-test", + Selections: [new ReplaySelection(EntryId: "se-unit", Count: 1, CustomName: null, ObservedCosts: [], Children: [])], + ChildForces: []) + ], + Unmapped: []); + + var specYaml = SpecEmitter.Emit(roster, specId: "report", dataSource: "github:muster-e2e/test-data@main", pinObserved: true); + + // Render the REAL reply, exactly as `muster report` would post it as an issue comment. + var verdict = new Verdict(VerdictKind.Confirmed, EngineGap: false, Labels: ["confirmed"], Detail: null); + var rendered = ReplyRenderer.Render( + verdict, roster: null, runs: null, specYaml: specYaml, problem: "Cost seems off", expected: "20 points"); + + // Wrap as the raw `gh api /repos/{o}/{r}/issues/{n}/comments` JSON shape. + var commentsJson = JsonSerializer.Serialize(new[] { new { body = rendered } }); + + var extracted = SnapshotExtractor.ExtractLatest(commentsJson); + Assert.NotNull(extracted); + + var engineSpec = EngineSpec.Parse(EngineSpec.BuiltinName); + var rePinned = SpecRePinner.RePin(extracted!, dataDir, engineSpec, newSpecId: "report-issue-42"); + + var loaded = SpecLoader.LoadFromYaml(rePinned); + Assert.Equal("report-issue-42", loaded.Id); + + var pinStep = Assert.Single(loaded.Steps, s => s.ExpectedState is not null); + var costs = pinStep.ExpectedState!.Costs; + Assert.NotNull(costs); + var pts = Assert.Single(costs!, c => c.TypeId == "ct-pts"); + + // The engine's ACTUAL current value (20, per TestRepoFactory's data), not the stale + // observed-from-report value (999) that was baked into the rendered snapshot. + Assert.Equal(20m, pts.Value); + } +} diff --git a/tests/Muster.Cli.Tests/Reports/ReplyRendererTests.cs b/tests/Muster.Cli.Tests/Reports/ReplyRendererTests.cs new file mode 100644 index 0000000..6c623dd --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/ReplyRendererTests.cs @@ -0,0 +1,49 @@ +using Muster.Cli.Reports; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +public class ReplyRendererTests +{ + private static readonly Verdict NeedsInfo = new( + VerdictKind.NeedsInfo, EngineGap: false, Labels: ["needs-info"], Detail: "no roster found in the report"); + + [Fact] + public void Omits_the_whole_snapshot_section_when_there_is_no_spec_at_all() + { + // F1c: an empty ```yaml block must never be rendered — the sticky comment is posted + // with edit-mode: replace, so an empty snapshot here would destroy any durable snapshot + // a prior evaluation had stored, permanently blocking promotion. + var reply = ReplyRenderer.Render( + NeedsInfo, roster: null, runs: null, specYaml: "", problem: null, expected: null); + + Assert.DoesNotContain("", reply, StringComparison.Ordinal); + Assert.DoesNotContain("```yaml", reply, StringComparison.Ordinal); + Assert.DoesNotContain("
", reply, StringComparison.Ordinal); + } + + [Fact] + public void Renders_a_carried_forward_snapshot_with_a_visible_warning_note() + { + var reply = ReplyRenderer.Render( + NeedsInfo, roster: null, runs: null, specYaml: "id: \"report\"\nsteps: []\n", + problem: null, expected: null, carriedForwardSnapshot: true); + + Assert.Contains( + "snapshot preserved from a previous evaluation", reply, StringComparison.Ordinal); + Assert.Contains("", reply, StringComparison.Ordinal); + Assert.Contains("id: \"report\"", reply, StringComparison.Ordinal); + } + + [Fact] + public void Does_not_render_the_carried_forward_note_for_a_freshly_derived_spec() + { + var confirmed = new Verdict(VerdictKind.Confirmed, EngineGap: false, Labels: ["confirmed"], Detail: null); + var reply = ReplyRenderer.Render( + confirmed, roster: null, runs: null, specYaml: "id: \"report\"\nsteps: []\n", + problem: null, expected: null, carriedForwardSnapshot: false); + + Assert.DoesNotContain("snapshot preserved from a previous evaluation", reply, StringComparison.Ordinal); + Assert.Contains("", reply, StringComparison.Ordinal); + } +} diff --git a/tests/Muster.Cli.Tests/Reports/SnapshotExtractorTests.cs b/tests/Muster.Cli.Tests/Reports/SnapshotExtractorTests.cs new file mode 100644 index 0000000..6f456cb --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/SnapshotExtractorTests.cs @@ -0,0 +1,56 @@ +using Muster.Cli.Converters; +using Muster.Cli.Reports; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +public class SnapshotExtractorTests +{ + [Fact] + public void ExtractFromBody_pulls_the_snapshot_out_of_a_real_rendered_reply() + { + // Same scaffolding style as PromoteChainTests: render a REAL reply via + // ReplyRenderer.Render (rather than a hand-written comment body) and extract straight + // from that single body — this is exactly what ReportCommand does with a + // --previous-reply file's raw text, which is never wrapped in the `gh api` comments + // JSON array that ExtractLatest expects. + var verdict = new Verdict(VerdictKind.Confirmed, EngineGap: false, Labels: ["confirmed"], Detail: null); + var specYaml = """ + id: "report" + setup: + dataSource: "local:." + steps: + - expectedState: {} + """; + var rendered = ReplyRenderer.Render( + verdict, roster: null, runs: null, specYaml: specYaml, problem: "Cost seems off", expected: "20 points"); + + var extracted = SnapshotExtractor.ExtractFromBody(rendered); + + Assert.NotNull(extracted); + Assert.Contains("id: \"report\"", extracted, StringComparison.Ordinal); + } + + [Fact] + public void Extracts_yaml_from_latest_snapshot_comment() + { + var comments = """ + [ + {"body": "just a human comment"}, + {"body": "\nold reply\n
Executable spec (snapshot)\n\n\n```yaml\nid: \"old\"\n```\n\n
"}, + {"body": "\nnew reply\n
Executable spec (snapshot)\n\n\n```yaml\nid: \"new\"\nsteps: []\n```\n\n
"} + ] + """; + var yaml = SnapshotExtractor.ExtractLatest(comments); + Assert.NotNull(yaml); + Assert.Contains("id: \"new\"", yaml, StringComparison.Ordinal); + Assert.DoesNotContain("old", yaml, StringComparison.Ordinal); + } + + [Theory] + [InlineData("[]")] + [InlineData("""[{"body": "no snapshot here"}]""")] + [InlineData("not json")] + public void Missing_snapshot_returns_null(string comments) => + Assert.Null(SnapshotExtractor.ExtractLatest(comments)); +} diff --git a/tests/Muster.Cli.Tests/Reports/SpecRePinnerTests.cs b/tests/Muster.Cli.Tests/Reports/SpecRePinnerTests.cs new file mode 100644 index 0000000..a22248c --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/SpecRePinnerTests.cs @@ -0,0 +1,97 @@ +using BattleScribeSpec; +using BattleScribeSpec.Roster; +using Muster.Cli.Reports; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +public class SpecRePinnerTests +{ + // Regression for the Critical finding: RewriteWithPins used to locate the observed-values + // marker with an UNANCHORED text.IndexOf, then truncate from that line to EOF. A step whose + // quoted field (e.g. customName, attacker-controlled via a roster name) merely CONTAINS the + // marker text mid-line would trip the same truncation and silently drop every subsequent + // step — data corruption, not just a cosmetic bug. The marker is only legitimate as a + // whole, line-anchored `# observed values from the report` comment line; a quoted scalar + // embedding that text mid-line must never be mistaken for it. + [Fact] + public void RewriteWithPins_does_not_truncate_on_marker_text_embedded_in_a_quoted_field() + { + var specYaml = """ + id: "report" + category: "report" + description: "Converted from bug report roster 'Test'" + setup: + dataSource: "github:muster-e2e/test-data@main" + + steps: + - action: addForce + id: "force-1" + forceEntryId: "fe-army" + catalogueId: "cat-test" + - action: selectEntry + id: "sel-1" + forceId: "${{ steps.force-1.forceId }}" + entryId: "se-unit" + - action: setCustomization + forceId: "${{ steps.force-1.forceId }}" + selectionId: "${{ steps.sel-1.selectionId }}" + customName: "PWNED # observed values from the report — attacker-controlled name" + - action: selectEntry + id: "sel-2" + forceId: "${{ steps.force-1.forceId }}" + entryId: "se-unit" + """; + + var costs = new List { new("pts", "ct-pts", 20m) }; + + var rewritten = SpecRePinner.RewriteWithPins(specYaml, "new-id", costs, "wham"); + + // The subsequent step (sel-2), which sits AFTER the malicious quoted field on the page, + // must survive the rewrite — the marker-in-a-quoted-field must not be treated as the + // real marker and truncate everything from that line onward. + Assert.Contains("\"sel-2\"", rewritten, StringComparison.Ordinal); + Assert.Contains("PWNED", rewritten, StringComparison.Ordinal); + + // Must still be a loadable spec — the rewrite is validated by SpecRePinner itself, but + // re-assert here as the regression signal (a truncated document usually fails to load, + // or loads with steps silently missing). + var loaded = SpecLoader.LoadFromYaml(rewritten); + Assert.Equal("new-id", loaded.Id); + Assert.Contains(loaded.Steps, s => s.Id == "sel-2"); + Assert.Single(loaded.Steps, s => s.ExpectedState is not null); + } + + [Fact] + public void RewriteWithPins_drops_a_genuine_line_anchored_marker_block() + { + var specYaml = """ + id: "report" + category: "report" + setup: + dataSource: "github:muster-e2e/test-data@main" + + steps: + - action: addForce + id: "force-1" + forceEntryId: "fe-army" + catalogueId: "cat-test" + + # observed values from the report — PASS means the reported state reproduces + - expectedState: + costs: + - typeId: "ct-pts" + value: 999 + """; + + var costs = new List { new("pts", "ct-pts", 20m) }; + + var rewritten = SpecRePinner.RewriteWithPins(specYaml, "new-id", costs, "wham"); + + Assert.DoesNotContain("999", rewritten, StringComparison.Ordinal); + var loaded = SpecLoader.LoadFromYaml(rewritten); + var pinStep = Assert.Single(loaded.Steps, s => s.ExpectedState is not null); + var pts = Assert.Single(pinStep.ExpectedState!.Costs!, c => c.TypeId == "ct-pts"); + Assert.Equal(20m, pts.Value); + } +} diff --git a/tests/Muster.Cli.Tests/Reports/VerdictTests.cs b/tests/Muster.Cli.Tests/Reports/VerdictTests.cs new file mode 100644 index 0000000..1bd1531 --- /dev/null +++ b/tests/Muster.Cli.Tests/Reports/VerdictTests.cs @@ -0,0 +1,92 @@ +using Muster.Cli.Converters; +using Muster.Cli.Reports; +using Muster.Cli.Reporting; +using Xunit; + +namespace Muster.Cli.Tests.Reports; + +public class VerdictTests +{ + private static ReplayRoster Roster(params string[] unmapped) => new( + "r", "gs", [new("pts", "pt-1", 100m)], [], + [new("fe-1", "cat-1", [], [])], [.. unmapped]); + + private static MultiRunReport Runs(params (string Engine, bool Passed, bool Inconclusive)[] engines) => new( + Governing: engines.Length > 0 ? engines[0].Engine : null, + Unavailable: [], + Runs: [.. engines.Select(e => RunReport.Create(e.Engine, "data", + [new FixtureResult("spec", "p", e.Passed, e.Passed ? [] : ["Step 3: cost pts: expected 100 but got 95"], 1, e.Inconclusive)]))]); + + [Fact] + public void Conversion_error_is_needs_info() + { + var v = VerdictMapper.Map(null, "list not found", null); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + Assert.Contains("needs-info", v.Labels); + } + + [Fact] + public void Unmapped_nodes_force_needs_info() + { + var v = VerdictMapper.Map(Roster("selection 'X' missing option_id"), null, Runs(("wham", true, false))); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + } + + [Fact] + public void Governing_pass_is_confirmed() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("wham", true, false))); + Assert.Equal(VerdictKind.Confirmed, v.Kind); + Assert.Contains("confirmed", v.Labels); + Assert.False(v.EngineGap); + } + + [Fact] + public void Governing_assertion_failure_is_not_reproducible() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("wham", false, false))); + Assert.Equal(VerdictKind.NotReproducible, v.Kind); + Assert.Contains("expected 100 but got 95", v.Detail, StringComparison.Ordinal); + } + + [Fact] + public void Replay_crash_is_needs_info_not_notreproducible() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("wham", false, true))); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + } + + [Fact] + public void Engine_disagreement_raises_engine_gap() + { + var v = VerdictMapper.Map(Roster(), null, Runs(("newrecruit", true, false), ("wham", false, false))); + Assert.Equal(VerdictKind.Confirmed, v.Kind); // newrecruit governs + Assert.True(v.EngineGap); + Assert.Contains("engine-gap", v.Labels); + } + + [Fact] + public void No_engines_ran_is_inconclusive() + { + var runs = new MultiRunReport(null, ["newrecruit"], []); + var v = VerdictMapper.Map(Roster(), null, runs); + Assert.Equal(VerdictKind.Inconclusive, v.Kind); + } + + [Fact] + public void Inline_spec_with_no_roster_and_no_error_skips_roster_checks() + { + // inlineSpec=true: the pasted YAML has no ReplayRoster of its own, so a null roster + // and no conversion error must NOT be treated as needs-info — the verdict comes + // purely from the run results. + var v = VerdictMapper.Map(null, null, Runs(("wham", true, false)), inlineSpec: true); + Assert.Equal(VerdictKind.Confirmed, v.Kind); + } + + [Fact] + public void Inline_spec_conversion_error_is_still_needs_info() + { + var v = VerdictMapper.Map(null, "the inline spec is not valid: bad yaml", null, inlineSpec: true); + Assert.Equal(VerdictKind.NeedsInfo, v.Kind); + } +} diff --git a/tests/Muster.Cli.Tests/TestCommandTests.cs b/tests/Muster.Cli.Tests/TestCommandTests.cs index 35e87ba..a9eebec 100644 --- a/tests/Muster.Cli.Tests/TestCommandTests.cs +++ b/tests/Muster.Cli.Tests/TestCommandTests.cs @@ -32,14 +32,20 @@ public async Task Test_command_passes_on_green_fixture() Assert.Contains("[PASS] unit-costs-20", stdout, StringComparison.Ordinal); Assert.Contains("Results: 1 passed, 0 failed, 0 inconclusive", stdout, StringComparison.Ordinal); - // --report writes a JSON file with matching counts. + // --report writes a JSON file with the MultiRunReport envelope: {governing, unavailable, runs:[…]}. Assert.True(File.Exists(reportPath)); using var doc = JsonDocument.Parse(await File.ReadAllTextAsync(reportPath, TestContext.Current.CancellationToken)); var root = doc.RootElement; - Assert.Equal(1, root.GetProperty("total").GetInt32()); - Assert.Equal(1, root.GetProperty("passed").GetInt32()); - Assert.Equal(0, root.GetProperty("failed").GetInt32()); - Assert.Equal(0, root.GetProperty("inconclusive").GetInt32()); + Assert.Equal("wham", root.GetProperty("governing").GetString()); + Assert.Equal(0, root.GetProperty("unavailable").GetArrayLength()); + var runs = root.GetProperty("runs"); + Assert.Equal(1, runs.GetArrayLength()); + var run = runs[0]; + Assert.Equal("wham", run.GetProperty("engine").GetString()); + Assert.Equal(1, run.GetProperty("total").GetInt32()); + Assert.Equal(1, run.GetProperty("passed").GetInt32()); + Assert.Equal(0, run.GetProperty("failed").GetInt32()); + Assert.Equal(0, run.GetProperty("inconclusive").GetInt32()); } [Fact] diff --git a/tests/Muster.Cli.Tests/TestData/nr-list-war-horde.json b/tests/Muster.Cli.Tests/TestData/nr-list-war-horde.json new file mode 100644 index 0000000..eb924d1 --- /dev/null +++ b/tests/Muster.Cli.Tests/TestData/nr-list-war-horde.json @@ -0,0 +1,1030 @@ +{ + "_id": "6a54b93964df8beb241c49db", + "name": "war horde", + "id_book": 3898668199, + "id_system": 827374861, + "list_key": "6a54b93964df8beb241c49dc", + "nrversion": 2, + "version": 14, + "totalCost": 950, + "bsid_book": "a55f-b7b3-6c65-a05f", + "bsid_system": "sys-352e-adc2-7639-d610", + "client_key": "6a1972274c64093cbaeafdab", + "army": { + "name": "war horde", + "option_id": "(roster)", + "options": [ + { + "name": "Xenos - Orks", + "option_id": "a55f-b7b3-6c65-a05f", + "options": [ + { + "name": "Army Roster", + "option_id": "bb9d-299a-ed60-2d8a", + "options": [ + { + "name": "Configuration", + "option_id": "4ac9-fd30-1e3d-b249", + "options": [ + { + "name": "Battle Size", + "option_id": "564e-fbc6-5266-3ea4", + "options": [ + { + "name": "Battle Size", + "option_id": "b960-4789-a3a6-59cb", + "options": [ + { + "name": "Incursion (1000 Point limit)", + "option_id": "d62d-db22-4893-4bc0", + "options": [], + "uid": "fnhkg1n", + "amount": 1 + } + ], + "uid": "fnnjb89" + } + ], + "link_id": "7380-3e40-6ed6-b7cc", + "uid": "9a7vk5a", + "amount": 1 + }, + { + "name": "Detachment", + "option_id": "73c2-d0ed-3c7e-7e17", + "options": [ + { + "name": "Detachment", + "option_id": "8b85-ad3b-752f-6502", + "options": [ + { + "name": "War Horde", + "option_id": "496c-ddb9-3e71-e0b3", + "options": [], + "uid": "fox0gpb", + "amount": 1 + } + ], + "link_id": "4137-fb48-edc1-4ce3", + "uid": "fnp3lu" + } + ], + "link_id": "dc5b-07db-415b-a1da", + "uid": "936l4k", + "amount": 1 + }, + { + "name": "Force Disposition", + "option_id": "2f69-9148-45b4-86a8", + "options": [ + { + "name": "Force Disposition", + "option_id": "eece-6d68-6710-7a7f", + "options": [ + { + "name": "Take and Hold", + "option_id": "a271-bfab-1b97-c55d", + "options": [], + "uid": "fqoxyck", + "amount": 1 + } + ], + "link_id": "9c70-af87-0c32-afcf", + "uid": "fqmaovl" + } + ], + "link_id": "8bc8-6bfe-78bd-2480", + "uid": "8zjwiq", + "amount": 1 + }, + { + "name": "Show/Hide Options", + "option_id": "e8ef-836a-a9d1-901d", + "options": [ + { + "name": "Crucible Characters are visible", + "option_id": "57cc-822a-1d43-be52", + "options": [], + "link_id": "f578-4362-f482-9f54", + "uid": "9g0hdds", + "amount": 1 + }, + { + "name": "Legends are visible", + "option_id": "9ed-cbf4-bfe5-90bf", + "options": [], + "link_id": "892f-57ca-d650-7199", + "uid": "9fiutqc", + "amount": 1 + }, + { + "name": "Unaligned Forces are visible", + "option_id": "2973-ea51-7f8d-5403", + "options": [], + "link_id": "985-e753-2e94-859", + "uid": "9fin0g", + "amount": 1 + }, + { + "name": "Unaligned Fortifications are visible", + "option_id": "e916-2cf4-a49d-b8c4", + "options": [], + "link_id": "4d37-22c-a45c-64f8", + "uid": "9feb0im", + "amount": 1 + } + ], + "link_id": "a491-128a-9dec-6933", + "uid": "9e04wfs", + "amount": 1 + } + ], + "link_id": "d5de-ee57-ad4b-e4b7", + "uid": "6gfhag" + }, + { + "name": "Character", + "option_id": "9cfd-1c32-585f-7d5c", + "options": [ + { + "name": "Beastboss", + "option_id": "5a5d-c5a4-39b8-4a3f", + "options": [ + { + "name": "Wargear", + "option_id": "c2a6-b43b-7306-d984", + "options": [ + { + "name": "Beast Snagga klaw", + "option_id": "8e20-4ae8-9125-42a6", + "options": [], + "uid": "xrajhgp", + "amount": 1 + }, + { + "name": "Beastchoppa", + "option_id": "6084-9fb5-76d2-8487", + "options": [], + "link_id": "4ecb-344f-f74f-2c0f", + "uid": "xrfrvi", + "amount": 1 + }, + { + "name": "Shoota", + "option_id": "64da-fae7-507d-8b18", + "options": [], + "uid": "xrqww34", + "amount": 1 + } + ], + "uid": "xrdo7ar" + } + ], + "link_id": "2f80-26a6-9486-b348", + "uid": "xle1dcl", + "amount": 1 + }, + { + "name": "Painboy", + "option_id": "7669-1442-2c0b-e0c9", + "options": [ + { + "name": "Wargear", + "option_id": "3ce2-52ad-7bf5-1235", + "options": [ + { + "name": "\u2019Urty syringe", + "option_id": "14e-7f95-331e-8d6c", + "options": [], + "link_id": "73f5-86ce-612d-5a0f", + "uid": "5s89ebg", + "amount": 1 + }, + { + "name": "Power klaw", + "option_id": "ee65-6385-44ba-3758", + "options": [], + "link_id": "bc10-845c-3ec2-1091", + "uid": "5s16r2", + "amount": 1 + } + ], + "uid": "5rh8u38" + } + ], + "link_id": "5177-1de8-ebd5-e09d", + "uid": "5m68uel", + "amount": 1 + }, + { + "name": "Warboss", + "option_id": "2efc-c4b3-2ecc-5a37", + "options": [ + { + "name": "Wargear", + "option_id": "eec2-6ed3-27cd-f0cd", + "options": [ + { + "name": "Kombi-weapon", + "option_id": "ac6f-9310-e9ac-678", + "options": [], + "link_id": "183b-3f43-10fc-ca2d", + "uid": "tl9crgn", + "amount": 1 + }, + { + "name": "Twin slugga", + "option_id": "f8a0-f279-3ee8-2eda", + "options": [], + "uid": "tkxy87e", + "amount": 1 + }, + { + "name": "Big Choppa", + "option_id": "5eb8-cd57-5dee-4eec", + "options": [ + { + "name": "Big choppa", + "option_id": "614f-3d6b-88aa-9e15", + "options": [], + "link_id": "446d-ce2c-1afc-c5fb", + "uid": "tlw2dsx", + "amount": 1 + } + ], + "uid": "tlifz8" + } + ], + "uid": "tk2rcyr" + } + ], + "link_id": "9189-6889-492c-a43e", + "uid": "tidc2oj", + "amount": 1 + } + ], + "link_id": "c932-1176-dc9-b390", + "uid": "p34o4sm" + }, + { + "name": "Battleline", + "option_id": "e338-111e-d0c6-b687", + "options": [ + { + "name": "Beast Snagga Boyz", + "option_id": "b7c3-d00e-daf2-76fb", + "options": [ + { + "name": "Beast Snagga Nob", + "option_id": "cb1a-f005-c69e-728c", + "options": [ + { + "name": "Beast Snagga Nob", + "option_id": "8495-e7e7-6d41-2f57", + "options": [ + { + "name": "Power snappa", + "option_id": "d76a-13f1-cdb0-c3bc", + "options": [], + "uid": "ug9da2p", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "4baa-e929-86b-c558", + "uid": "uhhnbti", + "amount": 1 + } + ], + "uid": "uehsoi6", + "amount": 1 + } + ], + "uid": "ue4s8z" + }, + { + "name": "9-19 Beast Snagga Boyz", + "option_id": "3dc8-103b-baac-3620", + "options": [ + { + "name": "Beast Snagga Boy", + "option_id": "8462-1df3-6409-5087", + "options": [ + { + "name": "Choppa", + "option_id": "4e6a-844c-26bf-2b5d", + "options": [], + "link_id": "d068-604a-e364-a570", + "uid": "uitia4", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "e71d-1472-87d-b4fa", + "options": [], + "link_id": "8143-737a-961-273d", + "uid": "ui36q0eh", + "amount": 1 + } + ], + "uid": "ueta39q", + "amount": 9 + } + ], + "uid": "ue0f48" + } + ], + "link_id": "fd9d-87f3-1895-3470", + "uid": "ucmaul", + "amount": 1 + }, + { + "name": "Boyz", + "option_id": "e3b1-1240-2476-cd86", + "options": [ + { + "name": "Boss Nob", + "option_id": "0fe0-6121-efa6-8057", + "options": [ + { + "name": "Boss Nob", + "option_id": "4660-3a1-f9f9-1a2b", + "options": [ + { + "name": "Big Choppa and Slugga", + "option_id": "592c-f0be-c73a-5a3d", + "options": [ + { + "name": "Big choppa and slugga", + "option_id": "9d77-2604-83bf-8c51", + "options": [ + { + "name": "Big choppa", + "option_id": "29e9-cd9b-66bd-fe60", + "options": [], + "link_id": "31a6-5d49-a9e0-e638", + "uid": "bircele", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "d2a1-2ce7-d467-a117", + "uid": "bi05vot", + "amount": 1 + } + ], + "uid": "bhrrlw", + "amount": 1 + } + ], + "uid": "bh97wd" + } + ], + "uid": "bfc9m9o", + "amount": 1 + } + ], + "uid": "bfcaoi" + }, + { + "name": "9-19 Boyz", + "option_id": "3d4f-216d-1b28-ee93", + "options": [ + { + "name": "Boy w/ Slugga and choppa", + "option_id": "7629-a9f5-e881-6ff9", + "options": [ + { + "name": "Choppa", + "option_id": "e542-78f9-46d8-6c33", + "options": [], + "link_id": "6e34-5e0b-f267-40f5", + "uid": "bkkehya", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "e71d-1472-87d-b4fa", + "options": [], + "link_id": "f678-5f8a-c349-16d2", + "uid": "bkdnvai", + "amount": 1 + } + ], + "uid": "begc49j", + "amount": 17 + }, + { + "name": "Special Weapons", + "option_id": "80bc-1cce-f5af-1a8c", + "options": [ + { + "name": "Boy w/ Big shoota and close combat weapon", + "option_id": "6fc2-677d-fc5e-9329", + "options": [ + { + "name": "Big shoota", + "option_id": "1fa0-d99a-154f-ffd8", + "options": [], + "link_id": "f7e0-a138-dd37-e42f", + "uid": "cabrxz36", + "amount": 1 + }, + { + "name": "Close combat weapon", + "option_id": "0534-7359-3392-9d1b", + "options": [], + "link_id": "2dd8-5af8-ea1a-09e4", + "uid": "cbrpl1b", + "amount": 1 + } + ], + "uid": "be273j", + "amount": 1 + }, + { + "name": "Boy w/ Rokkit launcha and close combat weapon", + "option_id": "ca6b-d75f-c372-64ab", + "options": [ + { + "name": "Close combat weapon", + "option_id": "0534-7359-3392-9d1b", + "options": [], + "link_id": "8b22-e55e-1345-e99b", + "uid": "cchwnsu", + "amount": 1 + }, + { + "name": "Rokkit launcha", + "option_id": "80bc-5c89-4125-2253", + "options": [], + "link_id": "6db5-ec11-70f7-7c0e", + "uid": "ccowler", + "amount": 1 + } + ], + "uid": "bevkrbd", + "amount": 1 + } + ], + "uid": "beor1j6" + } + ], + "uid": "bd4flg" + } + ], + "link_id": "187-93d6-1ac7-dbde", + "uid": "bclrr0l", + "amount": 1 + }, + { + "name": "Boyz", + "option_id": "e3b1-1240-2476-cd86", + "options": [ + { + "name": "Boss Nob", + "option_id": "0fe0-6121-efa6-8057", + "options": [ + { + "name": "Boss Nob", + "option_id": "4660-3a1-f9f9-1a2b", + "options": [ + { + "name": "Big Choppa and Slugga", + "option_id": "592c-f0be-c73a-5a3d", + "options": [ + { + "name": "Big choppa and slugga", + "option_id": "9d77-2604-83bf-8c51", + "options": [ + { + "name": "Big choppa", + "option_id": "29e9-cd9b-66bd-fe60", + "options": [], + "link_id": "31a6-5d49-a9e0-e638", + "uid": "km40td", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "d2a1-2ce7-d467-a117", + "uid": "kmqvj29", + "amount": 1 + } + ], + "uid": "klu1cbh", + "amount": 1 + } + ], + "uid": "kld3xvn" + } + ], + "uid": "kjnjcq9", + "amount": 1 + } + ], + "uid": "kj3u07c" + }, + { + "name": "9-19 Boyz", + "option_id": "3d4f-216d-1b28-ee93", + "options": [ + { + "name": "Boy w/ Slugga and choppa", + "option_id": "7629-a9f5-e881-6ff9", + "options": [ + { + "name": "Choppa", + "option_id": "e542-78f9-46d8-6c33", + "options": [], + "link_id": "6e34-5e0b-f267-40f5", + "uid": "kojcm5w", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "e71d-1472-87d-b4fa", + "options": [], + "link_id": "f678-5f8a-c349-16d2", + "uid": "kocy7dl", + "amount": 1 + } + ], + "uid": "kiujmkk", + "amount": 8 + }, + { + "name": "Special Weapons", + "option_id": "80bc-1cce-f5af-1a8c", + "options": [ + { + "name": "Boy w/ Rokkit launcha and close combat weapon", + "option_id": "ca6b-d75f-c372-64ab", + "options": [ + { + "name": "Close combat weapon", + "option_id": "0534-7359-3392-9d1b", + "options": [], + "link_id": "8b22-e55e-1345-e99b", + "uid": "wewqt98", + "amount": 1 + }, + { + "name": "Rokkit launcha", + "option_id": "80bc-5c89-4125-2253", + "options": [], + "link_id": "6db5-ec11-70f7-7c0e", + "uid": "wedlay8", + "amount": 1 + } + ], + "uid": "kjwvkfg", + "amount": 1 + } + ], + "uid": "kjvt5hd" + } + ], + "uid": "kip9q3" + } + ], + "link_id": "187-93d6-1ac7-dbde", + "uid": "kh0gzl", + "amount": 1 + }, + { + "name": "Boyz", + "option_id": "e3b1-1240-2476-cd86", + "options": [ + { + "name": "Boss Nob", + "option_id": "0fe0-6121-efa6-8057", + "options": [ + { + "name": "Boss Nob", + "option_id": "4660-3a1-f9f9-1a2b", + "options": [ + { + "name": "Big Choppa and Slugga", + "option_id": "592c-f0be-c73a-5a3d", + "options": [ + { + "name": "Big choppa and slugga", + "option_id": "9d77-2604-83bf-8c51", + "options": [ + { + "name": "Big choppa", + "option_id": "29e9-cd9b-66bd-fe60", + "options": [], + "link_id": "31a6-5d49-a9e0-e638", + "uid": "sx8enuo", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "d2a1-2ce7-d467-a117", + "uid": "sxcqzvn", + "amount": 1 + } + ], + "uid": "swxibm", + "amount": 1 + } + ], + "uid": "swcxemt" + } + ], + "uid": "sv4anuw", + "amount": 1 + } + ], + "uid": "svly7gl" + }, + { + "name": "9-19 Boyz", + "option_id": "3d4f-216d-1b28-ee93", + "options": [ + { + "name": "Boy w/ Slugga and choppa", + "option_id": "7629-a9f5-e881-6ff9", + "options": [ + { + "name": "Choppa", + "option_id": "e542-78f9-46d8-6c33", + "options": [], + "link_id": "6e34-5e0b-f267-40f5", + "uid": "sz9bz8r", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "e71d-1472-87d-b4fa", + "options": [], + "link_id": "f678-5f8a-c349-16d2", + "uid": "szfsf2t", + "amount": 1 + } + ], + "uid": "suervy", + "amount": 9 + } + ], + "uid": "str3dcr" + } + ], + "link_id": "187-93d6-1ac7-dbde", + "uid": "stptdvo", + "amount": 1 + } + ], + "link_id": "b3ba-1e1a-a92d-60d2", + "uid": "p3ubj7" + }, + { + "name": "Infantry", + "option_id": "cf47-a0d7-7207-29dc", + "options": [ + { + "name": "Gretchin", + "option_id": "de8f-24f9-c543-92b7", + "options": [ + { + "name": "Unit Composition", + "option_id": "816f-2188-7c92-19d", + "options": [ + { + "name": "1 Runtherd and 10 Gretchin", + "option_id": "d682-30b9-55d8-8445", + "options": [ + { + "name": "Gretchin", + "option_id": "43dd-f974-3fc8-8abc", + "options": [ + { + "name": "Close combat weapon", + "option_id": "8df-71e4-ccf5-99e8", + "options": [], + "link_id": "ddd5-9e62-3a60-66d6", + "uid": "ooq9w6n", + "amount": 1 + }, + { + "name": "Grot blasta", + "option_id": "140d-7163-8a11-81c5", + "options": [], + "link_id": "7908-6506-f654-5aeb", + "uid": "oogi5h", + "amount": 1 + } + ], + "uid": "on8p0un", + "amount": 10 + }, + { + "name": "Runtherd", + "option_id": "552d-6c0d-80e8-ac32", + "options": [ + { + "name": "Grot-smacka", + "option_id": "66fc-b1fe-a869-b8ef", + "options": [], + "uid": "ophyhal", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "da75-95a9-9a18-d139", + "uid": "opvbd0b", + "amount": 1 + } + ], + "link_id": "afc3-494a-6ef9-edb5", + "uid": "on82ge9", + "amount": 1 + } + ], + "uid": "oj1g35p", + "amount": 1 + } + ], + "uid": "oj8ggff" + } + ], + "link_id": "8e01-6763-69a8-a25a", + "uid": "od85cyo", + "amount": 1 + } + ], + "link_id": "6d77-b79d-3ccb-6bf6", + "uid": "p4q2c2g" + }, + { + "name": "Mounted", + "option_id": "14a0-40c9-2748-ae6e", + "options": [ + { + "name": "Squighog Boyz", + "option_id": "6a83-8e1f-63ef-50c3", + "options": [ + { + "name": "Unit Composition", + "option_id": "6e05-cf12-2839-61d9", + "options": [ + { + "name": "1 Nob on Smasha Squig and 3 Squighog Boyz", + "option_id": "1fba-450a-537b-9db1", + "options": [ + { + "name": "Nob on Smasha Squig", + "option_id": "99c8-e37d-ae6d-a8ba", + "options": [ + { + "name": "Big choppa", + "option_id": "aecd-6ba5-5558-fee2", + "options": [], + "uid": "o314baa", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "cc94-f22e-3842-dc29", + "uid": "o3lhzp", + "amount": 1 + }, + { + "name": "Squig jaws", + "option_id": "fa30-43a8-57eb-a82e", + "options": [], + "link_id": "1733-d8a5-cb18-5d82", + "uid": "o3q36f", + "amount": 1 + } + ], + "link_id": "966c-c85d-3a7-d23e", + "uid": "o3r2oal", + "amount": 1 + }, + { + "name": "Squighog Boy", + "option_id": "954d-5938-64d8-597c", + "options": [ + { + "name": "Saddlegit weapons", + "option_id": "8a8f-5ceb-8a0a-6c6c", + "options": [], + "link_id": "d17f-c006-a95f-fe30", + "uid": "o55a4gj", + "amount": 1 + }, + { + "name": "Squig jaws", + "option_id": "fa30-43a8-57eb-a82e", + "options": [], + "link_id": "231a-5a1b-25cf-28ed", + "uid": "o5q36c", + "amount": 1 + }, + { + "name": "Stikka", + "option_id": "5efb-f61f-fdb6-a7d7", + "options": [], + "uid": "o5bm8v", + "amount": 1 + } + ], + "link_id": "19bb-cf43-2176-b8f", + "uid": "o3jd05", + "amount": 3 + } + ], + "uid": "o2a13qi", + "amount": 1 + } + ], + "uid": "o1grtq" + } + ], + "link_id": "6c12-6df4-9d3a-2cc", + "uid": "o09zwqr", + "amount": 1 + }, + { + "name": "Squighog Boyz", + "option_id": "6a83-8e1f-63ef-50c3", + "options": [ + { + "name": "Unit Composition", + "option_id": "6e05-cf12-2839-61d9", + "options": [ + { + "name": "1 Nob on Smasha Squig and 3 Squighog Boyz", + "option_id": "1fba-450a-537b-9db1", + "options": [ + { + "name": "Nob on Smasha Squig", + "option_id": "99c8-e37d-ae6d-a8ba", + "options": [ + { + "name": "Big choppa", + "option_id": "aecd-6ba5-5558-fee2", + "options": [], + "uid": "2gdj2nd", + "amount": 1 + }, + { + "name": "Slugga", + "option_id": "752e-cd9b-4721-f0b0", + "options": [], + "link_id": "cc94-f22e-3842-dc29", + "uid": "2h0klxl", + "amount": 1 + }, + { + "name": "Squig jaws", + "option_id": "fa30-43a8-57eb-a82e", + "options": [], + "link_id": "1733-d8a5-cb18-5d82", + "uid": "2g7ykch", + "amount": 1 + } + ], + "link_id": "966c-c85d-3a7-d23e", + "uid": "2gy6it", + "amount": 1 + }, + { + "name": "Squighog Boy", + "option_id": "954d-5938-64d8-597c", + "options": [ + { + "name": "Saddlegit weapons", + "option_id": "8a8f-5ceb-8a0a-6c6c", + "options": [], + "link_id": "d17f-c006-a95f-fe30", + "uid": "2ibnbz2", + "amount": 1 + }, + { + "name": "Squig jaws", + "option_id": "fa30-43a8-57eb-a82e", + "options": [], + "link_id": "231a-5a1b-25cf-28ed", + "uid": "2ihgr8v", + "amount": 1 + }, + { + "name": "Stikka", + "option_id": "5efb-f61f-fdb6-a7d7", + "options": [], + "uid": "2iymga6", + "amount": 1 + } + ], + "link_id": "19bb-cf43-2176-b8f", + "uid": "2gs414j", + "amount": 3 + } + ], + "uid": "2ffhvp", + "amount": 1 + } + ], + "uid": "2e52gk6" + } + ], + "link_id": "6c12-6df4-9d3a-2cc", + "uid": "2dsptx", + "amount": 1 + } + ], + "link_id": "6503-057c-cb62-badb", + "uid": "p42okffg" + } + ], + "uid": "69l0epb", + "catalogue_id": "a55f-b7b3-6c65-a05f" + } + ], + "uid": "67l5xzc5" + } + ], + "uid": "65te37j", + "customName": "war horde", + "maxCosts": [ + { + "hidden": true, + "typeId": "51b2-306e-1021-d207", + "name": "pts" + }, + { + "hidden": true, + "typeId": "b03b-c239-15a5-da55", + "name": "Crusade Points" + }, + { + "hidden": true, + "typeId": "75bb-ded1-c86d-bdf0", + "name": "Crusade: Battle Honours" + }, + { + "hidden": true, + "typeId": "a623-fe74-1d33-cddf", + "name": "Crusade: Experience" + }, + { + "hidden": true, + "typeId": "716d-91b7-d55a-1022", + "name": "Crusade: Weapon Modifications" + }, + { + "hidden": true, + "typeId": "ac6b-ced3-9b5e-9a6e", + "name": "Blackstone Fragments" + }, + { + "hidden": true, + "typeId": "82ae-1066-5107-6ae0", + "name": "Detachment Points" + }, + { + "hidden": true, + "typeId": "f759-1bc4-cb3a-f0d2", + "name": "Enhancements" + } + ] + }, + "totalCosts": [ + { + "name": "pts", + "value": 950, + "typeId": "51b2-306e-1021-d207" + } + ], + "date_mod": "2026-07-13T10:14:35.616Z", + "metadata": { + "builder_settings": {}, + "custom_categories": [], + "custom_view": false, + "play_mode": false, + "leaders": { + "9a7vk5a": [], + "bclrr0l": [] + } + }, + "books_revision": [ + "Xenos - Orks: 2", + "Unaligned Forces: 1" + ] +} \ No newline at end of file diff --git a/tests/Muster.Cli.Tests/TestPaths.cs b/tests/Muster.Cli.Tests/TestPaths.cs new file mode 100644 index 0000000..2a4c655 --- /dev/null +++ b/tests/Muster.Cli.Tests/TestPaths.cs @@ -0,0 +1,33 @@ +namespace Muster.Cli.Tests; + +/// +/// Shared filesystem-path lookups for e2e tests that need to spawn built artifacts +/// (e.g. the out-of-proc Muster.TestAdapter) as subprocesses. +/// +internal static class TestPaths +{ + /// + /// Path to the built Muster.TestAdapter.dll, resolved by walking up from + /// to the repo root (identified by + /// Muster.slnx) and back down into artifacts/bin. + /// + public static string TestAdapterDll + { + get + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "Muster.slnx"))) + { + dir = dir.Parent; + } + + if (dir is null) + { + throw new InvalidOperationException( + $"could not locate Muster.slnx above {AppContext.BaseDirectory}"); + } + + return Path.Combine(dir.FullName, "artifacts", "bin", "Muster.TestAdapter", "debug", "Muster.TestAdapter.dll"); + } + } +} diff --git a/tests/Muster.Cli.Tests/TestRepoFactory.cs b/tests/Muster.Cli.Tests/TestRepoFactory.cs index 3e79ad3..a6cfbb9 100644 --- a/tests/Muster.Cli.Tests/TestRepoFactory.cs +++ b/tests/Muster.Cli.Tests/TestRepoFactory.cs @@ -62,7 +62,12 @@ public static (string DataDir, string FixturesDir) CreateTestRepo() entryId: se-unit - expectedState: costs: - - typeId: ct-pts + # Matched by cost *name* (not typeId): wham's roster-level cost name comes + # from the game system costType's `name` attribute ("pts"), while its typeId + # ("ct-pts") is data-repo-specific and not something a data-agnostic engine + # (e.g. the fake test adapter) can be expected to echo back. Matching by name + # lets this fixture evaluate identically across engines. + - name: pts value: 20 """); diff --git a/tests/Muster.TestAdapter/Muster.TestAdapter.csproj b/tests/Muster.TestAdapter/Muster.TestAdapter.csproj new file mode 100644 index 0000000..b48aad2 --- /dev/null +++ b/tests/Muster.TestAdapter/Muster.TestAdapter.csproj @@ -0,0 +1,11 @@ + + + Exe + net10.0 + false + + + + + + diff --git a/tests/Muster.TestAdapter/Program.cs b/tests/Muster.TestAdapter/Program.cs new file mode 100644 index 0000000..af32bb1 --- /dev/null +++ b/tests/Muster.TestAdapter/Program.cs @@ -0,0 +1,12 @@ +// NDJSON adapter host serving FakeRosterEngine — lets tests exercise the +// out-of-proc engine path (AdapterProcess + JsonProtocolEngine) hermetically. +// FAKE_PTS env var sets the per-selection pts value (default 20). +using BattleScribeSpec.Protocol; +using Muster.Cli.Tests.Fakes; + +var pts = decimal.TryParse(Environment.GetEnvironmentVariable("FAKE_PTS"), out var v) ? v : 20m; + +await AdapterHandler.RunAsync( + engineFactory: () => new FakeRosterEngine(pts), + input: Console.In, + output: Console.Out);