feat(ledger): persist ledger on unprotected ledger-state branch (your chosen path)

.github/workflows/scrape.yml

@@ -81,6 +81,25 @@ jobs:
           python -m pip install --upgrade pip
           pip install -e '.'
 
+      # Restore the processed-document ledger from the dedicated UNPROTECTED
+      # ledger-state branch, so coverage accounting persists across runs without
+      # touching protected main. github-actions[bot] can read/write ledger-state.
+      - name: Restore ledger from ledger-state
+        if: env.DRY_RUN != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.SCRAPE_BOT_TOKEN || secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p data/_meta
+          REPO="${{ github.repository }}"
+          CONTENT="$(gh api "repos/$REPO/contents/data/_meta/processed.json?ref=ledger-state" --jq .content 2>/dev/null || true)"
+          if [ -n "$CONTENT" ]; then
+            echo "$CONTENT" | base64 -d > data/_meta/processed.json
+            echo "restored ledger from ledger-state."
+          else
+            echo '{"version":1,"documents":{}}' > data/_meta/processed.json
+            echo "no ledger-state yet; starting fresh."
+          fi
+
       # Fetch -> extract -> sanitize -> project -> scope -> dedupe -> validate &
       # shard -> pii_guard. Writes data/ + logs/run_report.md (the PR body).
       - name: Run pipeline
@@ -114,34 +133,38 @@ jobs:
           python -m pipeline.validate --all
           python scripts/pii_guard.py data/
 
-      # Commit ONLY the processed-document ledger to main, as operational metadata,
-      # SEPARATE from any reviewed data. Opt-in via the LEDGER_TO_MAIN repo variable
-      # (the push needs a token whose account can bypass main protection — see
-      # CONTRIBUTING §Operations). Fenced three ways:
+      # Persist the updated processed-document ledger to the dedicated UNPROTECTED
+      # ledger-state branch (never merged; operational metadata only). No main-branch
+      # bypass needed. Fenced three ways:
       #   (a) schema + no-URL validation of the ledger,
       #   (b) pii_guard over data/_meta,
-      #   (c) the Contents API commits exactly ONE path, so it is structurally
-      #       incapable of touching anything but data/_meta/processed.json.
-      - name: Commit ledger to main
-        if: success() && env.DRY_RUN != 'true' && vars.LEDGER_TO_MAIN == 'true'
+      #   (c) the Contents API writes exactly ONE path, structurally incapable of
+      #       touching anything but data/_meta/processed.json.
+      - name: Persist ledger to ledger-state
+        if: success() && env.DRY_RUN != 'true'
         env:
           GH_TOKEN: ${{ secrets.SCRAPE_BOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
           LEDGER=data/_meta/processed.json
-          test -f "$LEDGER" || { echo "no ledger to commit"; exit 0; }
+          test -f "$LEDGER" || { echo "no ledger to persist"; exit 0; }
           python -m pipeline.validate --ledger "$LEDGER"     # (a) schema + no URL
           python scripts/pii_guard.py data/_meta/            # (b) PII gate
           REPO="${{ github.repository }}"
-          REMOTE="$(gh api "repos/$REPO/contents/$LEDGER?ref=main" --jq .content 2>/dev/null | base64 -d 2>/dev/null || true)"
+          if ! gh api "repos/$REPO/branches/ledger-state" >/dev/null 2>&1; then
+            MAIN_SHA="$(gh api "repos/$REPO/git/refs/heads/main" --jq .object.sha)"
+            gh api -X POST "repos/$REPO/git/refs" -f ref="refs/heads/ledger-state" -f sha="$MAIN_SHA" >/dev/null
+            echo "created ledger-state branch."
+          fi
+          REMOTE="$(gh api "repos/$REPO/contents/$LEDGER?ref=ledger-state" --jq .content 2>/dev/null | base64 -d 2>/dev/null || true)"
           if [ "$REMOTE" = "$(cat "$LEDGER")" ]; then
-            echo "ledger unchanged on main."; exit 0
+            echo "ledger unchanged."; exit 0
           fi
-          SHA="$(gh api "repos/$REPO/contents/$LEDGER?ref=main" --jq .sha 2>/dev/null || true)"
+          SHA="$(gh api "repos/$REPO/contents/$LEDGER?ref=ledger-state" --jq .sha 2>/dev/null || true)"
           gh api -X PUT "repos/$REPO/contents/$LEDGER" \
-            -f message="chore(meta): ledger update $(date +%F) run ${{ github.run_number }}" \
+            -f message="chore(ledger): update $(date +%F) run ${{ github.run_number }}" \
             -f content="$(base64 -w0 "$LEDGER")" \
-            -f branch=main ${SHA:+-f sha="$SHA"} >/dev/null    # (c) one path only
-          echo "ledger committed to main."
+            -f branch=ledger-state ${SHA:+-f sha="$SHA"} >/dev/null   # (c) one path only
+          echo "ledger persisted to ledger-state."
 
       # STAGED: push data/ to a review branch and open/update the PR. No publish.
       - name: Stage data for review
@@ -154,11 +177,9 @@ jobs:
           DATE="$(date +%F)"; BRANCH="data-staging"
           git checkout -B "$BRANCH"
           git add data/
-          # When the ledger is committed to main separately, keep it OUT of the
-          # review PR so the PR shows only reviewable case data.
-          if [ "${{ vars.LEDGER_TO_MAIN }}" = "true" ]; then
-            git reset -q -- data/_meta 2>/dev/null || true
-          fi
+          # The ledger lives on the ledger-state branch, never in the review PR —
+          # keep data/_meta out so the PR shows only reviewable case data.
+          git reset -q -- data/_meta 2>/dev/null || true
           if git diff --cached --quiet; then
             echo "No data changes to stage — nothing to review."
             exit 0

CONTRIBUTING.md

@@ -127,37 +127,30 @@ runs on its own, **data-review PRs merge only through the normal reviewed path**
 (required checks green + human review). Admin bypass is for emergencies only and
 every use must be noted in the PR.
 
-### The processed-document ledger on `main`
+### The processed-document ledger (the `ledger-state` branch)
 
 `data/_meta/processed.json` is the processed-document ledger — **operational
 metadata only** (a `sha256(url)` per document + outcome + dates; never a URL,
-never PII). Committing it to `main` each run lets coverage accounting work across
-runs independently of when data-review PRs are merged.
-
-- **Enable it** by setting the repo **variable** `LEDGER_TO_MAIN=true`. Off by
-  default, the ledger simply rides along in the review branch instead. Do NOT turn
-  it on until the push can succeed (below), or every run will fail on the push.
-- The push targets protected `main`, so the pushing actor must **bypass main
-  branch protection**. The fallback `GITHUB_TOKEN` (`github-actions[bot]`) **cannot**
-  be granted bypass — GitHub rejects adding the built-in Actions integration to a
-  ruleset's bypass list (HTTP 422). So `SCRAPE_BOT_TOKEN` must be either
-  **(a) a PAT for an account with the admin role** (already in the ruleset bypass
-  list via `RepositoryRole` admin), or **(b) a GitHub App installed on this repo**,
-  whose app id is then added to the ruleset bypass (`main-protection`, id 18731087)
-  with `bypass_mode: always` — a **push** bypass only; the `pull_request` +
-  `required_status_checks` rules stay enforced for everyone else.
-  Record the bypass grant (who/app, why, scope, date) in the PR that flips the var.
-- **Fencing (enforced by the workflow, not convention):** (a) the ledger is
-  validated against `schemas/ledger.schema.json` — `sha256` keys, an outcome enum,
-  ISO dates, `additionalProperties:false` — and scanned for URL-shaped strings;
-  (b) `pii_guard` runs over `data/_meta/`; (c) the commit uses the **Contents API**,
-  which writes exactly one path, so it is structurally incapable of touching
-  anything but `data/_meta/processed.json`. The reviewed data PR excludes
-  `data/_meta` when this is on.
+never PII). It persists on a dedicated, **unprotected** `ledger-state` branch so
+coverage accounting works across runs independently of when data-review PRs merge.
+`main` is never touched: `github-actions[bot]` cannot bypass its protection (GitHub
+rejects adding the built-in Actions integration to a ruleset bypass — HTTP 422),
+and an unprotected side-branch avoids needing to.
+
+- Each run **restores** the ledger from `ledger-state` before extraction and
+  **persists** the updated ledger back afterwards, both via the GitHub Contents
+  API — `github-actions[bot]` / `GITHUB_TOKEN` is enough, no bypass.
+- `ledger-state` is **never merged**; it is a storage branch, not code. Do not open
+  PRs from it. The reviewed data PR excludes `data/_meta`.
+- **Fenced three ways** before every persist: (a) validated against
+  `schemas/ledger.schema.json` — `sha256` keys, an outcome enum, ISO dates,
+  `additionalProperties:false` — and scanned for URL-shaped strings; (b) `pii_guard`
+  over `data/_meta/`; (c) the Contents API writes exactly one path, so it is
+  structurally incapable of touching anything but `data/_meta/processed.json`.
 - **Widening the launch window resets coverage:** a sexual-offence case outside the
   current `LAUNCH_STATES`/`LAUNCH_LOOKBACK_DAYS` is settled `out_of_window` (skipped
   from now on under that fixed window). If you later widen the states or lookback,
-  **delete `data/_meta/processed.json`** so those documents are re-examined.
+  **delete the `ledger-state` branch** so those documents are re-examined.
 
 ---
 

LAUNCH_REVIEW.md

@@ -115,13 +115,15 @@ signed off.
   outcomes (`published`/`out_of_scope`/`not_a_case`) settle; a quarantined or
   scope-filtered document is never settled and re-surfaces next run. Failing docs
   retry 3 runs, then park as `failed_permanent` with the URL logged for review.
-- **Committed to `main` as operational metadata** (opt-in `LEDGER_TO_MAIN=true`), in
-  a commit separate from any reviewed data, so cross-run coverage accounting works
-  regardless of PR-merge timing. Fenced three independent ways: (a) validated
+- **Persisted on a dedicated unprotected `ledger-state` branch** (restored before
+  each run, persisted after), so cross-run coverage accounting works regardless of
+  PR-merge timing without touching protected `main` (`github-actions[bot]` cannot be
+  granted ruleset bypass — GitHub 422). Fenced three independent ways: (a) validated
   against `schemas/ledger.schema.json` (sha256 keys, outcome enum, ISO dates,
   `additionalProperties:false`) + a no-URL scan; (b) `pii_guard` over `data/_meta/`;
-  (c) the commit uses the Contents API — structurally one path only. Provider errors
-  are now captured per run (`error_samples`) so an abort is diagnosable
+  (c) the Contents API writes exactly one path. A scope-filtered case settles
+  `out_of_window` (terminal for coverage); quarantined docs never settle. Provider
+  errors are captured per run (`error_samples`) so an abort is diagnosable
   (429 RESOURCE_EXHAUSTED vs 503 UNAVAILABLE).
 - **Evidence:** `test_ledger.py`, `test_orchestrator.py` (settled-skip +
   quarantine-re-surface), `test_validate.py::test_validate_ledger_*`. Hardened by a