From 8a8fc55393ff4fb61fd9dbb179344074a7208537 Mon Sep 17 00:00:00 2001 From: kiranpranay Date: Sat, 11 Jul 2026 00:09:27 +0530 Subject: [PATCH] feat(ledger): persist the ledger on an unprotected ledger-state branch (no main bypass) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit github-actions[bot] cannot be granted main-ruleset bypass (GitHub 422), so instead of committing the ledger to protected main, each run restores it from — and persists it to — a dedicated UNPROTECTED `ledger-state` branch via the Contents API. This delivers cross-run coverage accounting with the default GITHUB_TOKEN, no bypass and no loosening of main protection; main stays purely reviewed case data. - New "Restore ledger from ledger-state" step before extraction; "Persist ledger to ledger-state" step after Validate+PII guard (creates the branch on first run). - Same three-way fence: schema+no-URL validation, pii_guard over data/_meta, and the Contents API writing exactly one path. - The data-review PR always excludes data/_meta (the ledger is never in it). - Drops the LEDGER_TO_MAIN gating (unneeded — this just works). CONTRIBUTING §Operations + LAUNCH_REVIEW K updated. Python code unchanged (the pipeline still reads/writes the working-tree ledger; the workflow moves it to/from the branch). Ledger unit + schema tests still green. --- .github/workflows/scrape.yml | 61 ++++++++++++++++++++++++------------ CONTRIBUTING.md | 43 +++++++++++-------------- LAUNCH_REVIEW.md | 12 ++++--- 3 files changed, 66 insertions(+), 50 deletions(-) diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 62bb216..aa10dd9 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -81,6 +81,25 @@ jobs: python -m pip install --upgrade pip pip install -e '.' + # Restore the processed-document ledger from the dedicated UNPROTECTED + # ledger-state branch, so coverage accounting persists across runs without + # touching protected main. github-actions[bot] can read/write ledger-state. + - name: Restore ledger from ledger-state + if: env.DRY_RUN != 'true' + env: + GH_TOKEN: ${{ secrets.SCRAPE_BOT_TOKEN || secrets.GITHUB_TOKEN }} + run: | + mkdir -p data/_meta + REPO="${{ github.repository }}" + CONTENT="$(gh api "repos/$REPO/contents/data/_meta/processed.json?ref=ledger-state" --jq .content 2>/dev/null || true)" + if [ -n "$CONTENT" ]; then + echo "$CONTENT" | base64 -d > data/_meta/processed.json + echo "restored ledger from ledger-state." + else + echo '{"version":1,"documents":{}}' > data/_meta/processed.json + echo "no ledger-state yet; starting fresh." + fi + # Fetch -> extract -> sanitize -> project -> scope -> dedupe -> validate & # shard -> pii_guard. Writes data/ + logs/run_report.md (the PR body). - name: Run pipeline @@ -114,34 +133,38 @@ jobs: python -m pipeline.validate --all python scripts/pii_guard.py data/ - # Commit ONLY the processed-document ledger to main, as operational metadata, - # SEPARATE from any reviewed data. Opt-in via the LEDGER_TO_MAIN repo variable - # (the push needs a token whose account can bypass main protection — see - # CONTRIBUTING §Operations). Fenced three ways: + # Persist the updated processed-document ledger to the dedicated UNPROTECTED + # ledger-state branch (never merged; operational metadata only). No main-branch + # bypass needed. Fenced three ways: # (a) schema + no-URL validation of the ledger, # (b) pii_guard over data/_meta, - # (c) the Contents API commits exactly ONE path, so it is structurally - # incapable of touching anything but data/_meta/processed.json. - - name: Commit ledger to main - if: success() && env.DRY_RUN != 'true' && vars.LEDGER_TO_MAIN == 'true' + # (c) the Contents API writes exactly ONE path, structurally incapable of + # touching anything but data/_meta/processed.json. + - name: Persist ledger to ledger-state + if: success() && env.DRY_RUN != 'true' env: GH_TOKEN: ${{ secrets.SCRAPE_BOT_TOKEN || secrets.GITHUB_TOKEN }} run: | LEDGER=data/_meta/processed.json - test -f "$LEDGER" || { echo "no ledger to commit"; exit 0; } + test -f "$LEDGER" || { echo "no ledger to persist"; exit 0; } python -m pipeline.validate --ledger "$LEDGER" # (a) schema + no URL python scripts/pii_guard.py data/_meta/ # (b) PII gate REPO="${{ github.repository }}" - REMOTE="$(gh api "repos/$REPO/contents/$LEDGER?ref=main" --jq .content 2>/dev/null | base64 -d 2>/dev/null || true)" + if ! gh api "repos/$REPO/branches/ledger-state" >/dev/null 2>&1; then + MAIN_SHA="$(gh api "repos/$REPO/git/refs/heads/main" --jq .object.sha)" + gh api -X POST "repos/$REPO/git/refs" -f ref="refs/heads/ledger-state" -f sha="$MAIN_SHA" >/dev/null + echo "created ledger-state branch." + fi + REMOTE="$(gh api "repos/$REPO/contents/$LEDGER?ref=ledger-state" --jq .content 2>/dev/null | base64 -d 2>/dev/null || true)" if [ "$REMOTE" = "$(cat "$LEDGER")" ]; then - echo "ledger unchanged on main."; exit 0 + echo "ledger unchanged."; exit 0 fi - SHA="$(gh api "repos/$REPO/contents/$LEDGER?ref=main" --jq .sha 2>/dev/null || true)" + SHA="$(gh api "repos/$REPO/contents/$LEDGER?ref=ledger-state" --jq .sha 2>/dev/null || true)" gh api -X PUT "repos/$REPO/contents/$LEDGER" \ - -f message="chore(meta): ledger update $(date +%F) run ${{ github.run_number }}" \ + -f message="chore(ledger): update $(date +%F) run ${{ github.run_number }}" \ -f content="$(base64 -w0 "$LEDGER")" \ - -f branch=main ${SHA:+-f sha="$SHA"} >/dev/null # (c) one path only - echo "ledger committed to main." + -f branch=ledger-state ${SHA:+-f sha="$SHA"} >/dev/null # (c) one path only + echo "ledger persisted to ledger-state." # STAGED: push data/ to a review branch and open/update the PR. No publish. - name: Stage data for review @@ -154,11 +177,9 @@ jobs: DATE="$(date +%F)"; BRANCH="data-staging" git checkout -B "$BRANCH" git add data/ - # When the ledger is committed to main separately, keep it OUT of the - # review PR so the PR shows only reviewable case data. - if [ "${{ vars.LEDGER_TO_MAIN }}" = "true" ]; then - git reset -q -- data/_meta 2>/dev/null || true - fi + # The ledger lives on the ledger-state branch, never in the review PR — + # keep data/_meta out so the PR shows only reviewable case data. + git reset -q -- data/_meta 2>/dev/null || true if git diff --cached --quiet; then echo "No data changes to stage — nothing to review." exit 0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77236f8..286d64a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,37 +127,30 @@ runs on its own, **data-review PRs merge only through the normal reviewed path** (required checks green + human review). Admin bypass is for emergencies only and every use must be noted in the PR. -### The processed-document ledger on `main` +### The processed-document ledger (the `ledger-state` branch) `data/_meta/processed.json` is the processed-document ledger — **operational metadata only** (a `sha256(url)` per document + outcome + dates; never a URL, -never PII). Committing it to `main` each run lets coverage accounting work across -runs independently of when data-review PRs are merged. - -- **Enable it** by setting the repo **variable** `LEDGER_TO_MAIN=true`. Off by - default, the ledger simply rides along in the review branch instead. Do NOT turn - it on until the push can succeed (below), or every run will fail on the push. -- The push targets protected `main`, so the pushing actor must **bypass main - branch protection**. The fallback `GITHUB_TOKEN` (`github-actions[bot]`) **cannot** - be granted bypass — GitHub rejects adding the built-in Actions integration to a - ruleset's bypass list (HTTP 422). So `SCRAPE_BOT_TOKEN` must be either - **(a) a PAT for an account with the admin role** (already in the ruleset bypass - list via `RepositoryRole` admin), or **(b) a GitHub App installed on this repo**, - whose app id is then added to the ruleset bypass (`main-protection`, id 18731087) - with `bypass_mode: always` — a **push** bypass only; the `pull_request` + - `required_status_checks` rules stay enforced for everyone else. - Record the bypass grant (who/app, why, scope, date) in the PR that flips the var. -- **Fencing (enforced by the workflow, not convention):** (a) the ledger is - validated against `schemas/ledger.schema.json` — `sha256` keys, an outcome enum, - ISO dates, `additionalProperties:false` — and scanned for URL-shaped strings; - (b) `pii_guard` runs over `data/_meta/`; (c) the commit uses the **Contents API**, - which writes exactly one path, so it is structurally incapable of touching - anything but `data/_meta/processed.json`. The reviewed data PR excludes - `data/_meta` when this is on. +never PII). It persists on a dedicated, **unprotected** `ledger-state` branch so +coverage accounting works across runs independently of when data-review PRs merge. +`main` is never touched: `github-actions[bot]` cannot bypass its protection (GitHub +rejects adding the built-in Actions integration to a ruleset bypass — HTTP 422), +and an unprotected side-branch avoids needing to. + +- Each run **restores** the ledger from `ledger-state` before extraction and + **persists** the updated ledger back afterwards, both via the GitHub Contents + API — `github-actions[bot]` / `GITHUB_TOKEN` is enough, no bypass. +- `ledger-state` is **never merged**; it is a storage branch, not code. Do not open + PRs from it. The reviewed data PR excludes `data/_meta`. +- **Fenced three ways** before every persist: (a) validated against + `schemas/ledger.schema.json` — `sha256` keys, an outcome enum, ISO dates, + `additionalProperties:false` — and scanned for URL-shaped strings; (b) `pii_guard` + over `data/_meta/`; (c) the Contents API writes exactly one path, so it is + structurally incapable of touching anything but `data/_meta/processed.json`. - **Widening the launch window resets coverage:** a sexual-offence case outside the current `LAUNCH_STATES`/`LAUNCH_LOOKBACK_DAYS` is settled `out_of_window` (skipped from now on under that fixed window). If you later widen the states or lookback, - **delete `data/_meta/processed.json`** so those documents are re-examined. + **delete the `ledger-state` branch** so those documents are re-examined. --- diff --git a/LAUNCH_REVIEW.md b/LAUNCH_REVIEW.md index 62bd02e..b3d7c26 100644 --- a/LAUNCH_REVIEW.md +++ b/LAUNCH_REVIEW.md @@ -115,13 +115,15 @@ signed off. outcomes (`published`/`out_of_scope`/`not_a_case`) settle; a quarantined or scope-filtered document is never settled and re-surfaces next run. Failing docs retry 3 runs, then park as `failed_permanent` with the URL logged for review. -- **Committed to `main` as operational metadata** (opt-in `LEDGER_TO_MAIN=true`), in - a commit separate from any reviewed data, so cross-run coverage accounting works - regardless of PR-merge timing. Fenced three independent ways: (a) validated +- **Persisted on a dedicated unprotected `ledger-state` branch** (restored before + each run, persisted after), so cross-run coverage accounting works regardless of + PR-merge timing without touching protected `main` (`github-actions[bot]` cannot be + granted ruleset bypass — GitHub 422). Fenced three independent ways: (a) validated against `schemas/ledger.schema.json` (sha256 keys, outcome enum, ISO dates, `additionalProperties:false`) + a no-URL scan; (b) `pii_guard` over `data/_meta/`; - (c) the commit uses the Contents API — structurally one path only. Provider errors - are now captured per run (`error_samples`) so an abort is diagnosable + (c) the Contents API writes exactly one path. A scope-filtered case settles + `out_of_window` (terminal for coverage); quarantined docs never settle. Provider + errors are captured per run (`error_samples`) so an abort is diagnosable (429 RESOURCE_EXHAUSTED vs 503 UNAVAILABLE). - **Evidence:** `test_ledger.py`, `test_orchestrator.py` (settled-skip + quarantine-re-surface), `test_validate.py::test_validate_ledger_*`. Hardened by a