Automate Artifact Registry image cleanup + purge dead image repos

.github/scripts/cleanup_app_engine_versions.sh

@@ -18,6 +18,11 @@
 #      staging, and legacy timestamp-named versions), to stay under App Engine's
 #      210-versions-per-service limit. Prod is kept deeper because only prod
 #      versions are meaningful rollback targets.
+#   3. DELETEs the Artifact Registry image for each deleted version so the
+#      `gae-flexible` repo stays in lockstep with the retained versions (App
+#      Engine Flex builds one image per version — `<service>.<version-id>` — and
+#      nothing else prunes them, so the repo grows without bound). Best-effort
+#      per image; set CLEANUP_APP_ENGINE_IMAGES=0 to skip.
 #
 # Stopping (not deleting) preserves rollback: a stopped version can be brought
 # back with `gcloud app versions start <v>` then `gcloud app services set-traffic
@@ -37,6 +42,13 @@ KEEP_STOPPED_PROD="${KEEP_STOPPED_PROD:-10}"
 KEEP_STOPPED_STAGING="${KEEP_STOPPED_STAGING:-3}"
 DRY_RUN="${DRY_RUN:-0}"
 
+# Delete each deleted version's App Engine Flex image from Artifact Registry so
+# the image repo does not grow one image per version forever.
+CLEANUP_APP_ENGINE_IMAGES="${CLEANUP_APP_ENGINE_IMAGES:-1}"
+AR_LOCATION="${AR_LOCATION:-us-central1}"
+AR_IMAGE_PROJECT="${AR_IMAGE_PROJECT:-${APP_ENGINE_PROJECT:-policyengine-api}}"
+AR_IMAGE_REPO="${AR_IMAGE_REPO:-gae-flexible}"
+
 common_args=("--service=${APP_ENGINE_SERVICE}")
 if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
   common_args+=("--project=${APP_ENGINE_PROJECT}")
@@ -156,3 +168,23 @@ if [[ "${#to_delete[@]}" -gt 0 ]]; then
     gcloud app versions delete "${to_delete[@]}" "${common_args[@]}" --quiet
   fi
 fi
+
+# Delete each deleted version's App Engine Flex image so the image repo stays in
+# lockstep with the retained versions. Best-effort per image: a version may
+# predate this repo (older images live in the legacy us.gcr.io) or its image may
+# already be gone, so individual misses are ignored.
+if [[ "${CLEANUP_APP_ENGINE_IMAGES}" == "1" && "${#to_delete[@]}" -gt 0 ]]; then
+  image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_IMAGE_PROJECT}/${AR_IMAGE_REPO}"
+  img_done=0
+  for v in "${to_delete[@]}"; do
+    [[ -z "${v}" ]] && continue
+    image="${image_repo}/${APP_ENGINE_SERVICE}.${v}"
+    if [[ "${DRY_RUN}" == "1" ]]; then
+      echo "[dry-run] would delete image ${image}"
+    elif gcloud artifacts docker images delete "${image}" --delete-tags --quiet \
+      >/dev/null 2>&1; then
+      img_done=$((img_done + 1))
+    fi
+  done
+  [[ "${DRY_RUN}" == "1" ]] || echo "Deleted ${img_done}/${#to_delete[@]} gae-flexible image(s)."
+fi

.github/scripts/cleanup_cloud_run_images.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# Keep only the most recent Cloud Run images in Artifact Registry.
+#
+# The Cloud Run deploy pushes a new image on every release and nothing prunes
+# them, so the repo grows without bound. This keeps the newest KEEP image
+# versions (by createTime) and deletes the rest.
+#
+# Note: deleting an image that an older Cloud Run *revision* still references
+# stops that revision from starting again. Cloud Run here is the non-primary
+# migration candidate, so losing rollback to revisions older than KEEP deploys
+# is acceptable; KEEP=15 leaves a generous window.
+#
+# Images are sorted in-script (by createTime) rather than trusting gcloud's
+# Artifact Registry `--sort-by`, which has proven unreliable. Set DRY_RUN=1 to
+# print the plan without changing anything. Written for bash 3.2.
+#
+# The target repo (region/project/repository) comes from cloud_run_env.sh — the
+# same config the deploy scripts source — so this can never prune a different
+# repo than the pipeline actually pushes to.
+
+set -euo pipefail
+
+source .github/scripts/cloud_run_env.sh
+cloud_run_set_defaults
+
+KEEP="${KEEP:-15}"
+DRY_RUN="${DRY_RUN:-0}"
+AR_LOCATION="${AR_LOCATION:-${CLOUD_RUN_REGION}}"
+AR_PROJECT="${AR_PROJECT:-${CLOUD_RUN_PROJECT}}"
+AR_REPO="${AR_REPO:-${CLOUD_RUN_ARTIFACT_REPOSITORY}}"
+
+image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_PROJECT}/${AR_REPO}"
+
+# Deletable refs (<image>@<digest>), newest first — sorted here by createTime
+# (ISO timestamps sort chronologically), one row per image version.
+refs=()
+while IFS='|' read -r _ctime pkg ver; do
+  [[ -n "${ver}" ]] && refs+=("${pkg}@${ver}")
+done < <(
+  gcloud artifacts docker images list "${image_repo}" \
+    --format="value[separator='|'](createTime,package,version)" \
+    | sort -r
+)
+
+total="${#refs[@]}"
+to_delete=()
+if [[ "${total}" -gt "${KEEP}" ]]; then
+  to_delete=("${refs[@]:${KEEP}}")
+fi
+
+echo "Repo:      ${image_repo}"
+echo "Images:    ${total} (keeping newest ${KEEP})"
+echo "Deleting:  ${#to_delete[@]}"
+
+deleted=0
+for ref in "${to_delete[@]:-}"; do
+  [[ -z "${ref}" ]] && continue
+  if [[ "${DRY_RUN}" == "1" ]]; then
+    echo "[dry-run] would delete ${ref}"
+  elif gcloud artifacts docker images delete "${ref}" --delete-tags --quiet \
+    >/dev/null 2>&1; then
+    deleted=$((deleted + 1))
+  fi
+done
+[[ "${DRY_RUN}" == "1" ]] || echo "Deleted ${deleted}/${#to_delete[@]} image(s)."

.github/workflows/push.yml

@@ -599,6 +599,33 @@ jobs:
       - name: Wait for Cloud Run production service health
         run: bash .github/scripts/health_check.sh "${{ steps.cloud_run_service_url.outputs.url }}/readiness-check"
 
+  cleanup-cloud-run-images:
+    name: Clean up old Cloud Run images
+    runs-on: ubuntu-latest
+    needs: deploy-cloud-run-candidate
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    environment: production
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: GCP authentication
+        uses: "google-github-actions/auth@v2"
+        with:
+          workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
+      - name: Set up GCloud
+        uses: "google-github-actions/setup-gcloud@v2"
+      # The Cloud Run deploy pushes a new image every release and nothing prunes
+      # them, so the repo grows without bound. Keep the 15 most recent image
+      # versions (default in the script) and delete older ones.
+      - name: Prune old Cloud Run images
+        run: bash .github/scripts/cleanup_cloud_run_images.sh
+
   docker:
     name: Docker
     runs-on: ubuntu-latest

changelog.d/artifact-registry-image-cleanup.changed.md

@@ -0,0 +1 @@
+Prune Artifact Registry images automatically so the image repos stop growing without bound. After each deploy the pipeline keeps the 15 most recent Cloud Run images and deletes each removed App Engine version's Flex (`gae-flexible`) image. This bounds the ongoing growth of Artifact Registry storage (the repos had reached ~628 GiB / ~$45/mo).

tests/unit/test_app_engine_cleanup_scripts.py

@@ -18,6 +18,7 @@
 REPO = Path(__file__).resolve().parents[2]
 CLEANUP_SCRIPT = ".github/scripts/cleanup_app_engine_versions.sh"
 STOP_SCRIPT = ".github/scripts/stop_app_engine_version.sh"
+CR_IMAGES_SCRIPT = ".github/scripts/cleanup_cloud_run_images.sh"
 
 # A stub `gcloud` that answers `app versions list` from MOCK_* env vars (each a
 # newline-joined, newest-first id list) and appends any stop/delete calls to
@@ -28,6 +29,8 @@
   case "$a" in --filter=*) filter="${a#--filter=}";; esac
 done
 case "$*" in
+  *"artifacts docker images list"*) [ -n "${MOCK_AR_IMAGES:-}" ] && printf '%s\\n' "$MOCK_AR_IMAGES";;
+  *"artifacts docker images delete"*) echo "IMG_DELETE $*" >> "$GCLOUD_CALLS";;
   *"versions list"*)
     case "$filter" in
       *"traffic_split>0"*) [ -n "${MOCK_TRAFFIC:-}" ] && printf '%s\\n' "$MOCK_TRAFFIC";;
@@ -48,6 +51,7 @@ def _run(
     serving: list[str] | None = None,
     traffic: list[str] | None = None,
     stopped: list[str] | None = None,
+    ar_images: list[str] | None = None,
     extra_env: dict[str, str] | None = None,
 ) -> tuple[subprocess.CompletedProcess[str], Path]:
     stub_dir = tmp_path / "bin"
@@ -68,6 +72,7 @@ def _run(
         "MOCK_SERVING": "\n".join(serving or []),
         "MOCK_TRAFFIC": "\n".join(traffic or []),
         "MOCK_STOPPED": "\n".join(stopped or []),
+        "MOCK_AR_IMAGES": "\n".join(ar_images or []),
     }
     env.update(extra_env or {})
 
@@ -94,13 +99,99 @@ def _list_after(label: str, stdout: str) -> list[str]:
 
 
 def test_cleanup_scripts_are_shell_syntax_valid():
-    for script in (CLEANUP_SCRIPT, STOP_SCRIPT):
+    for script in (CLEANUP_SCRIPT, STOP_SCRIPT, CR_IMAGES_SCRIPT):
         result = subprocess.run(
             ["bash", "-n", script], cwd=REPO, capture_output=True, text=True
         )
         assert result.returncode == 0, f"{script}: {result.stderr}"
 
 
+# --- cloud run image cleanup (keep 15) -----------------------------------
+
+_CR_PKG = (
+    "us-central1-docker.pkg.dev/policyengine-api/policyengine-api/policyengine-api"
+)
+
+
+def _ar_rows(digests: list[str], package: str = _CR_PKG) -> list[str]:
+    """`createTime|package|version` rows, oldest first (day 01 = oldest). The
+    script sorts by createTime descending itself, so order here is irrelevant."""
+    return [
+        f"2026-01-{i + 1:02d}T00:00:00|{package}|sha256:{d}"
+        for i, d in enumerate(digests)
+    ]
+
+
+def test_cloud_run_images_keeps_newest_15(tmp_path):
+    digests = [f"d{i:02d}" for i in range(18)]  # d00 oldest ... d17 newest
+    result, calls = _run(CR_IMAGES_SCRIPT, tmp_path, ar_images=_ar_rows(digests))
+    assert result.returncode == 0, result.stderr
+    assert "Images:    18 (keeping newest 15)" in result.stdout
+    assert "Deleting:  3" in result.stdout
+    # The 3 oldest digests are the ones deleted; the newest is kept.
+    for old in ("sha256:d00", "sha256:d01", "sha256:d02"):
+        assert f"would delete {_CR_PKG}@{old}" in result.stdout
+    assert "sha256:d17" not in result.stdout.split("Deleting:")[1]
+    assert calls.read_text() == ""  # DRY_RUN performs no mutations
+
+
+def test_cloud_run_images_no_delete_when_within_keep(tmp_path):
+    result, _ = _run(
+        CR_IMAGES_SCRIPT, tmp_path, ar_images=_ar_rows([f"d{i}" for i in range(10)])
+    )
+    assert result.returncode == 0, result.stderr
+    assert "Deleting:  0" in result.stdout
+    assert "would delete" not in result.stdout
+
+
+def test_cloud_run_images_respects_keep_override(tmp_path):
+    result, _ = _run(
+        CR_IMAGES_SCRIPT,
+        tmp_path,
+        ar_images=_ar_rows([f"d{i}" for i in range(10)]),
+        extra_env={"KEEP": "5"},
+    )
+    assert "keeping newest 5" in result.stdout
+    assert "Deleting:  5" in result.stdout
+
+
+# --- app engine cleanup also prunes each deleted version's image ----------
+
+
+def test_app_engine_cleanup_deletes_gae_flexible_image_per_deleted_version(tmp_path):
+    result, _ = _run(
+        CLEANUP_SCRIPT,
+        tmp_path,
+        serving=["prod-110", "prod-108"],
+        traffic=["prod-110"],
+        stopped=[f"prod-{n}" for n in range(106, 84, -2)]  # 11 prod -> 1 deleted
+        + ["staging-90", "staging-88", "staging-86", "staging-84"],  # 1 deleted
+    )
+    assert result.returncode == 0, result.stderr
+    deleted = _list_after("Deleting", result.stdout)
+    assert deleted, "expected some versions to be deleted"
+    # AR_IMAGE_PROJECT defaults to APP_ENGINE_PROJECT, which _run sets.
+    repo = "us-central1-docker.pkg.dev/test-project/gae-flexible/default."
+    # Every deleted version has a matching gae-flexible image deletion line.
+    for v in deleted:
+        assert f"would delete image {repo}{v}" in result.stdout
+    # No image deletion for a kept (live/warm) version.
+    assert f"{repo}prod-110" not in result.stdout
+
+
+def test_app_engine_cleanup_image_pruning_can_be_disabled(tmp_path):
+    result, _ = _run(
+        CLEANUP_SCRIPT,
+        tmp_path,
+        serving=["prod-110", "prod-108"],
+        traffic=["prod-110"],
+        stopped=[f"prod-{n}" for n in range(106, 84, -2)],
+        extra_env={"CLEANUP_APP_ENGINE_IMAGES": "0"},
+    )
+    assert result.returncode == 0, result.stderr
+    assert "would delete image" not in result.stdout
+
+
 # --- cleanup: stop selection ---------------------------------------------