From 28be00abfcf1707d6155322bd9cd4ee3270d49a2 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 7 Jul 2026 22:25:35 +0200 Subject: [PATCH 1/3] Automate Artifact Registry image cleanup + purge dead repos The image repos accumulated ~628 GiB that nothing prunes (~$45/mo storage). Ongoing (in the deploy pipeline): - cleanup_app_engine_versions.sh: when it deletes an App Engine version, also delete that version's gae-flexible Flex image, so the repo stays in lockstep with the retained versions. Version-aware because a blind keep-N would delete a kept stopped-prod version's image (staging/prod images interleave). - cleanup_cloud_run_images.sh + a cleanup-cloud-run-images job: keep the 15 most recent Cloud Run images, delete older. Sorts in-script by createTime since gcloud's Artifact Registry --sort-by proved unreliable. One-time: - purge_stale_artifact_registry.sh (interactive): after a safety pre-check, delete the dead legacy repos (us.gcr.io ~540 GiB, gcr.io, cloud-run-source- deploy), prune gae-flexible orphans, and prune Cloud Run to 15. Tests extended with a stubbed gcloud artifacts. Cuts AR storage to a few $/mo. Co-Authored-By: Claude Opus 4.8 --- .../scripts/cleanup_app_engine_versions.sh | 32 ++++++ .github/scripts/cleanup_cloud_run_images.sh | 59 ++++++++++ .../scripts/purge_stale_artifact_registry.sh | 106 ++++++++++++++++++ .github/workflows/push.yml | 27 +++++ ...artifact-registry-image-cleanup.changed.md | 1 + tests/unit/test_app_engine_cleanup_scripts.py | 93 ++++++++++++++- 6 files changed, 317 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/cleanup_cloud_run_images.sh create mode 100755 .github/scripts/purge_stale_artifact_registry.sh create mode 100644 changelog.d/artifact-registry-image-cleanup.changed.md diff --git a/.github/scripts/cleanup_app_engine_versions.sh b/.github/scripts/cleanup_app_engine_versions.sh index af77a7978..5b41a613b 100755 --- a/.github/scripts/cleanup_app_engine_versions.sh +++ b/.github/scripts/cleanup_app_engine_versions.sh @@ -18,6 +18,11 @@ # staging, and legacy timestamp-named versions), to stay under App Engine's # 210-versions-per-service limit. Prod is kept deeper because only prod # versions are meaningful rollback targets. +# 3. DELETEs the Artifact Registry image for each deleted version so the +# `gae-flexible` repo stays in lockstep with the retained versions (App +# Engine Flex builds one image per version — `.` — and +# nothing else prunes them, so the repo grows without bound). Best-effort +# per image; set CLEANUP_APP_ENGINE_IMAGES=0 to skip. # # Stopping (not deleting) preserves rollback: a stopped version can be brought # back with `gcloud app versions start ` then `gcloud app services set-traffic @@ -37,6 +42,13 @@ KEEP_STOPPED_PROD="${KEEP_STOPPED_PROD:-10}" KEEP_STOPPED_STAGING="${KEEP_STOPPED_STAGING:-3}" DRY_RUN="${DRY_RUN:-0}" +# Delete each deleted version's App Engine Flex image from Artifact Registry so +# the image repo does not grow one image per version forever. +CLEANUP_APP_ENGINE_IMAGES="${CLEANUP_APP_ENGINE_IMAGES:-1}" +AR_LOCATION="${AR_LOCATION:-us-central1}" +AR_IMAGE_PROJECT="${AR_IMAGE_PROJECT:-${APP_ENGINE_PROJECT:-policyengine-api}}" +AR_IMAGE_REPO="${AR_IMAGE_REPO:-gae-flexible}" + common_args=("--service=${APP_ENGINE_SERVICE}") if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then common_args+=("--project=${APP_ENGINE_PROJECT}") @@ -156,3 +168,23 @@ if [[ "${#to_delete[@]}" -gt 0 ]]; then gcloud app versions delete "${to_delete[@]}" "${common_args[@]}" --quiet fi fi + +# Delete each deleted version's App Engine Flex image so the image repo stays in +# lockstep with the retained versions. Best-effort per image: a version may +# predate this repo (older images live in the legacy us.gcr.io) or its image may +# already be gone, so individual misses are ignored. +if [[ "${CLEANUP_APP_ENGINE_IMAGES}" == "1" && "${#to_delete[@]}" -gt 0 ]]; then + image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_IMAGE_PROJECT}/${AR_IMAGE_REPO}" + img_done=0 + for v in "${to_delete[@]}"; do + [[ -z "${v}" ]] && continue + image="${image_repo}/${APP_ENGINE_SERVICE}.${v}" + if [[ "${DRY_RUN}" == "1" ]]; then + echo "[dry-run] would delete image ${image}" + elif gcloud artifacts docker images delete "${image}" --delete-tags --quiet \ + >/dev/null 2>&1; then + img_done=$((img_done + 1)) + fi + done + [[ "${DRY_RUN}" == "1" ]] || echo "Deleted ${img_done}/${#to_delete[@]} gae-flexible image(s)." +fi diff --git a/.github/scripts/cleanup_cloud_run_images.sh b/.github/scripts/cleanup_cloud_run_images.sh new file mode 100755 index 000000000..d7e215a50 --- /dev/null +++ b/.github/scripts/cleanup_cloud_run_images.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Keep only the most recent Cloud Run images in Artifact Registry. +# +# The Cloud Run deploy pushes a new image on every release and nothing prunes +# them, so the repo grows without bound. This keeps the newest KEEP image +# versions (by createTime) and deletes the rest. +# +# Note: deleting an image that an older Cloud Run *revision* still references +# stops that revision from starting again. Cloud Run here is the non-primary +# migration candidate, so losing rollback to revisions older than KEEP deploys +# is acceptable; KEEP=15 leaves a generous window. +# +# Images are sorted in-script (by createTime) rather than trusting gcloud's +# Artifact Registry `--sort-by`, which has proven unreliable. Set DRY_RUN=1 to +# print the plan without changing anything. Written for bash 3.2. + +set -euo pipefail + +KEEP="${KEEP:-15}" +DRY_RUN="${DRY_RUN:-0}" +AR_LOCATION="${AR_LOCATION:-us-central1}" +AR_PROJECT="${AR_PROJECT:-${CLOUD_RUN_PROJECT:-policyengine-api}}" +AR_REPO="${AR_REPO:-${CLOUD_RUN_ARTIFACT_REPOSITORY:-policyengine-api}}" + +image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_PROJECT}/${AR_REPO}" + +# Deletable refs (@), newest first — sorted here by createTime +# (ISO timestamps sort chronologically), one row per image version. +refs=() +while IFS='|' read -r _ctime pkg ver; do + [[ -n "${ver}" ]] && refs+=("${pkg}@${ver}") +done < <( + gcloud artifacts docker images list "${image_repo}" \ + --format="value[separator='|'](createTime,package,version)" \ + | sort -r +) + +total="${#refs[@]}" +to_delete=() +if [[ "${total}" -gt "${KEEP}" ]]; then + to_delete=("${refs[@]:${KEEP}}") +fi + +echo "Repo: ${image_repo}" +echo "Images: ${total} (keeping newest ${KEEP})" +echo "Deleting: ${#to_delete[@]}" + +deleted=0 +for ref in "${to_delete[@]:-}"; do + [[ -z "${ref}" ]] && continue + if [[ "${DRY_RUN}" == "1" ]]; then + echo "[dry-run] would delete ${ref}" + elif gcloud artifacts docker images delete "${ref}" --delete-tags --quiet \ + >/dev/null 2>&1; then + deleted=$((deleted + 1)) + fi +done +[[ "${DRY_RUN}" == "1" ]] || echo "Deleted ${deleted}/${#to_delete[@]} image(s)." diff --git a/.github/scripts/purge_stale_artifact_registry.sh b/.github/scripts/purge_stale_artifact_registry.sh new file mode 100755 index 000000000..c86ac5048 --- /dev/null +++ b/.github/scripts/purge_stale_artifact_registry.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# One-time purge of stale Artifact Registry storage in policyengine-api. +# +# The image repos accumulated ~628 GiB that is never cleaned up. This removes +# the dead legacy repos and prunes the live repos to their retention windows. +# INTERACTIVE: prints what it will do and asks before each destructive step. +# Safe to re-run (idempotent). +# +# 1. Pre-check: abort if any App Engine version's image lives in a repo we +# are about to delete. +# 2. Delete dead repos wholesale: us.gcr.io, gcr.io, cloud-run-source-deploy. +# 3. Prune gae-flexible: delete images whose App Engine version no longer +# exists (orphans left by already-deleted versions). +# 4. Prune the Cloud Run repo to the newest 15 (cleanup_cloud_run_images.sh). +# +# Usage: PROJECT=policyengine-api bash .github/scripts/purge_stale_artifact_registry.sh +# +# Written for bash 3.2. + +set -euo pipefail + +PROJECT="${PROJECT:-policyengine-api}" +LOCATION="${LOCATION:-us-central1}" +SERVICE="${SERVICE:-default}" +GAE_REPO="gae-flexible" +DEAD_REPOS=("us.gcr.io" "gcr.io" "cloud-run-source-deploy") +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +confirm() { + local reply + read -r -p "$1 (y/N) " reply + [[ "${reply}" =~ ^[Yy]$ ]] +} + +echo "=== Purge stale Artifact Registry storage in ${PROJECT} ===" +echo "" + +# --- 1. Pre-check: dead repos are not referenced by any live version --------- +echo "[1/4] Checking no App Engine version references the repos to be deleted..." +version_images="$(gcloud app versions list --project="${PROJECT}" --service="${SERVICE}" \ + --format="value(deployment.container.image)" 2>/dev/null || true)" +for repo in "${DEAD_REPOS[@]}"; do + if printf '%s\n' "${version_images}" | grep -q "/${repo}/"; then + echo "ABORT: an App Engine version still references '${repo}'. Not safe." >&2 + exit 1 + fi +done +echo " OK — no live version references ${DEAD_REPOS[*]}." +echo "" + +# --- 2. Delete the dead repos wholesale -------------------------------------- +for repo in "${DEAD_REPOS[@]}"; do + if ! gcloud artifacts repositories describe "${repo}" --location="${LOCATION}" \ + --project="${PROJECT}" >/dev/null 2>&1; then + echo "[2/4] Repo '${repo}' not found (already gone) — skipping." + continue + fi + size="$(gcloud artifacts repositories describe "${repo}" --location="${LOCATION}" \ + --project="${PROJECT}" --format="value(sizeBytes.size())" 2>/dev/null || echo '?')" + echo "[2/4] Dead repo '${repo}' (${size})." + if confirm " Delete repository '${repo}' entirely?"; then + gcloud artifacts repositories delete "${repo}" --location="${LOCATION}" \ + --project="${PROJECT}" --quiet + echo " deleted ${repo}." + else + echo " skipped ${repo}." + fi +done +echo "" + +# --- 3. Prune gae-flexible orphans (images whose version is gone) ------------ +echo "[3/4] Pruning ${GAE_REPO} images whose App Engine version no longer exists..." +existing_versions="$(gcloud app versions list --project="${PROJECT}" --service="${SERVICE}" \ + --format="value(id)" 2>/dev/null || true)" +orphans=() +while IFS= read -r pkg; do + [[ -z "${pkg}" ]] && continue + base="${pkg##*/}" # "." + [[ "${base}" == "${SERVICE}."* ]] || continue + vid="${base#"${SERVICE}."}" + printf '%s\n' "${existing_versions}" | grep -qxF "${vid}" || orphans+=("${pkg}") +done < <( + gcloud artifacts docker images list \ + "${LOCATION}-docker.pkg.dev/${PROJECT}/${GAE_REPO}" \ + --format="value(package)" 2>/dev/null | sort -u +) +echo " ${#orphans[@]} orphaned image(s) (version deleted)." +if [[ "${#orphans[@]}" -gt 0 ]] && confirm " Delete ${#orphans[@]} orphaned ${GAE_REPO} image(s)?"; then + for pkg in "${orphans[@]}"; do + if gcloud artifacts docker images delete "${pkg}" --delete-tags --quiet >/dev/null 2>&1; then + echo " deleted ${pkg##*/}" + else + echo " skip ${pkg##*/}" + fi + done +fi +echo "" + +# --- 4. Prune the Cloud Run repo to the newest 15 ---------------------------- +echo "[4/4] Pruning the Cloud Run image repo to the newest 15..." +if confirm " Run cleanup_cloud_run_images.sh (keep 15)?"; then + CLOUD_RUN_PROJECT="${PROJECT}" bash "${SCRIPT_DIR}/cleanup_cloud_run_images.sh" +fi +echo "" +echo "Done." diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index facdd54b2..50c8c693f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -599,6 +599,33 @@ jobs: - name: Wait for Cloud Run production service health run: bash .github/scripts/health_check.sh "${{ steps.cloud_run_service_url.outputs.url }}/readiness-check" + cleanup-cloud-run-images: + name: Clean up old Cloud Run images + runs-on: ubuntu-latest + needs: deploy-cloud-run-candidate + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + environment: production + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: GCP authentication + uses: "google-github-actions/auth@v2" + with: + workload_identity_provider: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}" + service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" + - name: Set up GCloud + uses: "google-github-actions/setup-gcloud@v2" + # The Cloud Run deploy pushes a new image every release and nothing prunes + # them, so the repo grows without bound. Keep the 15 most recent image + # versions (default in the script) and delete older ones. + - name: Prune old Cloud Run images + run: bash .github/scripts/cleanup_cloud_run_images.sh + docker: name: Docker runs-on: ubuntu-latest diff --git a/changelog.d/artifact-registry-image-cleanup.changed.md b/changelog.d/artifact-registry-image-cleanup.changed.md new file mode 100644 index 000000000..76d5f7dcb --- /dev/null +++ b/changelog.d/artifact-registry-image-cleanup.changed.md @@ -0,0 +1 @@ +Prune Artifact Registry images automatically so the image repos stop growing without bound. After each deploy the pipeline keeps the 15 most recent Cloud Run images and deletes each removed App Engine version's Flex (`gae-flexible`) image. Adds a one-time `purge_stale_artifact_registry.sh` to delete the dead legacy image repos (`us.gcr.io`, `gcr.io`, `cloud-run-source-deploy`) and orphaned `gae-flexible` images. This cuts Artifact Registry storage (~628 GiB, ~$45/mo) to a few dollars. diff --git a/tests/unit/test_app_engine_cleanup_scripts.py b/tests/unit/test_app_engine_cleanup_scripts.py index cdf101c16..bde03faa4 100644 --- a/tests/unit/test_app_engine_cleanup_scripts.py +++ b/tests/unit/test_app_engine_cleanup_scripts.py @@ -18,6 +18,7 @@ REPO = Path(__file__).resolve().parents[2] CLEANUP_SCRIPT = ".github/scripts/cleanup_app_engine_versions.sh" STOP_SCRIPT = ".github/scripts/stop_app_engine_version.sh" +CR_IMAGES_SCRIPT = ".github/scripts/cleanup_cloud_run_images.sh" # A stub `gcloud` that answers `app versions list` from MOCK_* env vars (each a # newline-joined, newest-first id list) and appends any stop/delete calls to @@ -28,6 +29,8 @@ case "$a" in --filter=*) filter="${a#--filter=}";; esac done case "$*" in + *"artifacts docker images list"*) [ -n "${MOCK_AR_IMAGES:-}" ] && printf '%s\\n' "$MOCK_AR_IMAGES";; + *"artifacts docker images delete"*) echo "IMG_DELETE $*" >> "$GCLOUD_CALLS";; *"versions list"*) case "$filter" in *"traffic_split>0"*) [ -n "${MOCK_TRAFFIC:-}" ] && printf '%s\\n' "$MOCK_TRAFFIC";; @@ -48,6 +51,7 @@ def _run( serving: list[str] | None = None, traffic: list[str] | None = None, stopped: list[str] | None = None, + ar_images: list[str] | None = None, extra_env: dict[str, str] | None = None, ) -> tuple[subprocess.CompletedProcess[str], Path]: stub_dir = tmp_path / "bin" @@ -68,6 +72,7 @@ def _run( "MOCK_SERVING": "\n".join(serving or []), "MOCK_TRAFFIC": "\n".join(traffic or []), "MOCK_STOPPED": "\n".join(stopped or []), + "MOCK_AR_IMAGES": "\n".join(ar_images or []), } env.update(extra_env or {}) @@ -94,13 +99,99 @@ def _list_after(label: str, stdout: str) -> list[str]: def test_cleanup_scripts_are_shell_syntax_valid(): - for script in (CLEANUP_SCRIPT, STOP_SCRIPT): + for script in (CLEANUP_SCRIPT, STOP_SCRIPT, CR_IMAGES_SCRIPT): result = subprocess.run( ["bash", "-n", script], cwd=REPO, capture_output=True, text=True ) assert result.returncode == 0, f"{script}: {result.stderr}" +# --- cloud run image cleanup (keep 15) ----------------------------------- + +_CR_PKG = ( + "us-central1-docker.pkg.dev/policyengine-api/policyengine-api/policyengine-api" +) + + +def _ar_rows(digests: list[str], package: str = _CR_PKG) -> list[str]: + """`createTime|package|version` rows, oldest first (day 01 = oldest). The + script sorts by createTime descending itself, so order here is irrelevant.""" + return [ + f"2026-01-{i + 1:02d}T00:00:00|{package}|sha256:{d}" + for i, d in enumerate(digests) + ] + + +def test_cloud_run_images_keeps_newest_15(tmp_path): + digests = [f"d{i:02d}" for i in range(18)] # d00 oldest ... d17 newest + result, calls = _run(CR_IMAGES_SCRIPT, tmp_path, ar_images=_ar_rows(digests)) + assert result.returncode == 0, result.stderr + assert "Images: 18 (keeping newest 15)" in result.stdout + assert "Deleting: 3" in result.stdout + # The 3 oldest digests are the ones deleted; the newest is kept. + for old in ("sha256:d00", "sha256:d01", "sha256:d02"): + assert f"would delete {_CR_PKG}@{old}" in result.stdout + assert "sha256:d17" not in result.stdout.split("Deleting:")[1] + assert calls.read_text() == "" # DRY_RUN performs no mutations + + +def test_cloud_run_images_no_delete_when_within_keep(tmp_path): + result, _ = _run( + CR_IMAGES_SCRIPT, tmp_path, ar_images=_ar_rows([f"d{i}" for i in range(10)]) + ) + assert result.returncode == 0, result.stderr + assert "Deleting: 0" in result.stdout + assert "would delete" not in result.stdout + + +def test_cloud_run_images_respects_keep_override(tmp_path): + result, _ = _run( + CR_IMAGES_SCRIPT, + tmp_path, + ar_images=_ar_rows([f"d{i}" for i in range(10)]), + extra_env={"KEEP": "5"}, + ) + assert "keeping newest 5" in result.stdout + assert "Deleting: 5" in result.stdout + + +# --- app engine cleanup also prunes each deleted version's image ---------- + + +def test_app_engine_cleanup_deletes_gae_flexible_image_per_deleted_version(tmp_path): + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-110", "prod-108"], + traffic=["prod-110"], + stopped=[f"prod-{n}" for n in range(106, 84, -2)] # 11 prod -> 1 deleted + + ["staging-90", "staging-88", "staging-86", "staging-84"], # 1 deleted + ) + assert result.returncode == 0, result.stderr + deleted = _list_after("Deleting", result.stdout) + assert deleted, "expected some versions to be deleted" + # AR_IMAGE_PROJECT defaults to APP_ENGINE_PROJECT, which _run sets. + repo = "us-central1-docker.pkg.dev/test-project/gae-flexible/default." + # Every deleted version has a matching gae-flexible image deletion line. + for v in deleted: + assert f"would delete image {repo}{v}" in result.stdout + # No image deletion for a kept (live/warm) version. + assert f"{repo}prod-110" not in result.stdout + + +def test_app_engine_cleanup_image_pruning_can_be_disabled(tmp_path): + result, _ = _run( + CLEANUP_SCRIPT, + tmp_path, + serving=["prod-110", "prod-108"], + traffic=["prod-110"], + stopped=[f"prod-{n}" for n in range(106, 84, -2)], + extra_env={"CLEANUP_APP_ENGINE_IMAGES": "0"}, + ) + assert result.returncode == 0, result.stderr + assert "would delete image" not in result.stdout + + # --- cleanup: stop selection --------------------------------------------- From 1395dd4755d3a28fb7e851059bd27e33e7a2383f Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 7 Jul 2026 22:34:23 +0200 Subject: [PATCH 2/3] Drop the one-time purge script from the PR The backlog purge (deleting the dead legacy image repos + gae-flexible orphans) is a one-time manual operation, run out-of-band rather than shipped in the repo. This PR keeps only the ongoing automated pruning. Co-Authored-By: Claude Opus 4.8 --- .../scripts/purge_stale_artifact_registry.sh | 106 ------------------ ...artifact-registry-image-cleanup.changed.md | 2 +- 2 files changed, 1 insertion(+), 107 deletions(-) delete mode 100755 .github/scripts/purge_stale_artifact_registry.sh diff --git a/.github/scripts/purge_stale_artifact_registry.sh b/.github/scripts/purge_stale_artifact_registry.sh deleted file mode 100755 index c86ac5048..000000000 --- a/.github/scripts/purge_stale_artifact_registry.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -# One-time purge of stale Artifact Registry storage in policyengine-api. -# -# The image repos accumulated ~628 GiB that is never cleaned up. This removes -# the dead legacy repos and prunes the live repos to their retention windows. -# INTERACTIVE: prints what it will do and asks before each destructive step. -# Safe to re-run (idempotent). -# -# 1. Pre-check: abort if any App Engine version's image lives in a repo we -# are about to delete. -# 2. Delete dead repos wholesale: us.gcr.io, gcr.io, cloud-run-source-deploy. -# 3. Prune gae-flexible: delete images whose App Engine version no longer -# exists (orphans left by already-deleted versions). -# 4. Prune the Cloud Run repo to the newest 15 (cleanup_cloud_run_images.sh). -# -# Usage: PROJECT=policyengine-api bash .github/scripts/purge_stale_artifact_registry.sh -# -# Written for bash 3.2. - -set -euo pipefail - -PROJECT="${PROJECT:-policyengine-api}" -LOCATION="${LOCATION:-us-central1}" -SERVICE="${SERVICE:-default}" -GAE_REPO="gae-flexible" -DEAD_REPOS=("us.gcr.io" "gcr.io" "cloud-run-source-deploy") -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -confirm() { - local reply - read -r -p "$1 (y/N) " reply - [[ "${reply}" =~ ^[Yy]$ ]] -} - -echo "=== Purge stale Artifact Registry storage in ${PROJECT} ===" -echo "" - -# --- 1. Pre-check: dead repos are not referenced by any live version --------- -echo "[1/4] Checking no App Engine version references the repos to be deleted..." -version_images="$(gcloud app versions list --project="${PROJECT}" --service="${SERVICE}" \ - --format="value(deployment.container.image)" 2>/dev/null || true)" -for repo in "${DEAD_REPOS[@]}"; do - if printf '%s\n' "${version_images}" | grep -q "/${repo}/"; then - echo "ABORT: an App Engine version still references '${repo}'. Not safe." >&2 - exit 1 - fi -done -echo " OK — no live version references ${DEAD_REPOS[*]}." -echo "" - -# --- 2. Delete the dead repos wholesale -------------------------------------- -for repo in "${DEAD_REPOS[@]}"; do - if ! gcloud artifacts repositories describe "${repo}" --location="${LOCATION}" \ - --project="${PROJECT}" >/dev/null 2>&1; then - echo "[2/4] Repo '${repo}' not found (already gone) — skipping." - continue - fi - size="$(gcloud artifacts repositories describe "${repo}" --location="${LOCATION}" \ - --project="${PROJECT}" --format="value(sizeBytes.size())" 2>/dev/null || echo '?')" - echo "[2/4] Dead repo '${repo}' (${size})." - if confirm " Delete repository '${repo}' entirely?"; then - gcloud artifacts repositories delete "${repo}" --location="${LOCATION}" \ - --project="${PROJECT}" --quiet - echo " deleted ${repo}." - else - echo " skipped ${repo}." - fi -done -echo "" - -# --- 3. Prune gae-flexible orphans (images whose version is gone) ------------ -echo "[3/4] Pruning ${GAE_REPO} images whose App Engine version no longer exists..." -existing_versions="$(gcloud app versions list --project="${PROJECT}" --service="${SERVICE}" \ - --format="value(id)" 2>/dev/null || true)" -orphans=() -while IFS= read -r pkg; do - [[ -z "${pkg}" ]] && continue - base="${pkg##*/}" # "." - [[ "${base}" == "${SERVICE}."* ]] || continue - vid="${base#"${SERVICE}."}" - printf '%s\n' "${existing_versions}" | grep -qxF "${vid}" || orphans+=("${pkg}") -done < <( - gcloud artifacts docker images list \ - "${LOCATION}-docker.pkg.dev/${PROJECT}/${GAE_REPO}" \ - --format="value(package)" 2>/dev/null | sort -u -) -echo " ${#orphans[@]} orphaned image(s) (version deleted)." -if [[ "${#orphans[@]}" -gt 0 ]] && confirm " Delete ${#orphans[@]} orphaned ${GAE_REPO} image(s)?"; then - for pkg in "${orphans[@]}"; do - if gcloud artifacts docker images delete "${pkg}" --delete-tags --quiet >/dev/null 2>&1; then - echo " deleted ${pkg##*/}" - else - echo " skip ${pkg##*/}" - fi - done -fi -echo "" - -# --- 4. Prune the Cloud Run repo to the newest 15 ---------------------------- -echo "[4/4] Pruning the Cloud Run image repo to the newest 15..." -if confirm " Run cleanup_cloud_run_images.sh (keep 15)?"; then - CLOUD_RUN_PROJECT="${PROJECT}" bash "${SCRIPT_DIR}/cleanup_cloud_run_images.sh" -fi -echo "" -echo "Done." diff --git a/changelog.d/artifact-registry-image-cleanup.changed.md b/changelog.d/artifact-registry-image-cleanup.changed.md index 76d5f7dcb..8f4198e7e 100644 --- a/changelog.d/artifact-registry-image-cleanup.changed.md +++ b/changelog.d/artifact-registry-image-cleanup.changed.md @@ -1 +1 @@ -Prune Artifact Registry images automatically so the image repos stop growing without bound. After each deploy the pipeline keeps the 15 most recent Cloud Run images and deletes each removed App Engine version's Flex (`gae-flexible`) image. Adds a one-time `purge_stale_artifact_registry.sh` to delete the dead legacy image repos (`us.gcr.io`, `gcr.io`, `cloud-run-source-deploy`) and orphaned `gae-flexible` images. This cuts Artifact Registry storage (~628 GiB, ~$45/mo) to a few dollars. +Prune Artifact Registry images automatically so the image repos stop growing without bound. After each deploy the pipeline keeps the 15 most recent Cloud Run images and deletes each removed App Engine version's Flex (`gae-flexible`) image. This bounds the ongoing growth of Artifact Registry storage (the repos had reached ~628 GiB / ~$45/mo). From 08176b48d1a7b49369e2ba0fff9db81ec86b1b54 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 7 Jul 2026 23:54:21 +0200 Subject: [PATCH 3/3] Source cloud_run_env.sh in Cloud Run image cleanup Derive the prune target's region/project/repository from cloud_run_env.sh (the same config the deploy scripts source) instead of re-declaring the defaults. This removes the config duplication so the cleanup can never drift to a different repo than the pipeline pushes to. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cleanup_cloud_run_images.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/scripts/cleanup_cloud_run_images.sh b/.github/scripts/cleanup_cloud_run_images.sh index d7e215a50..0f9156e8c 100755 --- a/.github/scripts/cleanup_cloud_run_images.sh +++ b/.github/scripts/cleanup_cloud_run_images.sh @@ -14,14 +14,21 @@ # Images are sorted in-script (by createTime) rather than trusting gcloud's # Artifact Registry `--sort-by`, which has proven unreliable. Set DRY_RUN=1 to # print the plan without changing anything. Written for bash 3.2. +# +# The target repo (region/project/repository) comes from cloud_run_env.sh — the +# same config the deploy scripts source — so this can never prune a different +# repo than the pipeline actually pushes to. set -euo pipefail +source .github/scripts/cloud_run_env.sh +cloud_run_set_defaults + KEEP="${KEEP:-15}" DRY_RUN="${DRY_RUN:-0}" -AR_LOCATION="${AR_LOCATION:-us-central1}" -AR_PROJECT="${AR_PROJECT:-${CLOUD_RUN_PROJECT:-policyengine-api}}" -AR_REPO="${AR_REPO:-${CLOUD_RUN_ARTIFACT_REPOSITORY:-policyengine-api}}" +AR_LOCATION="${AR_LOCATION:-${CLOUD_RUN_REGION}}" +AR_PROJECT="${AR_PROJECT:-${CLOUD_RUN_PROJECT}}" +AR_REPO="${AR_REPO:-${CLOUD_RUN_ARTIFACT_REPOSITORY}}" image_repo="${AR_LOCATION}-docker.pkg.dev/${AR_PROJECT}/${AR_REPO}"