From ce4dd6cfdc630c55d3f8d4c9441a3257c9dc948f Mon Sep 17 00:00:00 2001 From: fredcamaral Date: Thu, 6 Aug 2026 07:25:00 +0000 Subject: [PATCH 1/4] ci(scripts): keep every released image public on Docker Hub :construction_worker: Docker Hub creates a repository on first push with the organization's default visibility, which is private, and the shared release pipeline never changes it. midaz-tracer, midaz-tracer-migrations and midaz-ledger-migrations therefore shipped 4.0.0-beta.x unpullable, so an anonymous helm install of the midaz chart ImagePullBackOffs on every v4 component. Add a dockerhub-visibility workflow that pre-creates the release images as public and flips any that are private, gate the release pipeline on it so a new image never exists private, and make it workflow_dispatch-able to repair the repositories that already went out private. The image list is derived from gitops_yaml_key_mappings, the pipeline's own registry of published images, so it cannot drift from what is actually pushed. Co-authored-by: Codesmith --- .github/workflows/dockerhub-visibility.yml | 29 +++++ .github/workflows/release.yml | 12 ++ scripts/ensure-dockerhub-public.sh | 122 +++++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 .github/workflows/dockerhub-visibility.yml create mode 100755 scripts/ensure-dockerhub-public.sh diff --git a/.github/workflows/dockerhub-visibility.yml b/.github/workflows/dockerhub-visibility.yml new file mode 100644 index 000000000..61c5622d9 --- /dev/null +++ b/.github/workflows/dockerhub-visibility.yml @@ -0,0 +1,29 @@ +name: Docker Hub Visibility + +# Docker Hub creates a repository on first push with the organization's default +# visibility (private), and the shared release pipeline never changes it, so every +# new image ships unpullable until someone flips it by hand. Callable from the +# release pipeline as a pre-push gate, and dispatchable to repair the images that +# already went out private. +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + ensure-public: + name: Ensure released images are public + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Ensure Docker Hub repositories are public + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_IMAGE_PUSH_TOKEN }} + run: ./scripts/ensure-dockerhub-public.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cb1e2a3f..1809048b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,17 @@ permissions: packages: write jobs: + # Gate the build on Docker Hub visibility: repositories are created on first push + # with the organization's private default, so without this an image is only public + # if someone remembered to flip it by hand. Runs before the push so a brand-new + # image never exists private, and derives its image list from the + # gitops_yaml_key_mappings below. + dockerhub-visibility: + uses: ./.github/workflows/dockerhub-visibility.yml + secrets: inherit + pipeline: + needs: dockerhub-visibility uses: LerianStudio/github-actions-shared-workflows/.github/workflows/go-release.yml@v1.46.5 with: enable_changelog: ${{ github.ref == 'refs/heads/main' }} @@ -65,6 +75,8 @@ jobs: # Distinct schema from helm_values_key_mappings: keys carry a .tag suffix and # values are full dotted YAML paths (.image.tag). The *-migrations tags nest # under the parent service block (.ledger/.tracer .migrations.image.tag). + # This is also the image list the dockerhub-visibility job reads, so an image + # added here is kept public automatically; one added without a mapping is not. gitops_yaml_key_mappings: '{"midaz-ledger.tag": ".ledger.image.tag", "midaz-tracer.tag": ".tracer.image.tag", "midaz-ledger-migrations.tag": ".ledger.migrations.image.tag", "midaz-tracer-migrations.tag": ".tracer.migrations.image.tag"}' enable_apidog_e2e: true enable_e2e_tests: true diff --git a/scripts/ensure-dockerhub-public.sh b/scripts/ensure-dockerhub-public.sh new file mode 100755 index 000000000..68ca3420e --- /dev/null +++ b/scripts/ensure-dockerhub-public.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Copyright (c) 2026 Lerian Studio. All rights reserved. +# Use of this source code is governed by the Elastic License 2.0 +# that can be found in the LICENSE file. + +# Ensure every image the release pipeline publishes exists on Docker Hub and is public. +# +# Docker Hub creates a repository on first push using the organization's default +# visibility, which is private. Nothing in the release pipeline flips it, so each new +# image ships unpullable until someone changes it by hand: midaz-tracer, +# midaz-tracer-migrations and midaz-ledger-migrations all answer denied/unauthorized +# today, which breaks any anonymous `helm install` of the midaz chart. +# +# This script is idempotent: it pre-creates missing repositories as public and flips +# existing private ones. Run it before the images are pushed so a first release never +# lands private. +# +# Env: DOCKERHUB_USERNAME, DOCKERHUB_TOKEN (Docker Hub PAT), optional DOCKERHUB_NAMESPACE +# and RELEASE_WORKFLOW. + +set -euo pipefail + +API="https://hub.docker.com/v2" +NAMESPACE="${DOCKERHUB_NAMESPACE:-lerianstudio}" +RELEASE_WORKFLOW="${RELEASE_WORKFLOW:-.github/workflows/release.yml}" +ATTEMPTS=3 + +: "${DOCKERHUB_USERNAME:?DOCKERHUB_USERNAME is required}" +: "${DOCKERHUB_TOKEN:?DOCKERHUB_TOKEN is required}" + +BODY=$(mktemp) +trap 'rm -f "$BODY"' EXIT + +# hub_call [json-payload] +# Writes the response body to $BODY, prints the HTTP status, and retries transport +# errors and 5xx so a flaky Docker Hub cannot fail a release on its own. +hub_call() { + local method="$1" url="$2" payload="${3:-}" status="" attempt=1 + + while [ "$attempt" -le "$ATTEMPTS" ]; do + if [ -n "$payload" ]; then + status=$(curl -sS -o "$BODY" -w '%{http_code}' -X "$method" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "$payload" "$url" .tag" key per image), so deriving the list from it cannot drift from what the +# pipeline actually pushes. It is a single-line, single-quoted JSON scalar; bail out +# rather than guess if that ever stops holding. +mappings=$(sed -n "s/^[[:space:]]*gitops_yaml_key_mappings:[[:space:]]*'\(.*\)'[[:space:]]*$/\1/p" "$RELEASE_WORKFLOW") +images=$(printf '%s' "$mappings" | jq -er 'keys[] | sub("\\.tag$"; "")' 2>/dev/null | sort -u || true) + +if [ -z "$images" ]; then + echo "error: could not read gitops_yaml_key_mappings from ${RELEASE_WORKFLOW}" >&2 + exit 1 +fi + +TOKEN=$(jq -n --arg u "$DOCKERHUB_USERNAME" --arg p "$DOCKERHUB_TOKEN" '{username: $u, password: $p}' | + curl -sS -X POST -H 'Content-Type: application/json' -d @- "${API}/users/login/" | + jq -r '.token // empty') + +if [ -z "$TOKEN" ]; then + echo "error: Docker Hub login failed for ${DOCKERHUB_USERNAME}" >&2 + exit 1 +fi + +failed=0 + +while read -r image; do + [ -n "$image" ] || continue + + repo="${NAMESPACE}/${image}" + status=$(hub_call GET "${API}/repositories/${repo}/") + + case "$status" in + 200) + if [ "$(jq -r '.is_private' "$BODY")" = "true" ]; then + status=$(hub_call PATCH "${API}/repositories/${repo}/" '{"is_private": false}') + if [ "$status" = "200" ]; then + echo "${repo}: was private, now public" + else + echo "error: ${repo}: could not make public (HTTP ${status})" >&2 + failed=1 + fi + else + echo "${repo}: already public" + fi + ;; + 404) + payload=$(jq -n --arg ns "$NAMESPACE" --arg name "$image" \ + '{namespace: $ns, name: $name, is_private: false}') + status=$(hub_call POST "${API}/repositories/" "$payload") + if [ "$status" = "201" ]; then + echo "${repo}: created as public" + else + echo "error: ${repo}: could not create (HTTP ${status})" >&2 + failed=1 + fi + ;; + *) + echo "error: ${repo}: unexpected response (HTTP ${status})" >&2 + failed=1 + ;; + esac +done <<<"$images" + +exit "$failed" From a3c0a71c0750f518a0a1d632dc361d7364a26b9c Mon Sep 17 00:00:00 2001 From: fredcamaral Date: Thu, 6 Aug 2026 07:28:23 +0000 Subject: [PATCH 2/4] build(deps): bump lib-commons to v6.7.0 :arrow_up: Co-authored-by: Codesmith --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d03eab56f..05561d6f2 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/LerianStudio/lib-commons/v6 v6.5.1 + github.com/LerianStudio/lib-commons/v6 v6.7.0 github.com/LerianStudio/lib-observability/v2 v2.1.1 github.com/LerianStudio/lib-service-discovery v1.1.0 github.com/LerianStudio/lib-streaming/v2 v2.0.0 diff --git a/go.sum b/go.sum index f62a71029..bece9788c 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/LerianStudio/lib-auth/v3 v3.3.0 h1:aDpJnAeER6CjBipggA7mVK951anOQzvkzqa8YzZgb0Q= github.com/LerianStudio/lib-auth/v3 v3.3.0/go.mod h1:mKU9zG1AeHE7Ux3iu1n+hZlzLA9KHDa5s+vC+XtbOM4= -github.com/LerianStudio/lib-commons/v6 v6.5.1 h1:05EdXikUpTza6zRMOoO4UzNezakPY/zg2VHmGk6zW10= -github.com/LerianStudio/lib-commons/v6 v6.5.1/go.mod h1:yo2QHBbXXwKNw1Wygip+Z/AANpIunyuXnEAZZT8hGQg= +github.com/LerianStudio/lib-commons/v6 v6.7.0 h1:qXqltprs5gX9RDTrCAa4DOsrNJ9zc5X60LtxYEjWcCA= +github.com/LerianStudio/lib-commons/v6 v6.7.0/go.mod h1:yo2QHBbXXwKNw1Wygip+Z/AANpIunyuXnEAZZT8hGQg= github.com/LerianStudio/lib-observability v1.1.0 h1:e/OrIoTKo8gI1RKXgKDyDDKcrddR4beh53al5ZmB6vQ= github.com/LerianStudio/lib-observability v1.1.0/go.mod h1:PBtmXygWXmcsTnyNLBetyYb/OtsWq6WT9fSJvoppeUQ= github.com/LerianStudio/lib-observability/v2 v2.1.1 h1:YpzgylqUqPg5g1FGpcf561SRbrgKT8wfUoD236CMGHo= From 27cd6e938221ed8218a33644e7f9734ed01b345f Mon Sep 17 00:00:00 2001 From: fredcamaral Date: Thu, 6 Aug 2026 07:33:44 +0000 Subject: [PATCH 3/4] ci(scripts): harden the Docker Hub visibility gate :construction_worker: Co-authored-by: Codesmith --- .github/workflows/dockerhub-visibility.yml | 5 ++ .github/workflows/release.yml | 4 +- scripts/ensure-dockerhub-public.sh | 72 ++++++++++++++-------- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/.github/workflows/dockerhub-visibility.yml b/.github/workflows/dockerhub-visibility.yml index 61c5622d9..d6b60da90 100644 --- a/.github/workflows/dockerhub-visibility.yml +++ b/.github/workflows/dockerhub-visibility.yml @@ -7,6 +7,11 @@ name: Docker Hub Visibility # already went out private. on: workflow_call: + secrets: + DOCKER_USERNAME: + required: true + DOCKERHUB_IMAGE_PUSH_TOKEN: + required: true workflow_dispatch: permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1809048b2..3a39aefe4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,9 @@ jobs: # gitops_yaml_key_mappings below. dockerhub-visibility: uses: ./.github/workflows/dockerhub-visibility.yml - secrets: inherit + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKERHUB_IMAGE_PUSH_TOKEN: ${{ secrets.DOCKERHUB_IMAGE_PUSH_TOKEN }} pipeline: needs: dockerhub-visibility diff --git a/scripts/ensure-dockerhub-public.sh b/scripts/ensure-dockerhub-public.sh index 68ca3420e..f5b200697 100755 --- a/scripts/ensure-dockerhub-public.sh +++ b/scripts/ensure-dockerhub-public.sh @@ -30,23 +30,31 @@ ATTEMPTS=3 : "${DOCKERHUB_TOKEN:?DOCKERHUB_TOKEN is required}" BODY=$(mktemp) -trap 'rm -f "$BODY"' EXIT +PAYLOAD_FILE=$(mktemp) +trap 'rm -f "$BODY" "$PAYLOAD_FILE"' EXIT # hub_call [json-payload] # Writes the response body to $BODY, prints the HTTP status, and retries transport -# errors and 5xx so a flaky Docker Hub cannot fail a release on its own. +# errors and 5xx so a flaky Docker Hub cannot fail a release on its own. The payload +# goes through a file so secrets (the login call) never appear on curl's argv, and +# curl's exit code is captured separately: on a transport error curl already prints +# "000" for %{http_code}, so appending a fallback would produce "000000" and dodge +# the retry branch. hub_call() { local method="$1" url="$2" payload="${3:-}" status="" attempt=1 + local args=(-sS -o "$BODY" -w '%{http_code}' --connect-timeout 10 --max-time 60 -X "$method") + + if [ -n "${TOKEN:-}" ]; then + args+=(-H "Authorization: Bearer ${TOKEN}") + fi + if [ -n "$payload" ]; then + printf '%s' "$payload" >"$PAYLOAD_FILE" + args+=(-H 'Content-Type: application/json' -d "@${PAYLOAD_FILE}") + fi while [ "$attempt" -le "$ATTEMPTS" ]; do - if [ -n "$payload" ]; then - status=$(curl -sS -o "$BODY" -w '%{http_code}' -X "$method" \ - -H "Authorization: Bearer ${TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "$payload" "$url" /dev/null || true) -if [ -z "$TOKEN" ]; then - echo "error: Docker Hub login failed for ${DOCKERHUB_USERNAME}" >&2 +if [ "$status" != "200" ] || [ -z "$TOKEN" ]; then + echo "error: Docker Hub login failed for ${DOCKERHUB_USERNAME} (HTTP ${status})" >&2 exit 1 fi @@ -89,17 +99,29 @@ while read -r image; do case "$status" in 200) - if [ "$(jq -r '.is_private' "$BODY")" = "true" ]; then - status=$(hub_call PATCH "${API}/repositories/${repo}/" '{"is_private": false}') - if [ "$status" = "200" ]; then - echo "${repo}: was private, now public" - else - echo "error: ${repo}: could not make public (HTTP ${status})" >&2 + # Only trust an explicit boolean. A missing field or a malformed body means the + # visibility was never confirmed, and this script's job is verification, so that + # must fail rather than pass as "already public". (Not `.is_private // "unknown"`: + # jq's // treats false as empty, which would flag every public repo as unknown.) + is_private=$(jq -r '.is_private | if type == "boolean" then tostring else "unknown" end' "$BODY" 2>/dev/null || echo "unknown") + case "$is_private" in + true) + status=$(hub_call PATCH "${API}/repositories/${repo}/" '{"is_private": false}') + if [ "$status" = "200" ]; then + echo "${repo}: was private, now public" + else + echo "error: ${repo}: could not make public (HTTP ${status})" >&2 + failed=1 + fi + ;; + false) + echo "${repo}: already public" + ;; + *) + echo "error: ${repo}: response did not report is_private" >&2 failed=1 - fi - else - echo "${repo}: already public" - fi + ;; + esac ;; 404) payload=$(jq -n --arg ns "$NAMESPACE" --arg name "$image" \ From 9e56e8f4bf4d371e1aa626e9ec5de249f6f7f021 Mon Sep 17 00:00:00 2001 From: fredcamaral Date: Thu, 6 Aug 2026 07:34:34 +0000 Subject: [PATCH 4/4] ci(workflows): serialize and time-box the visibility job :construction_worker: Co-authored-by: Codesmith --- .github/workflows/dockerhub-visibility.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/dockerhub-visibility.yml b/.github/workflows/dockerhub-visibility.yml index d6b60da90..6592bee53 100644 --- a/.github/workflows/dockerhub-visibility.yml +++ b/.github/workflows/dockerhub-visibility.yml @@ -21,6 +21,13 @@ jobs: ensure-public: name: Ensure released images are public runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + # Serialize runs: a release run and a manual repair run racing the same 404 both + # POST /repositories/ and the loser fails on a non-201. Job-level (not workflow-level) + # because a called reusable workflow only honors concurrency on its jobs. + concurrency: + group: dockerhub-visibility-${{ github.repository }} + cancel-in-progress: false steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1