ci(openapi-diff): Compare API schemas with the PR's merge-base

.github/workflows/openapi-diff.yml

@@ -11,15 +11,26 @@ concurrency:
 # hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
 env:
   SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3
+  # Shared by the jobs below: the artifact the build job hands to the compare
+  # jobs, and where it puts the PR's RPC schema and the baseline schemas.
+  DIFF_ARTIFACT: openapi-diff
+  RPC_SCHEMA: .artifacts/rpc_method_schema.json
+  BASELINE_DIR: .artifacts/openapi-baseline
 
 jobs:
-  check-diff:
-    name: (Optional) Shows the difference between the prod and dev schema
+  # Builds this PR's public spec and RPC schema, and fetches the ones on master
+  # where the PR branched off, for the jobs below to compare.
+  build:
+    name: Build the API schemas to compare
     runs-on: ubuntu-26.04-arm
     timeout-minutes: 90
     permissions:
+      # Artifacts of the openapi workflow on master hold the baselines.
+      actions: read
       contents: read
       pull-requests: read
+    outputs:
+      api_docs: ${{ steps.changes.outputs.api_docs }}
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
@@ -36,39 +47,78 @@ jobs:
           mode: migrations
         if: steps.changes.outputs.api_docs == 'true'
 
-      - name: Checkout getsentry/sentry-api-schema
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-        if: steps.changes.outputs.api_docs == 'true'
-        with:
-          ref: 'main'
-          repository: getsentry/sentry-api-schema
-          path: sentry-api-schema
-
       - uses: ./.github/actions/setup-node-pnpm
+        if: steps.changes.outputs.api_docs == 'true'
 
       - name: Build OpenAPI Derefed JSON
         if: steps.changes.outputs.api_docs == 'true'
-        run: |
-          make build-api-docs
+        run: make build-api-docs
 
-      - name: Compare OpenAPI Derefed JSON
+      - name: Build RPC method schema
         if: steps.changes.outputs.api_docs == 'true'
-        run: |
-          npx json-diff@0.5.4 --color sentry-api-schema/openapi-derefed.json tests/apidocs/openapi-derefed.json
+        run: make build-rpc-schema
 
-      - name: Dump RPC schema for new version
-        id: newschema
+      - name: Fetch the merge-base's schemas
         if: steps.changes.outputs.api_docs == 'true'
-        run: |
-          mkdir schemas
-          sentry rpcschema --loglevel=WARNING > rpc_method_schema.json
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: .github/workflows/scripts/openapi-diff.sh baseline "$BASELINE_DIR"
 
-      # Runs oasdiff directly from the getsentry/image-mirror copy instead of
-      # oasdiff/oasdiff-action, which builds FROM tufin/oasdiff:main on Docker Hub.
-      - name: Output RPC schema comparison
+      - name: Upload the schemas to compare
         if: steps.changes.outputs.api_docs == 'true'
-        run: |
-          docker run --rm -v "$PWD:$PWD" -w "$PWD" \
-            ghcr.io/getsentry/image-mirror-tufin-oasdiff:v1.33.0 \
-            breaking sentry-api-schema/rpc_method_schema.json ./rpc_method_schema.json \
-            --fail-on WARN --format githubactions
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: ${{ env.DIFF_ARTIFACT }}
+          path: |
+            tests/apidocs/openapi-derefed.json
+            ${{ env.RPC_SCHEMA }}
+            ${{ env.BASELINE_DIR }}/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 7
+
+  changelog:
+    name: Public API changelog
+    needs: build
+    if: needs.build.outputs.api_docs == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          sparse-checkout: .github/workflows/scripts
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: ${{ env.DIFF_ARTIFACT }}
+
+      # Writes the changelog to the job summary. It is information for the
+      # author and reviewers, so it never fails the build.
+      - name: Summarize public API changes
+        run: .github/workflows/scripts/openapi-diff.sh changelog "$BASELINE_DIR/openapi-derefed.json" tests/apidocs/openapi-derefed.json
+
+  rpc-breaking:
+    name: RPC schema breaking changes
+    needs: build
+    if: needs.build.outputs.api_docs == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          sparse-checkout: .github/workflows/scripts
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: ${{ env.DIFF_ARTIFACT }}
+
+      # Control and cell silos deploy separately, so an RPC change must keep
+      # working with callers built from the merge-base.
+      - name: Check RPC schema for breaking changes
+        run: .github/workflows/scripts/openapi-diff.sh rpc-breaking "$BASELINE_DIR/rpc_method_schema.json" "$RPC_SCHEMA"

.github/workflows/openapi.yml

@@ -63,6 +63,19 @@ jobs:
         run: |
           sentry rpcschema --loglevel=WARNING > rpc_method_schema.json
 
+      # openapi-diff compares each pull request with these, from the master
+      # commit it branched off.
+      - name: Upload the API schemas
+        if: steps.changes.outputs.api_docs == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: api-schemas
+          path: |
+            tests/apidocs/openapi-derefed.json
+            rpc_method_schema.json
+          if-no-files-found: error
+          retention-days: 30
+
       - name: Copy artifacts into getsentry/sentry-api-schema
         if: steps.changes.outputs.api_docs == 'true'
         run: |

.github/workflows/scripts/openapi-diff.sh

@@ -0,0 +1,174 @@
+#!/bin/sh
+# Compare a pull request's API schemas with the ones on master where it
+# branched off. Used by .github/workflows/openapi-diff.yml and
+# `make diff-api-docs`.
+#
+#   openapi-diff.sh baseline DIR
+#       Write the merge-base's openapi-derefed.json and rpc_method_schema.json
+#       to DIR, and a one-line description of where they came from to
+#       DIR/source.txt. Tries, in order:
+#         1. the `api-schemas` artifact of the openapi workflow run for the
+#            merge-base, or for the closest earlier master commit that has one;
+#         2. the newest `api@` GitHub release;
+#         3. getsentry/sentry-api-schema main, while it still receives the spec.
+#       With BASE_SHA and HEAD_SHA set (CI), the merge-base comes from the
+#       GitHub API. Otherwise it is `git merge-base origin/master HEAD`.
+#   openapi-diff.sh changelog BASE_SPEC PR_SPEC
+#       Write an oasdiff changelog as Markdown to $GITHUB_STEP_SUMMARY, or to
+#       stdout when that is unset. Never fails the build: a missing baseline
+#       or an oasdiff error becomes a note in the summary.
+#   openapi-diff.sh rpc-breaking BASE_SCHEMA PR_SCHEMA
+#       Fail on RPC method schema changes that break callers.
+#
+# Environment: GH_TOKEN (or a logged-in gh), GITHUB_REPOSITORY (defaults to
+# getsentry/sentry), BASE_SHA and HEAD_SHA (optional).
+set -eu
+
+repo=${GITHUB_REPOSITORY:-getsentry/sentry}
+artifact=api-schemas
+oasdiff_image=ghcr.io/getsentry/image-mirror-tufin-oasdiff:v1.33.0
+legacy_raw=https://raw.githubusercontent.com/getsentry/sentry-api-schema/main
+# How many master commits before the merge-base to search for an artifact.
+# Every commit that can change the schemas uploads one.
+max_commits=30
+# GitHub rejects step summaries over 1 MiB.
+max_summary_bytes=900000
+
+merge_base() {
+  if [ -n "${BASE_SHA:-}" ] && [ -n "${HEAD_SHA:-}" ]; then
+    timeout 60 gh api "repos/$repo/compare/$BASE_SHA...$HEAD_SHA" --jq .merge_base_commit.sha
+  else
+    git merge-base origin/master HEAD
+  fi
+}
+
+# Print the id of the `api-schemas` artifact uploaded for a master commit, or
+# nothing when that commit's openapi run did not upload one.
+artifact_for_commit() {
+  run_ids=$(timeout 60 gh api \
+    "repos/$repo/actions/workflows/openapi.yml/runs?head_sha=$1&event=push&status=success" \
+    --jq '.workflow_runs[].id') || return 1
+  for run_id in $run_ids; do
+    artifact_id=$(timeout 60 gh api "repos/$repo/actions/runs/$run_id/artifacts?name=$artifact" \
+      --jq '[.artifacts[] | select(.expired | not)][0].id // empty') || return 1
+    if [ -n "$artifact_id" ]; then
+      printf '%s\n' "$artifact_id"
+      return 0
+    fi
+  done
+}
+
+from_artifact() {
+  dest=$1
+  mb=$(merge_base) || return 1
+  commits=$(timeout 60 gh api "repos/$repo/commits?sha=$mb&per_page=$max_commits" --jq '.[].sha') || return 1
+  for sha in $commits; do
+    artifact_id=$(artifact_for_commit "$sha") || return 1
+    if [ -n "$artifact_id" ]; then
+      zip="$dest/$artifact.zip"
+      timeout 300 gh api "repos/$repo/actions/artifacts/$artifact_id/zip" > "$zip" || return 1
+      unzip -o -q "$zip" -d "$dest/artifact" || return 1
+      # The files keep their repository paths, which differ between versions
+      # of the openapi workflow, so look them up by name.
+      for name in openapi-derefed.json rpc_method_schema.json; do
+        file=$(find "$dest/artifact" -type f -name "$name") || return 1
+        if [ -z "$file" ]; then
+          echo "The $artifact artifact for $sha has no $name." >&2
+          return 1
+        fi
+        mv "$file" "$dest/$name" || return 1
+      done
+      rm -r "$zip" "$dest/artifact"
+      printf 'the openapi workflow artifact for %s (merge-base %s)\n' "$sha" "$mb" > "$dest/source.txt"
+      return 0
+    fi
+  done
+  echo "No $artifact artifact for the $max_commits master commits up to the merge-base $mb." >&2
+  return 1
+}
+
+from_release() {
+  dest=$1
+  tag=$(timeout 60 gh api "repos/$repo/releases?per_page=100" \
+    --jq '[.[] | select(.tag_name | startswith("api@"))][0].tag_name // empty') || return 1
+  if [ -z "$tag" ]; then
+    echo "No api@ release yet." >&2
+    return 1
+  fi
+  timeout 300 gh release download "$tag" --repo "$repo" --dir "$dest" --clobber \
+    --pattern openapi-derefed.json --pattern rpc_method_schema.json || return 1
+  printf 'the %s release, which can be newer than the merge-base\n' "$tag" > "$dest/source.txt"
+}
+
+from_legacy_repo() {
+  dest=$1
+  for name in openapi-derefed.json rpc_method_schema.json; do
+    timeout 300 curl --fail --silent --show-error --location -o "$dest/$name" "$legacy_raw/$name" || return 1
+  done
+  echo 'getsentry/sentry-api-schema main, which can be newer than the merge-base' > "$dest/source.txt"
+}
+
+baseline() {
+  out=$1
+  # Each source starts from an empty directory, so a source that fails
+  # halfway leaves nothing behind for the next one.
+  work="$out.partial"
+  for source in from_artifact from_release from_legacy_repo; do
+    rm -rf "$work"
+    mkdir -p "$work"
+    if "$source" "$work"; then
+      rm -rf "$out"
+      mv "$work" "$out"
+      description=$(cat "$out/source.txt")
+      echo "Baseline: $description"
+      return 0
+    fi
+    echo "$source found no baseline, trying the next source." >&2
+  done
+  rm -rf "$work"
+  echo "::error::Found no baseline API schemas."
+  return 1
+}
+
+oasdiff() {
+  docker run --rm -v "$PWD:$PWD" -w "$PWD" "$oasdiff_image" "$@"
+}
+
+changelog() {
+  summary=${GITHUB_STEP_SUMMARY:-/dev/stdout}
+  description=$(cat "$(dirname "$1")/source.txt" 2>/dev/null) || description='an unknown baseline'
+  {
+    echo '## Public API changes'
+    echo
+    if [ ! -f "$1" ]; then
+      echo 'There is no baseline spec to compare with; see the build job log.'
+    elif ! report=$(oasdiff changelog "$1" "$2" --format markdown); then
+      echo 'oasdiff could not compare the specs; see this job log.'
+    else
+      echo "Compared with $description."
+      echo
+      if [ "${#report}" -gt "$max_summary_bytes" ]; then
+        printf '%s\n' "$report" | head -c "$max_summary_bytes"
+        echo
+        echo "The changelog is cut off here; run \`make diff-api-docs\` for the rest."
+      else
+        printf '%s\n' "$report"
+      fi
+    fi
+  } >> "$summary"
+}
+
+rpc_breaking() {
+  oasdiff breaking "$1" "$2" --fail-on WARN --format githubactions
+}
+
+command=${1:-}
+case $command in
+  baseline) baseline "$2" ;;
+  changelog) changelog "$2" "$3" ;;
+  rpc-breaking) rpc_breaking "$2" "$3" ;;
+  *)
+    echo "usage: $0 baseline DIR | changelog BASE_SPEC PR_SPEC | rpc-breaking BASE_SCHEMA PR_SCHEMA" >&2
+    exit 2
+    ;;
+esac

.pre-commit-config.yaml

@@ -167,3 +167,7 @@ repos:
         types: [file]
         files: ^\.envrc$
         args: [--external-sources]
+      - id: shellcheck
+        name: shellcheck (POSIX sh)
+        files: ^\.github/workflows/scripts/openapi-diff\.sh$
+        args: [--shell=sh, --enable=check-extra-masked-returns]

Makefile

@@ -48,6 +48,11 @@ build-api-docs: build-deprecated-docs build-spectacular-docs
 	@echo "--> Dereference the json schema for ease of use"
 	pnpm run deref-api-docs
 
+# The internal RPC method schema, which CI's openapi-diff checks for breaking changes.
+build-rpc-schema:
+	@mkdir -p .artifacts
+	sentry rpcschema --loglevel=WARNING > .artifacts/rpc_method_schema.json
+
 # Opt-in private schema for local review; never included in the public docs build.
 build-investigation-api-docs: build-deprecated-docs
 	@mkdir -p .artifacts/investigation-api-docs
@@ -63,9 +68,13 @@ api-owner-map:
 watch-api-docs:
 	@cd api-docs/ && node ./watch.ts
 
+# Compares tests/apidocs/openapi-derefed.json (from build-api-docs) with the
+# master spec where this branch started, the same way CI's openapi-diff does.
+# Needs gh (logged in) and docker.
 diff-api-docs:
-	@echo "--> diffing local api docs against sentry-api-schema/openapi-derefed.json"
-	pnpm run diff-docs
+	@echo "--> diffing local api docs against the merge-base's spec"
+	.github/workflows/scripts/openapi-diff.sh baseline .artifacts/openapi-baseline
+	.github/workflows/scripts/openapi-diff.sh changelog .artifacts/openapi-baseline/openapi-derefed.json tests/apidocs/openapi-derefed.json
 
 build: locale