Fix #6186: Flaky proguard in CI

.github/workflows/build_tests.yml

@@ -18,12 +18,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_oppia_aab:
-    name: Build Oppia AAB (${{ matrix.flavor }} flavor)
+  build_oppia_dev_aab:
+    name: Build Oppia AAB (dev flavor)
     runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        flavor: [dev, alpha, beta, ga]
     env:
       CACHE_DIRECTORY: ~/.bazel_cache
     steps:
@@ -90,10 +87,121 @@ jobs:
       - name: Check Bazel environment
         run: bazel info
 
-      - name: Build Oppia ${{ matrix.flavor }} AAB
+      - name: Build Oppia dev AAB
+        run: bazel build -- //:oppia_dev
+
+  # This job pre-builds the deployable (pre-Proguard-map) AABs for all Proguard-enabled flavors in
+  # one shot. Building them together is more memory-efficient since Bazel shares intermediate
+  # artifacts across flavors. The resulting Bazel disk cache is uploaded as a workflow artifact so
+  # that the finalize jobs can skip all heavy compilation & Proguard work.
+  build_proguarded_bases:
+    name: Build Proguarded base AABs (alpha, beta, ga)
+    runs-on: ubuntu-24.04
+    env:
+      CACHE_DIRECTORY: ~/.bazel_cache
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Set up Bazel
+        uses: abhinavsingh/setup-bazel@v3
+        with:
+          version: 6.5.0
+
+      - name: Set up build environment
+        uses: ./.github/actions/set-up-android-bazel-build-environment
+
+      - uses: actions/cache@v5
+        id: cache
+        with:
+          path: ${{ env.CACHE_DIRECTORY }}
+          key: ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-binary-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-binary-
+            ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-tests-
+            ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-
+
+      - name: Ensure cache size
+        env:
+          BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }}
         run: |
-          if [[ "${{ matrix.flavor }}" == "dev" ]]; then
-            bazel build -- //:oppia_${{ matrix.flavor }}
-          else
-            bazel build --compilation_mode=opt -- //:oppia_${{ matrix.flavor }}
+          EXPANDED_BAZEL_CACHE_PATH="${BAZEL_CACHE_DIR/#\~/$HOME}"
+          CACHE_SIZE_MB=$(du -smc $EXPANDED_BAZEL_CACHE_PATH | grep total | cut -f1)
+          echo "Total size of Bazel cache (rounded up to MBs): $CACHE_SIZE_MB"
+          if [[ "$CACHE_SIZE_MB" -gt 4500 ]]; then
+            echo "Cache exceeds cut-off; resetting it (will result in a slow build)"
+            rm -rf $EXPANDED_BAZEL_CACHE_PATH
           fi
+
+      - name: Configure Bazel to use a local cache
+        env:
+          BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }}
+        run: |
+          EXPANDED_BAZEL_CACHE_PATH="${BAZEL_CACHE_DIR/#\~/$HOME}"
+          echo "Using $EXPANDED_BAZEL_CACHE_PATH as Bazel's cache path"
+          echo "build --disk_cache=$EXPANDED_BAZEL_CACHE_PATH" >> $HOME/.bazelrc
+        shell: bash
+
+      - name: Check Bazel environment
+        run: bazel info
+
+      - name: Build all deployable AABs
+        run: |
+          for flavor in alpha beta ga; do
+            echo "=== Building //:oppia_${flavor}_deployable ==="
+            bazel build --compilation_mode=opt -- //:oppia_${flavor}_deployable
+            echo "=== Shutting down Bazel to free memory ==="
+            bazel shutdown
+          done
+
+      - name: Upload Bazel cache for finalize jobs
+        uses: actions/upload-artifact@v7
+        with:
+          name: bazel-cache-proguarded
+          path: ~/.bazel_cache
+          retention-days: 1
+
+  # Each finalize job downloads the Bazel cache produced by build_proguarded_bases and runs only the
+  # lightweight final packaging step (bundling the Proguard map into the already-built deployable
+  # AAB). This starts with a fresh JVM so there is no accumulated memory pressure from the heavy
+  # compilation + Proguard work, which avoids OOM crashes (error code 14).
+  finalize_proguarded_aab:
+    name: Finalize Oppia AAB (${{ matrix.flavor }} flavor)
+    needs: build_proguarded_bases
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        flavor: [alpha, beta, ga]
+    env:
+      CACHE_DIRECTORY: ~/.bazel_cache
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Set up Bazel
+        uses: abhinavsingh/setup-bazel@v3
+        with:
+          version: 6.5.0
+
+      - name: Set up build environment
+        uses: ./.github/actions/set-up-android-bazel-build-environment
+
+      - name: Download Bazel cache from base build
+        uses: actions/download-artifact@v8
+        with:
+          name: bazel-cache-proguarded
+          path: ~/.bazel_cache
+
+      - name: Configure Bazel to use the downloaded cache
+        run: |
+          echo "Using $HOME/.bazel_cache as Bazel's cache path"
+          echo "build --disk_cache=$HOME/.bazel_cache" >> $HOME/.bazelrc
+        shell: bash
+
+      - name: Check Bazel environment
+        run: bazel info
+
+      - name: Build Oppia ${{ matrix.flavor }} final AAB
+        run: bazel build --compilation_mode=opt -- //:oppia_${{ matrix.flavor }}

.github/workflows/stats.yml

@@ -29,8 +29,11 @@ jobs:
           CURRENT_OPEN_PR_INFO="$(gh pr list --json number,baseRefName,headRefName,headRepository,headRepositoryOwner | tr -d '[:space:]')"
           echo "matrix={\"prInfo\": $CURRENT_OPEN_PR_INFO}" >> "$GITHUB_OUTPUT"
 
-  build_stats:
-    name: Build Stats
+  # This job builds the deployable (pre-Proguard-map) AABs for both head and base branches, then
+  # uploads the Bazel disk cache so that the finalize job can build finals with a fresh JVM,
+  # avoiding OOM crashes (error code 14).
+  build_stats_deployables:
+    name: Build Stats (deployable AABs)
     needs: find_open_pull_requests
     runs-on: ubuntu-24.04
     # Reduce parallelization due to high build times, and allow individual PRs to fail.
@@ -204,6 +207,171 @@ jobs:
           ref: ${{ env.PR_HEAD_REF_NAME }}
           path: head
 
+      - name: Build deployable AABs (feature branch)
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        run: |
+          cd head
+          git log -n 1
+          for flavor in alpha beta ga; do
+            echo "=== Building //:oppia_${flavor}_deployable ==="
+            bazel build -- //:oppia_${flavor}_deployable
+            echo "=== Shutting down Bazel to free memory ==="
+            bazel shutdown
+          done
+
+      - name: Build deployable AABs (base branch)
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        run: |
+          cd base
+          git log -n 1
+          for flavor in alpha beta ga; do
+            echo "=== Building //:oppia_${flavor}_deployable ==="
+            bazel build -- //:oppia_${flavor}_deployable
+            echo "=== Shutting down Bazel to free memory ==="
+            bazel shutdown
+          done
+
+      - name: Upload Bazel cache for finalize job
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: bazel-cache-stats-pr-${{ matrix.prInfo.number }}
+          path: ~/.bazel_cache
+          retention-days: 1
+
+  # Each finalize job downloads the Bazel cache produced by build_stats_deployables and runs only
+  # the lightweight final packaging steps (bundling the Proguard map into the already-built
+  # deployable AABs). This starts with a fresh JVM so there is no accumulated memory pressure.
+  finalize_stats_and_report:
+    name: Finalize Stats & Report
+    needs: [find_open_pull_requests, build_stats_deployables]
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix: ${{ fromJson(needs.find_open_pull_requests.outputs.matrix) }}
+    env:
+      ENABLE_CACHING: false
+      CACHE_DIRECTORY: ~/.bazel_cache
+    steps:
+      # Re-check whether this PR has new commits (lightweight GitHub API calls).
+      - name: Find the latest APK & AAB Analysis Comment
+        uses: actions/github-script@v8
+        id: find_latest_apk_aab_comment
+        with:
+           script: |
+             const comments = await github.paginate(github.rest.issues.listComments, {
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               issue_number: ${{ matrix.prInfo.number }},
+             });
+
+             for (let i = comments.length - 1; i >= 0; i--) {
+               if (comments[i].body.includes("# APK & AAB differences analysis")) {
+                 const latestComment = comments[i];
+                 const commentBody = latestComment.body;
+                 const commentDate = latestComment.created_at;
+
+                 require('fs').writeFileSync('latest_aab_comment_body.log', commentBody, 'utf8');
+                 core.setOutput("latest_aab_comment_created_at", commentDate);
+
+                 return
+               }
+             }
+
+      - name: Track Commits following the latest APK & AAB Analysis Comment
+        uses: actions/github-script@v8
+        id: track_commits
+        with:
+           script: |
+             const commits = await github.paginate(github.rest.pulls.listCommits, {
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               pull_number: ${{ matrix.prInfo.number }},
+             });
+
+             const latestCommentDate = "${{ steps.find_latest_apk_aab_comment.outputs.latest_aab_comment_created_at }}";
+             const recentCommits = commits.filter(commit => {
+               const commitDate = new Date(commit.commit.committer.date);
+               return commitDate > new Date("${{ steps.find_latest_apk_aab_comment.outputs.latest_aab_comment_created_at }}");
+             });
+
+             const recentCommitsCount = recentCommits.length;
+             if(recentCommitsCount > 0 || !latestCommentDate) {
+               core.setOutput("new_commits", "true");
+             } else {
+               console.log("No new commits since the last APK & AAB report; Skipping redundant analysis.");
+               core.setOutput("new_commits", "false");
+             }
+
+      - name: Compute PR head owner/repo reference
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        env:
+          PR_HEAD_REPO: ${{ matrix.prInfo.headRepository.name }}
+          PR_HEAD_REPO_OWNER: ${{ matrix.prInfo.headRepositoryOwner.login }}
+        run: |
+          echo "PR_HEAD=$PR_HEAD_REPO_OWNER/$PR_HEAD_REPO" >> "$GITHUB_ENV"
+
+      - name: Set up Bazel
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        uses: abhinavsingh/setup-bazel@v3
+        with:
+          version: 6.5.0
+
+      - name: Download Bazel cache from deployable build
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        uses: actions/download-artifact@v8
+        with:
+          name: bazel-cache-stats-pr-${{ matrix.prInfo.number }}
+          path: ~/.bazel_cache
+
+      - name: Configure Bazel to use the downloaded cache
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        run: |
+          echo "Using $HOME/.bazel_cache as Bazel's cache path"
+          echo "build --disk_cache=$HOME/.bazel_cache" >> $HOME/.bazelrc
+        shell: bash
+
+      # This checks out the actual true develop branch separately to ensure that the stats check is
+      # run from the latest develop rather than the base branch (which might be different for
+      # chained PRs).
+      - name: Check out develop repository
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        uses: actions/checkout@v6
+        with:
+          path: develop
+
+      - name: Set up build environment
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        uses: ./develop/.github/actions/set-up-android-bazel-build-environment
+
+      - name: Check Bazel environment
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        run: |
+          cd develop
+          bazel info
+
+      - name: Check out base repository and branch
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        env:
+          PR_BASE_REF_NAME: ${{ matrix.prInfo.baseRefName }}
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          ref: ${{ env.PR_BASE_REF_NAME }}
+          path: base
+
+      - name: Check out head repository and branch
+        if: ${{ steps.track_commits.outputs.new_commits == 'true' }}
+        env:
+          PR_HEAD_REF_NAME: ${{ matrix.prInfo.headRefName }}
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          repository: ${{ env.PR_HEAD }}
+          ref: ${{ env.PR_HEAD_REF_NAME }}
+          path: head
+
       # Note that Bazel is shutdown between builds since multiple Bazel servers will otherwise end
       # up being active (due to multiple repositories being used) and this can quickly overwhelm CI
       # worker resources.