diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index 2e8aa4bbf12..9019f125d93 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -18,12 +18,9 @@ concurrency: cancel-in-progress: true jobs: - build_oppia_aab: - name: Build Oppia AAB (${{ matrix.flavor }} flavor) + build_oppia_dev_aab: + name: Build Oppia AAB (dev flavor) runs-on: ubuntu-24.04 - strategy: - matrix: - flavor: [dev, alpha, beta, ga] env: CACHE_DIRECTORY: ~/.bazel_cache steps: @@ -90,10 +87,121 @@ jobs: - name: Check Bazel environment run: bazel info - - name: Build Oppia ${{ matrix.flavor }} AAB + - name: Build Oppia dev AAB + run: bazel build -- //:oppia_dev + + # This job pre-builds the deployable (pre-Proguard-map) AABs for all Proguard-enabled flavors in + # one shot. Building them together is more memory-efficient since Bazel shares intermediate + # artifacts across flavors. The resulting Bazel disk cache is uploaded as a workflow artifact so + # that the finalize jobs can skip all heavy compilation & Proguard work. + build_proguarded_bases: + name: Build Proguarded base AABs (alpha, beta, ga) + runs-on: ubuntu-24.04 + env: + CACHE_DIRECTORY: ~/.bazel_cache + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Bazel + uses: abhinavsingh/setup-bazel@v3 + with: + version: 6.5.0 + + - name: Set up build environment + uses: ./.github/actions/set-up-android-bazel-build-environment + + - uses: actions/cache@v5 + id: cache + with: + path: ${{ env.CACHE_DIRECTORY }} + key: ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-binary-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-binary- + ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel-tests- + ${{ runner.os }}-${{ env.CACHE_DIRECTORY }}-bazel- + + - name: Ensure cache size + env: + BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }} run: | - if [[ "${{ matrix.flavor }}" == "dev" ]]; then - bazel build -- //:oppia_${{ matrix.flavor }} - else - bazel build --compilation_mode=opt -- //:oppia_${{ matrix.flavor }} + EXPANDED_BAZEL_CACHE_PATH="${BAZEL_CACHE_DIR/#\~/$HOME}" + CACHE_SIZE_MB=$(du -smc $EXPANDED_BAZEL_CACHE_PATH | grep total | cut -f1) + echo "Total size of Bazel cache (rounded up to MBs): $CACHE_SIZE_MB" + if [[ "$CACHE_SIZE_MB" -gt 4500 ]]; then + echo "Cache exceeds cut-off; resetting it (will result in a slow build)" + rm -rf $EXPANDED_BAZEL_CACHE_PATH fi + + - name: Configure Bazel to use a local cache + env: + BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }} + run: | + EXPANDED_BAZEL_CACHE_PATH="${BAZEL_CACHE_DIR/#\~/$HOME}" + echo "Using $EXPANDED_BAZEL_CACHE_PATH as Bazel's cache path" + echo "build --disk_cache=$EXPANDED_BAZEL_CACHE_PATH" >> $HOME/.bazelrc + shell: bash + + - name: Check Bazel environment + run: bazel info + + - name: Build all deployable AABs + run: | + for flavor in alpha beta ga; do + echo "=== Building //:oppia_${flavor}_deployable ===" + bazel build --compilation_mode=opt -- //:oppia_${flavor}_deployable + echo "=== Shutting down Bazel to free memory ===" + bazel shutdown + done + + - name: Upload Bazel cache for finalize jobs + uses: actions/upload-artifact@v7 + with: + name: bazel-cache-proguarded + path: ~/.bazel_cache + retention-days: 1 + + # Each finalize job downloads the Bazel cache produced by build_proguarded_bases and runs only the + # lightweight final packaging step (bundling the Proguard map into the already-built deployable + # AAB). This starts with a fresh JVM so there is no accumulated memory pressure from the heavy + # compilation + Proguard work, which avoids OOM crashes (error code 14). + finalize_proguarded_aab: + name: Finalize Oppia AAB (${{ matrix.flavor }} flavor) + needs: build_proguarded_bases + runs-on: ubuntu-24.04 + strategy: + matrix: + flavor: [alpha, beta, ga] + env: + CACHE_DIRECTORY: ~/.bazel_cache + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Bazel + uses: abhinavsingh/setup-bazel@v3 + with: + version: 6.5.0 + + - name: Set up build environment + uses: ./.github/actions/set-up-android-bazel-build-environment + + - name: Download Bazel cache from base build + uses: actions/download-artifact@v8 + with: + name: bazel-cache-proguarded + path: ~/.bazel_cache + + - name: Configure Bazel to use the downloaded cache + run: | + echo "Using $HOME/.bazel_cache as Bazel's cache path" + echo "build --disk_cache=$HOME/.bazel_cache" >> $HOME/.bazelrc + shell: bash + + - name: Check Bazel environment + run: bazel info + + - name: Build Oppia ${{ matrix.flavor }} final AAB + run: bazel build --compilation_mode=opt -- //:oppia_${{ matrix.flavor }} diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index 35d84893d1e..51845408d99 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -29,8 +29,11 @@ jobs: CURRENT_OPEN_PR_INFO="$(gh pr list --json number,baseRefName,headRefName,headRepository,headRepositoryOwner | tr -d '[:space:]')" echo "matrix={\"prInfo\": $CURRENT_OPEN_PR_INFO}" >> "$GITHUB_OUTPUT" - build_stats: - name: Build Stats + # This job builds the deployable (pre-Proguard-map) AABs for both head and base branches, then + # uploads the Bazel disk cache so that the finalize job can build finals with a fresh JVM, + # avoiding OOM crashes (error code 14). + build_stats_deployables: + name: Build Stats (deployable AABs) needs: find_open_pull_requests runs-on: ubuntu-24.04 # Reduce parallelization due to high build times, and allow individual PRs to fail. @@ -204,6 +207,171 @@ jobs: ref: ${{ env.PR_HEAD_REF_NAME }} path: head + - name: Build deployable AABs (feature branch) + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + run: | + cd head + git log -n 1 + for flavor in alpha beta ga; do + echo "=== Building //:oppia_${flavor}_deployable ===" + bazel build -- //:oppia_${flavor}_deployable + echo "=== Shutting down Bazel to free memory ===" + bazel shutdown + done + + - name: Build deployable AABs (base branch) + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + run: | + cd base + git log -n 1 + for flavor in alpha beta ga; do + echo "=== Building //:oppia_${flavor}_deployable ===" + bazel build -- //:oppia_${flavor}_deployable + echo "=== Shutting down Bazel to free memory ===" + bazel shutdown + done + + - name: Upload Bazel cache for finalize job + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: bazel-cache-stats-pr-${{ matrix.prInfo.number }} + path: ~/.bazel_cache + retention-days: 1 + + # Each finalize job downloads the Bazel cache produced by build_stats_deployables and runs only + # the lightweight final packaging steps (bundling the Proguard map into the already-built + # deployable AABs). This starts with a fresh JVM so there is no accumulated memory pressure. + finalize_stats_and_report: + name: Finalize Stats & Report + needs: [find_open_pull_requests, build_stats_deployables] + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJson(needs.find_open_pull_requests.outputs.matrix) }} + env: + ENABLE_CACHING: false + CACHE_DIRECTORY: ~/.bazel_cache + steps: + # Re-check whether this PR has new commits (lightweight GitHub API calls). + - name: Find the latest APK & AAB Analysis Comment + uses: actions/github-script@v8 + id: find_latest_apk_aab_comment + with: + script: | + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ matrix.prInfo.number }}, + }); + + for (let i = comments.length - 1; i >= 0; i--) { + if (comments[i].body.includes("# APK & AAB differences analysis")) { + const latestComment = comments[i]; + const commentBody = latestComment.body; + const commentDate = latestComment.created_at; + + require('fs').writeFileSync('latest_aab_comment_body.log', commentBody, 'utf8'); + core.setOutput("latest_aab_comment_created_at", commentDate); + + return + } + } + + - name: Track Commits following the latest APK & AAB Analysis Comment + uses: actions/github-script@v8 + id: track_commits + with: + script: | + const commits = await github.paginate(github.rest.pulls.listCommits, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: ${{ matrix.prInfo.number }}, + }); + + const latestCommentDate = "${{ steps.find_latest_apk_aab_comment.outputs.latest_aab_comment_created_at }}"; + const recentCommits = commits.filter(commit => { + const commitDate = new Date(commit.commit.committer.date); + return commitDate > new Date("${{ steps.find_latest_apk_aab_comment.outputs.latest_aab_comment_created_at }}"); + }); + + const recentCommitsCount = recentCommits.length; + if(recentCommitsCount > 0 || !latestCommentDate) { + core.setOutput("new_commits", "true"); + } else { + console.log("No new commits since the last APK & AAB report; Skipping redundant analysis."); + core.setOutput("new_commits", "false"); + } + + - name: Compute PR head owner/repo reference + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + env: + PR_HEAD_REPO: ${{ matrix.prInfo.headRepository.name }} + PR_HEAD_REPO_OWNER: ${{ matrix.prInfo.headRepositoryOwner.login }} + run: | + echo "PR_HEAD=$PR_HEAD_REPO_OWNER/$PR_HEAD_REPO" >> "$GITHUB_ENV" + + - name: Set up Bazel + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + uses: abhinavsingh/setup-bazel@v3 + with: + version: 6.5.0 + + - name: Download Bazel cache from deployable build + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + uses: actions/download-artifact@v8 + with: + name: bazel-cache-stats-pr-${{ matrix.prInfo.number }} + path: ~/.bazel_cache + + - name: Configure Bazel to use the downloaded cache + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + run: | + echo "Using $HOME/.bazel_cache as Bazel's cache path" + echo "build --disk_cache=$HOME/.bazel_cache" >> $HOME/.bazelrc + shell: bash + + # This checks out the actual true develop branch separately to ensure that the stats check is + # run from the latest develop rather than the base branch (which might be different for + # chained PRs). + - name: Check out develop repository + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + uses: actions/checkout@v6 + with: + path: develop + + - name: Set up build environment + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + uses: ./develop/.github/actions/set-up-android-bazel-build-environment + + - name: Check Bazel environment + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + run: | + cd develop + bazel info + + - name: Check out base repository and branch + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + env: + PR_BASE_REF_NAME: ${{ matrix.prInfo.baseRefName }} + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ env.PR_BASE_REF_NAME }} + path: base + + - name: Check out head repository and branch + if: ${{ steps.track_commits.outputs.new_commits == 'true' }} + env: + PR_HEAD_REF_NAME: ${{ matrix.prInfo.headRefName }} + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: ${{ env.PR_HEAD }} + ref: ${{ env.PR_HEAD_REF_NAME }} + path: head + # Note that Bazel is shutdown between builds since multiple Bazel servers will otherwise end # up being active (due to multiple repositories being used) and this can quickly overwhelm CI # worker resources.