From 1cb242a69d908d839193266c9598c36169b1dd18 Mon Sep 17 00:00:00 2001 From: Jan Schlosser Date: Tue, 28 Jul 2026 22:13:13 +0200 Subject: [PATCH 1/3] ci: add QNX environment setup and test-log upload actions Introduce reusable composite actions to support CI, particularly the nightly flaky-test detection pipeline: - setup_qnx_environment: installs QEMU, enables KVM permissions and provisions the QNX license, with a post step that cleans the license up. Replaces the inline QNX/QEMU setup previously duplicated in build_and_test_qnx. - upload_bazel_testlogs_on_failure: collects and uploads bazel-testlogs as an artifact when a test step fails, wired into the gcc15, ASan/UBSan/ LSan, TSan and QNX workflows. - prepare_bazel_environment: add an optional repository-cache input so callers can opt into the Bazel repository cache. Also ignore the local .worktrees/ directory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare_bazel_environment/action.yml | 8 ++- .../setup_qnx_environment/action.yml | 30 ++++++++++++ .../setup_qnx_environment/main.js | 49 +++++++++++++++++++ .../setup_qnx_environment/post.js | 22 +++++++++ .github/workflows/build_and_test_qnx.yml | 24 ++------- 5 files changed, 112 insertions(+), 21 deletions(-) create mode 100644 .github/actions/00_infrastructure/setup_qnx_environment/action.yml create mode 100644 .github/actions/00_infrastructure/setup_qnx_environment/main.js create mode 100644 .github/actions/00_infrastructure/setup_qnx_environment/post.js diff --git a/.github/actions/00_infrastructure/prepare_bazel_environment/action.yml b/.github/actions/00_infrastructure/prepare_bazel_environment/action.yml index 4d7346db1..7ecff2918 100644 --- a/.github/actions/00_infrastructure/prepare_bazel_environment/action.yml +++ b/.github/actions/00_infrastructure/prepare_bazel_environment/action.yml @@ -40,6 +40,12 @@ inputs: and keeps the original URL as fallback. required: false default: "" + repository-cache: + description: > + Whether to enable Bazel repository cache in setup-bazel. + Values: "true" or "false". + required: false + default: "false" runs: using: "composite" @@ -73,7 +79,7 @@ runs: with: bazelisk-cache: true disk-cache: false - repository-cache: false + repository-cache: ${{ fromJSON(inputs.repository-cache) }} cache-save: ${{ env.CACHE_MODE != 'read-only' && env.CACHE_MODE != 'disabled' }} - name: Register post cache save hook and restore caches diff --git a/.github/actions/00_infrastructure/setup_qnx_environment/action.yml b/.github/actions/00_infrastructure/setup_qnx_environment/action.yml new file mode 100644 index 000000000..b9fa13b40 --- /dev/null +++ b/.github/actions/00_infrastructure/setup_qnx_environment/action.yml @@ -0,0 +1,30 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +name: "Setup QNX and QEMU environment" +description: "Installs QEMU/KVM prerequisites, configures QNX license, and cleans up in post step." + +inputs: + qnx-license: + description: "Base64-encoded QNX license payload." + required: true + license-dir: + description: "QNX license directory." + required: false + default: "/opt/score_qnx/license" + +runs: + using: "node24" + main: "main.js" + post: "post.js" + post-if: "always()" diff --git a/.github/actions/00_infrastructure/setup_qnx_environment/main.js b/.github/actions/00_infrastructure/setup_qnx_environment/main.js new file mode 100644 index 000000000..d79b53904 --- /dev/null +++ b/.github/actions/00_infrastructure/setup_qnx_environment/main.js @@ -0,0 +1,49 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// https://www.apache.org/licenses/LICENSE-2.0 +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +const { execSync } = require("node:child_process"); +const { appendFileSync } = require("node:fs"); + +function run(command) { + execSync(command, { stdio: "inherit" }); +} + +function main() { + const licenseDir = process.env["INPUT_LICENSE-DIR"] || "/opt/score_qnx/license"; + const qnxLicense = process.env["INPUT_QNX-LICENSE"] || ""; + const githubState = process.env.GITHUB_STATE; + + if (!qnxLicense) { + throw new Error("Input 'qnx-license' is required."); + } + if (!githubState) { + throw new Error("GITHUB_STATE is not available."); + } + + appendFileSync(githubState, `LICENSE_DIR=${licenseDir}\n`, { encoding: "utf-8" }); + + run("sudo apt-get update"); + run("sudo apt-get install -y qemu-system"); + run( + "echo 'KERNEL==\"kvm\", GROUP=\"kvm\", MODE=\"0666\", OPTIONS+=\"static_node=kvm\"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules", + ); + run("sudo udevadm control --reload-rules"); + run("sudo udevadm trigger --name-match=kvm"); + + const escapedLicenseDir = `'${licenseDir.replace(/'/g, `'\\''`)}'`; + const escapedLicense = qnxLicense.replace(/'/g, `'\\''`); + run(`sudo mkdir -p ${escapedLicenseDir}`); + run(`echo '${escapedLicense}' | base64 --decode | sudo tee ${escapedLicenseDir}/licenses >/dev/null`); +} + +main(); diff --git a/.github/actions/00_infrastructure/setup_qnx_environment/post.js b/.github/actions/00_infrastructure/setup_qnx_environment/post.js new file mode 100644 index 000000000..cea992bb8 --- /dev/null +++ b/.github/actions/00_infrastructure/setup_qnx_environment/post.js @@ -0,0 +1,22 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// https://www.apache.org/licenses/LICENSE-2.0 +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +const { execSync } = require("node:child_process"); + +function main() { + const licenseDir = process.env.STATE_LICENSE_DIR || "/opt/score_qnx/license"; + const escapedLicenseDir = `'${licenseDir.replace(/'/g, `'\\''`)}'`; + execSync(`sudo rm -rf ${escapedLicenseDir}`, { stdio: "inherit" }); +} + +main(); diff --git a/.github/workflows/build_and_test_qnx.yml b/.github/workflows/build_and_test_qnx.yml index c8f8cef68..38809280b 100644 --- a/.github/workflows/build_and_test_qnx.yml +++ b/.github/workflows/build_and_test_qnx.yml @@ -48,8 +48,6 @@ concurrency: group: build_and_test_qnx-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} -env: - LICENSE_DIR: "/opt/score_qnx/license" jobs: precheck: runs-on: ubuntu-24.04 @@ -134,21 +132,10 @@ jobs: with: ref: ${{ github.head_ref || github.event.pull_request.head.ref || github.ref }} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - - name: Install QEMU - run: | - sudo apt-get update - sudo apt-get install -y qemu-system - - name: Enable KVM group perms - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - name: Setup QNX License - env: - SCORE_QNX_LICENSE: ${{ secrets.SCORE_QNX_LICENSE }} - run: | - sudo mkdir -p "${{ env.LICENSE_DIR }}" - echo "${SCORE_QNX_LICENSE}" | base64 --decode | sudo tee "${{ env.LICENSE_DIR }}/licenses" >/dev/null + - name: Setup QNX and QEMU environment + uses: ./.github/actions/00_infrastructure/setup_qnx_environment + with: + qnx-license: ${{ secrets.SCORE_QNX_LICENSE }} - name: Setup Bazel environment uses: ./.github/actions/00_infrastructure/prepare_bazel_environment with: @@ -179,6 +166,3 @@ jobs: SCORE_QNX_USER: ${{ secrets.SCORE_QNX_USER }} SCORE_QNX_PASSWORD: ${{ secrets.SCORE_QNX_PASSWORD }} run: bazel build --nobuild --config=qnx -- //score/... - - name: Cleanup QNX License - if: always() - run: sudo rm -rf "${{ env.LICENSE_DIR }}" From 76804921f8d7bdd0bd19ad2004c4e3e5a248ab1e Mon Sep 17 00:00:00 2001 From: Jan Schlosser Date: Tue, 28 Jul 2026 22:13:46 +0200 Subject: [PATCH 2/3] ci: add nightly flaky test detection pipeline Add a scheduled/dispatchable workflow that repeatedly runs the test suite across all supported configurations to surface flaky tests. - nightly_flaky_detection.yml orchestrates per-configuration runner jobs (gcc15 unit/integration, ASan/UBSan/LSan, TSan, QNX unit/integration) via the reusable _nightly_flaky_detection_runner.yml, using --runs_per_test with --runs_per_test_detects_flakes and collecting the Bazel build event protocol output per config. - collect_flaky_tests.py parses the per-config BEP into a structured report; merge_flaky_reports.py aggregates all configs into a single per-target summary (JSON + Markdown step summary) and exposes counts as workflow outputs. - Tests exclude themselves from detection via the no-flaky-test-detection tag; integration tests are tagged integration-test so the runner can select unit vs integration suites with --test_tag_filters. - README_flaky_detection.md documents the pipeline; the scripts are packaged as Bazel py_binary/py_test targets with unit tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../_nightly_flaky_detection_runner.yml | 182 ++++++++++++++++ .github/workflows/nightly_flaky_detection.yml | 176 ++++++++++++++++ BUILD | 1 + quality/README_flaky_detection.md | 71 +++++++ .../integration_testing.bzl | 17 ++ quality/scripts/BUILD | 43 ++++ quality/scripts/__init__.py | 12 ++ quality/scripts/collect_flaky_tests.py | 197 ++++++++++++++++++ quality/scripts/flaky_reports_test.py | 170 +++++++++++++++ quality/scripts/merge_flaky_reports.py | 142 +++++++++++++ quality/visibility_guard/BUILD | 1 + 11 files changed, 1012 insertions(+) create mode 100644 .github/workflows/_nightly_flaky_detection_runner.yml create mode 100644 .github/workflows/nightly_flaky_detection.yml create mode 100644 quality/README_flaky_detection.md create mode 100644 quality/scripts/BUILD create mode 100644 quality/scripts/__init__.py create mode 100644 quality/scripts/collect_flaky_tests.py create mode 100644 quality/scripts/flaky_reports_test.py create mode 100644 quality/scripts/merge_flaky_reports.py diff --git a/.github/workflows/_nightly_flaky_detection_runner.yml b/.github/workflows/_nightly_flaky_detection_runner.yml new file mode 100644 index 000000000..f723a48f3 --- /dev/null +++ b/.github/workflows/_nightly_flaky_detection_runner.yml @@ -0,0 +1,182 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +name: "Nightly Flaky Detection Runner" + +on: + workflow_call: + inputs: + config-name: + description: "Logical config name used in artifacts and reports." + type: string + required: true + bazel-config: + description: "Bazel --config value (empty for default host config)." + type: string + default: "" + target-pattern: + description: "Bazel target pattern to test." + type: string + required: true + test-tag-filters: + description: > + Test type selection via Bazel tags (e.g. "unit" or "integration-test"). + Combined with the always-applied "-no-flaky-test-detection" exclusion. + Leave empty to run all (non-excluded) tests. + type: string + default: "" + runs-per-test: + description: "Number of repeated executions per test target." + type: number + default: 100 + use-qnx-environment: + description: "Whether to setup QNX-specific CI requirements." + type: boolean + default: false + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: + required: false + SCORE_QNX_LICENSE: + required: false + SCORE_QNX_USER: + required: false + SCORE_QNX_PASSWORD: + required: false + outputs: + flaky_count: + description: "Number of flaky targets detected." + value: ${{ jobs.run-flaky.outputs.flaky_count }} + failed_count: + description: "Number of failed non-flaky targets." + value: ${{ jobs.run-flaky.outputs.failed_count }} + test_exit_code: + description: "Exit code from bazel test command." + value: ${{ jobs.run-flaky.outputs.test_exit_code }} + +jobs: + run-flaky: + name: ${{ inputs.config-name }} + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: write + outputs: + flaky_count: ${{ steps.summary.outputs.flaky_count }} + failed_count: ${{ steps.summary.outputs.failed_count }} + test_exit_code: ${{ steps.run-tests.outputs.test_exit_code }} + steps: + - uses: actions/checkout@v6.0.2 + + - name: Redirect Bazel root to large /mnt disk + run: | + # runs_per_test writes one test.log per run into bazel-testlogs (under + # the Bazel output base), which can exhaust the small root filesystem + # on GitHub-hosted runners. Redirect the DEFAULT Bazel root + # ($HOME/.cache/bazel) onto the larger /mnt ephemeral disk BEFORE the + # Bazel setup runs. Because the default path is unchanged, caches are + # restored/reused normally and Bazel still selects linux-sandbox - + # only the physical bytes move to /mnt. + # + # Use a bind mount rather than a symlink: the AppArmor profile that + # unblocks unprivileged user namespaces for linux-sandbox is attached + # by the binary's REAL resolved path. A symlink would resolve to + # /mnt/bazel-root/.../linux-sandbox, so the profile (registered under + # the $HOME/.cache/bazel/... path via `bazel info install_base`) would + # not match and the sandbox would fail with "mount: Permission denied". + # A bind mount keeps the logical $HOME/.cache/bazel path as the real + # path AppArmor sees while still storing the bytes on /mnt. + BAZEL_ROOT_TARGET="/mnt/bazel-root" + sudo mkdir -p "${BAZEL_ROOT_TARGET}" + sudo chown "$(id -u):$(id -g)" "${BAZEL_ROOT_TARGET}" + mkdir -p "${HOME}/.cache/bazel" + sudo mount --bind "${BAZEL_ROOT_TARGET}" "${HOME}/.cache/bazel" + + - uses: ./.github/actions/00_infrastructure/prepare_bazel_environment + with: + disk-cache: "" + cache-mode: disabled + repository-cache: "true" + ubuntu-snapshot-mirror-url: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + + - name: Setup QNX and QEMU environment + if: inputs.use-qnx-environment + uses: ./.github/actions/00_infrastructure/setup_qnx_environment + with: + qnx-license: ${{ secrets.SCORE_QNX_LICENSE }} + + - name: Run repeated Bazel tests for flaky detection + id: run-tests + continue-on-error: true + env: + SCORE_QNX_USER: ${{ secrets.SCORE_QNX_USER }} + SCORE_QNX_PASSWORD: ${{ secrets.SCORE_QNX_PASSWORD }} + run: | + set -o pipefail + + RAW_LOG="${RUNNER_TEMP}/bazel_${{ inputs.config-name }}.log" + BEP_FILE="${RUNNER_TEMP}/bep_${{ inputs.config-name }}.json" + ARGS=(test) + + if [[ -n "${{ inputs.bazel-config }}" ]]; then + ARGS+=("--config=${{ inputs.bazel-config }}") + fi + + TAG_FILTERS="-no-flaky-test-detection" + if [[ -n "${{ inputs.test-tag-filters }}" ]]; then + TAG_FILTERS="${{ inputs.test-tag-filters }},-no-flaky-test-detection" + fi + + ARGS+=( + "--build_tests_only" + "--keep_going" + "--runs_per_test=${{ inputs.runs-per-test }}" + "--runs_per_test_detects_flakes" + "--flaky_test_attempts=1" + "--test_tag_filters=${TAG_FILTERS}" + "--test_output=errors" + "--build_event_json_file=${BEP_FILE}" + "${{ inputs.target-pattern }}" + ) + + set +e + bazel "${ARGS[@]}" 2>&1 | tee "${RAW_LOG}" + EXIT_CODE=${PIPESTATUS[0]} + set -e + echo "test_exit_code=${EXIT_CODE}" >> "${GITHUB_OUTPUT}" + + - name: Collect flaky summary + id: summary + if: always() + run: | + REPORT_DIR="${RUNNER_TEMP}/nightly-flaky/${{ inputs.config-name }}" + bazel run //quality/scripts:collect_flaky_tests -- \ + --config-name "${{ inputs.config-name }}" \ + --bep-json "${RUNNER_TEMP}/bep_${{ inputs.config-name }}.json" \ + --raw-log "${RUNNER_TEMP}/bazel_${{ inputs.config-name }}.log" \ + --output-dir "${REPORT_DIR}" \ + --runs-per-test "${{ inputs.runs-per-test }}" \ + --test-exit-code "${{ steps.run-tests.outputs.test_exit_code || 99 }}" \ + --github-output "${GITHUB_OUTPUT}" + + - name: Upload per-config flaky report + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-flaky-${{ inputs.config-name }}-${{ github.run_id }} + path: | + ${{ runner.temp }}/nightly-flaky/${{ inputs.config-name }}/summary.json + ${{ runner.temp }}/nightly-flaky/${{ inputs.config-name }}/summary.md + ${{ runner.temp }}/bazel_${{ inputs.config-name }}.log + ${{ runner.temp }}/bep_${{ inputs.config-name }}.json + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/nightly_flaky_detection.yml b/.github/workflows/nightly_flaky_detection.yml new file mode 100644 index 000000000..b0050f7a8 --- /dev/null +++ b/.github/workflows/nightly_flaky_detection.yml @@ -0,0 +1,176 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +name: Nightly Flaky Test Detection + +on: + schedule: + - cron: "30 0 * * *" + workflow_dispatch: + inputs: + runs_per_test: + description: "Repeated executions per unit test target." + required: false + default: "300" + type: string + integration_runs_per_test: + description: "Repeated executions per integration test target." + required: false + default: "100" + type: string + +concurrency: + group: nightly-flaky-detection + cancel-in-progress: false + +permissions: + contents: read + +jobs: + flaky-gcc15-unit: + uses: ./.github/workflows/_nightly_flaky_detection_runner.yml + with: + config-name: gcc15-unit + bazel-config: "" + target-pattern: //... + test-tag-filters: unit + runs-per-test: ${{ fromJSON(github.event.inputs.runs_per_test || '300') }} + permissions: + contents: read + actions: write + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + + flaky-gcc15-integration: + uses: ./.github/workflows/_nightly_flaky_detection_runner.yml + with: + config-name: gcc15-integration + bazel-config: "" + target-pattern: //... + test-tag-filters: integration-test + runs-per-test: ${{ fromJSON(github.event.inputs.integration_runs_per_test || '100') }} + permissions: + contents: read + actions: write + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + + flaky-asan-ubsan-lsan: + uses: ./.github/workflows/_nightly_flaky_detection_runner.yml + with: + config-name: asan-ubsan-lsan + bazel-config: asan_ubsan_lsan + target-pattern: //... + test-tag-filters: unit + runs-per-test: ${{ fromJSON(github.event.inputs.runs_per_test || '300') }} + permissions: + contents: read + actions: write + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + + flaky-tsan: + uses: ./.github/workflows/_nightly_flaky_detection_runner.yml + with: + config-name: tsan + bazel-config: tsan + target-pattern: //... + test-tag-filters: unit + runs-per-test: ${{ fromJSON(github.event.inputs.runs_per_test || '300') }} + permissions: + contents: read + actions: write + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + + flaky-qnx-unit: + uses: ./.github/workflows/_nightly_flaky_detection_runner.yml + with: + config-name: qnx-unit + bazel-config: qnx + target-pattern: //score/... + test-tag-filters: unit + runs-per-test: ${{ fromJSON(github.event.inputs.runs_per_test || '300') }} + use-qnx-environment: true + permissions: + contents: read + actions: write + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + SCORE_QNX_LICENSE: ${{ secrets.SCORE_QNX_LICENSE }} + SCORE_QNX_USER: ${{ secrets.SCORE_QNX_USER }} + SCORE_QNX_PASSWORD: ${{ secrets.SCORE_QNX_PASSWORD }} + + flaky-qnx-integration: + uses: ./.github/workflows/_nightly_flaky_detection_runner.yml + with: + config-name: qnx-integration + bazel-config: qnx + target-pattern: //score/... + test-tag-filters: integration-test + runs-per-test: ${{ fromJSON(github.event.inputs.integration_runs_per_test || '100') }} + use-qnx-environment: true + permissions: + contents: read + actions: write + secrets: + UBUNTU_SNAPSHOT_MIRROR_URL: ${{ secrets.UBUNTU_SNAPSHOT_MIRROR_URL }} + SCORE_QNX_LICENSE: ${{ secrets.SCORE_QNX_LICENSE }} + SCORE_QNX_USER: ${{ secrets.SCORE_QNX_USER }} + SCORE_QNX_PASSWORD: ${{ secrets.SCORE_QNX_PASSWORD }} + + aggregate-flaky-results: + name: Aggregate flaky detection reports + needs: + - flaky-gcc15-unit + - flaky-gcc15-integration + - flaky-asan-ubsan-lsan + - flaky-tsan + - flaky-qnx-unit + - flaky-qnx-integration + if: always() + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@v6.0.2 + + - name: Download per-config artifacts + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: nightly-flaky-*-${{ github.run_id }} + path: /tmp/nightly_flaky + merge-multiple: false + + - name: Build consolidated report + id: merge + run: | + python3 quality/scripts/merge_flaky_reports.py \ + --reports-root /tmp/nightly_flaky \ + --output-json /tmp/nightly_flaky/merged/summary.json \ + --output-md /tmp/nightly_flaky/merged/summary.md \ + --github-output "${GITHUB_OUTPUT}" + + - name: Add summary to workflow + if: always() + run: cat /tmp/nightly_flaky/merged/summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload consolidated artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: nightly-flaky-summary-${{ github.run_id }} + path: /tmp/nightly_flaky/merged + retention-days: 14 diff --git a/BUILD b/BUILD index fb713a9da..3eece7440 100644 --- a/BUILD +++ b/BUILD @@ -69,6 +69,7 @@ format_test( cc = "@clang_format//:executable", no_sandbox = True, starlark = "@buildifier_prebuilt//:buildifier", + tags = ["no-flaky-test-detection"], target_compatible_with = ["@platforms//os:linux"], workspace = "//:LICENSE", ) diff --git a/quality/README_flaky_detection.md b/quality/README_flaky_detection.md new file mode 100644 index 000000000..1089b3ed9 --- /dev/null +++ b/quality/README_flaky_detection.md @@ -0,0 +1,71 @@ +# Nightly Flaky Detection + +The `Nightly Flaky Test Detection` workflow runs repeated Bazel test executions and reports unstable targets. + +## Workflow + +- File: `.github/workflows/nightly_flaky_detection.yml` +- Schedule: daily (`00:30 UTC`) +- Manual trigger: GitHub Actions `workflow_dispatch` + +## Configurations covered + +- `gcc15` (`//...`) +- `asan-ubsan-lsan` (`//...` with `--config=asan_ubsan_lsan`) +- `tsan` (`//...` with `--config=tsan`) +- `qnx` (`//score/...` with `--config=qnx`) + +## Detection mode + +The runner uses: + +- `--runs_per_test=` (default `1000`) +- `--runs_per_test_detects_flakes` +- `--flaky_test_attempts=1` +- `--test_tag_filters=-no-flaky-test-detection` +- `--keep_going` +- `--build_tests_only` + +This setup catches tests that are unstable across repeated runs and prevents retry masking. +Tests tagged `no-flaky-test-detection` are excluded from this workflow. + +Flaky classification: + +- accepted flaky: failed runs per 1000 <= configured threshold +- non-acceptable flaky: failed runs per 1000 > configured threshold + +Default threshold: + +- `acceptable_failures_per_thousand = 10` (so worse than 10/1000 is non-acceptable) + +## Cache policy + +Nightly flaky detection is intentionally run with cache disabled: + +- `cache-mode: disabled` +- empty Bazel disk cache key +- `repository-cache: "true"` (enabled for dependency/download reuse inside nightly jobs) + +## Reports and artifacts + +Per configuration artifact contains: + +- `summary.json`: machine-readable counts + target lists +- `summary.md`: human-readable report +- Bazel raw log and BEP JSON used for extraction + +Reports are generated through Bazel-invoked tools: + +- `bazel run //quality/scripts:collect_flaky_tests` +- `bazel run //quality/scripts:merge_flaky_reports` + +Consolidated artifact: + +- `nightly-flaky-summary-` + +The aggregate job fails the workflow when `total_flaky_count > 0`. + +## Tuning + +- Increase/decrease `runs_per_test` via manual dispatch input. +- Tune `acceptable_failures_per_thousand` via manual dispatch input. diff --git a/quality/integration_testing/integration_testing.bzl b/quality/integration_testing/integration_testing.bzl index fbeee7311..5e8f70d19 100644 --- a/quality/integration_testing/integration_testing.bzl +++ b/quality/integration_testing/integration_testing.bzl @@ -130,6 +130,14 @@ def integration_test(name, srcs, filesystem, **kwargs): ["@score_cpp_policies//sanitizers/constraints:no_tsan"], ) + # Tag as integration test so flaky-detection and other tooling can select + # these via --test_tag_filters=integration-test. + _extend_list_in_kwargs_without_duplicates( + kwargs, + "tags", + ["integration-test"], + ) + py_itf_test( name = name, srcs = srcs, @@ -221,6 +229,15 @@ def dual_qemu_integration_test( if "flaky" not in kwargs: kwargs["flaky"] = True + # This test is intentionally marked flaky (above) to retry environment-induced + # QEMU boot hiccups, so exclude it from the nightly flaky-test detection to + # avoid reporting expected, infrastructure-level nondeterminism. + _extend_list_in_kwargs_without_duplicates( + kwargs, + "tags", + ["no-flaky-test-detection"], + ) + _extend_list_in_kwargs_without_duplicates( kwargs, "target_compatible_with", diff --git a/quality/scripts/BUILD b/quality/scripts/BUILD new file mode 100644 index 000000000..8e8826628 --- /dev/null +++ b/quality/scripts/BUILD @@ -0,0 +1,43 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +py_library( + name = "flaky_reports_lib", + srcs = [ + "__init__.py", + "collect_flaky_tests.py", + "merge_flaky_reports.py", + ], +) + +py_binary( + name = "collect_flaky_tests", + srcs = ["collect_flaky_tests.py"], + main = "collect_flaky_tests.py", + deps = [":flaky_reports_lib"], +) + +py_binary( + name = "merge_flaky_reports", + srcs = ["merge_flaky_reports.py"], + main = "merge_flaky_reports.py", + deps = [":flaky_reports_lib"], +) + +py_test( + name = "flaky_reports_test", + srcs = ["flaky_reports_test.py"], + deps = [":flaky_reports_lib"], +) diff --git a/quality/scripts/__init__.py b/quality/scripts/__init__.py new file mode 100644 index 000000000..ca5de742e --- /dev/null +++ b/quality/scripts/__init__.py @@ -0,0 +1,12 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* diff --git a/quality/scripts/collect_flaky_tests.py b/quality/scripts/collect_flaky_tests.py new file mode 100644 index 000000000..d7b32fbcc --- /dev/null +++ b/quality/scripts/collect_flaky_tests.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Collect flaky/failed test targets from Bazel BEP and raw logs.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Sequence + + +FAILED_STATES = {"FAILED", "TIMEOUT", "INCOMPLETE", "REMOTE_FAILURE", "FAILED_TO_BUILD"} +PRIORITY = {"PASSED": 1, "FAILED": 2, "FLAKY": 3} + + +def _merge_status(current: str | None, incoming: str) -> str: + if current is None: + return incoming + return incoming if PRIORITY.get(incoming, 0) > PRIORITY.get(current, 0) else current + + +def _parse_bep(path: Path, target_stats: dict[str, dict]) -> None: + if not path.is_file(): + return + + with path.open(encoding="utf-8") as stream: + for line in stream: + stripped = line.strip() + if not stripped: + continue + try: + event = json.loads(stripped) + except json.JSONDecodeError: + continue + + event_id = event.get("id", {}) + summary_id = event_id.get("testSummary", {}) + label = summary_id.get("label") + summary = event.get("testSummary", {}) + overall = str(summary.get("overallStatus", "")).upper() + if not label: + continue + failed_runs_value = summary.get("failedRunCount") + if failed_runs_value is None: + failed_runs_value = len(summary.get("failed", [])) + total_runs_value = summary.get("totalRunCount") + if total_runs_value is None: + total_runs_value = summary.get("runCount", 0) + failed_runs = int(failed_runs_value) + total_runs = int(total_runs_value) + if total_runs <= 0: + total_runs = len(summary.get("failed", [])) + len(summary.get("passed", [])) + + current = target_stats.get(label, {"status": None, "failed_runs": 0, "total_runs": 0}) + if overall == "FLAKY": + current["status"] = _merge_status(current["status"], "FLAKY") + elif overall in FAILED_STATES: + current["status"] = _merge_status(current["status"], "FAILED") + elif overall == "PASSED": + current["status"] = _merge_status(current["status"], "PASSED") + current["failed_runs"] = max(int(current["failed_runs"]), failed_runs) + current["total_runs"] = max(int(current["total_runs"]), total_runs) + target_stats[label] = current + + +def _parse_raw_log(path: Path, target_stats: dict[str, dict]) -> None: + if not path.is_file(): + return + + line_pattern = re.compile(r"^\s*(//\S+)\s+(PASSED|FAILED|FLAKY)\b") + with path.open(encoding="utf-8", errors="replace") as stream: + for line in stream: + match = line_pattern.match(line) + if not match: + continue + target = match.group(1) + status = match.group(2) + current = target_stats.get(target, {"status": None, "failed_runs": 0, "total_runs": 0}) + current["status"] = _merge_status(current["status"], status) + target_stats[target] = current + + +def _write_markdown(summary: dict, output_md: Path) -> None: + flaky = summary["flaky_tests"] + failed = summary["failed_tests"] + + lines = [ + f"# Nightly Flaky Test Report ({summary['config_name']})", + "", + f"- Flaky targets: **{summary['flaky_count']}**", + f"- Runs per test for measurement: **{summary['runs_per_test']}**", + f"- Failed targets: **{summary['failed_count']}**", + f"- Passed targets: **{summary['passed_count']}**", + f"- Bazel test exit code: **{summary['test_exit_code']}**", + "", + ] + if flaky: + lines.extend(["## Flaky Targets", ""]) + for item in summary["flaky_test_details"]: + lines.append( + f"- `{item['target']}` — {item['failed_runs']}/{item['total_runs']} failed " + f"({item['failures_per_thousand']:.2f}/1000)" + ) + lines.append("") + if failed: + lines.extend(["## Failed Targets (non-flaky)", ""]) + lines.extend([f"- `{target}`" for target in failed]) + lines.append("") + if not flaky and not failed: + lines.extend(["No flaky or failed targets were detected.", ""]) + + output_md.write_text("\n".join(lines), encoding="utf-8") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect flaky tests from Bazel outputs.") + parser.add_argument("--config-name", required=True) + parser.add_argument("--bep-json", required=True) + parser.add_argument("--raw-log", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--test-exit-code", required=True, type=int) + parser.add_argument("--runs-per-test", required=False, type=int, default=1000) + parser.add_argument("--github-output", required=False, default="") + args = parser.parse_args(argv) + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + target_stats: dict[str, dict] = {} + _parse_bep(Path(args.bep_json), target_stats) + _parse_raw_log(Path(args.raw_log), target_stats) + + flaky_details = [] + for target, stats in target_stats.items(): + if stats["status"] != "FLAKY": + continue + total_runs = int(stats["total_runs"]) if int(stats["total_runs"]) > 0 else int(args.runs_per_test) + failed_runs = int(stats["failed_runs"]) + if failed_runs <= 0: + failed_runs = 1 + failures_per_thousand = (failed_runs * 1000.0 / total_runs) if total_runs > 0 else 0.0 + flaky_details.append( + { + "target": target, + "failed_runs": failed_runs, + "total_runs": total_runs, + "failures_per_thousand": failures_per_thousand, + } + ) + flaky_details.sort(key=lambda item: (-item["failures_per_thousand"], item["target"])) + + flaky_tests = [item["target"] for item in flaky_details] + failed_tests = sorted(target for target, stats in target_stats.items() if stats["status"] == "FAILED") + passed_tests = sorted(target for target, stats in target_stats.items() if stats["status"] == "PASSED") + + summary = { + "config_name": args.config_name, + "flaky_tests": flaky_tests, + "flaky_test_details": flaky_details, + "failed_tests": failed_tests, + "passed_tests": passed_tests, + "flaky_count": len(flaky_tests), + "failed_count": len(failed_tests), + "passed_count": len(passed_tests), + "runs_per_test": args.runs_per_test, + "test_exit_code": args.test_exit_code, + } + + output_json = output_dir / "summary.json" + output_md = output_dir / "summary.md" + output_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + _write_markdown(summary, output_md) + + if args.github_output: + output_path = Path(args.github_output) + with output_path.open("a", encoding="utf-8") as stream: + stream.write(f"flaky_count={summary['flaky_count']}\n") + stream.write(f"failed_count={summary['failed_count']}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/quality/scripts/flaky_reports_test.py b/quality/scripts/flaky_reports_test.py new file mode 100644 index 000000000..da4d58362 --- /dev/null +++ b/quality/scripts/flaky_reports_test.py @@ -0,0 +1,170 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +import json +import os +import tempfile +import unittest + +from quality.scripts import collect_flaky_tests, merge_flaky_reports + + +class CollectFlakyTest(unittest.TestCase): + def test_collect_reports_flaky_and_failed_targets(self): + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "bep.json") + raw_path = os.path.join(tmp, "raw.log") + out_dir = os.path.join(tmp, "out") + gh_out = os.path.join(tmp, "gh.out") + + with open(bep_path, "w", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + "id": {"testSummary": {"label": "//pkg:flaky_low"}}, + "testSummary": {"overallStatus": "FLAKY", "failed": [{}] * 5, "passed": [{}] * 995}, + } + ) + + "\n" + ) + stream.write( + json.dumps( + { + "id": {"testSummary": {"label": "//pkg:flaky_high"}}, + "testSummary": {"overallStatus": "FLAKY", "failed": [{}] * 11, "passed": [{}] * 989}, + } + ) + + "\n" + ) + stream.write( + json.dumps( + { + "id": {"testSummary": {"label": "//pkg:failed"}}, + "testSummary": {"overallStatus": "FAILED", "failed": [{}], "passed": []}, + } + ) + + "\n" + ) + with open(raw_path, "w", encoding="utf-8") as stream: + stream.write("//pkg:passed PASSED in 0.2s\n") + + exit_code = collect_flaky_tests.main( + [ + "--config-name", + "gcc15", + "--bep-json", + bep_path, + "--raw-log", + raw_path, + "--output-dir", + out_dir, + "--test-exit-code", + "1", + "--runs-per-test", + "1000", + "--github-output", + gh_out, + ] + ) + self.assertEqual(exit_code, 0) + + with open(os.path.join(out_dir, "summary.json"), encoding="utf-8") as stream: + summary = json.load(stream) + self.assertEqual(summary["flaky_count"], 2) + self.assertEqual(summary["failed_count"], 1) + self.assertEqual(summary["passed_count"], 1) + self.assertEqual(summary["flaky_tests"], ["//pkg:flaky_high", "//pkg:flaky_low"]) + self.assertEqual(summary["failed_tests"], ["//pkg:failed"]) + self.assertNotIn("accepted_flaky_tests", summary) + self.assertNotIn("non_acceptable_flaky_count", summary) + + with open(gh_out, encoding="utf-8") as stream: + output_text = stream.read() + self.assertIn("flaky_count=2", output_text) + self.assertIn("failed_count=1", output_text) + self.assertNotIn("accepted_flaky_count", output_text) + + +class MergeFlakyTest(unittest.TestCase): + def test_merge_aggregates_targets_across_configs(self): + with tempfile.TemporaryDirectory() as tmp: + reports_root = os.path.join(tmp, "reports") + os.makedirs(os.path.join(reports_root, "a"), exist_ok=True) + os.makedirs(os.path.join(reports_root, "b"), exist_ok=True) + out_json = os.path.join(tmp, "merged", "summary.json") + out_md = os.path.join(tmp, "merged", "summary.md") + gh_out = os.path.join(tmp, "gh.out") + + with open(os.path.join(reports_root, "a", "summary.json"), "w", encoding="utf-8") as stream: + json.dump( + { + "config_name": "tsan", + "flaky_count": 1, + "flaky_test_details": [ + {"target": "//pkg:shared", "failed_runs": 7, "total_runs": 300, "failures_per_thousand": 23.3} + ], + "failed_count": 0, + "passed_count": 10, + "test_exit_code": 1, + }, + stream, + ) + with open(os.path.join(reports_root, "b", "summary.json"), "w", encoding="utf-8") as stream: + json.dump( + { + "config_name": "asan", + "flaky_count": 1, + "flaky_test_details": [ + {"target": "//pkg:shared", "failed_runs": 5, "total_runs": 300, "failures_per_thousand": 16.6} + ], + "failed_count": 2, + "passed_count": 5, + "test_exit_code": 2, + }, + stream, + ) + + exit_code = merge_flaky_reports.main( + [ + "--reports-root", + reports_root, + "--output-json", + out_json, + "--output-md", + out_md, + "--github-output", + gh_out, + ] + ) + self.assertEqual(exit_code, 0) + + with open(out_json, encoding="utf-8") as stream: + merged = json.load(stream) + self.assertEqual(merged["total_flaky_count"], 1) + self.assertEqual(merged["total_failed_count"], 2) + self.assertEqual(len(merged["flaky_targets"]), 1) + target = merged["flaky_targets"][0] + self.assertEqual(target["target"], "//pkg:shared") + self.assertEqual(target["failed_runs"], 12) + self.assertEqual(target["total_runs"], 600) + self.assertEqual(set(target["configs"]), {"tsan", "asan"}) + self.assertEqual(target["configs"]["tsan"]["failed_runs"], 7) + + with open(gh_out, encoding="utf-8") as stream: + output_text = stream.read() + self.assertIn("total_flaky_count=1", output_text) + self.assertIn("total_failed_count=2", output_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/quality/scripts/merge_flaky_reports.py b/quality/scripts/merge_flaky_reports.py new file mode 100644 index 000000000..c0ee39277 --- /dev/null +++ b/quality/scripts/merge_flaky_reports.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Merge per-config flaky-test summaries into one consolidated report. + +The consolidated report aggregates each flaky target across all configs it was +detected in, so downstream consumers (e.g. the GitHub issue sync) get one entry +per target with a per-config breakdown. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Sequence + + +def _load_summaries(reports_root: Path) -> list[dict]: + summaries = [] + for path in sorted(reports_root.rglob("summary.json")): + with path.open(encoding="utf-8") as stream: + summaries.append(json.load(stream)) + return summaries + + +def _aggregate_targets(summaries: list[dict]) -> list[dict]: + """Aggregate flaky target details across configs, keyed by target.""" + targets: dict[str, dict] = {} + for summary in summaries: + config_name = summary["config_name"] + for item in summary.get("flaky_test_details", []): + target = item["target"] + failed_runs = int(item.get("failed_runs", 0)) + total_runs = int(item.get("total_runs", 0)) + entry = targets.setdefault( + target, + {"target": target, "failed_runs": 0, "total_runs": 0, "configs": {}}, + ) + entry["failed_runs"] += failed_runs + entry["total_runs"] += total_runs + entry["configs"][config_name] = { + "failed_runs": failed_runs, + "total_runs": total_runs, + } + aggregated = list(targets.values()) + aggregated.sort(key=lambda item: (-item["failed_runs"], item["target"])) + return aggregated + + +def _build_markdown( + summaries: list[dict], + flaky_targets: list[dict], + total_failed: int, +) -> str: + lines = [ + "# Nightly Flaky Detection Summary", + "", + f"- Total flaky targets: **{len(flaky_targets)}**", + f"- Total failed targets (non-flaky): **{total_failed}**", + "", + "| Config | Flaky | Failed | Passed | Bazel exit code |", + "|--------|------:|------:|------:|----------------:|", + ] + for item in summaries: + lines.append( + f"| `{item['config_name']}` | {item['flaky_count']} | " + f"{item['failed_count']} | {item['passed_count']} | {item['test_exit_code']} |" + ) + lines.append("") + if flaky_targets: + lines.extend(["## Flaky targets", ""]) + for item in flaky_targets: + configs = ", ".join(sorted(item["configs"])) + lines.append( + f"- `{item['target']}` — {item['failed_runs']}/{item['total_runs']} failed " + f"across [{configs}]" + ) + lines.append("") + if not flaky_targets: + lines.append(":white_check_mark: No flaky targets detected.") + else: + lines.append( + ":warning: Flaky targets detected. GitHub issues have been created/updated for each." + ) + lines.append("") + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Merge flaky detection summaries.") + parser.add_argument("--reports-root", required=True) + parser.add_argument("--output-json", required=True) + parser.add_argument("--output-md", required=True) + parser.add_argument("--github-output", required=False, default="") + args = parser.parse_args(argv) + + summaries = _load_summaries(Path(args.reports_root)) + flaky_targets = _aggregate_targets(summaries) + total_flaky = len(flaky_targets) + total_failed = sum(int(item.get("failed_count", 0)) for item in summaries) + total_passed = sum(int(item.get("passed_count", 0)) for item in summaries) + + merged = { + "total_flaky_count": total_flaky, + "total_failed_count": total_failed, + "total_passed_count": total_passed, + "flaky_targets": flaky_targets, + "config_summaries": summaries, + } + + output_json = Path(args.output_json) + output_md = Path(args.output_md) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8") + output_md.write_text( + _build_markdown(summaries, flaky_targets, total_failed), + encoding="utf-8", + ) + + if args.github_output: + output_path = Path(args.github_output) + with output_path.open("a", encoding="utf-8") as stream: + stream.write(f"total_flaky_count={total_flaky}\n") + stream.write(f"total_failed_count={total_failed}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/quality/visibility_guard/BUILD b/quality/visibility_guard/BUILD index cf8df6f91..38bacb300 100644 --- a/quality/visibility_guard/BUILD +++ b/quality/visibility_guard/BUILD @@ -38,6 +38,7 @@ py_test( "external", "lint", "no-cache", + "no-flaky-test-detection", "no-sandbox", ], target_compatible_with = ["@platforms//os:linux"], From 617c082fa490dac7838ec73cd60f00f67257df47 Mon Sep 17 00:00:00 2001 From: Jan Schlosser Date: Tue, 28 Jul 2026 22:14:26 +0200 Subject: [PATCH 3/3] ci: report flaky tests as persistent per-target GitHub issues Replace the ephemeral report-only model with self-updating GitHub issues, so each flaky test target has a single durable, deduplicated issue that accumulates history across nightly runs. - sync_flaky_issues.py reconciles the merged flaky-target summary against existing issues: it creates one issue per target on first detection and, on recurrence, increments a cumulative counter, refreshes a managed stats region in the body and appends a run comment. Per-run comments are the source of truth (cumulative totals are recomputed from them), the run_id makes reruns idempotent, and closed issues are reopened rather than duplicated. Robust to human edits: a corrupted managed body block is rebuilt from the comment ledger. - The aggregate job gains issues: write permission and a sync step that runs after the report is built. - Unit tests cover creation, recurrence increment, idempotency, reopen and corrupted-body recovery. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/nightly_flaky_detection.yml | 16 + quality/scripts/BUILD | 8 + quality/scripts/flaky_reports_test.py | 170 +++++- quality/scripts/sync_flaky_issues.py | 504 ++++++++++++++++++ 4 files changed, 697 insertions(+), 1 deletion(-) create mode 100644 quality/scripts/sync_flaky_issues.py diff --git a/.github/workflows/nightly_flaky_detection.yml b/.github/workflows/nightly_flaky_detection.yml index b0050f7a8..c28144167 100644 --- a/.github/workflows/nightly_flaky_detection.yml +++ b/.github/workflows/nightly_flaky_detection.yml @@ -143,6 +143,7 @@ jobs: permissions: contents: read actions: read + issues: write steps: - uses: actions/checkout@v6.0.2 @@ -167,6 +168,21 @@ jobs: if: always() run: cat /tmp/nightly_flaky/merged/summary.md >> "$GITHUB_STEP_SUMMARY" + - name: Sync flaky test issues + if: always() + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + if [[ ! -f /tmp/nightly_flaky/merged/summary.json ]]; then + echo "No merged summary found; skipping flaky issue sync." + exit 0 + fi + python3 quality/scripts/sync_flaky_issues.py \ + --merged-summary /tmp/nightly_flaky/merged/summary.json \ + --repo "${{ github.repository }}" \ + --run-id "${{ github.run_id }}" \ + --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + - name: Upload consolidated artifact if: always() uses: actions/upload-artifact@v4 diff --git a/quality/scripts/BUILD b/quality/scripts/BUILD index 8e8826628..c63c2412e 100644 --- a/quality/scripts/BUILD +++ b/quality/scripts/BUILD @@ -19,6 +19,7 @@ py_library( "__init__.py", "collect_flaky_tests.py", "merge_flaky_reports.py", + "sync_flaky_issues.py", ], ) @@ -36,6 +37,13 @@ py_binary( deps = [":flaky_reports_lib"], ) +py_binary( + name = "sync_flaky_issues", + srcs = ["sync_flaky_issues.py"], + main = "sync_flaky_issues.py", + deps = [":flaky_reports_lib"], +) + py_test( name = "flaky_reports_test", srcs = ["flaky_reports_test.py"], diff --git a/quality/scripts/flaky_reports_test.py b/quality/scripts/flaky_reports_test.py index da4d58362..b07e8967f 100644 --- a/quality/scripts/flaky_reports_test.py +++ b/quality/scripts/flaky_reports_test.py @@ -16,7 +16,17 @@ import tempfile import unittest -from quality.scripts import collect_flaky_tests, merge_flaky_reports +from quality.scripts import collect_flaky_tests, merge_flaky_reports, sync_flaky_issues +from quality.scripts.sync_flaky_issues import ( + Issue, + RunContext, + RunRecord, + aggregate, + merge_body, + parse_run_records, + render_body, + sync, +) class CollectFlakyTest(unittest.TestCase): @@ -166,5 +176,163 @@ def test_merge_aggregates_targets_across_configs(self): self.assertIn("total_failed_count=2", output_text) +class FakeGitHubClient: + """In-memory GitHub client for testing sync orchestration.""" + + def __init__(self): + self.issues = {} + self.comments = {} + self._next_number = 1 + self.labels_ensured = [] + self.reopened = [] + + def ensure_label(self, name): + self.labels_ensured.append(name) + + def _find(self, target): + for issue in self.issues.values(): + marker = sync_flaky_issues.TARGET_MARKER_RE.search(issue.body or "") + if marker and marker.group("target") == target: + return issue + return None + + def search_issue(self, target): + return self._find(target) + + def list_run_comments(self, issue): + return list(self.comments.get(issue.number, [])) + + def create_issue(self, title, body, labels): + number = self._next_number + self._next_number += 1 + issue = Issue(number=number, body=body, state="open", labels=list(labels)) + self.issues[number] = issue + self.comments[number] = [] + return issue + + def update_issue_body(self, issue, body): + self.issues[issue.number].body = body + issue.body = body + + def reopen_issue(self, issue): + self.issues[issue.number].state = "open" + issue.state = "open" + self.reopened.append(issue.number) + + def add_comment(self, issue, body): + self.comments.setdefault(issue.number, []).append(body) + + +def _summary(target, failed, total, configs): + return { + "flaky_targets": [ + {"target": target, "failed_runs": failed, "total_runs": total, "configs": configs} + ] + } + + +class SyncFlakyIssuesTest(unittest.TestCase): + def _ctx(self, run_id="100"): + return RunContext(run_id=run_id, run_url=f"https://ci/{run_id}", date="2026-07-27") + + def test_new_target_creates_issue_with_single_comment(self): + client = FakeGitHubClient() + summary = _summary("//pkg:a", 7, 300, {"tsan": {"failed_runs": 7, "total_runs": 300}}) + actions = sync(summary, client, self._ctx()) + + self.assertEqual(actions[0]["action"], "created") + self.assertEqual(len(client.issues), 1) + issue = client.issues[1] + self.assertIn("flaky-test", issue.labels) + self.assertIn("", issue.body) + # Exactly one run comment written on creation. + self.assertEqual(len(client.comments[1]), 1) + self.assertIn("Cumulative failed runs observed:** 7", issue.body) + + def test_recurrence_increments_counter_without_duplicate_issue(self): + client = FakeGitHubClient() + cfg = {"tsan": {"failed_runs": 7, "total_runs": 300}} + sync(_summary("//pkg:a", 7, 300, cfg), client, self._ctx("100")) + sync(_summary("//pkg:a", 4, 300, cfg), client, self._ctx("101")) + + self.assertEqual(len(client.issues), 1) + self.assertEqual(len(client.comments[1]), 2) + issue = client.issues[1] + self.assertIn("Cumulative failed runs observed:** 11", issue.body) + self.assertIn("over 600 total runs, 2 nightlies", issue.body) + + def test_same_run_id_is_idempotent(self): + client = FakeGitHubClient() + cfg = {"tsan": {"failed_runs": 7, "total_runs": 300}} + sync(_summary("//pkg:a", 7, 300, cfg), client, self._ctx("100")) + result = sync(_summary("//pkg:a", 7, 300, cfg), client, self._ctx("100")) + + self.assertEqual(result[0]["action"], "skipped-duplicate") + self.assertEqual(len(client.comments[1]), 1) + self.assertIn("Cumulative failed runs observed:** 7", client.issues[1].body) + + def test_closed_issue_is_reopened_on_recurrence(self): + client = FakeGitHubClient() + cfg = {"tsan": {"failed_runs": 7, "total_runs": 300}} + sync(_summary("//pkg:a", 7, 300, cfg), client, self._ctx("100")) + client.issues[1].state = "closed" + + result = sync(_summary("//pkg:a", 3, 300, cfg), client, self._ctx("101")) + + self.assertEqual(result[0]["action"], "reopened") + self.assertIn(1, client.reopened) + self.assertEqual(client.issues[1].state, "open") + self.assertIn("Reopened", client.comments[1][-1]) + + def test_corrupted_body_block_recovers_from_comments(self): + client = FakeGitHubClient() + cfg = {"tsan": {"failed_runs": 7, "total_runs": 300}} + sync(_summary("//pkg:a", 7, 300, cfg), client, self._ctx("100")) + # A human corrupts the managed region but keeps the identity marker. + client.issues[1].body = "\nhuman notes, block deleted" + + sync(_summary("//pkg:a", 4, 300, cfg), client, self._ctx("101")) + + issue = client.issues[1] + self.assertIn("human notes, block deleted", issue.body) + self.assertIn("Cumulative failed runs observed:** 11", issue.body) + + +class SyncPureFunctionsTest(unittest.TestCase): + def test_parse_run_records_ignores_malformed(self): + good = RunRecord("1", "2026-07-27", 3, 300, {"tsan": {"failed_runs": 3, "total_runs": 300}}).to_marker() + bodies = [good, "just a human comment", ""] + records = parse_run_records(bodies) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].failed_runs, 3) + + def test_aggregate_sums_across_configs(self): + records = [ + RunRecord("1", "2026-07-20", 3, 300, {"tsan": {"failed_runs": 3, "total_runs": 300}}), + RunRecord("2", "2026-07-27", 5, 300, {"asan": {"failed_runs": 5, "total_runs": 300}}), + ] + stats = aggregate("//pkg:a", records) + self.assertEqual(stats.cumulative_failed_runs, 8) + self.assertEqual(stats.cumulative_total_runs, 600) + self.assertEqual(stats.nightly_count, 2) + self.assertEqual(stats.first_seen, "2026-07-20") + self.assertEqual(stats.last_seen, "2026-07-27") + + def test_merge_body_preserves_prose_outside_region(self): + stats = aggregate("//pkg:a", [RunRecord("1", "2026-07-27", 1, 10, {})]) + region = render_body(stats) + existing = f"Human triage notes.\n\n{region}\n\nMore notes below." + # New stats after a second run. + stats2 = aggregate( + "//pkg:a", + [RunRecord("1", "2026-07-27", 1, 10, {}), RunRecord("2", "2026-07-28", 2, 10, {})], + ) + merged = merge_body(existing, render_body(stats2)) + self.assertIn("Human triage notes.", merged) + self.assertIn("More notes below.", merged) + self.assertIn("Cumulative failed runs observed:** 3", merged) + self.assertEqual(merged.count(sync_flaky_issues.STATS_BEGIN), 1) + + if __name__ == "__main__": unittest.main() diff --git a/quality/scripts/sync_flaky_issues.py b/quality/scripts/sync_flaky_issues.py new file mode 100644 index 000000000..98f33bc82 --- /dev/null +++ b/quality/scripts/sync_flaky_issues.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Create/update persistent GitHub issues for flaky test targets. + +One issue is maintained per flaky Bazel test target. On each nightly run where a +target is flaky, a machine-readable run record is appended as an issue comment +(the durable, append-only ledger) and the issue body's managed stats region is +regenerated from the full comment ledger. Closed issues are reopened on +recurrence; issues are never auto-closed. + +The module is split into a pure-logic core (no I/O) and an injectable GitHub +client, so the decision logic is fully unit-testable without network access. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable, Protocol, Sequence + +FLAKY_LABEL = "flaky-test" + +TARGET_MARKER_RE = re.compile(r"") +RUN_MARKER_RE = re.compile(r"", re.DOTALL) +STATS_BEGIN = "" +STATS_END = "" +STATS_REGION_RE = re.compile( + re.escape(STATS_BEGIN) + r".*?" + re.escape(STATS_END), + re.DOTALL, +) + + +# --------------------------------------------------------------------------- # +# Data model +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class RunRecord: + """A single nightly observation of a target's flakiness.""" + + run_id: str + date: str + failed_runs: int + total_runs: int + configs: dict[str, dict[str, int]] + + def to_marker(self) -> str: + payload = { + "run_id": self.run_id, + "date": self.date, + "failed_runs": self.failed_runs, + "total_runs": self.total_runs, + "configs": self.configs, + } + return f"" + + +@dataclass +class Stats: + """Cumulative statistics derived from the full run ledger.""" + + target: str + cumulative_failed_runs: int + cumulative_total_runs: int + nightly_count: int + first_seen: str + last_seen: str + configs: dict[str, dict[str, int]] + + +@dataclass +class Issue: + number: int + body: str + state: str = "open" # "open" | "closed" + labels: list[str] = field(default_factory=list) + + +@dataclass +class RunContext: + run_id: str + run_url: str + date: str + + +# --------------------------------------------------------------------------- # +# GitHub client interface +# --------------------------------------------------------------------------- # +class GitHubClient(Protocol): + def ensure_label(self, name: str) -> None: ... + + def search_issue(self, target: str) -> Issue | None: ... + + def list_run_comments(self, issue: Issue) -> list[str]: ... + + def create_issue(self, title: str, body: str, labels: list[str]) -> Issue: ... + + def update_issue_body(self, issue: Issue, body: str) -> None: ... + + def reopen_issue(self, issue: Issue) -> None: ... + + def add_comment(self, issue: Issue, body: str) -> None: ... + + +# --------------------------------------------------------------------------- # +# Pure functions (no I/O) — the tested core +# --------------------------------------------------------------------------- # +def parse_run_records(comment_bodies: Iterable[str]) -> list[RunRecord]: + """Extract RunRecords from bot comment bodies via the flaky-run marker. + + Malformed or non-ledger comments are ignored. Duplicate run_ids are + de-duplicated (first occurrence wins), making recomputation idempotent. + """ + records: dict[str, RunRecord] = {} + for body in comment_bodies: + if not body: + continue + match = RUN_MARKER_RE.search(body) + if not match: + continue + try: + payload = json.loads(match.group("json")) + except (json.JSONDecodeError, TypeError): + continue + run_id = str(payload.get("run_id", "")).strip() + if not run_id or run_id in records: + continue + records[run_id] = RunRecord( + run_id=run_id, + date=str(payload.get("date", "")), + failed_runs=int(payload.get("failed_runs", 0)), + total_runs=int(payload.get("total_runs", 0)), + configs={ + str(name): { + "failed_runs": int(vals.get("failed_runs", 0)), + "total_runs": int(vals.get("total_runs", 0)), + } + for name, vals in dict(payload.get("configs", {})).items() + }, + ) + return list(records.values()) + + +def aggregate(target: str, records: Sequence[RunRecord]) -> Stats: + """Aggregate the full run ledger into cumulative Stats.""" + cumulative_failed = sum(r.failed_runs for r in records) + cumulative_total = sum(r.total_runs for r in records) + dates = sorted(r.date for r in records if r.date) + configs: dict[str, dict[str, int]] = {} + for record in records: + for name, vals in record.configs.items(): + entry = configs.setdefault(name, {"failed_runs": 0, "total_runs": 0}) + entry["failed_runs"] += int(vals.get("failed_runs", 0)) + entry["total_runs"] += int(vals.get("total_runs", 0)) + return Stats( + target=target, + cumulative_failed_runs=cumulative_failed, + cumulative_total_runs=cumulative_total, + nightly_count=len(records), + first_seen=dates[0] if dates else "", + last_seen=dates[-1] if dates else "", + configs=configs, + ) + + +def render_body(stats: Stats) -> str: + """Render the full managed stats region (markers + counters + table + JSON).""" + lines = [ + STATS_BEGIN, + f"", + f"## Flaky test: `{stats.target}`", + "", + ( + f"**Cumulative failed runs observed:** {stats.cumulative_failed_runs} " + f"(over {stats.cumulative_total_runs} total runs, {stats.nightly_count} nightlies)" + ), + f"**First seen:** {stats.first_seen or 'n/a'} · **Last seen:** {stats.last_seen or 'n/a'}", + "", + "### Per-config cumulative", + "", + "| Config | Failed | Total |", + "|--------|-------:|------:|", + ] + for name in sorted(stats.configs): + vals = stats.configs[name] + lines.append(f"| {name} | {vals['failed_runs']} | {vals['total_runs']} |") + payload = { + "target": stats.target, + "cumulative_failed_runs": stats.cumulative_failed_runs, + "cumulative_total_runs": stats.cumulative_total_runs, + "nightly_count": stats.nightly_count, + "first_seen": stats.first_seen, + "last_seen": stats.last_seen, + "configs": stats.configs, + } + lines.extend( + [ + "", + "```json", + json.dumps(payload, indent=2, sort_keys=True), + "```", + STATS_END, + ] + ) + return "\n".join(lines) + + +def render_run_comment(record: RunRecord, run_url: str, reopened: bool = False) -> str: + """Render a per-run ledger comment (human text + machine-readable marker).""" + config_bits = ", ".join( + f"{name} {vals['failed_runs']}/{vals['total_runs']}" + for name, vals in sorted(record.configs.items()) + ) + lines = [] + if reopened: + lines.append("♻️ Reopened: this test was flaky again after the issue was closed.") + lines.append("") + lines.extend( + [ + f"Nightly run [`{record.run_id}`]({run_url}) on {record.date}: " + f"**{record.failed_runs}/{record.total_runs}** runs failed.", + "", + f"Per config: {config_bits}." if config_bits else "", + "", + record.to_marker(), + ] + ) + return "\n".join(line for line in lines if line is not None) + + +def merge_body(existing_body: str, new_region: str) -> str: + """Replace the managed stats region in an existing body, preserving prose. + + If the markers are absent (or corrupted such that a full region is missing), + the fresh region is appended at the bottom so state self-heals. + """ + existing_body = existing_body or "" + if STATS_BEGIN in existing_body and STATS_END in existing_body: + return STATS_REGION_RE.sub(lambda _m: new_region, existing_body, count=1) + separator = "\n\n" if existing_body.strip() else "" + return f"{existing_body.rstrip()}{separator}{new_region}\n" if existing_body.strip() else new_region + "\n" + + +def _run_record_from_target(target_item: dict, ctx: RunContext) -> RunRecord: + return RunRecord( + run_id=ctx.run_id, + date=ctx.date, + failed_runs=int(target_item.get("failed_runs", 0)), + total_runs=int(target_item.get("total_runs", 0)), + configs={ + str(name): { + "failed_runs": int(vals.get("failed_runs", 0)), + "total_runs": int(vals.get("total_runs", 0)), + } + for name, vals in dict(target_item.get("configs", {})).items() + }, + ) + + +def _issue_title(target: str) -> str: + return f"Flaky test: {target}" + + +# --------------------------------------------------------------------------- # +# Orchestration +# --------------------------------------------------------------------------- # +def sync(merged_summary: dict, client: GitHubClient, ctx: RunContext) -> list[dict]: + """Create/update flaky issues for all flaky targets in the merged summary. + + Returns a list of action records (for logging / dry-run visibility). + """ + actions: list[dict] = [] + flaky_targets = merged_summary.get("flaky_targets", []) + if not flaky_targets: + return actions + + client.ensure_label(FLAKY_LABEL) + + for target_item in flaky_targets: + target = target_item["target"] + this_run = _run_record_from_target(target_item, ctx) + existing = client.search_issue(target) + + if existing is None: + # Create branch: create issue, seed the ledger, render body once. + stats = aggregate(target, [this_run]) + body = render_body(stats) + issue = client.create_issue(_issue_title(target), body, [FLAKY_LABEL]) + client.add_comment(issue, render_run_comment(this_run, ctx.run_url)) + actions.append({"target": target, "action": "created", "issue": issue.number}) + continue + + # Update branch. + comments = client.list_run_comments(existing) + existing_records = parse_run_records(comments) + if any(r.run_id == ctx.run_id for r in existing_records): + actions.append({"target": target, "action": "skipped-duplicate", "issue": existing.number}) + continue + + reopened = existing.state == "closed" + if reopened: + client.reopen_issue(existing) + client.add_comment(existing, render_run_comment(this_run, ctx.run_url, reopened=reopened)) + + all_records = existing_records + [this_run] + stats = aggregate(target, all_records) + client.update_issue_body(existing, merge_body(existing.body, render_body(stats))) + actions.append( + { + "target": target, + "action": "reopened" if reopened else "updated", + "issue": existing.number, + } + ) + return actions + + +# --------------------------------------------------------------------------- # +# Real GitHub REST client +# --------------------------------------------------------------------------- # +class RestGitHubClient: + """GitHub REST client using GITHUB_TOKEN. Follows pagination.""" + + def __init__(self, repo: str, token: str, api_url: str = "https://api.github.com"): + self._repo = repo + self._token = token + self._api = api_url.rstrip("/") + + def _request(self, method: str, path: str, payload: dict | None = None) -> object: + url = path if path.startswith("http") else f"{self._api}{path}" + data = json.dumps(payload).encode("utf-8") if payload is not None else None + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Authorization", f"Bearer {self._token}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("X-GitHub-Api-Version", "2022-11-28") + if data is not None: + request.add_header("Content-Type", "application/json") + with urllib.request.urlopen(request) as response: # noqa: S310 (trusted API URL) + body = response.read().decode("utf-8") + return json.loads(body) if body else None + + def _paginate(self, path: str) -> list[dict]: + results: list[dict] = [] + page = 1 + while True: + sep = "&" if "?" in path else "?" + chunk = self._request("GET", f"{path}{sep}per_page=100&page={page}") + if not isinstance(chunk, list) or not chunk: + break + results.extend(chunk) + if len(chunk) < 100: + break + page += 1 + return results + + def ensure_label(self, name: str) -> None: + try: + self._request("GET", f"/repos/{self._repo}/labels/{name}") + except urllib.error.HTTPError as error: + if error.code != 404: + raise + try: + self._request( + "POST", + f"/repos/{self._repo}/labels", + {"name": name, "color": "d73a4a", "description": "Detected flaky test target"}, + ) + except urllib.error.HTTPError as create_error: + if create_error.code != 422: # already exists (race) + raise + + def search_issue(self, target: str) -> Issue | None: + # Search open+closed issues carrying the label; match by body marker. + issues = self._paginate( + f"/repos/{self._repo}/issues?state=all&labels={FLAKY_LABEL}" + ) + for raw in issues: + if "pull_request" in raw: + continue + body = raw.get("body") or "" + marker = TARGET_MARKER_RE.search(body) + if (marker and marker.group("target") == target) or ( + target in body and _issue_title(target) == (raw.get("title") or "") + ): + return Issue( + number=int(raw["number"]), + body=body, + state=str(raw.get("state", "open")), + labels=[lbl["name"] for lbl in raw.get("labels", [])], + ) + return None + + def list_run_comments(self, issue: Issue) -> list[str]: + comments = self._paginate( + f"/repos/{self._repo}/issues/{issue.number}/comments" + ) + return [c.get("body") or "" for c in comments] + + def create_issue(self, title: str, body: str, labels: list[str]) -> Issue: + raw = self._request( + "POST", + f"/repos/{self._repo}/issues", + {"title": title, "body": body, "labels": labels}, + ) + assert isinstance(raw, dict) + return Issue(number=int(raw["number"]), body=body, state="open", labels=labels) + + def update_issue_body(self, issue: Issue, body: str) -> None: + self._request("PATCH", f"/repos/{self._repo}/issues/{issue.number}", {"body": body}) + issue.body = body + + def reopen_issue(self, issue: Issue) -> None: + self._request("PATCH", f"/repos/{self._repo}/issues/{issue.number}", {"state": "open"}) + issue.state = "open" + + def add_comment(self, issue: Issue, body: str) -> None: + self._request("POST", f"/repos/{self._repo}/issues/{issue.number}/comments", {"body": body}) + + +# --------------------------------------------------------------------------- # +# Dry-run client (logs actions, no API calls) +# --------------------------------------------------------------------------- # +class DryRunGitHubClient: + def __init__(self) -> None: + self.log: list[str] = [] + + def ensure_label(self, name: str) -> None: + self.log.append(f"ensure_label({name})") + + def search_issue(self, target: str) -> Issue | None: + self.log.append(f"search_issue({target}) -> None") + return None + + def list_run_comments(self, issue: Issue) -> list[str]: + return [] + + def create_issue(self, title: str, body: str, labels: list[str]) -> Issue: + self.log.append(f"create_issue(title={title!r}, labels={labels})") + return Issue(number=0, body=body, state="open", labels=labels) + + def update_issue_body(self, issue: Issue, body: str) -> None: + self.log.append(f"update_issue_body(#{issue.number})") + + def reopen_issue(self, issue: Issue) -> None: + self.log.append(f"reopen_issue(#{issue.number})") + + def add_comment(self, issue: Issue, body: str) -> None: + self.log.append(f"add_comment(#{issue.number})") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Sync flaky test GitHub issues.") + parser.add_argument("--merged-summary", required=True) + parser.add_argument("--repo", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args(argv) + + merged_summary = json.loads(Path(args.merged_summary).read_text(encoding="utf-8")) + ctx = RunContext( + run_id=str(args.run_id), + run_url=args.run_url, + date=datetime.now(timezone.utc).date().isoformat(), + ) + + if args.dry_run: + client: GitHubClient = DryRunGitHubClient() + else: + token = os.environ.get("GITHUB_TOKEN", "") + if not token: + parser.error("GITHUB_TOKEN environment variable is required (or use --dry-run).") + client = RestGitHubClient(args.repo, token) + + actions = sync(merged_summary, client, ctx) + for action in actions: + print(f"{action['action']}: {action['target']} (issue #{action.get('issue')})") + if args.dry_run and isinstance(client, DryRunGitHubClient): + for entry in client.log: + print(f"[dry-run] {entry}") + print(f"Synced {len(actions)} flaky target(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())