Skip to content
CI/CD

Fix Windows wheel build quoting, and stop it from blocking other plat… #2229

Sign in to view logs

Fix Windows wheel build quoting, and stop it from blocking other plat…

Fix Windows wheel build quoting, and stop it from blocking other plat… #2229

Workflow file for this run

.github/workflows/python-package.yml at b3b4939
---
name: CI/CD
on:
push:
branches: [master]
pull_request:
branches: [master]
# Run the checks on merge-queue candidates too, so the merge queue enabled
# on `master` can validate each candidate as a `merge_group` event -- the
# required `check` job must run there or queued PRs stall.
merge_group:
release:
types: [created]
branches: [master]
env:
FORCE_COLOR: '1' # Make tools pretty.
PIP_DISABLE_PIP_VERSION_CHECK: '1'
PIP_NO_PYTHON_VERSION_WARNING: '1'
PYTHON_LATEST: '3.12'
# One run per pull request at a time: a new push supersedes the run of the
# commit it replaces, so the ~45 job-minutes of a full matrix are not spent
# validating a commit nobody will ever merge.
#
# Cancellation is deliberately limited to `pull_request`. Every other event
# gets its own group and always runs to completion: a cancelled `merge_group`
# candidate would stall the merge queue, a cancelled `release` would skip
# publishing, and `push` runs on master are what seed the caches everything
# else restores from.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
lint:
name: Check linting
runs-on: ubuntu-latest
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_LATEST }}
cache: pip
# Hash every requirements file the workflow installs from, not just
# the default `**/requirements.txt`. The pins this job actually
# installs live in test.txt, which pulls requirements.txt and several
# extras/ files in via `-r`; with the default pattern the key never
# moved when those changed, so the entry could neither be refreshed
# (a hit never re-saves) nor serve the packages that were added.
# Every job here uses the same list so they all share one entry per
# interpreter instead of each warming a partial one.
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
- run: |
python -m pip install build
python -m pip install -r requirements/test.txt
name: Install core libraries for build and install
- name: Run linting checks
run: scripts/check
test-pytest:
name: 'Python ${{ matrix.python-version }}/Cython: ${{ matrix.use-cython }}/Driver: ${{ matrix.kafka-driver }}'
runs-on: ubuntu-latest
timeout-minutes: 10 # Maybe we should remove this someday but the PyPy tests are acting strange
strategy:
# Complete all jobs even if one fails, allows us to see
# for example if a test fails only when Cython is enabled
fail-fast: false
matrix:
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
use-cython: ['true', 'false']
experimental: [false]
# aiokafka is the default driver and a core dependency, so it runs the
# full grid. The confluent driver lives behind the optional
# `ckafka` extra; give it a dedicated leg (below) so its tests --
# tests/unit/transport/drivers/test_confluent.py, otherwise skipped for
# a missing confluent_kafka -- actually run.
kafka-driver: ['aiokafka']
include:
# Python 3.15. `allow-prereleases` below resolves this to whatever
# pre-release the runner image publishes -- 3.15.0rc1 at the time of
# writing, and 3.15.0 final once it ships, with no change needed
# here. Same shape as a stable row: Cython on and off, plus a
# confluent leg.
#
# Advisory (`experimental: true` -> `continue-on-error`) for two
# reasons. The pre-release one is the same as everywhere else: rc2
# and final are still to come. The specific one is that faust does
# not import at all on 3.15 until mode-streaming ships the fix for
# `typing._eval_type`, whose `type_params` argument became a required
# positional in 3.15 -- every faust Record model resolves its
# annotations through `mode.utils.objects.annotations`, so collection
# dies before the first test runs. The fix is on mode's master; this
# leg goes green once a mode-streaming release carries it (bump the
# floor in requirements/requirements.txt then, and move these entries
# into the matrix above to make 3.15 required).
- python-version: '3.15'
use-cython: 'true'
experimental: true
kafka-driver: 'aiokafka'
- python-version: '3.15'
use-cython: 'false'
experimental: true
kafka-driver: 'aiokafka'
- python-version: '3.15'
use-cython: 'false'
experimental: true
kafka-driver: 'confluent'
# confluent driver: broker-less unit tests over a pure-Python
# wrapper, so one leg per Python version (Cython off) is enough.
- python-version: '3.10'
use-cython: 'false'
experimental: false
kafka-driver: 'confluent'
- python-version: '3.11'
use-cython: 'false'
experimental: false
kafka-driver: 'confluent'
- python-version: '3.12'
use-cython: 'false'
experimental: false
kafka-driver: 'confluent'
- python-version: '3.13'
use-cython: 'false'
experimental: false
kafka-driver: 'confluent'
- python-version: '3.14'
use-cython: 'false'
experimental: false
kafka-driver: 'confluent'
env:
USE_CYTHON: ${{ matrix.use-cython }}
continue-on-error: ${{ matrix.experimental }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
# Required by the 3.15 rows: a bare `3.15` matches stable releases
# only and fails with "Version 3.15 was not found in the local
# cache". Safe for every other row -- it widens `3.X` to
# `~3.X.0-0`, and a pre-release only wins when no stable release
# satisfies the spec, so 3.10-3.14 still resolve to their newest
# stable patch.
allow-prereleases: true
cache: pip
# See the lint job: test.txt alone leaves the key unchanged when the
# transitively-included pins move, and the confluent legs below
# install extras/ckafka.txt on top of it.
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
- name: Install dependencies
run: |
pip install -r requirements/test.txt
pip install .
if [ "${{ matrix.kafka-driver }}" = "confluent" ]; then
pip install -r requirements/extras/ckafka.txt
fi
- name: Build the Cython extensions in place
# `pip install .` above compiles the extensions into site-packages,
# where the tests never see them: pytest runs from the repository
# root, so `import faust` resolves to the source tree, and every
# accelerated import sits behind `try: ... except ImportError`. The
# fallback engaged silently, so these legs differed from the
# `use-cython: false` ones only in whether the build step succeeded --
# the compiled code itself was never executed by a single test.
#
# Building in place puts the .so files next to the .pyx files, which
# is what the source-tree import actually picks up.
if: matrix.use-cython == 'true'
run: USE_CYTHON=1 python setup.py build_ext --inplace
- name: Run tests
# FAUST_REQUIRE_CYTHON turns a silent fallback into a failure, so this
# leg cannot quietly go back to testing pure Python if the build stops
# producing importable extensions. See
# tests/unit/test_cython_parity.py.
env:
FAUST_REQUIRE_CYTHON: ${{ matrix.use-cython == 'true' && '1' || '' }}
run: |
if [ "${{ matrix.kafka-driver }}" = "confluent" ]; then
# Dedicated confluent leg: run just the confluent driver's unit
# tests (the aiokafka legs already cover the rest of the suite).
pytest tests/unit/transport/drivers/test_confluent.py
else
scripts/tests
fi
- name: Enforce coverage
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
test-freethreading:
name: 'Python ${{ matrix.python-version }} (free-threaded)'
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# The two interpreters `[tool.cibuildwheel]` publishes free-threaded
# wheels for. Keep the two lists in step: a version we ship a wheel
# for is a version this job has to cover.
python-version: ['3.13t', '3.14t']
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
- name: Install dependencies
# Not requirements/test.txt: parts of it cannot be built on a
# free-threaded interpreter at all (twine -> cffi, and hypothesis'
# PyO3 extension on 3.13t). freethreading.txt is that list minus the
# ones that fail, and documents each omission.
run: |
pip install -r requirements/freethreading.txt
pip install 'Cython>=3.1' setuptools setuptools_scm
# Editable, unlike the other jobs' `pip install .`. pytest runs from
# the repo root, so `import faust` resolves to the source tree either
# way -- but the suite also needs the distribution *metadata* to
# exist, because `faust/__init__.py` does
# `version("faust-streaming")` at import time. An editable install
# registers that metadata against the tree the tests actually import,
# instead of a second copy in site-packages that nothing loads.
USE_CYTHON=1 pip install -e . --no-build-isolation
- name: Build the Cython extensions in place
# The extensions have to sit next to the .pyx files or they are never
# imported: `faust/streams.py` and friends pull their accelerated
# implementation in behind `try: ... except ImportError`, so a missing
# .so silently falls back to pure Python and the job would test
# something other than what it thinks. This is also what lets
# tests/unit/test_free_threading.py import the extensions rather than
# skipping.
run: USE_CYTHON=1 python setup.py build_ext --inplace
- name: Verify the extensions did not silently re-enable the GIL
# Fails loudly if an extension is missing `freethreading_compatible`,
# rather than leaving it to a RuntimeWarning nobody reads. Runs
# before the suite so the cause is obvious when it breaks.
run: python -m pytest tests/unit/test_free_threading.py -v --no-cov
- name: Run tests
# PYTHON_GIL=0 keeps the GIL off for the whole run even if some
# *dependency* re-enables it (aiokafka's _crecords does, today), so
# the suite really is exercised without a GIL rather than quietly
# falling back to one.
env:
PYTHON_GIL: '0'
# As in the main matrix: fail rather than silently fall back to pure
# Python if the extensions stop being importable from the tree.
FAUST_REQUIRE_CYTHON: '1'
run: python -m pytest tests/unit tests/functional -q --no-cov
test-pypy:
name: 'Python pypy3.11/Cython: false'
runs-on: ubuntu-latest
# PyPy runs the pure-Python paths and is markedly slower than CPython, and
# this leg now runs the formerly-skipped tests too; give it headroom over
# the old 10-minute cap, which already clipped the run at ~96%.
timeout-minutes: 15
continue-on-error: true
env:
USE_CYTHON: 'false'
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: pypy3.11
cache: pip
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
- name: Install dependencies
id: install
continue-on-error: true
run: |
pip install -r requirements/test.txt
pip install .
- name: Run tests
if: steps.install.outcome == 'success'
# `scripts/tests` also runs bandit after pytest, and pytest coverage on
# PyPy is expensive enough to push this advisory leg into the timeout.
# Run pytest directly without coverage, and stop at the first failure so
# a red job reports the real test error instead of a timeout.
run: >
python -m pytest tests/unit tests/functional tests/integration
tests/meticulous tests/regression -x --no-cov
test-integration:
name: 'Integration (Kafka ${{ matrix.kafka-version }})'
runs-on: ubuntu-latest
timeout-minutes: 10
# Advisory for now: broker tests can be flaky while we stabilise them, so
# a red here must not block the required checks / the merge queue. This
# job is intentionally NOT in the `check` job's `needs`.
continue-on-error: true
strategy:
fail-fast: false
matrix:
# Test against both a 3.x broker and Kafka 4.x (KRaft-only, released
# 2025). The `KAFKA_CONTROLLER_QUORUM_VOTERS` static-voter config below
# works for both.
kafka-version: ['3.8.1', '4.0.0']
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.12'
cache: pip
# This job also installs extras/ckafka.txt, which the old
# test.txt-only key did not cover.
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
# Run Kafka as a plain container (not a GH `services:` container) so its
# very chatty broker log stays inside the container -- retrievable on
# demand via `docker logs` -- and the job log shows the pytest output.
- name: Start Kafka (KRaft, single node)
run: |
docker run -d --name kafka -p 9092:9092 \
-e KAFKA_NODE_ID=1 \
-e KAFKA_PROCESS_ROLES=broker,controller \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
-e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \
-e KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \
-e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
-e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \
-e KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 \
-e KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \
-e KAFKA_AUTO_CREATE_TOPICS_ENABLE=true \
apache/kafka:${{ matrix.kafka-version }}
- name: Install dependencies
run: |
pip install -r requirements/test.txt
pip install -r requirements/extras/ckafka.txt
pip install .
- name: Wait for Kafka to be ready
run: |
python - <<'PY'
import socket, time, sys
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
try:
with socket.create_connection(("localhost", 9092), 2):
print("Kafka port is open")
break
except OSError:
time.sleep(2)
else:
sys.exit("Kafka did not become reachable in time")
PY
- name: Run live-broker integration tests
env:
FAUST_TEST_BROKER: 'kafka://localhost:9092'
run: pytest tests/integration/broker -v -ra --tb=short --no-cov
# Only the tail of the broker log on failure, so it augments rather than
# buries the pytest output in the job log.
- name: Kafka broker logs (on failure)
if: failure()
run: docker logs --tail 40 kafka
test-redis-integration:
name: 'Redis cache integration'
runs-on: ubuntu-latest
timeout-minutes: 10
# Advisory for now, like the Kafka integration job: intentionally NOT in
# the `check` job's `needs`, so a red here does not block the merge queue.
continue-on-error: true
services:
redis:
image: redis:7
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 3s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: '3.12'
cache: pip
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
- name: Install dependencies
run: |
pip install -r requirements/test.txt
pip install .
# Runs the redis cache backend against the real server started above --
# the unit tests only ever mock the client.
- name: Run redis cache integration tests
env:
FAUST_TEST_REDIS: 'redis://localhost:6379'
run: pytest tests/integration/cache -v -ra --tb=short --no-cov
check: # This job does nothing and is only used for the branch protection
name: ✅ Ensure the required checks passing
if: always()
# test-freethreading gates too: `[tool.cibuildwheel]` publishes cp313t and
# cp314t wheels, and a wheel we ship should not be able to go out on a red
# run. (The integration jobs stay out of this list -- they are advisory.)
needs: [lint, test-pytest, test-freethreading]
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
build_wheels:
name: 📦 Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: check
if: github.event_name == 'release' && github.event.action == 'created'
strategy:
matrix:
# Modern, supported runner images (ubuntu-20.04 was retired in 2025).
# macos-15-intel -> x86_64 wheels, macos-14 -> arm64 wheels. Build
# config lives in pyproject.toml's [tool.cibuildwheel] (cp3*, auto64,
# Cython).
#
# windows-2022 is deliberately not in this list: a failure on that leg
# fails the whole `build_wheels` job (matrix legs are ANDed together
# for job success, independent of fail-fast), and `publish` needs
# `build_wheels` -- so one broken Windows build blocks every
# platform's release, not just Windows' (see #790). Re-add it once
# Windows wheel builds have proven stable again.
os: [ubuntu-latest, macos-15-intel, macos-14]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build wheels
# cibuildwheel 2.21.3 predates CPython 3.14, so `build = "cp3*"`
# stopped at cp313 and no 3.14 wheels were published (issue #715).
# 4.x builds 3.14 by default and still supports cp310-cp313.
#
# 4.2.0 is the first pin that knows about CPython 3.15, and it builds
# cp315 by default -- no `CIBW_ENABLE: cpython-prerelease` needed,
# because cibuildwheel gates a Python behind that group only until its
# first release candidate (the ABI is frozen at rc1, so a wheel built
# against 3.15.0rc1 is forward-compatible with 3.15.0 final). The
# `cp31?t-*` skip in pyproject.toml's [tool.cibuildwheel] keeps this
# to cp315, without cp315t, matching the other versions.
uses: pypa/cibuildwheel@v4.2.0
- uses: actions/upload-artifact@v4
with:
name: cibw-wheels-${{ matrix.os }}
path: ./wheelhouse/*.whl
if-no-files-found: error
build_sdist:
name: 📦 Build the source distribution
runs-on: ubuntu-latest
needs: check
if: github.event_name == 'release' && github.event.action == 'created'
steps:
- uses: actions/checkout@v4
name: Checkout source repository
with:
fetch-depth: 0
- uses: actions/setup-python@v5
with:
# Pin the interpreter (the step previously took whatever the runner
# image shipped) so the pip cache below has a stable key.
python-version: ${{ env.PYTHON_LATEST }}
cache: pip
cache-dependency-path: |
requirements/*.txt
requirements/extras/*.txt
- name: Build sdist
# Use `python -m build`, not the deprecated `setup.py sdist`, so the
# sdist is named with the normalized project name
# (`faust_streaming-*.tar.gz`). PyPI enforces PEP 625 and rejects the
# legacy hyphenated `faust-streaming-*.tar.gz`.
run: >
pip3 install build pkgconfig cython --upgrade &&
python3 -m build --sdist --outdir dist
- uses: actions/upload-artifact@v4
name: Upload build artifacts
with:
name: cibw-sdist
path: dist/*.tar.gz
if-no-files-found: error
publish:
name: 📦 Publish to PyPI
runs-on: ubuntu-latest
needs: [build_wheels, build_sdist]
permissions:
id-token: write
environment: pypi
if: github.event_name == 'release' && github.event.action == 'created'
steps:
- name: Download all build artifacts (sdist + every wheel)
uses: actions/download-artifact@v4
with:
pattern: cibw-*
merge-multiple: true
path: dist
- name: Publish package to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_API_TOKEN }}
skip-existing: true