[cuda.compute]: add CI job that builds c.parallel with thread sanitizer

c/parallel/CMakeLists.txt

@@ -8,6 +8,17 @@ option(
   "Build cccl.c.parallel standalone headers."
   OFF
 )
+# Instrument cccl.c.parallel's host code with ThreadSanitizer for the
+# free-threaded (3.14t) TSan CI lane. TSan is a host-only tool, so CUDA device
+# code is never instrumented -- only the host portion of each .cu (via
+# -Xcompiler). The libtsan runtime is resolved at load time (the TSan lane
+# LD_PRELOADs it) rather than linked-and-bundled, so the wheel build must
+# --exclude libtsan from auditwheel repair.
+option(
+  CCCL_C_PARALLEL_SANITIZE_THREAD
+  "Build cccl.c.parallel host code with ThreadSanitizer (-fsanitize=thread)."
+  OFF
+)
 
 # FIXME Ideally this would be handled by presets and install rules, but for now
 # consumers may override this to control the target location of cccl.c.parallel.
@@ -45,6 +56,20 @@ if (CCCL_C_PARALLEL_LIBRARY_OUTPUT_DIRECTORY)
   )
 endif()
 
+if (CCCL_C_PARALLEL_SANITIZE_THREAD)
+  # Host-only: -fsanitize=thread on C++ TUs, and forwarded to the host compiler
+  # (-Xcompiler, one flag per prefix) for the host side of each .cu. -g keeps
+  # file:line frames in TSan reports; -fno-omit-frame-pointer keeps complete stack
+  # traces through heavily inlined CUB/Thrust code.
+  target_compile_options(
+    cccl.c.parallel
+    PRIVATE
+      $<$<COMPILE_LANGUAGE:CXX>:-fsanitize=thread;-fno-omit-frame-pointer;-g>
+      $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-fsanitize=thread;-Xcompiler=-fno-omit-frame-pointer;-g>
+  )
+  target_link_options(cccl.c.parallel PRIVATE -fsanitize=thread)
+endif()
+
 cccl_get_cub()
 cccl_get_cudatoolkit()
 cccl_get_thrust()

ci/build_cuda_cccl_python.sh

@@ -90,6 +90,7 @@ for ctk in 12 13; do
         --env "GITHUB_RUN_ID=${GITHUB_RUN_ID:-}" \
         --env "JOB_ID=${JOB_ID:-}" \
         --env "CCCL_PYTHON_USE_V2=${CCCL_PYTHON_USE_V2:-}" \
+        --env "CCCL_C_PARALLEL_SANITIZE_THREAD=${CCCL_C_PARALLEL_SANITIZE_THREAD:-}" \
         --env "CCACHE_DIR=/root/.ccache" \
         --env "CPM_SOURCE_CACHE=/root/.cpm-cache" \
         "$image" \
@@ -133,6 +134,14 @@ echo "Found CUDA 13 wheel: $cu13_wheel"
 # Merge the wheels
 python python/cuda_cccl/merge_cuda_wheels.py "$cu12_wheel" "$cu13_wheel" --output-dir wheelhouse_merged
 
+# A ThreadSanitizer wheel links libtsan; keep it external (excluded) so it is
+# NOT bundled -- the TSan test lane LD_PRELOADs the runner's matching libtsan
+# instead. Harmless for normal builds (the .so has no libtsan dependency).
+tsan_exclude=()
+if [[ "${CCCL_C_PARALLEL_SANITIZE_THREAD:-}" =~ ^(1|true|TRUE|on|ON)$ ]]; then
+  tsan_exclude=(--exclude 'libtsan.so.2')
+fi
+
 # Install auditwheel and repair the merged wheel
 python -m pip install patchelf auditwheel
 for wheel in wheelhouse_merged/cuda_cccl-*.whl; do
@@ -145,6 +154,7 @@ for wheel in wheelhouse_merged/cuda_cccl-*.whl; do
         --exclude 'libcudart.so.12' \
         --exclude 'libcudart.so.13' \
         --exclude 'libcuda.so.1' \
+        "${tsan_exclude[@]}" \
         "$wheel" \
         --wheel-dir wheelhouse_final
 done

ci/build_cuda_cccl_python_tsan.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Thin wrapper around build_cuda_cccl_python.sh that builds the cuda_cccl wheel
+# with ThreadSanitizer instrumentation on the c.parallel (v1) host code, for the
+# free-threaded (3.14t) TSan nightly lane. The shared build script honors
+# CCCL_C_PARALLEL_SANITIZE_THREAD (passes -DCCCL_C_PARALLEL_SANITIZE_THREAD=ON to
+# the wheel build and --exclude libtsan from the auditwheel repair).
+set -euo pipefail
+
+ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+export CCCL_C_PARALLEL_SANITIZE_THREAD=1
+"$ci_dir/build_cuda_cccl_python.sh" "$@"

ci/build_cuda_cccl_wheel.sh

@@ -4,10 +4,22 @@ set -euo pipefail
 # Target script for `docker run` command in build_cuda_cccl_python.sh
 # The /workspace pathnames are hard-wired here.
 
-# Install GCC 13 toolset (needed for the build) and ccache (shared between
+# Toolchain version, pinned in one place. Every gcc-toolset package name and the
+# /opt/rh enable path below derive from it — the -fsanitize=thread libtsan must
+# come from the same toolset as the compiler, so these cannot drift apart.
+readonly gcc_toolset_version=13
+
+# Install the GCC toolset (needed for the build) and ccache (shared between
 # cu12 and cu13 builds via /root/.ccache bind-mount from the host).
 /workspace/ci/util/retry.sh 5 30 dnf -y install \
-  gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ ccache
+  "gcc-toolset-${gcc_toolset_version}-gcc" "gcc-toolset-${gcc_toolset_version}-gcc-c++" ccache
+
+# ThreadSanitizer builds link libcccl.c.parallel.so with -fsanitize=thread, which
+# the linker resolves via the toolset's libtsan.so. That runtime lives in a
+# separate package not pulled in by -gcc-c++, so install it for the TSan lane.
+if [[ "${CCCL_C_PARALLEL_SANITIZE_THREAD:-}" =~ ^(1|true|TRUE|on|ON)$ ]]; then
+  /workspace/ci/util/retry.sh 5 30 dnf -y install "gcc-toolset-${gcc_toolset_version}-libtsan-devel"
+fi
 
 # When the caller bind-mounts a ccache dir, wire it through to CMake. This
 # transparently caches every compile, so the second wheel build (cu13 after
@@ -20,7 +32,7 @@ if [[ -n "${CCACHE_DIR:-}" ]]; then
   ccache --version 2>&1 | head -1 || true
   ccache --show-stats 2>&1 | head -5 || true
 fi
-echo -e "#!/usr/bin/env bash\nsource /opt/rh/gcc-toolset-13/enable" >/etc/profile.d/enable_devtools.sh
+echo -e "#!/usr/bin/env bash\nsource /opt/rh/gcc-toolset-${gcc_toolset_version}/enable" >/etc/profile.d/enable_devtools.sh
 # shellcheck disable=SC1091
 source /etc/profile.d/enable_devtools.sh
 
@@ -91,6 +103,15 @@ if [[ "${CCCL_PYTHON_USE_V2:-}" =~ ^(1|true|TRUE|on|ON)$ ]]; then
   python -m pip install --upgrade 'cmake>=3.27'
 fi
 
+# When CCCL_C_PARALLEL_SANITIZE_THREAD is set (=1/true/on), instrument the
+# c.parallel (v1) host code with ThreadSanitizer for the free-threaded TSan
+# nightly lane. Host-only; the libtsan runtime stays external (the shared
+# build_cuda_cccl_python.sh --excludes it from auditwheel).
+if [[ "${CCCL_C_PARALLEL_SANITIZE_THREAD:-}" =~ ^(1|true|TRUE|on|ON)$ ]]; then
+  export CMAKE_ARGS="${CMAKE_ARGS:-} -DCCCL_C_PARALLEL_SANITIZE_THREAD=ON"
+  echo "Building wheel with ThreadSanitizer-instrumented c.parallel: CMAKE_ARGS=${CMAKE_ARGS}"
+fi
+
 # Build the wheel
 python -m pip wheel --no-deps --verbose --wheel-dir dist .
 

ci/matrix.yaml

@@ -96,6 +96,9 @@ workflows:
     # Free-threaded Python currently supports only the minimal cuda.compute extra on Linux.
     - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: '13.X', py_version: '3.14', gpu: 'l4', cxx: 'gcc13'}
     - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'}
+    # ThreadSanitizer runs per-PR (a dedicated instrumented build + FT sweep under the TSan
+    # runtime) so data races in the shared build/launch machinery are caught before merge.
+    - {jobs: ['test_py_compute_minimal'], project: 'python_tsan', ctk: '13.X', py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'}
     # CCCL packaging:
     - {jobs: ['test'], project: 'packaging', ctk: '12.0', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080', args: '-min-cmake'}
     - {jobs: ['test'], project: 'packaging', ctk: '12.X', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080'}
@@ -238,6 +241,10 @@ workflows:
     # so FT regressions (e.g. from dependency bumps) surface between PRs.
     - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X', '13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'}
     - {jobs: ['test_py_compute_minimal'], project: 'python_v2', ctk: '13.X', py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'}
+    # Free-threaded (3.14t) ThreadSanitizer lane: builds c.parallel v1 host code
+    # with -fsanitize=thread and runs the FT stress + parallel sweep under the
+    # TSan runtime.
+    - {jobs: ['test_py_compute_minimal'], project: 'python_tsan', ctk: '13.X', py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'}
     # CCCL packaging:
     - {jobs: ['test'], project: 'packaging', ctk: '12.0', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080', args: '-min-cmake'}
     - {jobs: ['test'], project: 'packaging', ctk: '12.X', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080'}
@@ -624,6 +631,17 @@ projects:
     job_map:
       build: ['build_py_wheel']
       test:  ['test_py_par', 'test_py_examples']
+  python_tsan:
+    name: "Python (cuda.compute free-threaded ThreadSanitizer)"
+    # Runs per-PR (and nightly). The producer build_py_wheel runs
+    # build_cuda_cccl_python_tsan.sh (c.parallel v1 host code built with
+    # -fsanitize=thread); test_py_compute_minimal runs
+    # test_cuda_compute_minimal_python_tsan.sh (the FT stress + sweep under the
+    # TSan runtime). Deliberately absent from the `python-wheels` publish
+    # workflow, so these instrumented wheels never reach PyPI.
+    job_map:
+      build: ['build_py_wheel']
+      test:  ['test_py_compute_minimal']
   cccl_c_parallel:
     name: 'CCCL C Parallel'
     stds: [20]

ci/test_cuda_compute_minimal_python.sh

@@ -44,4 +44,28 @@ if [[ "${py_version}" == "3.14t" ]]; then
     compute/test_free_threading_stress.py \
     compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable \
     compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load
+
+  # Broad thread-safety sweep (pytest-run-parallel): re-run the numba-free
+  # functional suite with each test executed concurrently across threads
+  # (barrier-synchronized start), stressing the process-wide build cache,
+  # single-flight coordination, and the Cython bindings from many threads at
+  # once. Complements test_free_threading_stress.py above, which targets specific
+  # shared-object scenarios by hand. -n 0 so the threads share one interpreter.
+  #
+  # --parallel-threads=2 matches CuPy's free-threading CI (the closest GPU
+  # precedent); a small fixed count bounds GPU-memory pressure from concurrent
+  # kernels and stays reproducible across runners, unlike =auto (the runner's
+  # logical-core count).
+  #
+  # pytest-run-parallel is only used by this sweep, so install it on the 3.14t
+  # path rather than for every minimal (e.g. non-free-threaded 3.14) run.
+  python -m pip install pytest-run-parallel
+
+  # Fail fast if the interpreter is not actually GIL-free (wrong build /
+  # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled from
+  # the start -- it would run threads GIL-serialized and pass vacuously. (A GIL
+  # *re-enabled mid-run* by a non-free-threaded import IS caught by the plugin,
+  # which is why we do not pass --ignore-gil-enabled.)
+  python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'"
+  python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py
 fi

ci/test_cuda_compute_minimal_python_tsan.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+# ThreadSanitizer variant of the minimal free-threaded cuda.compute lane.
+#
+# Installs the TSan-instrumented cuda_cccl wheel (produced by the `python_tsan`
+# build, which compiles c.parallel v1 host code with -fsanitize=thread) and runs
+# the free-threaded stress tests + the pytest-run-parallel sweep under the TSan
+# runtime. A real data race in c.parallel (e.g. build-owned state mutated at
+# launch and shared across threads) fails the nightly here instead of silently
+# corrupting results under some interleaving.
+#
+# Only meaningful on a free-threaded (3.14t) interpreter; the GIL would serialize
+# the threads and hide the very races we are looking for.
+
+set -euo pipefail
+
+ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(cd "$ci_dir/.." && pwd)"
+source "$ci_dir/pyenv_helper.sh"
+
+# Parse common arguments
+source "$ci_dir/util/python/common_arg_parser.sh"
+parse_python_args "$@"
+require_py_version "Usage: $0 -py-version <python_version>"
+
+if [[ "${py_version}" != *t ]]; then
+  echo "ERROR: the TSan lane requires a free-threaded (…t) interpreter; got '${py_version}'." >&2
+  echo "On a GIL interpreter the sweep serializes and TSan has no signal." >&2
+  exit 1
+fi
+
+# Instrument c.parallel when this script builds the wheel itself (local runs). In
+# CI the wheel is the pre-built `python_tsan` artifact, already instrumented; the
+# export is harmless there.
+export CCCL_C_PARALLEL_SANITIZE_THREAD=1
+
+cuda_major_version=$(nvcc --version | grep release | awk '{print $6}' | tr -d ',' | cut -d '.' -f 1 | cut -d 'V' -f 2)
+
+# Setup Python environment
+setup_python_env "${py_version}"
+
+# Fetch or build the TSan-instrumented cuda_cccl wheel. Under project
+# `python_tsan`, get_wheel_artifact_name.sh resolves to the distinct `-tsan`
+# artifact, so this never grabs an uninstrumented wheel.
+if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+  wheel_artifact_name=$("$ci_dir/util/workflow/get_wheel_artifact_name.sh")
+  "$ci_dir/util/artifacts/download.sh" "${wheel_artifact_name}" "${repo_root}/"
+  wheelhouse_dir="${repo_root}/wheelhouse"
+else
+  "$ci_dir/build_cuda_cccl_python.sh" -py-version "${py_version}"
+  wheelhouse_dir="${repo_root}/wheelhouse"
+fi
+
+# minimal-cu* extra intentionally avoids numba/numba-cuda (which re-enable the
+# GIL). pytest-run-parallel drives the concurrent sweep.
+CUDA_CCCL_WHEEL_PATH="$(ls "${wheelhouse_dir}"/cuda_cccl-*.whl)"
+python -m pip install "${CUDA_CCCL_WHEEL_PATH}[minimal-cu${cuda_major_version}]"
+python -m pip install pytest pytest-xdist pytest-run-parallel
+
+# The instrumented .so links libtsan but keeps it external (auditwheel --exclude),
+# so the TSan runtime must be present from process start -- LD_PRELOAD the
+# runner's libtsan (same soname/major as the gcc-13 build). Without preload the
+# .so fails to load ("cannot allocate memory in static TLS block").
+tsan_runtime="$(gcc -print-file-name=libtsan.so.2)"
+if [[ ! -e "${tsan_runtime}" ]]; then
+  echo "ERROR: libtsan.so.2 not found (gcc -print-file-name returned '${tsan_runtime}')." >&2
+  exit 1
+fi
+
+# setarch -R disables ASLR for the process tree. Required: TSan reserves fixed
+# shadow-memory regions and aborts ("unexpected memory mapping") when ASLR drops
+# something into them (google/sanitizers#1686). Uses personality(ADDR_NO_RANDOMIZE)
+# -- if a runner's seccomp profile blocks it, this call fails and must be allowed
+# (e.g. --security-opt seccomp=unconfined on the job's container).
+#
+# ignore_noninstrumented_modules=1: only c.parallel is instrumented, so ignore
+# races inside uninstrumented CPython / CUDA libs (avoids boundary false
+# positives). halt_on_error=1: stop at the first race -- it is usually the root
+# cause, and later reports are typically downstream noise.
+# exitcode=66: exit non-zero when any (unsuppressed) race is found, failing the
+# job even though pytest itself passes.
+run_under_tsan() {
+  setarch -R env \
+    LD_PRELOAD="${tsan_runtime}" \
+    TSAN_OPTIONS="ignore_noninstrumented_modules=1 halt_on_error=1 history_size=7 exitcode=66" \
+    "$@"
+}
+
+# Fail fast if the interpreter is not actually GIL-free (wrong build /
+# PYTHON_GIL=1): the sweep would run GIL-serialized and pass vacuously.
+run_under_tsan python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; TSan sweep has no signal'"
+
+cd "${repo_root}/python/cuda_cccl/tests/"
+
+# Hand-written free-threaded stress scenarios (they spawn their own worker
+# threads) + the serialization node-ids, all sharing one interpreter (-n 0).
+run_under_tsan python -m pytest -n 0 -v \
+  compute/test_free_threading_stress.py \
+  compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable \
+  compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load
+
+# Broad sweep: run the numba-free functional suite with each test executed
+# concurrently across threads (barrier-synchronized), exercising many more
+# c.parallel algorithms under contention than the hand-written stress tests.
+# -n 0 so the threads share one interpreter; --parallel-threads=2 bounds
+# GPU-memory pressure and stays reproducible across runners.
+run_under_tsan python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py

ci/util/workflow/get_wheel_artifact_name.sh

@@ -58,6 +58,11 @@ done
 suffix=""
 if [[ "$project" == "python_v2" ]]; then
   suffix="-v2"
+elif [[ "$project" == "python_tsan" ]]; then
+  # ThreadSanitizer-instrumented wheel (free-threaded TSan nightly lane). Must
+  # be distinct so its build doesn't clobber the normal wheel and the TSan test
+  # job doesn't grab an uninstrumented one.
+  suffix="-tsan"
 fi
 
 echo "wheel-cccl${suffix}-$os-$arch-py$py_version"

python/cuda_cccl/tests/compute/conftest.py

@@ -88,14 +88,21 @@ def cuda_stream() -> Generator[Stream, None, None]:
 
 
 @pytest.fixture(scope="function", autouse=True)
-def verify_sass(request, monkeypatch):
+def verify_sass(request):
     if request.node.get_closest_marker("no_verify_sass"):
         return
 
     if not check_ldl_stl_in_sass:
-        print("not checking sass")
         return
 
+    # Pull monkeypatch dynamically rather than as a fixture parameter so this
+    # autouse fixture does not add monkeypatch to every test's static fixture
+    # closure. pytest-run-parallel treats monkeypatch as thread-unsafe based on
+    # that closure, so a parameter here would serialize the entire free-threaded
+    # parallel sweep -- even though this fixture only patches on the opt-in
+    # SASS-check path (check_ldl_stl_in_sass, off by default and in CI).
+    monkeypatch = request.getfixturevalue("monkeypatch")
+
     import cuda.compute._cccl_interop
 
     monkeypatch.setattr(
@@ -121,12 +128,42 @@ def guarded_import(name, *args, **kwargs):
 
 
 def pytest_collection_modifyitems(config, items):
+    """Runs after pytest collects the tests. Makes a test marked no_numba fail
+    if it imports numba, and skips a test marked serialization when running on
+    the v2 (HostJIT) backend."""
     serialization_skip = pytest.mark.skip(
         reason="serialization not supported on v2 (HostJIT) backend"
     )
+
+    # Tests marked no_numba must not import numba. We enforce that by attaching
+    # the raise_on_numba_import fixture defined above to each one; it raises if
+    # numba is imported.
+    #
+    # We skip attaching it during a real pytest-run-parallel sweep of more than
+    # one thread: the fixture uses monkeypatch, which pytest-run-parallel
+    # serializes as thread-unsafe, so attaching it to every no_numba test would
+    # make the whole sweep run serially and defeat its purpose. A single-threaded
+    # run has no sweep to protect, so we attach it and keep the check.
+    #
+    # config.getoption gives the --parallel-threads value as an int for the
+    # default but a str when passed on the command line; normalize to str and
+    # count the run as parallel only for an explicit number > 1:
+    #
+    #   pytest ...                          -> 1       single-threaded, attach
+    #   pytest --parallel-threads=1 ...     -> "1"     single-threaded, attach
+    #   pytest --parallel-threads=8 ...     -> "8"     parallel, skip (CI sweep)
+    #   pytest --parallel-threads=auto ...  -> "auto"  single-threaded, attach
+    #
+    # "auto" counts as single-threaded on purpose: it can resolve to one CPU, so
+    # keeping the check is safer than dropping it on a non-parallel run. isdigit
+    # also stops int() from raising on the non-numeric "auto".
+    parallel_threads = str(config.getoption("parallel_threads", 1))
+    running_parallel = parallel_threads.isdigit() and int(parallel_threads) > 1
     for item in items:
-        if item.get_closest_marker("no_numba"):
+        # no_numba: add raise_on_numba_import unless we skip it for the sweep
+        if item.get_closest_marker("no_numba") and not running_parallel:
             if "raise_on_numba_import" not in item.fixturenames:
                 item.fixturenames.append("raise_on_numba_import")
+        # serialization is unsupported on v2 (HostJIT); skip those tests there
         if USING_V2 and item.get_closest_marker("serialization"):
             item.add_marker(serialization_skip)