diff --git a/ci/test_cuda_cccl_examples_python.sh b/ci/test_cuda_cccl_examples_python.sh index 6377bcc5fa0..cb440c3843c 100755 --- a/ci/test_cuda_cccl_examples_python.sh +++ b/ci/test_cuda_cccl_examples_python.sh @@ -21,10 +21,32 @@ else "$ci_dir/build_cuda_cccl_python.sh" -py-version "${py_version}" fi -# Install cuda_cccl, plus CuPy which the cuda.compute examples require +# Install cuda_cccl, plus CuPy which the cuda.compute examples require, plus +# pytest-benchmark for the host-overhead benchmark smoke test below. (cuda-bench, +# for the throughput smoke, is installed best-effort further down since it does +# not always ship a wheel for the newest Python.) CUDA_CCCL_WHEEL_PATH="$(ls /home/coder/cccl/wheelhouse/cuda_cccl-*.whl)" -python -m pip install "${CUDA_CCCL_WHEEL_PATH}[test-cu${cuda_major_version}]" "cupy-cuda${cuda_major_version}x" +python -m pip install "${CUDA_CCCL_WHEEL_PATH}[test-cu${cuda_major_version}]" "cupy-cuda${cuda_major_version}x" pytest-benchmark # Run tests for parallel module cd "/home/coder/cccl/python/cuda_cccl/tests/" python -m pytest -n 6 test_examples.py + +# Smoke-test the host-overhead benchmark harness: run every benchmark case +# exactly once (pass/fail only, no timing) so harness rot fails CI here instead +# of silently surviving until someone runs the perf suite. +cd "/home/coder/cccl/python/cuda_cccl/benchmarks/compute/host/" +python -m pytest -v --benchmark-disable . + +# Smoke-test the throughput (nvbench) benchmarks the same way. --profile runs +# each configuration once (no sampling); --quick uses the reduced quick_configs +# axes (one dtype, smallest size) so every benchmark harness still imports, +# registers, launches, and completes. cuda-bench does not always ship a wheel for +# the newest Python, so install it best-effort and skip (with a warning) rather +# than failing the lane when it is unavailable. +if python -m pip install "cuda-bench[cu${cuda_major_version}]" pyyaml; then + cd "/home/coder/cccl/python/cuda_cccl/benchmarks/compute/" + python run_benchmarks.py --py --profile --quick +else + echo "::warning::cuda-bench has no wheel for Python ${py_version}; skipping the throughput benchmark smoke test." +fi diff --git a/ci/test_cuda_coop_python.sh b/ci/test_cuda_coop_python.sh index 5d5983a2d61..7cdbb4559c6 100755 --- a/ci/test_cuda_coop_python.sh +++ b/ci/test_cuda_coop_python.sh @@ -21,10 +21,24 @@ else "$ci_dir/build_cuda_cccl_python.sh" -py-version "${py_version}" fi -# Install cuda_cccl +# Install cuda_cccl. (cuda-bench, for the benchmark smoke test below, is +# installed best-effort further down since it does not always ship a wheel for +# the newest Python.) CUDA_CCCL_WHEEL_PATH="$(ls /home/coder/cccl/wheelhouse/cuda_cccl-*.whl)" python -m pip install "${CUDA_CCCL_WHEEL_PATH}[test-cu${cuda_major_version}]" # Run tests for coop module cd "/home/coder/cccl/python/cuda_cccl/tests/" python -m pytest -n auto -v coop/_experimental/ + +# Smoke-test the coop benchmark: --profile runs each configuration once (no +# sampling) so a benchmark that stops building/running fails CI here instead of +# rotting until someone runs the perf suite. cuda-bench does not always ship a +# wheel for the newest Python, so install it best-effort and skip (with a +# warning) rather than failing the lane when it is unavailable. +if python -m pip install "cuda-bench[cu${cuda_major_version}]"; then + cd "/home/coder/cccl/python/cuda_cccl/benchmarks/coop/_experimental/" + python bench_warp_reduce.py --profile +else + echo "::warning::cuda-bench has no wheel for Python ${py_version}; skipping the coop benchmark smoke test." +fi diff --git a/ci/windows/test_cuda_cccl_examples_python.ps1 b/ci/windows/test_cuda_cccl_examples_python.ps1 index 45eeb889bc5..5becf63a5fe 100644 --- a/ci/windows/test_cuda_cccl_examples_python.ps1 +++ b/ci/windows/test_cuda_cccl_examples_python.ps1 @@ -22,9 +22,10 @@ ${wheelPath} = Get-CudaCcclWheel # $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit # must be checked explicitly or a failed pip/pytest is masked by a later # successful command and the job passes green. -& $python -m pip install -U pip pytest pytest-xdist +# pytest-benchmark is for the host-benchmark smoke test below. +& $python -m pip install -U pip pytest pytest-xdist pytest-benchmark if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist" + throw "Failed to install pytest / pytest-xdist / pytest-benchmark" } # CuPy is required by the cuda.compute examples and is not part of the test extras & $python -m pip install "${wheelPath}[test-cu$cudaMajor]" "cupy-cuda${cudaMajor}x" @@ -40,3 +41,18 @@ try { } } finally { Pop-Location } + +# Smoke-test the host-overhead benchmark harness: run every benchmark case +# exactly once (pass/fail only, no timing) so harness rot fails CI here instead +# of silently surviving until someone runs the perf suite. --benchmark-disable +# makes pytest-benchmark invoke each benchmarked callable a single time. This +# lane already installs cupy + numba (for the examples), which the benchmark +# suite also needs, so only pytest-benchmark is added above. +Push-Location (Join-Path $repoRoot "python/cuda_cccl/benchmarks/compute/host") +try { + & $python -m pytest -v --benchmark-disable . + if ($LASTEXITCODE -ne 0) { + throw "host benchmark smoke test failed" + } +} +finally { Pop-Location } diff --git a/python/cuda_cccl/benchmarks/compute/host/host_benchmark_cases.py b/python/cuda_cccl/benchmarks/compute/host/host_benchmark_cases.py index ceaa51709d6..29fb1d920a0 100644 --- a/python/cuda_cccl/benchmarks/compute/host/host_benchmark_cases.py +++ b/python/cuda_cccl/benchmarks/compute/host/host_benchmark_cases.py @@ -35,20 +35,47 @@ class HostBenchmarkCase: class NoopBuildResult: - """Proxy that skips native compute while preserving wrapper host work.""" + """Proxy over a loaded build result that skips the native kernel launch. + + The host-overhead benchmarks want to measure everything ``__call__`` does on + the host — iterator/op state updates, stream validation, build-result + resolution — without paying for (or allocating correct temp storage for) the + GPU kernel. Wrapping the loaded build result intercepts every kernel-launch + entry point with a fabricated return, while delegating any other attribute + (e.g. ``determinism``, which reduce reads to pick its entry point) to the + real result. + """ + + # Every build-result method an algorithm's __call__ dispatches its kernel + # through. Keep in sync with cuda/compute/algorithms/ (grep for + # ``loaded_build_result.compute``). A new entry point missing here would run + # the real kernel on the benchmark's 1-byte temp storage and fail loudly — + # which the CI smoke test (rounds=iterations=1) exists to catch. + _COMPUTE_METHODS = frozenset( + { + "compute", + "compute_even", + "compute_nondeterministic", + "compute_inclusive", + "compute_inclusive_no_init", + "compute_inclusive_future_value", + "compute_exclusive", + "compute_exclusive_future_value", + } + ) def __init__(self, real_build_result: Any, return_kind: NoopReturnKind): self._real_build_result = real_build_result self._return_kind = return_kind def __getattr__(self, name: str) -> Any: - return getattr(self._real_build_result, name) + if name in NoopBuildResult._COMPUTE_METHODS: - def compute(self, *args, **kwargs): - return _noop_return(self._return_kind) + def noop_compute(*args, **kwargs): + return _noop_return(self._return_kind) - def compute_even(self, *args, **kwargs): - return _noop_return(self._return_kind) + return noop_compute + return getattr(self._real_build_result, name) def _noop_return(return_kind: NoopReturnKind): @@ -64,18 +91,20 @@ def _noop_return(return_kind: NoopReturnKind): def patch_wrapper_to_skip_native_compute( wrapper: Any, return_kind: NoopReturnKind ) -> None: - """Patch a cached wrapper so measured calls skip native compute.""" - if hasattr(wrapper, "build_result"): - wrapper.build_result = NoopBuildResult(wrapper.build_result, return_kind) - - if hasattr(wrapper, "device_reduce_fn"): - wrapper.device_reduce_fn = lambda *args, **kwargs: _noop_return(return_kind) - - if hasattr(wrapper, "device_scan_fn"): - wrapper.device_scan_fn = lambda *args, **kwargs: _noop_return(return_kind) - - if hasattr(wrapper, "partitioner"): - patch_wrapper_to_skip_native_compute(wrapper.partitioner, return_kind) + """Patch a cached wrapper so measured calls skip the native kernel launch. + + Default-build wrappers resolve, on every ``__call__``, to the loaded build + result bound at construction (``_bound_build_result``; see + ``resolve_build_result``). Replacing it with a NoopBuildResult is therefore + enough: the per-call re-resolution and any re-binding of the compute fn both + read through it. ``_Select`` owns no build result of its own — it delegates + to a nested three-way-partition wrapper — so recurse into ``partitioner``. + """ + if (bound := getattr(wrapper, "_bound_build_result", None)) is not None: + wrapper._bound_build_result = NoopBuildResult(bound, return_kind) + + if (partitioner := getattr(wrapper, "partitioner", None)) is not None: + patch_wrapper_to_skip_native_compute(partitioner, return_kind) def make_tiny_temp_storage() -> cp.ndarray: diff --git a/python/cuda_cccl/benchmarks/compute/run_benchmarks.py b/python/cuda_cccl/benchmarks/compute/run_benchmarks.py index 75ffd545a7d..c32a3e9089a 100755 --- a/python/cuda_cccl/benchmarks/compute/run_benchmarks.py +++ b/python/cuda_cccl/benchmarks/compute/run_benchmarks.py @@ -18,6 +18,7 @@ --py Only run Python benchmarks --cpp Only run C++ benchmarks --quick, -q Run with reduced parameter set for fast testing + --profile Run each config once (nvbench profile mode), no sampling -h, --help Show this help message Benchmark names follow CUB structure: @@ -39,8 +40,6 @@ import sys from pathlib import Path -import yaml - # ============================================================================ # Configuration # ============================================================================ @@ -156,6 +155,8 @@ def print_section(msg: str) -> None: def load_quick_configs() -> dict: """Load quick mode configurations from YAML file.""" + import yaml # only needed for --quick + with open(QUICK_CONFIG_FILE) as f: return yaml.safe_load(f) @@ -263,6 +264,7 @@ def run_benchmark( run_cpp: bool, quick_mode: bool, quick_configs: dict, + profile: bool, ) -> dict: """Run a single benchmark. @@ -324,6 +326,8 @@ def run_benchmark( cmd = [str(cpp_bin), "--json", str(cpp_result), "--devices", device] cmd.extend(cpp_axis_args) + if profile: + cmd.append("--profile") # Ensure the CUB build lib dir (containing libnvbench.so) is on the # dynamic linker search path for the child process. cpp_lib_dir = str(CUB_BENCH_DIR.parent / "lib") @@ -357,6 +361,8 @@ def run_benchmark( device, ] cmd.extend(py_axis_args) + if profile: + cmd.append("--profile") py_status = run_and_log(cmd, py_log) print(f" Results: {py_result}") print(f" Log: {py_log}") @@ -398,6 +404,13 @@ def main(): action="store_true", help="Run with reduced parameter set for fast testing", ) + parser.add_argument( + "--profile", + action="store_true", + help="Run each benchmark configuration once (nvbench profile mode) with " + "no sampling -- for smoke-testing that benchmarks execute without error, " + "not for measuring performance.", + ) args = parser.parse_args() # Determine what to run @@ -438,6 +451,7 @@ def main(): print(f" Run C++: {run_cpp}") print(f" Run Python: {run_py}") print(f" Quick Mode: {args.quick}") + print(f" Profile Mode: {args.profile}") print() # Check C++ binaries directory exists (if running C++) @@ -454,7 +468,7 @@ def main(): for bench in benchmarks_to_run: print_section(f"Benchmark: {bench}") results = run_benchmark( - bench, args.device, run_py, run_cpp, args.quick, quick_configs + bench, args.device, run_py, run_cpp, args.quick, quick_configs, args.profile ) all_results[bench] = results print() @@ -496,6 +510,19 @@ def main(): print() + # Exit non-zero if any benchmark failed so CI (e.g. the --profile smoke gate) + # catches benchmark rot instead of silently passing on printed warnings. + failed = [] + for bench in benchmarks_to_run: + results = all_results.get(bench, {}) + for key in ("cpp_status", "py_status"): + if results.get(key, {}).get("status") not in (None, "ok"): + failed.append(bench) + break + if failed: + print(f"ERROR: {len(failed)} benchmark(s) failed: {failed}") + sys.exit(1) + if __name__ == "__main__": main()