From 6f4423482f9c36a02f018d553e21f9dcfd1f5cc3 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Wed, 15 Jul 2026 18:58:28 -0500 Subject: [PATCH 1/8] add pytest run parallel --- ci/test_cuda_compute_minimal_python.sh | 22 ++++++++++++++++++- python/cuda_cccl/tests/compute/conftest.py | 22 ++++++++++++++++--- .../cuda_cccl/tests/compute/test_no_numba.py | 18 +++++++-------- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/ci/test_cuda_compute_minimal_python.sh b/ci/test_cuda_compute_minimal_python.sh index f0ec1d85760..3eef6324ed8 100755 --- a/ci/test_cuda_compute_minimal_python.sh +++ b/ci/test_cuda_compute_minimal_python.sh @@ -30,7 +30,7 @@ fi # full cu* extras because those pull in numba/numba-cuda. CUDA_CCCL_WHEEL_PATH="$(ls "${wheelhouse_dir}"/cuda_cccl-*.whl)" python -m pip install "${CUDA_CCCL_WHEEL_PATH}[minimal-cu${cuda_major_version}]" -python -m pip install pytest pytest-xdist +python -m pip install pytest pytest-xdist pytest-run-parallel cd "${repo_root}/python/cuda_cccl/tests/" python -m pytest -n 6 -v compute/test_no_numba.py @@ -44,4 +44,24 @@ if [[ "${py_version}" == "3.14t" ]]; then compute/test_free_threading_stress.py \ compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable \ compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load + + # Broad thread-safety sweep (pytest-run-parallel): re-run the numba-free + # functional suite with each test executed concurrently across threads + # (barrier-synchronized start), stressing the process-wide build cache, + # single-flight coordination, and the Cython bindings from many threads at + # once. Complements test_free_threading_stress.py above, which targets specific + # shared-object scenarios by hand. -n 0 so the threads share one interpreter. + # + # --parallel-threads=2 matches CuPy's free-threading CI (the closest GPU + # precedent); a small fixed count bounds GPU-memory pressure from concurrent + # kernels and stays reproducible across runners, unlike =auto (the runner's + # logical-core count). + # + # Fail fast if the interpreter is not actually GIL-free (wrong build / + # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled from + # the start -- it would run threads GIL-serialized and pass vacuously. (A GIL + # *re-enabled mid-run* by a non-free-threaded import IS caught by the plugin, + # which is why we do not pass --ignore-gil-enabled.) + python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'" + python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py fi diff --git a/python/cuda_cccl/tests/compute/conftest.py b/python/cuda_cccl/tests/compute/conftest.py index 57c73ed7de8..d195c46d6cc 100644 --- a/python/cuda_cccl/tests/compute/conftest.py +++ b/python/cuda_cccl/tests/compute/conftest.py @@ -88,14 +88,21 @@ def cuda_stream() -> Generator[Stream, None, None]: @pytest.fixture(scope="function", autouse=True) -def verify_sass(request, monkeypatch): +def verify_sass(request): if request.node.get_closest_marker("no_verify_sass"): return if not check_ldl_stl_in_sass: - print("not checking sass") return + # Pull monkeypatch dynamically rather than as a fixture parameter so this + # autouse fixture does not add monkeypatch to every test's static fixture + # closure. pytest-run-parallel treats monkeypatch as thread-unsafe based on + # that closure, so a parameter here would serialize the entire free-threaded + # parallel sweep -- even though this fixture only patches on the opt-in + # SASS-check path (check_ldl_stl_in_sass, off by default and in CI). + monkeypatch = request.getfixturevalue("monkeypatch") + import cuda.compute._cccl_interop monkeypatch.setattr( @@ -124,8 +131,17 @@ def pytest_collection_modifyitems(config, items): serialization_skip = pytest.mark.skip( reason="serialization not supported on v2 (HostJIT) backend" ) + # Under the pytest-run-parallel sweep (--parallel-threads > 1) skip the + # blanket raise_on_numba_import injection: it monkeypatches + # builtins.__import__, which pytest-run-parallel treats as thread-unsafe and + # would serialize every no_numba test, neutering the sweep. + # + # getoption returns the int 1 from the argparse default but a *string* for + # CLI-passed values ("2", "auto", "1"), so normalize to str before comparing + # -- a bare `> 1` would raise TypeError on the "2" string. + running_parallel = str(config.getoption("parallel_threads", 1)) != "1" for item in items: - if item.get_closest_marker("no_numba"): + if item.get_closest_marker("no_numba") and not running_parallel: if "raise_on_numba_import" not in item.fixturenames: item.fixturenames.append("raise_on_numba_import") if USING_V2 and item.get_closest_marker("serialization"): diff --git a/python/cuda_cccl/tests/compute/test_no_numba.py b/python/cuda_cccl/tests/compute/test_no_numba.py index 264cf117aa0..da605d0dab2 100644 --- a/python/cuda_cccl/tests/compute/test_no_numba.py +++ b/python/cuda_cccl/tests/compute/test_no_numba.py @@ -103,7 +103,11 @@ def _raw_negate_i16_op() -> RawOp: return _raw_op(source, "no_numba_negate_i16") -def test_import_numba_raises(): +def test_import_numba_raises(raise_on_numba_import): + # Request the guard explicitly rather than relying on the no_numba + # auto-injection (skipped under the parallel sweep, see conftest + # pytest_collection_modifyitems): this test exercises the guard directly, and + # the monkeypatch it depends on keeps it single-threaded under the sweep. with pytest.raises( ImportError, match="This test is marked 'no_numba' but attempted to import it" ): @@ -205,9 +209,8 @@ def test_binary_search_explicit_opkind_less(search, side): np.testing.assert_array_equal(d_out.copy_to_host(), expected) -def test_segmented_reduce_well_known_plus(monkeypatch): - monkeypatch.setattr(cuda.compute._cccl_interop, "_check_sass", False) - +@pytest.mark.no_verify_sass +def test_segmented_reduce_well_known_plus(): h_input = np.asarray([1, 2, 3, 4, 5, 6, 7, 8], dtype=np.uint32) h_starts = np.asarray([0, 3, 5], dtype=np.int32) h_ends = np.asarray([3, 5, 8], dtype=np.int32) @@ -288,11 +291,8 @@ def test_segmented_sort_keys(): np.testing.assert_array_equal(d_output.copy_to_host(), expected) -def test_unique_by_key_well_known_equal_to(monkeypatch): - cc_major, _ = cuda.compute._cccl_interop.CudaDevice().compute_capability - if cc_major >= 9: - monkeypatch.setattr(cuda.compute._cccl_interop, "_check_sass", False) - +@pytest.mark.no_verify_sass +def test_unique_by_key_well_known_equal_to(): h_keys = np.asarray([1, 1, 2, 2, 2, 3, 4, 4], dtype=np.int16) h_values = np.asarray([10, 11, 20, 21, 22, 30, 40, 41], dtype=np.int8) d_keys = DeviceArray.from_numpy(h_keys) From ec4557e8d3975c1226b513bf963191bf4f1bfa75 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Wed, 15 Jul 2026 19:19:42 -0500 Subject: [PATCH 2/8] address review --- ci/test_cuda_compute_minimal_python.sh | 6 +++++- python/cuda_cccl/tests/compute/conftest.py | 10 +++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ci/test_cuda_compute_minimal_python.sh b/ci/test_cuda_compute_minimal_python.sh index 3eef6324ed8..df3d8f8a5b6 100755 --- a/ci/test_cuda_compute_minimal_python.sh +++ b/ci/test_cuda_compute_minimal_python.sh @@ -30,7 +30,7 @@ fi # full cu* extras because those pull in numba/numba-cuda. CUDA_CCCL_WHEEL_PATH="$(ls "${wheelhouse_dir}"/cuda_cccl-*.whl)" python -m pip install "${CUDA_CCCL_WHEEL_PATH}[minimal-cu${cuda_major_version}]" -python -m pip install pytest pytest-xdist pytest-run-parallel +python -m pip install pytest pytest-xdist cd "${repo_root}/python/cuda_cccl/tests/" python -m pytest -n 6 -v compute/test_no_numba.py @@ -57,6 +57,10 @@ if [[ "${py_version}" == "3.14t" ]]; then # kernels and stays reproducible across runners, unlike =auto (the runner's # logical-core count). # + # pytest-run-parallel is only used by this sweep, so install it on the 3.14t + # path rather than for every minimal (e.g. non-free-threaded 3.14) run. + python -m pip install pytest-run-parallel + # Fail fast if the interpreter is not actually GIL-free (wrong build / # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled from # the start -- it would run threads GIL-serialized and pass vacuously. (A GIL diff --git a/python/cuda_cccl/tests/compute/conftest.py b/python/cuda_cccl/tests/compute/conftest.py index d195c46d6cc..40b5a8d581c 100644 --- a/python/cuda_cccl/tests/compute/conftest.py +++ b/python/cuda_cccl/tests/compute/conftest.py @@ -137,9 +137,13 @@ def pytest_collection_modifyitems(config, items): # would serialize every no_numba test, neutering the sweep. # # getoption returns the int 1 from the argparse default but a *string* for - # CLI-passed values ("2", "auto", "1"), so normalize to str before comparing - # -- a bare `> 1` would raise TypeError on the "2" string. - running_parallel = str(config.getoption("parallel_threads", 1)) != "1" + # CLI-passed values ("2", "auto"). Only bypass for an explicit numeric thread + # count > 1 (what CI passes): "auto" is left to inject the guard so a run the + # plugin executes single-threaded (e.g. "auto" resolving to 1 logical CPU) is + # not mistaken for a parallel run. (A bare `> 1` would also TypeError on the + # "2" string.) + parallel_threads = str(config.getoption("parallel_threads", 1)) + running_parallel = parallel_threads.isdigit() and int(parallel_threads) > 1 for item in items: if item.get_closest_marker("no_numba") and not running_parallel: if "raise_on_numba_import" not in item.fixturenames: From d86673144a14fc8edaaf82b981a28a1de4cc6cc8 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Wed, 15 Jul 2026 19:43:53 -0500 Subject: [PATCH 3/8] Add minimal testing to windows --- ci/matrix.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 259bba3c3cd..453e1841b50 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -93,9 +93,8 @@ workflows: - {jobs: ['test'], project: 'python', ctk: ['12.X', '13.X'], py_version: ['3.10'], gpu: 'l4', cxx: ['gcc13', 'msvc2022']} - {jobs: ['test'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14', gpu: 'l4', cxx: ['gcc13', 'msvc2022']} - {jobs: ['test'], project: 'python', py_version: '3.14', gpu: 'h100', cxx: 'gcc13'} - # Free-threaded Python currently supports only the minimal cuda.compute extra on Linux. - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: '13.X', py_version: '3.14', gpu: 'l4', cxx: 'gcc13'} - - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'} + - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: ['gcc13', 'msvc2022']} # CCCL packaging: - {jobs: ['test'], project: 'packaging', ctk: '12.0', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080', args: '-min-cmake'} - {jobs: ['test'], project: 'packaging', ctk: '12.X', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080'} From 9c64ab6c870c83a8bcfd292aac8bcd705341d246 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Thu, 16 Jul 2026 11:08:45 -0500 Subject: [PATCH 4/8] Add missing windows script --- .../test_cuda_compute_minimal_python.ps1 | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 ci/windows/test_cuda_compute_minimal_python.ps1 diff --git a/ci/windows/test_cuda_compute_minimal_python.ps1 b/ci/windows/test_cuda_compute_minimal_python.ps1 new file mode 100644 index 00000000000..779fb85cb2c --- /dev/null +++ b/ci/windows/test_cuda_compute_minimal_python.ps1 @@ -0,0 +1,69 @@ +Param( + [Parameter(Mandatory = $true)] + [Alias("py-version")] + [ValidatePattern("^\d+\.\d+t?$")] + [string]$PyVersion +) + +$ErrorActionPreference = "Stop" + +# Import shared helpers +Import-Module "$PSScriptRoot/build_common.psm1" +Import-Module "$PSScriptRoot/build_common_python.psm1" + +$python = Get-Python -Version $PyVersion +$cudaMajor = Get-CudaMajor + +$repoRoot = Get-RepoRoot + +$wheelPath = Get-CudaCcclWheel + +# Install cuda_cccl with the minimal CUDA extra. This intentionally avoids the +# full cu* extras because those pull in numba/numba-cuda. +& $python -m pip install -U pip pytest pytest-xdist +& $python -m pip install "$wheelPath[minimal-cu$cudaMajor]" + +Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") +try { + & $python -m pytest -n 6 -v compute/test_no_numba.py + + if ($PyVersion -eq "3.14t") { + # Select only tests that support the minimal extra so pytest does not + # collect tests that import numba-cuda and re-enable the GIL. These tests + # provide their own worker threads, so keep pytest itself in a single + # process. The serialization node-ids are module-skipped on the v2 + # backend today and will start running there automatically once v2 gains + # serialization support. + & $python -m pytest -n 0 -v ` + compute/test_free_threading_stress.py ` + compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable ` + compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load + + # Broad thread-safety sweep (pytest-run-parallel): re-run the numba-free + # functional suite with each test executed concurrently across threads + # (barrier-synchronized start), stressing the process-wide build cache, + # single-flight coordination, and the Cython bindings from many threads at + # once. Complements test_free_threading_stress.py above, which targets + # specific shared-object scenarios by hand. -n 0 so the threads share one + # interpreter. + # + # --parallel-threads=2 matches CuPy's free-threading CI (the closest GPU + # precedent); a small fixed count bounds GPU-memory pressure from + # concurrent kernels and stays reproducible across runners, unlike =auto + # (the runner's logical-core count). + # + # pytest-run-parallel is only used by this sweep, so install it on the + # 3.14t path rather than for every minimal (e.g. non-free-threaded 3.14) + # run. + & $python -m pip install pytest-run-parallel + + # Fail fast if the interpreter is not actually GIL-free (wrong build / + # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled + # from the start -- it would run threads GIL-serialized and pass + # vacuously. (A GIL *re-enabled mid-run* by a non-free-threaded import IS + # caught by the plugin, which is why we do not pass --ignore-gil-enabled.) + & $python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'" + & $python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py + } +} +finally { Pop-Location } From fe9526bd25a12e36b0497de28399738cedaf9034 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Tue, 21 Jul 2026 08:39:06 -0500 Subject: [PATCH 5/8] Address comments and update docs --- ci/matrix.yaml | 2 +- .../test_cuda_compute_minimal_python.ps1 | 26 +++++++++++++++++++ docs/python/compute/developer_overview.rst | 14 +++++----- docs/python/compute/index.rst | 2 +- docs/python/setup.rst | 7 +++-- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/ci/matrix.yaml b/ci/matrix.yaml index 453e1841b50..a838b482a76 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -235,7 +235,7 @@ workflows: - {jobs: ['test'], project: 'python', ctk: ['12.X', '13.X'], py_version: '3.14', gpu: 'h100', cxx: 'gcc13'} # Python free-threaded (3.14t) minimal lanes -- mirrors the pull_request rows # so FT regressions (e.g. from dependency bumps) surface between PRs. - - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X', '13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'} + - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X', '13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: ['gcc13', 'msvc2022']} - {jobs: ['test_py_compute_minimal'], project: 'python_v2', ctk: '13.X', py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'} # CCCL packaging: - {jobs: ['test'], project: 'packaging', ctk: '12.0', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080', args: '-min-cmake'} diff --git a/ci/windows/test_cuda_compute_minimal_python.ps1 b/ci/windows/test_cuda_compute_minimal_python.ps1 index 779fb85cb2c..bb5e9999aa5 100644 --- a/ci/windows/test_cuda_compute_minimal_python.ps1 +++ b/ci/windows/test_cuda_compute_minimal_python.ps1 @@ -18,14 +18,28 @@ $repoRoot = Get-RepoRoot $wheelPath = Get-CudaCcclWheel +# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; +# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit +# must be checked explicitly or a failed pip/pytest is masked by a later +# successful command and the job passes green. + # Install cuda_cccl with the minimal CUDA extra. This intentionally avoids the # full cu* extras because those pull in numba/numba-cuda. & $python -m pip install -U pip pytest pytest-xdist +if ($LASTEXITCODE -ne 0) { + throw "Failed to install pytest / pytest-xdist" +} & $python -m pip install "$wheelPath[minimal-cu$cudaMajor]" +if ($LASTEXITCODE -ne 0) { + throw "Failed to install cuda_cccl minimal extra" +} Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { & $python -m pytest -n 6 -v compute/test_no_numba.py + if ($LASTEXITCODE -ne 0) { + throw "test_no_numba.py failed" + } if ($PyVersion -eq "3.14t") { # Select only tests that support the minimal extra so pytest does not @@ -38,6 +52,9 @@ try { compute/test_free_threading_stress.py ` compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable ` compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load + if ($LASTEXITCODE -ne 0) { + throw "free-threading stress / serialization tests failed" + } # Broad thread-safety sweep (pytest-run-parallel): re-run the numba-free # functional suite with each test executed concurrently across threads @@ -56,6 +73,9 @@ try { # 3.14t path rather than for every minimal (e.g. non-free-threaded 3.14) # run. & $python -m pip install pytest-run-parallel + if ($LASTEXITCODE -ne 0) { + throw "Failed to install pytest-run-parallel" + } # Fail fast if the interpreter is not actually GIL-free (wrong build / # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled @@ -63,7 +83,13 @@ try { # vacuously. (A GIL *re-enabled mid-run* by a non-free-threaded import IS # caught by the plugin, which is why we do not pass --ignore-gil-enabled.) & $python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'" + if ($LASTEXITCODE -ne 0) { + throw "interpreter is not GIL-free; parallel sweep has no signal" + } & $python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py + if ($LASTEXITCODE -ne 0) { + throw "parallel-threads sweep failed" + } } } finally { Pop-Location } diff --git a/docs/python/compute/developer_overview.rst b/docs/python/compute/developer_overview.rst index 732335fa4a6..622ba92c4ae 100644 --- a/docs/python/compute/developer_overview.rst +++ b/docs/python/compute/developer_overview.rst @@ -546,13 +546,13 @@ The free-threading design is constrained by the following requirements: * Same-key concurrent cold builds should build once; waiters should receive the same result or observe the same exception. -The current free-threading support boundary is Linux with the -``minimal-cu12`` and ``minimal-cu13`` extras. These extras omit Numba and Numba -CUDA. Consequently, free-threaded support currently covers built-in ``OpKind`` -operations and externally compiled ``RawOp`` operations, but not Python-callable -operators or ``cuda.coop._experimental``. The full ``cu12`` and ``cu13`` extras -remain outside the support claim until the Numba CUDA dependency is replaced by -a free-threading-compatible implementation. +The current free-threading support boundary is the ``minimal-cu12`` and +``minimal-cu13`` extras. These extras omit Numba and Numba CUDA. Consequently, +free-threaded support currently covers built-in ``OpKind`` operations and +externally compiled ``RawOp`` operations, but not Python-callable operators or +``cuda.coop._experimental``. The full ``cu12`` and ``cu13`` extras remain +outside the support claim until the Numba CUDA dependency is replaced by a +free-threading-compatible implementation. CI runs ``test_free_threading_stress.py`` directly from the minimal test job. The v1 backend is covered across the supported CUDA 12 and 13 lanes, and a diff --git a/docs/python/compute/index.rst b/docs/python/compute/index.rst index c1458d0852b..ec8ae71105f 100644 --- a/docs/python/compute/index.rst +++ b/docs/python/compute/index.rst @@ -302,7 +302,7 @@ Free-threaded Python .. important:: - Free-threaded Python support is currently validated on Linux with the + Free-threaded Python support is currently validated with the ``minimal-cu12`` and ``minimal-cu13`` extras, which do not install Numba or Numba CUDA: diff --git a/docs/python/setup.rst b/docs/python/setup.rst index b64fec0dbde..dc8358b6187 100644 --- a/docs/python/setup.rst +++ b/docs/python/setup.rst @@ -50,10 +50,9 @@ For a minimal install without Numba (useful when you supply your own pip install cuda-cccl[minimal-cu13] # pip-installed CUDA toolkit pip install cuda-cccl[minimal-sysctk13] # system CUDA toolkit -Free-threaded Python support is currently validated on Linux with the -``minimal-cu12`` and ``minimal-cu13`` extras. The full ``cu12`` and ``cu13`` -extras depend on Numba CUDA and are not currently supported in free-threaded -Python. +Free-threaded Python support is currently validated with the ``minimal-cu12`` +and ``minimal-cu13`` extras. The full ``cu12`` and ``cu13`` extras depend on +Numba CUDA and are not currently supported in free-threaded Python. Install from conda-forge ~~~~~~~~~~~~~~~~~~~~~~~~~ From b9ce5eec015018058a744b3a8530da1110a0bdd2 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Fri, 24 Jul 2026 15:19:26 -0500 Subject: [PATCH 6/8] Pin to container --- ci/windows/test_cuda_compute_minimal_python.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/windows/test_cuda_compute_minimal_python.ps1 b/ci/windows/test_cuda_compute_minimal_python.ps1 index bb5e9999aa5..61609b03474 100644 --- a/ci/windows/test_cuda_compute_minimal_python.ps1 +++ b/ci/windows/test_cuda_compute_minimal_python.ps1 @@ -14,6 +14,10 @@ Import-Module "$PSScriptRoot/build_common_python.psm1" $python = Get-Python -Version $PyVersion $cudaMajor = Get-CudaMajor +# Pin cuda-toolkit to the container's CTK minor (CCCL_PYTHON_TEST_LATEST_CTK=1 +# opts out). See build_common_python.psm1. +Set-CtkPin + $repoRoot = Get-RepoRoot $wheelPath = Get-CudaCcclWheel From f86d95b5a58a7bcc70cd163b519c0f4f765893b5 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Fri, 24 Jul 2026 15:39:44 -0500 Subject: [PATCH 7/8] Move error checking code into common function --- ci/windows/build_common.psm1 | 22 ++++++++- ci/windows/build_cuda_cccl_python.ps1 | 25 ++-------- ci/windows/test_cuda_cccl_examples_python.ps1 | 24 ++-------- ci/windows/test_cuda_cccl_headers_python.ps1 | 19 ++------ .../test_cuda_compute_minimal_python.ps1 | 48 +++++-------------- ci/windows/test_cuda_compute_python.ps1 | 24 ++-------- 6 files changed, 49 insertions(+), 113 deletions(-) diff --git a/ci/windows/build_common.psm1 b/ci/windows/build_common.psm1 index bb7288b4247..7d6b9554403 100644 --- a/ci/windows/build_common.psm1 +++ b/ci/windows/build_common.psm1 @@ -209,5 +209,25 @@ function configure_and_build_preset { build_preset $BUILD_NAME $PRESET } -Export-ModuleMember -Function configure_preset, build_preset, test_preset, configure_and_build_preset +function Invoke-Checked { + <# + .SYNOPSIS + Runs a script block and throws if the last native command in it exits + non-zero. $ErrorActionPreference = "Stop" does not make native commands + (python/pip/pytest/...) throw, so their $LASTEXITCODE must be checked + explicitly; this wraps that boilerplate into one call. + .EXAMPLE + Invoke-Checked { & $python -m pip install pytest } "pip install failed" + #> + param( + [Parameter(Mandatory, Position = 0)][scriptblock]$ScriptBlock, + [Parameter(Position = 1)][string]$ErrorMessage = "Native command failed" + ) + & $ScriptBlock + if ($LASTEXITCODE -ne 0) { + throw "$ErrorMessage (exit code $LASTEXITCODE)" + } +} + +Export-ModuleMember -Function configure_preset, build_preset, test_preset, configure_and_build_preset, Invoke-Checked Export-ModuleMember -Variable BUILD_DIR, CL_VERSION diff --git a/ci/windows/build_cuda_cccl_python.ps1 b/ci/windows/build_cuda_cccl_python.ps1 index 65ea5b88f7f..274c29c306f 100644 --- a/ci/windows/build_cuda_cccl_python.ps1 +++ b/ci/windows/build_cuda_cccl_python.ps1 @@ -224,10 +224,7 @@ function Invoke-Cuda13NestedBuild { ) Write-Host ("About to invoke: docker " + ($dockerArgs -join ' ')) - & docker @dockerArgs - if ($LASTEXITCODE -ne 0) { - throw 'Nested CUDA 13 wheel build failed' - } + Invoke-Checked { & docker @dockerArgs } 'Nested CUDA 13 wheel build failed' } function Build-CudaCcclWheel { @@ -285,10 +282,7 @@ function Build-CudaCcclWheel { ) + $pipConfigArgs Write-Host ("python " + ($pythonArgs -join ' ')) - & $PythonExe @pythonArgs - if ($LASTEXITCODE -ne 0) { - throw "Wheel build failed for CUDA $Major" - } + Invoke-Checked { & $PythonExe @pythonArgs } "Wheel build failed for CUDA $Major" # Normalise the wheel filename (append .cu12/.cu13) and prune duplicates. $builtWheel = Get-OnePathMatch -Path $outDir ` @@ -363,19 +357,13 @@ if ($DoMerge) { Write-Host "Found CUDA 13 wheel: $Cu13Wheel" Write-Host 'Merging CUDA wheels...' - & $PythonExe -m pip install wheel | Write-Host - if ($LASTEXITCODE -ne 0) { - throw 'Failed to install wheel for merging' - } + Invoke-Checked { & $PythonExe -m pip install wheel | Write-Host } 'Failed to install wheel for merging' $WheelhouseMerged = Join-Path $RepoRoot 'wheelhouse_merged' ${null} = New-Item -ItemType Directory -Path $WheelhouseMerged -Force $mergePy = Join-Path $RepoRoot 'python/cuda_cccl/merge_cuda_wheels.py' - & $PythonExe $mergePy $Cu12Wheel $Cu13Wheel --output-dir $WheelhouseMerged - if ($LASTEXITCODE -ne 0) { - throw 'Merging wheels failed' - } + Invoke-Checked { & $PythonExe $mergePy $Cu12Wheel $Cu13Wheel --output-dir $WheelhouseMerged } 'Merging wheels failed' # Clean up the per‑major directories and move the merged wheel into the # final location. @@ -426,10 +414,7 @@ if ($env:GITHUB_ACTIONS -and -not $SkipUpload) { Write-Host "Wheel artifact name: $wheelArtifactName" $uploadCmd = "ci/util/artifacts/upload.sh $wheelArtifactName 'wheelhouse/.*'" - & bash -lc $uploadCmd - if ($LASTEXITCODE -ne 0) { - throw 'Wheel artifact upload failed' - } + Invoke-Checked { & bash -lc $uploadCmd } 'Wheel artifact upload failed' } finally { Pop-Location diff --git a/ci/windows/test_cuda_cccl_examples_python.ps1 b/ci/windows/test_cuda_cccl_examples_python.ps1 index ba26bd31e80..0ebabb59dbd 100644 --- a/ci/windows/test_cuda_cccl_examples_python.ps1 +++ b/ci/windows/test_cuda_cccl_examples_python.ps1 @@ -22,27 +22,14 @@ $repoRoot = Get-RepoRoot ${wheelPath} = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. # pytest-benchmark is for the host-benchmark smoke test below. -& $python -m pip install -U pip pytest pytest-xdist pytest-benchmark -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist / pytest-benchmark" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist pytest-benchmark } "Failed to install pytest / pytest-xdist / pytest-benchmark" # CuPy is required by the cuda.compute examples and is not part of the test extras -& $python -m pip install "${wheelPath}[test-cu$cudaMajor]" "cupy-cuda${cudaMajor}x" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl test extra / cupy" -} +Invoke-Checked { & $python -m pip install "${wheelPath}[test-cu$cudaMajor]" "cupy-cuda${cudaMajor}x" } "Failed to install cuda_cccl test extra / cupy" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n 6 test_examples.py - if ($LASTEXITCODE -ne 0) { - throw "examples tests failed" - } + Invoke-Checked { & $python -m pytest -n 6 test_examples.py } "examples tests failed" } finally { Pop-Location } @@ -54,9 +41,6 @@ finally { Pop-Location } # suite also needs, so only pytest-benchmark is added above. Push-Location (Join-Path $repoRoot "python/cuda_cccl/benchmarks/compute/host") try { - & $python -m pytest -v --benchmark-disable . - if ($LASTEXITCODE -ne 0) { - throw "host benchmark smoke test failed" - } + Invoke-Checked { & $python -m pytest -v --benchmark-disable . } "host benchmark smoke test failed" } finally { Pop-Location } diff --git a/ci/windows/test_cuda_cccl_headers_python.ps1 b/ci/windows/test_cuda_cccl_headers_python.ps1 index aa16dc894f7..18f41d943a1 100644 --- a/ci/windows/test_cuda_cccl_headers_python.ps1 +++ b/ci/windows/test_cuda_cccl_headers_python.ps1 @@ -22,24 +22,11 @@ $repoRoot = Get-RepoRoot ${wheelPath} = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. -& $python -m pip install -U pip pytest pytest-xdist -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist" -} -& $python -m pip install "${wheelPath}[test-cu$cudaMajor]" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl test extra" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist } "Failed to install pytest / pytest-xdist" +Invoke-Checked { & $python -m pip install "${wheelPath}[test-cu$cudaMajor]" } "Failed to install cuda_cccl test extra" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n auto -v headers/ - if ($LASTEXITCODE -ne 0) { - throw "headers tests failed" - } + Invoke-Checked { & $python -m pytest -n auto -v headers/ } "headers tests failed" } finally { Pop-Location } diff --git a/ci/windows/test_cuda_compute_minimal_python.ps1 b/ci/windows/test_cuda_compute_minimal_python.ps1 index 61609b03474..6001f53e53d 100644 --- a/ci/windows/test_cuda_compute_minimal_python.ps1 +++ b/ci/windows/test_cuda_compute_minimal_python.ps1 @@ -22,28 +22,14 @@ $repoRoot = Get-RepoRoot $wheelPath = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. - # Install cuda_cccl with the minimal CUDA extra. This intentionally avoids the # full cu* extras because those pull in numba/numba-cuda. -& $python -m pip install -U pip pytest pytest-xdist -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist" -} -& $python -m pip install "$wheelPath[minimal-cu$cudaMajor]" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl minimal extra" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist } "Failed to install pytest / pytest-xdist" +Invoke-Checked { & $python -m pip install "$wheelPath[minimal-cu$cudaMajor]" } "Failed to install cuda_cccl minimal extra" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n 6 -v compute/test_no_numba.py - if ($LASTEXITCODE -ne 0) { - throw "test_no_numba.py failed" - } + Invoke-Checked { & $python -m pytest -n 6 -v compute/test_no_numba.py } "test_no_numba.py failed" if ($PyVersion -eq "3.14t") { # Select only tests that support the minimal extra so pytest does not @@ -52,13 +38,12 @@ try { # process. The serialization node-ids are module-skipped on the v2 # backend today and will start running there automatically once v2 gains # serialization support. - & $python -m pytest -n 0 -v ` - compute/test_free_threading_stress.py ` - compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable ` - compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load - if ($LASTEXITCODE -ne 0) { - throw "free-threading stress / serialization tests failed" - } + Invoke-Checked { + & $python -m pytest -n 0 -v ` + compute/test_free_threading_stress.py ` + compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable ` + compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load + } "free-threading stress / serialization tests failed" # Broad thread-safety sweep (pytest-run-parallel): re-run the numba-free # functional suite with each test executed concurrently across threads @@ -76,24 +61,15 @@ try { # pytest-run-parallel is only used by this sweep, so install it on the # 3.14t path rather than for every minimal (e.g. non-free-threaded 3.14) # run. - & $python -m pip install pytest-run-parallel - if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest-run-parallel" - } + Invoke-Checked { & $python -m pip install pytest-run-parallel } "Failed to install pytest-run-parallel" # Fail fast if the interpreter is not actually GIL-free (wrong build / # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled # from the start -- it would run threads GIL-serialized and pass # vacuously. (A GIL *re-enabled mid-run* by a non-free-threaded import IS # caught by the plugin, which is why we do not pass --ignore-gil-enabled.) - & $python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'" - if ($LASTEXITCODE -ne 0) { - throw "interpreter is not GIL-free; parallel sweep has no signal" - } - & $python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py - if ($LASTEXITCODE -ne 0) { - throw "parallel-threads sweep failed" - } + Invoke-Checked { & $python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'" } "interpreter is not GIL-free; parallel sweep has no signal" + Invoke-Checked { & $python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py } "parallel-threads sweep failed" } } finally { Pop-Location } diff --git a/ci/windows/test_cuda_compute_python.ps1 b/ci/windows/test_cuda_compute_python.ps1 index b4dd1744e76..21a002b8b23 100644 --- a/ci/windows/test_cuda_compute_python.ps1 +++ b/ci/windows/test_cuda_compute_python.ps1 @@ -22,28 +22,12 @@ $repoRoot = Get-RepoRoot $wheelPath = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. -& $python -m pip install -U pip pytest pytest-xdist -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist" -} -& $python -m pip install "$wheelPath[test-cu$cudaMajor]" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl test extra" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist } "Failed to install pytest / pytest-xdist" +Invoke-Checked { & $python -m pip install "$wheelPath[test-cu$cudaMajor]" } "Failed to install cuda_cccl test extra" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n 6 -v compute/ -m "not large and not free_threading" - if ($LASTEXITCODE -ne 0) { - throw "compute tests (not large) failed" - } - & $python -m pytest -n 0 -v compute/ -m "large and not free_threading" - if ($LASTEXITCODE -ne 0) { - throw "compute tests (large) failed" - } + Invoke-Checked { & $python -m pytest -n 6 -v compute/ -m "not large and not free_threading" } "compute tests (not large) failed" + Invoke-Checked { & $python -m pytest -n 0 -v compute/ -m "large and not free_threading" } "compute tests (large) failed" } finally { Pop-Location } From bf971dce20a8d2438e4684985f8079a93d9604c8 Mon Sep 17 00:00:00 2001 From: Nader Al Awar Date: Fri, 24 Jul 2026 15:53:51 -0500 Subject: [PATCH 8/8] address review --- ci/windows/build_cuda_cccl_python.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/windows/build_cuda_cccl_python.ps1 b/ci/windows/build_cuda_cccl_python.ps1 index 274c29c306f..e151c673f4b 100644 --- a/ci/windows/build_cuda_cccl_python.ps1 +++ b/ci/windows/build_cuda_cccl_python.ps1 @@ -30,7 +30,7 @@ .PARAMETER Cuda13Image Optional. The Docker image name used for a nested build of the CUDA 13 wheel when the outer container defaults to CUDA 12.9. The default value - matches the RAPIDS dev‑container image that contains the required + matches the RAPIDS dev-container image that contains the required toolchain: `rapidsai/devcontainers:26.06-cuda13.0-cl14.44-windows2022`. .PARAMETER SkipUpload @@ -194,12 +194,12 @@ function Invoke-Cuda13NestedBuild { } Write-Host "DooD appears to be working, continuing..." - # Detect outer‑container resources so we can set sensible limits. + # Detect outer-container resources so we can set sensible limits. $os = Get-WmiObject -Class Win32_OperatingSystem $totalGB = [math]::Floor($os.TotalVisibleMemorySize / 1MB) # KB -> GB $procCount = [Environment]::ProcessorCount - # Leave a little head‑room so the outer container doesn't starve + # Leave a little head-room so the outer container doesn't starve $memLimitGB = [math]::Max(2, [int]([math]::Floor($totalGB * 0.9))) $cpuCount = [math]::Max(2, $procCount) @@ -257,7 +257,7 @@ function Build-CudaCcclWheel { throw "nvcc not found at $NvccForMajor" } - # Convert Windows paths to Unix‑style for CMake + # Convert Windows paths to Unix-style for CMake $NvccUnix = Convert-ToUnixPath $NvccForMajor $CudaUnix = Convert-ToUnixPath $CudaPathForMajor @@ -338,7 +338,7 @@ finally { } -# Merge the two major‑version wheels (if both were built). This will fail if +# Merge the two major-version wheels (if both were built). This will fail if # either wheel can't be found. This only runs on the outer (non-nested) # container image. if ($DoMerge) { @@ -365,7 +365,7 @@ if ($DoMerge) { $mergePy = Join-Path $RepoRoot 'python/cuda_cccl/merge_cuda_wheels.py' Invoke-Checked { & $PythonExe $mergePy $Cu12Wheel $Cu13Wheel --output-dir $WheelhouseMerged } 'Merging wheels failed' - # Clean up the per‑major directories and move the merged wheel into the + # Clean up the per-major directories and move the merged wheel into the # final location. Get-ChildItem $Wheelhouse -Filter '*.whl' | ForEach-Object {