diff --git a/ci/matrix.yaml b/ci/matrix.yaml index dcaf806eb68..8d7ecd07876 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -93,9 +93,8 @@ workflows: - {jobs: ['test'], project: 'python', ctk: ['12.X', '13.X'], py_version: ['3.10'], gpu: 'l4', cxx: ['gcc13', 'msvc2022']} - {jobs: ['test'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14', gpu: 'l4', cxx: ['gcc13', 'msvc2022']} - {jobs: ['test'], project: 'python', py_version: '3.14', gpu: 'h100', cxx: 'gcc13'} - # Free-threaded Python currently supports only the minimal cuda.compute extra on Linux. - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: '13.X', py_version: '3.14', gpu: 'l4', cxx: 'gcc13'} - - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'} + - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X','13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: ['gcc13', 'msvc2022']} # Deliberately unpinned: CCCL_PYTHON_TEST_LATEST_CTK skips the CTK pin so pip # resolves the latest minor -- catching breakage a plain `pip install # cuda-cccl[cu12]/[cu13]` would hit before the container CTK bumps. @@ -240,7 +239,7 @@ workflows: - {jobs: ['test'], project: 'python', ctk: ['12.X', '13.X'], py_version: '3.14', gpu: 'h100', cxx: 'gcc13'} # Python free-threaded (3.14t) minimal lanes -- mirrors the pull_request rows # so FT regressions (e.g. from dependency bumps) surface between PRs. - - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X', '13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'} + - {jobs: ['test_py_compute_minimal'], project: 'python', ctk: ['12.X', '13.0', '13.X'], py_version: '3.14t', gpu: 'l4', cxx: ['gcc13', 'msvc2022']} - {jobs: ['test_py_compute_minimal'], project: 'python_v2', ctk: '13.X', py_version: '3.14t', gpu: 'l4', cxx: 'gcc13'} # CCCL packaging: - {jobs: ['test'], project: 'packaging', ctk: '12.0', cxx: ['gcc10', 'clang14'], gpu: 'rtx2080', args: '-min-cmake'} diff --git a/ci/windows/build_common.psm1 b/ci/windows/build_common.psm1 index bb7288b4247..7d6b9554403 100644 --- a/ci/windows/build_common.psm1 +++ b/ci/windows/build_common.psm1 @@ -209,5 +209,25 @@ function configure_and_build_preset { build_preset $BUILD_NAME $PRESET } -Export-ModuleMember -Function configure_preset, build_preset, test_preset, configure_and_build_preset +function Invoke-Checked { + <# + .SYNOPSIS + Runs a script block and throws if the last native command in it exits + non-zero. $ErrorActionPreference = "Stop" does not make native commands + (python/pip/pytest/...) throw, so their $LASTEXITCODE must be checked + explicitly; this wraps that boilerplate into one call. + .EXAMPLE + Invoke-Checked { & $python -m pip install pytest } "pip install failed" + #> + param( + [Parameter(Mandatory, Position = 0)][scriptblock]$ScriptBlock, + [Parameter(Position = 1)][string]$ErrorMessage = "Native command failed" + ) + & $ScriptBlock + if ($LASTEXITCODE -ne 0) { + throw "$ErrorMessage (exit code $LASTEXITCODE)" + } +} + +Export-ModuleMember -Function configure_preset, build_preset, test_preset, configure_and_build_preset, Invoke-Checked Export-ModuleMember -Variable BUILD_DIR, CL_VERSION diff --git a/ci/windows/build_cuda_cccl_python.ps1 b/ci/windows/build_cuda_cccl_python.ps1 index 65ea5b88f7f..e151c673f4b 100644 --- a/ci/windows/build_cuda_cccl_python.ps1 +++ b/ci/windows/build_cuda_cccl_python.ps1 @@ -30,7 +30,7 @@ .PARAMETER Cuda13Image Optional. The Docker image name used for a nested build of the CUDA 13 wheel when the outer container defaults to CUDA 12.9. The default value - matches the RAPIDS dev‑container image that contains the required + matches the RAPIDS dev-container image that contains the required toolchain: `rapidsai/devcontainers:26.06-cuda13.0-cl14.44-windows2022`. .PARAMETER SkipUpload @@ -194,12 +194,12 @@ function Invoke-Cuda13NestedBuild { } Write-Host "DooD appears to be working, continuing..." - # Detect outer‑container resources so we can set sensible limits. + # Detect outer-container resources so we can set sensible limits. $os = Get-WmiObject -Class Win32_OperatingSystem $totalGB = [math]::Floor($os.TotalVisibleMemorySize / 1MB) # KB -> GB $procCount = [Environment]::ProcessorCount - # Leave a little head‑room so the outer container doesn't starve + # Leave a little head-room so the outer container doesn't starve $memLimitGB = [math]::Max(2, [int]([math]::Floor($totalGB * 0.9))) $cpuCount = [math]::Max(2, $procCount) @@ -224,10 +224,7 @@ function Invoke-Cuda13NestedBuild { ) Write-Host ("About to invoke: docker " + ($dockerArgs -join ' ')) - & docker @dockerArgs - if ($LASTEXITCODE -ne 0) { - throw 'Nested CUDA 13 wheel build failed' - } + Invoke-Checked { & docker @dockerArgs } 'Nested CUDA 13 wheel build failed' } function Build-CudaCcclWheel { @@ -260,7 +257,7 @@ function Build-CudaCcclWheel { throw "nvcc not found at $NvccForMajor" } - # Convert Windows paths to Unix‑style for CMake + # Convert Windows paths to Unix-style for CMake $NvccUnix = Convert-ToUnixPath $NvccForMajor $CudaUnix = Convert-ToUnixPath $CudaPathForMajor @@ -285,10 +282,7 @@ function Build-CudaCcclWheel { ) + $pipConfigArgs Write-Host ("python " + ($pythonArgs -join ' ')) - & $PythonExe @pythonArgs - if ($LASTEXITCODE -ne 0) { - throw "Wheel build failed for CUDA $Major" - } + Invoke-Checked { & $PythonExe @pythonArgs } "Wheel build failed for CUDA $Major" # Normalise the wheel filename (append .cu12/.cu13) and prune duplicates. $builtWheel = Get-OnePathMatch -Path $outDir ` @@ -344,7 +338,7 @@ finally { } -# Merge the two major‑version wheels (if both were built). This will fail if +# Merge the two major-version wheels (if both were built). This will fail if # either wheel can't be found. This only runs on the outer (non-nested) # container image. if ($DoMerge) { @@ -363,21 +357,15 @@ if ($DoMerge) { Write-Host "Found CUDA 13 wheel: $Cu13Wheel" Write-Host 'Merging CUDA wheels...' - & $PythonExe -m pip install wheel | Write-Host - if ($LASTEXITCODE -ne 0) { - throw 'Failed to install wheel for merging' - } + Invoke-Checked { & $PythonExe -m pip install wheel | Write-Host } 'Failed to install wheel for merging' $WheelhouseMerged = Join-Path $RepoRoot 'wheelhouse_merged' ${null} = New-Item -ItemType Directory -Path $WheelhouseMerged -Force $mergePy = Join-Path $RepoRoot 'python/cuda_cccl/merge_cuda_wheels.py' - & $PythonExe $mergePy $Cu12Wheel $Cu13Wheel --output-dir $WheelhouseMerged - if ($LASTEXITCODE -ne 0) { - throw 'Merging wheels failed' - } + Invoke-Checked { & $PythonExe $mergePy $Cu12Wheel $Cu13Wheel --output-dir $WheelhouseMerged } 'Merging wheels failed' - # Clean up the per‑major directories and move the merged wheel into the + # Clean up the per-major directories and move the merged wheel into the # final location. Get-ChildItem $Wheelhouse -Filter '*.whl' | ForEach-Object { @@ -426,10 +414,7 @@ if ($env:GITHUB_ACTIONS -and -not $SkipUpload) { Write-Host "Wheel artifact name: $wheelArtifactName" $uploadCmd = "ci/util/artifacts/upload.sh $wheelArtifactName 'wheelhouse/.*'" - & bash -lc $uploadCmd - if ($LASTEXITCODE -ne 0) { - throw 'Wheel artifact upload failed' - } + Invoke-Checked { & bash -lc $uploadCmd } 'Wheel artifact upload failed' } finally { Pop-Location diff --git a/ci/windows/test_cuda_cccl_examples_python.ps1 b/ci/windows/test_cuda_cccl_examples_python.ps1 index ba26bd31e80..0ebabb59dbd 100644 --- a/ci/windows/test_cuda_cccl_examples_python.ps1 +++ b/ci/windows/test_cuda_cccl_examples_python.ps1 @@ -22,27 +22,14 @@ $repoRoot = Get-RepoRoot ${wheelPath} = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. # pytest-benchmark is for the host-benchmark smoke test below. -& $python -m pip install -U pip pytest pytest-xdist pytest-benchmark -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist / pytest-benchmark" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist pytest-benchmark } "Failed to install pytest / pytest-xdist / pytest-benchmark" # CuPy is required by the cuda.compute examples and is not part of the test extras -& $python -m pip install "${wheelPath}[test-cu$cudaMajor]" "cupy-cuda${cudaMajor}x" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl test extra / cupy" -} +Invoke-Checked { & $python -m pip install "${wheelPath}[test-cu$cudaMajor]" "cupy-cuda${cudaMajor}x" } "Failed to install cuda_cccl test extra / cupy" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n 6 test_examples.py - if ($LASTEXITCODE -ne 0) { - throw "examples tests failed" - } + Invoke-Checked { & $python -m pytest -n 6 test_examples.py } "examples tests failed" } finally { Pop-Location } @@ -54,9 +41,6 @@ finally { Pop-Location } # suite also needs, so only pytest-benchmark is added above. Push-Location (Join-Path $repoRoot "python/cuda_cccl/benchmarks/compute/host") try { - & $python -m pytest -v --benchmark-disable . - if ($LASTEXITCODE -ne 0) { - throw "host benchmark smoke test failed" - } + Invoke-Checked { & $python -m pytest -v --benchmark-disable . } "host benchmark smoke test failed" } finally { Pop-Location } diff --git a/ci/windows/test_cuda_cccl_headers_python.ps1 b/ci/windows/test_cuda_cccl_headers_python.ps1 index aa16dc894f7..18f41d943a1 100644 --- a/ci/windows/test_cuda_cccl_headers_python.ps1 +++ b/ci/windows/test_cuda_cccl_headers_python.ps1 @@ -22,24 +22,11 @@ $repoRoot = Get-RepoRoot ${wheelPath} = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. -& $python -m pip install -U pip pytest pytest-xdist -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist" -} -& $python -m pip install "${wheelPath}[test-cu$cudaMajor]" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl test extra" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist } "Failed to install pytest / pytest-xdist" +Invoke-Checked { & $python -m pip install "${wheelPath}[test-cu$cudaMajor]" } "Failed to install cuda_cccl test extra" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n auto -v headers/ - if ($LASTEXITCODE -ne 0) { - throw "headers tests failed" - } + Invoke-Checked { & $python -m pytest -n auto -v headers/ } "headers tests failed" } finally { Pop-Location } diff --git a/ci/windows/test_cuda_compute_minimal_python.ps1 b/ci/windows/test_cuda_compute_minimal_python.ps1 new file mode 100644 index 00000000000..6001f53e53d --- /dev/null +++ b/ci/windows/test_cuda_compute_minimal_python.ps1 @@ -0,0 +1,75 @@ +Param( + [Parameter(Mandatory = $true)] + [Alias("py-version")] + [ValidatePattern("^\d+\.\d+t?$")] + [string]$PyVersion +) + +$ErrorActionPreference = "Stop" + +# Import shared helpers +Import-Module "$PSScriptRoot/build_common.psm1" +Import-Module "$PSScriptRoot/build_common_python.psm1" + +$python = Get-Python -Version $PyVersion +$cudaMajor = Get-CudaMajor + +# Pin cuda-toolkit to the container's CTK minor (CCCL_PYTHON_TEST_LATEST_CTK=1 +# opts out). See build_common_python.psm1. +Set-CtkPin + +$repoRoot = Get-RepoRoot + +$wheelPath = Get-CudaCcclWheel + +# Install cuda_cccl with the minimal CUDA extra. This intentionally avoids the +# full cu* extras because those pull in numba/numba-cuda. +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist } "Failed to install pytest / pytest-xdist" +Invoke-Checked { & $python -m pip install "$wheelPath[minimal-cu$cudaMajor]" } "Failed to install cuda_cccl minimal extra" + +Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") +try { + Invoke-Checked { & $python -m pytest -n 6 -v compute/test_no_numba.py } "test_no_numba.py failed" + + if ($PyVersion -eq "3.14t") { + # Select only tests that support the minimal extra so pytest does not + # collect tests that import numba-cuda and re-enable the GIL. These tests + # provide their own worker threads, so keep pytest itself in a single + # process. The serialization node-ids are module-skipped on the v2 + # backend today and will start running there automatically once v2 gains + # serialization support. + Invoke-Checked { + & $python -m pytest -n 0 -v ` + compute/test_free_threading_stress.py ` + compute/test_multi_cc_serialization.py::test_aot_build_result_load_failure_is_shared_and_retryable ` + compute/test_multi_cc_serialization.py::test_aot_serialization_waits_for_canonical_first_load + } "free-threading stress / serialization tests failed" + + # Broad thread-safety sweep (pytest-run-parallel): re-run the numba-free + # functional suite with each test executed concurrently across threads + # (barrier-synchronized start), stressing the process-wide build cache, + # single-flight coordination, and the Cython bindings from many threads at + # once. Complements test_free_threading_stress.py above, which targets + # specific shared-object scenarios by hand. -n 0 so the threads share one + # interpreter. + # + # --parallel-threads=2 matches CuPy's free-threading CI (the closest GPU + # precedent); a small fixed count bounds GPU-memory pressure from + # concurrent kernels and stays reproducible across runners, unlike =auto + # (the runner's logical-core count). + # + # pytest-run-parallel is only used by this sweep, so install it on the + # 3.14t path rather than for every minimal (e.g. non-free-threaded 3.14) + # run. + Invoke-Checked { & $python -m pip install pytest-run-parallel } "Failed to install pytest-run-parallel" + + # Fail fast if the interpreter is not actually GIL-free (wrong build / + # PYTHON_GIL=1): pytest-run-parallel does NOT catch a GIL that is enabled + # from the start -- it would run threads GIL-serialized and pass + # vacuously. (A GIL *re-enabled mid-run* by a non-free-threaded import IS + # caught by the plugin, which is why we do not pass --ignore-gil-enabled.) + Invoke-Checked { & $python -c "import sys; assert not sys._is_gil_enabled(), 'GIL is enabled; parallel sweep has no signal'" } "interpreter is not GIL-free; parallel sweep has no signal" + Invoke-Checked { & $python -m pytest -n 0 -v --parallel-threads=2 compute/test_no_numba.py } "parallel-threads sweep failed" + } +} +finally { Pop-Location } diff --git a/ci/windows/test_cuda_compute_python.ps1 b/ci/windows/test_cuda_compute_python.ps1 index b4dd1744e76..21a002b8b23 100644 --- a/ci/windows/test_cuda_compute_python.ps1 +++ b/ci/windows/test_cuda_compute_python.ps1 @@ -22,28 +22,12 @@ $repoRoot = Get-RepoRoot $wheelPath = Get-CudaCcclWheel -# Native commands (python.exe / pip / pytest) only set $LASTEXITCODE on failure; -# $ErrorActionPreference = "Stop" does not make them throw, so a non-zero exit -# must be checked explicitly or a failed pip/pytest is masked by a later -# successful command and the job passes green. -& $python -m pip install -U pip pytest pytest-xdist -if ($LASTEXITCODE -ne 0) { - throw "Failed to install pytest / pytest-xdist" -} -& $python -m pip install "$wheelPath[test-cu$cudaMajor]" -if ($LASTEXITCODE -ne 0) { - throw "Failed to install cuda_cccl test extra" -} +Invoke-Checked { & $python -m pip install -U pip pytest pytest-xdist } "Failed to install pytest / pytest-xdist" +Invoke-Checked { & $python -m pip install "$wheelPath[test-cu$cudaMajor]" } "Failed to install cuda_cccl test extra" Push-Location (Join-Path $repoRoot "python/cuda_cccl/tests") try { - & $python -m pytest -n 6 -v compute/ -m "not large and not free_threading" - if ($LASTEXITCODE -ne 0) { - throw "compute tests (not large) failed" - } - & $python -m pytest -n 0 -v compute/ -m "large and not free_threading" - if ($LASTEXITCODE -ne 0) { - throw "compute tests (large) failed" - } + Invoke-Checked { & $python -m pytest -n 6 -v compute/ -m "not large and not free_threading" } "compute tests (not large) failed" + Invoke-Checked { & $python -m pytest -n 0 -v compute/ -m "large and not free_threading" } "compute tests (large) failed" } finally { Pop-Location } diff --git a/docs/python/compute/developer_overview.rst b/docs/python/compute/developer_overview.rst index 835ac5e1689..94a3c12928a 100644 --- a/docs/python/compute/developer_overview.rst +++ b/docs/python/compute/developer_overview.rst @@ -546,13 +546,13 @@ The free-threading design is constrained by the following requirements: * Same-key concurrent cold builds should build once; waiters should receive the same result or observe the same exception. -The current free-threading support boundary is Linux with the -``minimal-cu12`` and ``minimal-cu13`` extras. These extras omit Numba and Numba -CUDA. Consequently, free-threaded support currently covers built-in ``OpKind`` -operations and externally compiled ``RawOp`` operations, but not Python-callable -operators. The full ``cu12`` and ``cu13`` extras remain outside the support claim -until the Numba CUDA dependency is replaced by a free-threading-compatible -implementation. +The current free-threading support boundary is the ``minimal-cu12`` and +``minimal-cu13`` extras. These extras omit Numba and Numba CUDA. Consequently, +free-threaded support currently covers built-in ``OpKind`` operations and +externally compiled ``RawOp`` operations, but not Python-callable operators. +The full ``cu12`` and ``cu13`` extras remain +outside the support claim until the Numba CUDA dependency is replaced by a +free-threading-compatible implementation. CI runs ``test_free_threading_stress.py`` directly from the minimal test job. The v1 backend is covered across the supported CUDA 12 and 13 lanes, and a diff --git a/docs/python/compute/index.rst b/docs/python/compute/index.rst index d40740331a6..24b272ab29b 100644 --- a/docs/python/compute/index.rst +++ b/docs/python/compute/index.rst @@ -302,7 +302,7 @@ Free-threaded Python .. important:: - Free-threaded Python support is currently validated on Linux with the + Free-threaded Python support is currently validated with the ``minimal-cu12`` and ``minimal-cu13`` extras, which do not install Numba or Numba CUDA: diff --git a/docs/python/setup.rst b/docs/python/setup.rst index c54502dcec0..f6183947272 100644 --- a/docs/python/setup.rst +++ b/docs/python/setup.rst @@ -50,10 +50,9 @@ For a minimal install without Numba (useful when you supply your own pip install cuda-cccl[minimal-cu13] # pip-installed CUDA toolkit pip install cuda-cccl[minimal-sysctk13] # system CUDA toolkit -Free-threaded Python support is currently validated on Linux with the -``minimal-cu12`` and ``minimal-cu13`` extras. The full ``cu12`` and ``cu13`` -extras depend on Numba CUDA and are not currently supported in free-threaded -Python. +Free-threaded Python support is currently validated with the ``minimal-cu12`` +and ``minimal-cu13`` extras. The full ``cu12`` and ``cu13`` extras depend on +Numba CUDA and are not currently supported in free-threaded Python. Install from conda-forge ~~~~~~~~~~~~~~~~~~~~~~~~~